From 79bee588d3433891a6600adfed7b3f5aab02b8aa Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Sun, 8 Nov 2026 21:58:17 +0530 Subject: [PATCH 1/3] feat(kin): owner stop switch (LEAD-11) Owner-only kin.org.pause/resume (plus kin.org.get) stop the running lead wake-up, workers and their delegated tasks, hold queued events, and on resume enqueue one resume event and expire events queued over 24 hours. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/kin/KinServer.ts | 9 +- apps/server/src/kin/chat/layer.ts | 5 +- apps/server/src/kin/chat/rpcHandlers.test.ts | 74 ++++ apps/server/src/kin/chat/rpcHandlers.ts | 35 +- apps/server/src/kin/lead/KinBudget.ts | 13 +- .../server/src/kin/lead/KinEventQueue.test.ts | 1 + apps/server/src/kin/lead/KinStop.test.ts | 193 +++++++++ apps/server/src/kin/lead/KinStop.ts | 365 ++++++++++++++++++ apps/server/src/kin/lead/WakeupRunner.test.ts | 4 + .../src/kin/lead/WakeupRunner.testkit.ts | 51 ++- apps/server/src/kin/lead/WakeupRunner.ts | 36 +- apps/server/src/kin/lead/Workers.test.ts | 9 +- apps/server/src/kin/lead/Workers.ts | 60 ++- apps/server/src/kin/lead/layer.ts | 6 + apps/server/src/kin/lead/orgState.ts | 31 ++ docs/kin/specs/lead.md | 46 ++- .../src/state/commandPermissions.test.ts | 18 + packages/contracts/src/kinLead.test.ts | 8 +- packages/contracts/src/kinLead.ts | 18 +- packages/contracts/src/kinRpcs.ts | 42 +- 20 files changed, 953 insertions(+), 71 deletions(-) create mode 100644 apps/server/src/kin/lead/KinStop.test.ts create mode 100644 apps/server/src/kin/lead/KinStop.ts create mode 100644 apps/server/src/kin/lead/orgState.ts diff --git a/apps/server/src/kin/KinServer.ts b/apps/server/src/kin/KinServer.ts index d61dcc1cc503..5a0146d804e8 100644 --- a/apps/server/src/kin/KinServer.ts +++ b/apps/server/src/kin/KinServer.ts @@ -7,8 +7,15 @@ import { KinChatService } from "./chat/KinChatService.ts"; import { KinChatHub } from "./chat/KinChatHub.ts"; import { KinBudget } from "./lead/KinBudget.ts"; import { KinEventQueue } from "./lead/KinEventQueue.ts"; +import { KinStop } from "./lead/KinStop.ts"; -export type Services = KinIdentity | KinChatService | KinBudget | KinEventQueue | KinChatHub; +export type Services = + | KinIdentity + | KinChatService + | KinBudget + | KinEventQueue + | KinChatHub + | KinStop; export class KinServer extends Context.Service< KinServer, diff --git a/apps/server/src/kin/chat/layer.ts b/apps/server/src/kin/chat/layer.ts index c0f0d522e979..99c40d7d8eda 100644 --- a/apps/server/src/kin/chat/layer.ts +++ b/apps/server/src/kin/chat/layer.ts @@ -3,6 +3,7 @@ import * as NoopJudge from "../floor/NoopJudge.ts"; import * as Layer from "effect/Layer"; import * as Budget from "../lead/KinBudget.ts"; import * as Queue from "../lead/KinEventQueue.ts"; +import * as Stop from "../lead/KinStop.ts"; import * as Cache from "../memory/MemoryCache.ts"; import * as KinPersistence from "../persistence/KinPersistence.ts"; import * as Repository from "./KinChatRepository.ts"; @@ -12,7 +13,9 @@ import { KinIdentity, KinIdentityConfig } from "./KinIdentity.ts"; export const layer = Layer.mergeAll( Service.layer, - Budget.layer.pipe(Layer.provideMerge(Queue.layer), Layer.provide(Cache.layer)), + Layer.mergeAll(Budget.layer.pipe(Layer.provide(Cache.layer)), Stop.layer).pipe( + Layer.provideMerge(Queue.layer), + ), KinIdentity.layer.pipe(Layer.provide(KinIdentityConfig.layer)), ).pipe( Layer.provide(JudgeShadow.layer.pipe(Layer.provide(NoopJudge.layer))), diff --git a/apps/server/src/kin/chat/rpcHandlers.test.ts b/apps/server/src/kin/chat/rpcHandlers.test.ts index 88cd07d9aea0..fc3ac29be4e9 100644 --- a/apps/server/src/kin/chat/rpcHandlers.test.ts +++ b/apps/server/src/kin/chat/rpcHandlers.test.ts @@ -748,6 +748,80 @@ it.effect( }).pipe(Effect.provide(root.pipe(Layer.provideMerge(Sqlite.layerMemory)))), ); +it.effect("lets only the owner stop and resume Kin over RPC; members may read the state", () => + Effect.gen(function* () { + const { person, repo } = yield* seed; + const clientFor = ( + handlers: Effect.Success>, + scopes: ReadonlyArray<(typeof AuthStandardClientScopes)[number]> = AuthStandardClientScopes, + ) => + RpcTest.makeClient(group).pipe( + Effect.provide( + Layer.merge( + group.toLayer({ ...handlers, "server.probe": () => Effect.succeed({}) }), + RpcAuthorization.layer(scopes), + ), + ), + ); + const handlers = yield* makeKinHandlers(session); + const client = yield* clientFor(handlers); + const input = { orgId: person.orgId }; + const paused = yield* client["kin.org.pause"]({ ...input, reason: "Going offline" }); + assert.include(paused, { paused: true, pausedBy: person.personId, reason: "Going offline" }); + assert.deepStrictEqual(yield* client["kin.org.pause"](input), paused); + assert.deepStrictEqual(yield* client["kin.org.get"](input), paused); + assert.include( + yield* client["kin.org.pause"]({ orgId: KinOrgId.make("foreign") }).pipe(Effect.flip), + { reason: "not_found" }, + ); + + const memberId = KinPersonId.make("stop-member"); + yield* repo.insertPerson({ + orgId: person.orgId, + personId: memberId, + handle: "stop-member", + displayName: "Member", + githubLogin: null, + createdAt: "1970-01-01T00:00:00.000Z", + }); + const kin = yield* Kin.KinServer; + const services = Option.getOrThrow(kin.services); + const identity = Context.get(services, KinIdentity); + const member = yield* clientFor( + yield* makeKinHandlers(session).pipe( + Effect.provideService(Kin.KinServer, { + ...kin, + services: Option.some( + Context.add(services, KinIdentity, { + ...identity, + resolve: () => + Effect.succeed([{ _tag: "person", orgId: person.orgId, personId: memberId }]), + }), + ), + }), + ), + ); + assert.deepStrictEqual(yield* member["kin.org.get"](input), paused); + assert.include(yield* member["kin.org.resume"](input).pipe(Effect.flip), { + reason: "not_allowed", + }); + assert.include(yield* member["kin.org.pause"](input).pipe(Effect.flip), { + reason: "not_allowed", + }); + const readOnly = yield* clientFor(handlers, [AuthOrchestrationReadScope]); + for (const method of ["kin.org.pause", "kin.org.resume"] as const) + assert.strictEqual( + (yield* readOnly[method](input).pipe(Effect.flip))._tag, + "EnvironmentAuthorizationError", + ); + assert.isTrue((yield* readOnly["kin.org.get"](input)).paused); + + const resumed = yield* client["kin.org.resume"](input); + assert.isFalse(resumed.paused); + assert.deepStrictEqual(yield* client["kin.org.resume"](input), resumed); + }).pipe(Effect.provide(root.pipe(Layer.provideMerge(Sqlite.layerMemory)))), +); + it.effect("replays offline edits, deletions and widget revisions once with afterChangeSeq", () => Effect.gen(function* () { const { chat, repo, person, direct } = yield* seed; diff --git a/apps/server/src/kin/chat/rpcHandlers.ts b/apps/server/src/kin/chat/rpcHandlers.ts index 82fd9f79ea70..655c41d473b4 100644 --- a/apps/server/src/kin/chat/rpcHandlers.ts +++ b/apps/server/src/kin/chat/rpcHandlers.ts @@ -1,4 +1,4 @@ -import { KinChatError, KinChatAndBudgetRpcs } from "@t3tools/contracts"; +import { KinChatError, KinRpcs } from "@t3tools/contracts"; import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; import * as Schema from "effect/Schema"; @@ -8,10 +8,11 @@ import * as RpcGroup from "effect/rpc/RpcGroup"; import type { AuthenticatedSession } from "../../auth/EnvironmentAuth.ts"; import { KinServer } from "../layer.ts"; import { KinBudget } from "../lead/KinBudget.ts"; +import { KinStop } from "../lead/KinStop.ts"; import { KinIdentity } from "./KinIdentity.ts"; import { KinChatService } from "./KinChatService.ts"; -const group = RpcGroup.make(...KinChatAndBudgetRpcs); +const group = RpcGroup.make(...KinRpcs); const unavailable = new KinChatError({ reason: "unavailable", message: "Kin is unavailable." }); const disabled = new KinChatError({ reason: "disabled", @@ -33,6 +34,7 @@ export const makeKinChatHandlers = Effect.fn("makeKinChatHandlers")(function* ( return { identity: Context.get(kin.value.services.value, KinIdentity), budget: Context.get(kin.value.services.value, KinBudget), + stop: Context.get(kin.value.services.value, KinStop), chat: Context.get(kin.value.services.value, KinChatService), }; }); @@ -61,6 +63,35 @@ export const makeKinChatHandlers = Effect.fn("makeKinChatHandlers")(function* ( Effect.mapError(mapError), Effect.catchDefect(defect), ), + "kin.org.get": (input) => + services.pipe( + Effect.flatMap(({ identity, stop }) => + identity + .resolve(session) + .pipe(Effect.flatMap((people) => stop.view(people, input.orgId))), + ), + Effect.tapError(logInfra), + Effect.mapError(mapError), + Effect.catchDefect(defect), + ), + "kin.org.pause": (input) => + services.pipe( + Effect.flatMap(({ identity, stop }) => + identity.resolve(session).pipe(Effect.flatMap((people) => stop.pause(people, input))), + ), + Effect.tapError(logInfra), + Effect.mapError(mapError), + Effect.catchDefect(defect), + ), + "kin.org.resume": (input) => + services.pipe( + Effect.flatMap(({ identity, stop }) => + identity.resolve(session).pipe(Effect.flatMap((people) => stop.resume(people, input))), + ), + Effect.tapError(logInfra), + Effect.mapError(mapError), + Effect.catchDefect(defect), + ), "kin.orgs.list": () => services.pipe( Effect.flatMap(({ identity, chat }) => diff --git a/apps/server/src/kin/lead/KinBudget.ts b/apps/server/src/kin/lead/KinBudget.ts index 757855e9a56f..f61011c6b3db 100644 --- a/apps/server/src/kin/lead/KinBudget.ts +++ b/apps/server/src/kin/lead/KinBudget.ts @@ -25,6 +25,7 @@ import * as Chat from "../chat/KinChatRepository.ts"; import * as Hub from "../chat/KinChatHub.ts"; import * as MemoryCache from "../memory/MemoryCache.ts"; import * as Queue from "./KinEventQueue.ts"; +import { isOrgOwner } from "./orgState.ts"; type Priority = "P0" | "P1" | "P2" | "P3" | "P4"; export type Usage = { @@ -263,14 +264,10 @@ const make = Effect.gen(function* () { yield* ensure(input.orgId, day); const person = people.find((p) => p.orgId === input.orgId); if (person === undefined) return yield* notFound; - // The reserved bootstrap owner is authoritative before a lead is configured. - const owner = - yield* sql`SELECT 1 FROM kin_people p WHERE p.org_id = ${input.orgId} AND p.person_id = ${person.personId} - AND ((p.org_id = 'kin-default' AND p.person_id = 'kin-default-owner' - AND NOT EXISTS (SELECT 1 FROM kin_leads WHERE org_id = p.org_id)) OR EXISTS ( - SELECT 1 FROM kin_leads l JOIN kin_agents a ON a.org_id = l.org_id AND a.agent_id = l.agent_id - WHERE l.org_id = p.org_id AND a.owner_person_id = p.person_id))`; - if (owner.length === 0) return yield* notAllowed; + const owner = yield* isOrgOwner(input.orgId, person.personId).pipe( + Effect.provideService(SqlClient.SqlClient, sql), + ); + if (!owner) return yield* notAllowed; const json = yield* encodeJson(limits); yield* sql`INSERT INTO kin_budget_limits (org_id, limits_json) VALUES (${input.orgId}, ${json}) ON CONFLICT DO NOTHING RETURNING org_id`; yield* sql`UPDATE kin_budget_limits SET limits_json = ${json} WHERE org_id = ${input.orgId}`; diff --git a/apps/server/src/kin/lead/KinEventQueue.test.ts b/apps/server/src/kin/lead/KinEventQueue.test.ts index 65ec358124f4..8e420a8453a4 100644 --- a/apps/server/src/kin/lead/KinEventQueue.test.ts +++ b/apps/server/src/kin/lead/KinEventQueue.test.ts @@ -71,6 +71,7 @@ const event = Effect.fn("test.event")(function* ( itemIds: [], routine: "owner_check", pausedAt: DateTime.formatIso(DateTime.makeUnsafe(now)), + expired: { count: 0, events: [] }, subject: { type: "reminder", itemId: "reminder" }, lateByMs: 0, workerId: id, diff --git a/apps/server/src/kin/lead/KinStop.test.ts b/apps/server/src/kin/lead/KinStop.test.ts new file mode 100644 index 000000000000..cb37a8eae00e --- /dev/null +++ b/apps/server/src/kin/lead/KinStop.test.ts @@ -0,0 +1,193 @@ +import { assert, describe, it } from "@effect/vitest"; +import { + CommandId, + KinPersonId, + KinWakeupId, + MessageId, + ProjectId, + ProviderInstanceId, + ThreadId, +} from "@t3tools/contracts"; +import * as Clock from "effect/Clock"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as Option from "effect/Option"; +import * as DateTime from "effect/DateTime"; +import * as SqlClient from "effect/sql/SqlClient"; +import * as TestClock from "effect/testing/TestClock"; + +import * as EffectWorker from "../../orchestration-v2/EffectWorker.ts"; +import * as ThreadLaunch from "../../orchestration-v2/ThreadLaunchService.ts"; +import * as ThreadManagement from "../../orchestration-v2/ThreadManagementService.ts"; +import { provideDeterministicTestRuntime } from "../../orchestration-v2/testkit/DeterministicRuntime.ts"; +import * as KinEventQueue from "./KinEventQueue.ts"; +import * as KinStop from "./KinStop.ts"; +import * as WakeupRunner from "./WakeupRunner.ts"; +import { agentId, event, harness, orgId, seed } from "./WakeupRunner.testkit.ts"; + +const owner = [{ _tag: "person" as const, orgId, personId: KinPersonId.make("owner") }]; + +describe("KinStop", () => { + it.effect( + "stops a running wake-up and worker, holds their events and stays paused when pressed twice", + () => + Effect.gen(function* () { + const entered = yield* Deferred.make(); + yield* Effect.gen(function* () { + yield* seed; + const sql = yield* SqlClient.SqlClient; + const queue = yield* KinEventQueue.KinEventQueue; + const runner = yield* WakeupRunner.WakeupRunner; + const stop = yield* KinStop.KinStop; + const threads = yield* ThreadManagement.ThreadManagementService; + const launch = yield* ThreadLaunch.ThreadLaunchService; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const delegated: Array = []; + yield* stop.attach( + yield* KinStop.makeFanOut(Option.none()).pipe( + Effect.provideService(ThreadManagement.ThreadManagementService, { + ...threads, + stopDelegatedTasks: (input) => + Effect.sync(() => delegated.push(input.threadId)).pipe( + Effect.andThen(threads.stopDelegatedTasks(input)), + ), + }), + ), + ); + + yield* queue.enqueue(event("work")); + const wakeup = yield* runner.runOnce(orgId, agentId).pipe(Effect.forkChild); + yield* TestClock.adjust("20 seconds"); + yield* Deferred.await(entered); + const [running] = yield* sql<{ + wakeupId: string; + threadId: string; + }>`SELECT wakeup_id AS "wakeupId", thread_id AS "threadId" FROM kin_wakeups WHERE org_id = ${orgId} ORDER BY wakeup_id`; + const leadThread = ThreadId.make(running!.threadId); + + const workerThread = ThreadId.make("kin-worker-stop-test"); + yield* launch.launch({ + commandId: CommandId.make("kin-worker:stop-test"), + threadId: workerThread, + projectId: ProjectId.make("kin:org"), + title: "Worker", + modelSelection: { + instanceId: ProviderInstanceId.make("codex"), + model: "gpt-5.1-codex", + }, + runtimeMode: "full-access", + interactionMode: "default", + workspaceStrategy: { type: "root" }, + initialMessage: { + messageId: MessageId.make("kin-worker-msg:stop-test"), + text: "Work", + attachments: [], + }, + createdBy: "agent", + creationSource: "server", + }); + yield* sql`INSERT INTO kin_workers (worker_id, org_id, agent_id, wakeup_id, kind, thread_id, model_json, provenance_json, status, started_at) + VALUES ('worker', ${orgId}, ${agentId}, ${running!.wakeupId}, 'build', ${workerThread}, '{}', '{"_tag":"system"}', 'running', 0)`; + yield* worker.drain(); + const live = (threadId: ThreadId) => + threads + .getThreadProjection(threadId) + .pipe(Effect.map((projection) => projection.runs.at(-1)?.status)); + assert.include(ThreadManagement.ACTIVE_RUN_STATUSES, yield* live(workerThread)); + + const paused = yield* stop.pause(owner, { orgId, reason: "Testing" }); + assert.include(paused, { paused: true, pausedBy: "owner", reason: "Testing" }); + yield* worker.drain(); + yield* Fiber.join(wakeup); + + assert.strictEqual(yield* live(leadThread), "interrupted"); + assert.strictEqual(yield* live(workerThread), "interrupted"); + assert.sameMembers(delegated, [leadThread, workerThread]); + assert.deepStrictEqual( + yield* sql`SELECT status FROM kin_wakeups WHERE org_id = ${orgId} ORDER BY wakeup_id`, + [{ status: "released" }], + ); + const now = yield* Clock.currentTimeMillis; + assert.deepStrictEqual( + yield* sql`SELECT state, attempts, not_before FROM kin_events WHERE org_id = ${orgId} ORDER BY event_id`, + [{ state: "queued", attempts: 0, not_before: now + KinStop.PAUSE_POLL_INTERVAL_MS }], + ); + + assert.deepStrictEqual(yield* stop.pause(owner, { orgId }), paused); + assert.deepStrictEqual( + yield* sql`SELECT body FROM kin_messages WHERE org_id = ${orgId} ORDER BY seq`, + [{ body: KinStop.PAUSED_NOTICE }], + ); + + yield* queue.enqueue(event("held")); + yield* TestClock.adjust("2 minutes"); + yield* runner.runOnce(orgId, agentId); + assert.lengthOf( + yield* sql`SELECT 1 FROM kin_wakeups WHERE org_id = ${orgId} ORDER BY wakeup_id`, + 1, + ); + assert.deepStrictEqual( + yield* sql`SELECT state FROM kin_events WHERE org_id = ${orgId} ORDER BY event_id`, + [{ state: "queued" }, { state: "queued" }], + ); + }).pipe(Effect.provide(harness({ entered, untilInterrupted: true }))); + }).pipe(provideDeterministicTestRuntime), + ); + + it.effect("resumes once with one resume event and expires events queued over a day ago", () => + Effect.gen(function* () { + yield* seed; + const sql = yield* SqlClient.SqlClient; + const queue = yield* KinEventQueue.KinEventQueue; + const stop = yield* KinStop.KinStop; + const paused = yield* stop.pause(owner, { orgId }); + assert.isTrue(yield* stop.isPaused(orgId)); + yield* queue.enqueue(event("old")); + yield* TestClock.adjust("25 hours"); + yield* queue.enqueue(event("fresh")); + + const resumed = yield* stop.resume(owner, { orgId }); + assert.deepStrictEqual(resumed, { + orgId, + paused: false, + pausedAt: null, + pausedBy: null, + reason: null, + }); + assert.deepStrictEqual(yield* stop.resume(owner, { orgId }), resumed); + assert.isFalse(yield* stop.isPaused(orgId)); + assert.deepStrictEqual( + yield* sql`SELECT event_id, state FROM kin_events WHERE org_id = ${orgId} AND tag != 'resume' ORDER BY event_id`, + [ + { event_id: "fresh", state: "queued" }, + { event_id: "old", state: "expired" }, + ], + ); + const resumes = yield* sql<{ + json: string; + priority: string; + }>`SELECT json, priority FROM kin_events WHERE org_id = ${orgId} AND tag = 'resume' ORDER BY event_id`; + assert.lengthOf(resumes, 1); + assert.strictEqual(resumes[0]!.priority, "P0"); + assert.isTrue(paused.paused); + assert.include(JSON.parse(resumes[0]!.json), { + _tag: "resume", + pausedAt: paused.pausedAt === null ? null : DateTime.formatIso(paused.pausedAt), + }); + assert.deepStrictEqual(JSON.parse(resumes[0]!.json).expired, { + count: 1, + events: [{ eventId: "old", tag: "routine", receivedAt: "2026-11-08T00:00:00.000Z" }], + }); + + yield* TestClock.adjust("2 seconds"); + const batch = yield* Effect.scoped( + queue.claim({ orgId, agentId, wakeupId: KinWakeupId.make("resumed"), model: "test" }), + ); + assert.deepStrictEqual( + batch.map((item) => item._tag), + ["resume", "routine"], + ); + }).pipe(Effect.provide(harness()), provideDeterministicTestRuntime), + ); +}); diff --git a/apps/server/src/kin/lead/KinStop.ts b/apps/server/src/kin/lead/KinStop.ts new file mode 100644 index 000000000000..7a66e110d14a --- /dev/null +++ b/apps/server/src/kin/lead/KinStop.ts @@ -0,0 +1,365 @@ +import { + KinChatError, + KinChatId, + KinEvent, + KinMessageId, + KinOrgState, + ThreadId, + type KinMessage, + type KinOrgId, + type KinOrgPauseInput, + type KinOrgResumeInput, +} from "@t3tools/contracts"; +import * as Clock from "effect/Clock"; +import * as Context from "effect/Context"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; +import type * as Scope from "effect/Scope"; +import * as SqlClient from "effect/sql/SqlClient"; +import { isSqlError } from "effect/sql/SqlError"; + +import * as ThreadManagement from "../../orchestration-v2/ThreadManagementService.ts"; +import { PersistenceSqlError, toPersistenceSqlError } from "../../persistence/Errors.ts"; +import * as Hub from "../chat/KinChatHub.ts"; +import type { PersonIdentity } from "../chat/KinIdentity.ts"; +import * as Chat from "../chat/KinChatRepository.ts"; +import type * as Routines from "../triggers/routines.ts"; +import * as Queue from "./KinEventQueue.ts"; +import { isOrgOwner, isOrgPaused } from "./orgState.ts"; +import { stopThreadTree } from "./Workers.ts"; + +/** A claim that races a pause returns its events for this long, so a paused org does not spin. */ +export const PAUSE_POLL_INTERVAL_MS = 5_000; +/** Resume expires events queued longer ago than this instead of replaying them. */ +const PAUSE_EXPIRY_MS = 24 * 60 * 60 * 1_000; +export const PAUSED_NOTICE = "Kin is paused. Nothing runs until you resume."; +const STOP_REASON = "Stopped by the Kin stop switch."; +const EXPIRED_LISTED = 50; + +export class KinStopFanOutError extends Schema.TaggedError()( + "KinStopFanOutError", + { orgId: Schema.String, cause: Schema.Defect() }, +) {} +/** Interrupts work in the lead runtime, which owns the thread services the WS side cannot reach. */ +export type KinStopFanOut = { + readonly pause: (orgId: KinOrgId, pausedAt: number) => Effect.Effect; + readonly resume: (orgId: KinOrgId) => Effect.Effect; +}; +type StopError = KinChatError | PersistenceSqlError; + +const notFound = new KinChatError({ reason: "not_found", message: "Kin organization not found." }); +const notAllowed = new KinChatError({ + reason: "not_allowed", + message: "Only the organization owner can stop or resume Kin.", +}); +const invalid = new KinChatError({ reason: "invalid_state", message: "Invalid Kin stop data." }); +const unavailable = new KinChatError({ + reason: "unavailable", + message: "Kin is paused, but some running work could not be stopped. Try again.", +}); +const isChatError = Schema.is(KinChatError); +const isPersistenceError = Schema.is(PersistenceSqlError); +const mapError = (cause: unknown): StopError => + isSqlError(cause) + ? toPersistenceSqlError("KinStop")(cause) + : isChatError(cause) || isPersistenceError(cause) + ? cause + : invalid; +const decodeState = Schema.decodeUnknownEffect(KinOrgState); +const decodeEvent = Schema.decodeUnknownEffect(KinEvent); +const iso = (millis: number) => DateTime.formatIso(DateTime.makeUnsafe(millis)); + +export class KinStop extends Context.Service< + KinStop, + { + readonly isPaused: (orgId: KinOrgId) => Effect.Effect; + readonly view: ( + people: ReadonlyArray, + orgId: KinOrgId, + ) => Effect.Effect; + /** Owner only. Repeating a pause keeps the first pause and retries the fan-out. */ + readonly pause: ( + people: ReadonlyArray, + input: KinOrgPauseInput, + ) => Effect.Effect; + /** Owner only. Resuming an org that is not paused changes nothing. */ + readonly resume: ( + people: ReadonlyArray, + input: KinOrgResumeInput, + ) => Effect.Effect; + readonly attach: (fanOut: KinStopFanOut) => Effect.Effect; + } +>()("t3/kin/lead/KinStop") {} + +const make = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const queue = yield* Queue.KinEventQueue; + const chat = yield* Chat.KinChatRepository; + const hub = yield* Hub.KinChatHub; + let attached: KinStopFanOut | undefined; + + const state = Effect.fn("KinStop.state")(function* (orgId: KinOrgId) { + const [row] = yield* sql<{ + paused: number; + pausedAt: number | null; + pausedBy: string | null; + reason: string | null; + }>`SELECT paused, paused_at AS "pausedAt", paused_by AS "pausedBy", reason + FROM kin_org_state WHERE org_id = ${orgId} ORDER BY org_id`; + return yield* decodeState( + row === undefined || row.paused === 0 || row.pausedAt === null + ? { orgId, paused: false, pausedAt: null, pausedBy: null, reason: null } + : { + orgId, + paused: true, + pausedAt: iso(row.pausedAt), + pausedBy: row.pausedBy, + reason: row.reason, + }, + ); + }); + const member = Effect.fn("KinStop.member")(function* ( + people: ReadonlyArray, + orgId: KinOrgId, + ) { + const person = people.find((candidate) => candidate.orgId === orgId); + if (person === undefined) return yield* notFound; + const rows = + yield* sql`SELECT 1 FROM kin_people WHERE org_id = ${orgId} AND person_id = ${person.personId} ORDER BY person_id`; + if (rows.length === 0) return yield* notFound; + return person; + }); + const owner = Effect.fn("KinStop.owner")(function* ( + people: ReadonlyArray, + orgId: KinOrgId, + ) { + const person = yield* member(people, orgId); + const allowed = yield* isOrgOwner(orgId, person.personId).pipe( + Effect.provideService(SqlClient.SqlClient, sql), + ); + if (!allowed) return yield* notAllowed; + return person; + }); + const notice = Effect.fn("KinStop.notice")(function* (orgId: KinOrgId, now: number) { + const chats = yield* sql<{ + chatId: string; + }>`SELECT DISTINCT owner_chat_id AS "chatId" FROM kin_leads WHERE org_id = ${orgId} ORDER BY owner_chat_id`; + return yield* Effect.forEach(chats, (row) => + chat.insertMessage({ + orgId, + chatId: KinChatId.make(row.chatId), + messageId: KinMessageId.make(`kin-pause:${now}:${row.chatId}`), + author: { _tag: "system" }, + kind: "text", + body: PAUSED_NOTICE, + intent: null, + mentions: [], + replyTo: null, + widgetId: null, + floorRunId: null, + status: "posted", + sentBy: null, + clientMessageId: null, + createdAt: iso(now), + editedAt: null, + }), + ); + }); + const fanOut = Effect.fn("KinStop.fanOut")(function* (orgId: KinOrgId, pausedAt: number) { + if (attached === undefined) return; + yield* attached.pause(orgId, pausedAt).pipe( + Effect.tapCause((cause) => Effect.logError("Kin stop fan-out failed", { orgId, cause })), + Effect.mapError(() => unavailable), + ); + }); + + const pause = Effect.fn("KinStop.pause")(function* ( + people: ReadonlyArray, + input: KinOrgPauseInput, + ) { + const result = yield* sql.withTransaction( + Effect.gen(function* () { + const person = yield* owner(people, input.orgId); + const now = yield* Clock.currentTimeMillis; + const reason = input.reason ?? null; + const inserted = + yield* sql`INSERT INTO kin_org_state (org_id, paused, paused_at, paused_by, reason) + VALUES (${input.orgId}, 1, ${now}, ${person.personId}, ${reason}) + ON CONFLICT DO NOTHING RETURNING org_id`; + const changed = + inserted.length > 0 + ? inserted + : yield* sql`UPDATE kin_org_state SET paused = 1, paused_at = ${now}, + paused_by = ${person.personId}, reason = ${reason} + WHERE org_id = ${input.orgId} AND paused = 0 RETURNING org_id`; + const messages: ReadonlyArray = + changed.length === 0 ? [] : yield* notice(input.orgId, now); + return { state: yield* state(input.orgId), messages }; + }), + ); + for (const message of result.messages) + yield* hub.publish({ type: "message.posted", orgId: input.orgId, message }); + if (result.state.paused) + yield* fanOut( + input.orgId, + DateTime.toEpochMillis(DateTime.makeUnsafe(result.state.pausedAt)), + ); + return result.state; + }, Effect.mapError(mapError)); + + const resume = Effect.fn("KinStop.resume")(function* ( + people: ReadonlyArray, + input: KinOrgResumeInput, + ) { + const resumed = yield* sql.withTransaction( + Effect.gen(function* () { + yield* owner(people, input.orgId); + const [cleared] = yield* sql<{ pausedAt: number }>`SELECT paused_at AS "pausedAt" + FROM kin_org_state WHERE org_id = ${input.orgId} AND paused = 1 ORDER BY org_id`; + if (cleared === undefined) return false; + yield* sql`UPDATE kin_org_state SET paused = 0, paused_at = NULL, paused_by = NULL, reason = NULL + WHERE org_id = ${input.orgId} AND paused = 1`; + const now = yield* Clock.currentTimeMillis; + const expired = yield* sql<{ + eventId: string; + agentId: string; + tag: string; + receivedAt: number; + }>`UPDATE kin_events SET state = 'expired' + WHERE org_id = ${input.orgId} AND state = 'queued' AND queued_at <= ${now - PAUSE_EXPIRY_MS} + RETURNING event_id AS "eventId", agent_id AS "agentId", tag, received_at AS "receivedAt"`; + const leads = yield* sql<{ + agentId: string; + }>`SELECT l.agent_id AS "agentId" FROM kin_leads l + JOIN kin_agents a ON a.org_id = l.org_id AND a.agent_id = l.agent_id + WHERE l.org_id = ${input.orgId} AND a.retired_at IS NULL ORDER BY l.agent_id`; + for (const lead of leads) { + const mine = expired + .filter((row) => row.agentId === lead.agentId) + .toSorted( + (a, b) => + a.receivedAt - b.receivedAt || + (a.eventId < b.eventId ? -1 : a.eventId > b.eventId ? 1 : 0), + ); + const eventId = `resume:${cleared.pausedAt}:${lead.agentId}`; + yield* queue.enqueue( + yield* decodeEvent({ + _tag: "resume", + orgId: input.orgId, + agentId: lead.agentId, + eventId, + receivedAt: iso(now), + dedupeKey: eventId, + provenance: { _tag: "system" }, + pausedAt: iso(cleared.pausedAt), + expired: { + count: mine.length, + events: mine.slice(0, EXPIRED_LISTED).map((row) => ({ + eventId: row.eventId, + tag: row.tag, + receivedAt: iso(row.receivedAt), + })), + }, + }), + ); + } + return true; + }), + ); + if (resumed && attached !== undefined) yield* attached.resume(input.orgId); + return yield* state(input.orgId); + }, Effect.mapError(mapError)); + + return KinStop.of({ + isPaused: Effect.fn("KinStop.isPaused")((orgId) => + isOrgPaused(orgId).pipe(Effect.provideService(SqlClient.SqlClient, sql)), + ), + view: Effect.fn("KinStop.view")(function* (people, orgId) { + yield* member(people, orgId); + return yield* state(orgId); + }, Effect.mapError(mapError)), + pause, + resume, + attach: Effect.fn("KinStop.attach")(function* (next) { + yield* Effect.acquireRelease( + Effect.sync(() => { + attached = next; + }), + () => + Effect.sync(() => { + if (attached === next) attached = undefined; + }), + ); + // A pause from before a restart still owes its fan-out to work recovered since. + const paused = yield* sql<{ + orgId: KinOrgId; + pausedAt: number; + }>`SELECT org_id AS "orgId", paused_at AS "pausedAt" FROM kin_org_state WHERE paused = 1 ORDER BY org_id`.pipe( + Effect.catchCause((cause) => + Effect.logError("Kin stop catch-up unavailable", { cause }).pipe(Effect.as([])), + ), + ); + for (const row of paused) yield* fanOut(row.orgId, row.pausedAt).pipe(Effect.ignore); + }), + }); +}); + +export const layer = Layer.effect(KinStop, make); + +/** The lead runtime's fan-out: lead runs, worker runs and their delegated tasks, then routines. */ +export const makeFanOut = Effect.fn("KinStop.makeFanOut")(function* ( + routines: Option.Option, +) { + const sql = yield* SqlClient.SqlClient; + const threads = yield* ThreadManagement.ThreadManagementService; + return { + pause: Effect.fn("KinStop.fanOut.pause")( + function* (orgId: KinOrgId, pausedAt: number) { + const rows = yield* sql<{ threadId: string }>` + SELECT thread_id AS "threadId" FROM kin_leads WHERE org_id = ${orgId} AND thread_id IS NOT NULL + UNION SELECT thread_id FROM kin_workers WHERE org_id = ${orgId} AND ended_at IS NULL + ORDER BY 1`; + yield* Effect.forEach( + rows, + (row) => + Effect.gen(function* () { + const threadId = ThreadId.make(row.threadId); + const projection = yield* threads.getThreadProjection(threadId).pipe(Effect.option); + if (Option.isNone(projection)) return; + const run = projection.value.runs.at(-1)?.id ?? "idle"; + yield* stopThreadTree(threads, { + projection: projection.value, + prefix: "kin-stop", + tag: `${threadId}:${pausedAt}:${run}`, + reason: STOP_REASON, + }); + }), + { concurrency: "unbounded", discard: true }, + ); + if (Option.isSome(routines)) + yield* routines.value + .pauseAll(orgId) + .pipe( + Effect.catchCause((cause) => + Effect.logWarning("Kin routines could not pause", { orgId, cause }), + ), + ); + }, + (effect, orgId) => + effect.pipe(Effect.mapError((cause) => new KinStopFanOutError({ orgId, cause }))), + ), + resume: Effect.fn("KinStop.fanOut.resume")(function* (orgId: KinOrgId) { + if (Option.isSome(routines)) + yield* routines.value + .resumeAll(orgId) + .pipe( + Effect.catchCause((cause) => + Effect.logWarning("Kin routines could not resume", { orgId, cause }), + ), + ); + }), + } satisfies KinStopFanOut; +}); diff --git a/apps/server/src/kin/lead/WakeupRunner.test.ts b/apps/server/src/kin/lead/WakeupRunner.test.ts index 5c620452a067..c1fe6b1eda17 100644 --- a/apps/server/src/kin/lead/WakeupRunner.test.ts +++ b/apps/server/src/kin/lead/WakeupRunner.test.ts @@ -45,6 +45,7 @@ import * as ServerSettings from "../../serverSettings.ts"; import * as KinIssuedTokens from "../mcp/KinIssuedTokens.ts"; import * as KinLead from "./layer.ts"; import * as KinEventQueue from "./KinEventQueue.ts"; +import * as KinStop from "./KinStop.ts"; import * as WakeupRunner from "./WakeupRunner.ts"; import { KinLeadPrompt } from "./LeadPrompt.ts"; import * as KinPersistence from "../persistence/KinPersistence.ts"; @@ -193,6 +194,7 @@ describe("WakeupRunner", () => { ...event("urgent"), _tag: "resume", pausedAt: DateTime.makeUnsafe(0), + expired: { count: 0, events: [] }, }); yield* queue.enqueue(event("routine")); yield* Fiber.join(yield* run()); @@ -270,6 +272,7 @@ describe("WakeupRunner", () => { receivedAt: "1970-01-01T00:00:00.000Z", _tag: "resume", pausedAt: "1970-01-01T00:00:00.000Z", + expired: { count: 0, events: [] }, }), ); yield* Fiber.join(yield* run()); @@ -1104,6 +1107,7 @@ describe("WakeupRunner", () => { Layer.mock(ChatService.KinChatService)({}), Layer.mock(ChatHub.KinChatHub)({}), Layer.mock(Budget.KinBudget)({}), + Layer.mock(KinStop.KinStop)({}), Layer.mock(KinEventQueue.KinEventQueue)({}), Layer.mock(ThreadLaunch.ThreadLaunchService)({}), Layer.mock(ThreadManagement.ThreadManagementService)({}), diff --git a/apps/server/src/kin/lead/WakeupRunner.testkit.ts b/apps/server/src/kin/lead/WakeupRunner.testkit.ts index 850e0cc598f0..b4072c970c32 100644 --- a/apps/server/src/kin/lead/WakeupRunner.testkit.ts +++ b/apps/server/src/kin/lead/WakeupRunner.testkit.ts @@ -41,6 +41,7 @@ import * as Budget from "./KinBudget.ts"; import * as MemoryCache from "../memory/MemoryCache.ts"; import * as Hub from "../chat/KinChatHub.ts"; import * as WakeupRunner from "./WakeupRunner.ts"; +import * as KinStop from "./KinStop.ts"; import * as KinLeadRegistry from "./KinLeadRegistry.ts"; import * as KinEventQueue from "./KinEventQueue.ts"; import * as KinIssuedTokens from "../mcp/KinIssuedTokens.ts"; @@ -127,6 +128,8 @@ type HarnessOptions = { readonly settings?: Parameters[0]; /** Runtime requests are opened with this capability instead of `live`. */ readonly requestCapability?: "not_resumable"; + /** Each turn stays open until it is interrupted, then ends `interrupted`. */ + readonly untilInterrupted?: boolean; }; export function harness(options: HarnessOptions = {}) { @@ -147,6 +150,10 @@ export function orchestrationHarness(options: HarnessOptions = {}) { Effect.gen(function* () { const now = yield* DateTime.now; const events = yield* PubSub.unbounded(); + const open = new Map< + string, + Parameters[0] + >(); const runtime: ProviderAdapter.ProviderAdapterV2SessionRuntime = { instanceId, driver, @@ -221,6 +228,10 @@ export function orchestrationHarness(options: HarnessOptions = {}) { }, }, }); + if (options.untilInterrupted) { + open.set(id, input); + return; + } if (options.approval !== undefined) yield* PubSub.publish(events, { type: "runtime_request.updated", @@ -302,7 +313,37 @@ export function orchestrationHarness(options: HarnessOptions = {}) { }), compactThread: (input) => runtime.startTurn(input), steerTurn: () => Effect.void, - interruptTurn: () => Effect.void, + interruptTurn: Effect.fn("test.interruptTurn")(function* ({ providerTurnId }) { + const turn = open.get(providerTurnId); + if (turn === undefined) return; + open.delete(providerTurnId); + const time = yield* DateTime.now; + yield* PubSub.publish(events, { + type: "provider_turn.updated", + driver, + providerTurn: { + id: providerTurnId, + providerThreadId: turn.providerThread.id, + nodeId: turn.rootNodeId, + runAttemptId: turn.attemptId, + nativeTurnRef: null, + ordinal: turn.runOrdinal, + status: "interrupted", + startedAt: time, + completedAt: time, + }, + }); + yield* PubSub.publish(events, { + type: "turn.terminal", + driver, + providerThreadId: turn.providerThread.id, + providerTurnId, + runOrdinal: turn.runOrdinal, + status: "interrupted", + failure: null, + threadDisposition: "reusable", + }); + }), respondToRuntimeRequest: (response) => options.approval === undefined ? Effect.void @@ -367,6 +408,14 @@ export function orchestrationHarness(options: HarnessOptions = {}) { ); const kin = Layer.mergeAll( KinEventQueue.layer, + KinStop.layer.pipe( + Layer.provide( + Layer.mergeAll( + KinEventQueue.layer, + Hub.layer.pipe(Layer.provideMerge(KinChatRepository.layer)), + ), + ), + ), Budget.layer.pipe( Layer.provide( Layer.mergeAll( diff --git a/apps/server/src/kin/lead/WakeupRunner.ts b/apps/server/src/kin/lead/WakeupRunner.ts index 3c0303d343c9..21374651a87f 100644 --- a/apps/server/src/kin/lead/WakeupRunner.ts +++ b/apps/server/src/kin/lead/WakeupRunner.ts @@ -37,6 +37,8 @@ import * as KinEventQueue from "./KinEventQueue.ts"; import * as KinBudget from "./KinBudget.ts"; import * as KinLeadRegistry from "./KinLeadRegistry.ts"; import { KinLeadPrompt } from "./LeadPrompt.ts"; +import { PAUSE_POLL_INTERVAL_MS } from "./KinStop.ts"; +import { isOrgPaused } from "./orgState.ts"; const ownerCheckDispatcherReady = false; @@ -111,6 +113,8 @@ const make = Effect.gen(function* () { const projects = yield* ProjectService.ProjectService; const launch = yield* ThreadLaunchService.ThreadLaunchService; const threads = yield* ThreadManagementService.ThreadManagementService; + const paused = (orgId: KinOrgId) => + isOrgPaused(orgId).pipe(Effect.provideService(SqlClient.SqlClient, sql)); const threadActivity = Effect.fn("WakeupRunner.threadActivity")(function* (orgId: KinOrgId) { const rows = yield* sql<{ threadId: string; @@ -191,6 +195,8 @@ const make = Effect.gen(function* () { : result.run?.status === "completed" ? "completed" : "failed"; + // The stop switch interrupted this run: hand its events back without spending an attempt. + const stopped = status === "failed" && (yield* paused(row.orgId)); yield* Effect.gen(function* () { if (result.timedOut) { yield* threads.interruptThread({ @@ -226,7 +232,14 @@ const make = Effect.gen(function* () { ); } yield* capture(row); - yield* queue.finish({ orgId: row.orgId, wakeupId: row.wakeupId, status }); + if (stopped) + yield* queue.release({ + orgId: row.orgId, + wakeupId: row.wakeupId, + countAttempt: false, + notBefore: (yield* Clock.currentTimeMillis) + PAUSE_POLL_INTERVAL_MS, + }); + else yield* queue.finish({ orgId: row.orgId, wakeupId: row.wakeupId, status }); }).pipe( Effect.onExit((exit) => result.timedOut && Exit.isFailure(exit) @@ -299,8 +312,7 @@ const make = Effect.gen(function* () { function* (orgId: KinOrgId, agentId: KinAgentId) { const record = yield* lead(orgId, agentId); if (record.retiredAt !== null) return yield* retire(record); - const paused = yield* sql`SELECT 1 FROM kin_org_state WHERE org_id = ${orgId} AND paused = 1`; - if (paused.length > 0) return; + if (yield* paused(orgId)) return; if (record.threadId !== null) { const unfinished = yield* sql`SELECT 1 FROM orchestration_v2_projection_provider_turns WHERE thread_id = ${record.threadId} AND completed_at IS NULL LIMIT 1`; @@ -341,14 +353,12 @@ const make = Effect.gen(function* () { ); row = { ...row, startedAt: yield* Clock.currentTimeMillis }; yield* Effect.gen(function* () { - const paused = - yield* sql`SELECT 1 FROM kin_org_state WHERE org_id = ${orgId} AND paused = 1`; - if (paused.length > 0) { + if (yield* paused(orgId)) { yield* queue.release({ orgId, wakeupId, countAttempt: false, - notBefore: (yield* Clock.currentTimeMillis) + 5_000, + notBefore: (yield* Clock.currentTimeMillis) + PAUSE_POLL_INTERVAL_MS, }); return; } @@ -490,6 +500,14 @@ const make = Effect.gen(function* () { return yield* new KinWakeupError({ orgId, agentId, cause: "missing_run" }); row = { ...row, runId: run.id }; yield* sql`UPDATE kin_wakeups SET run_id = ${run.id} WHERE org_id = ${orgId} AND wakeup_id = ${wakeupId} AND agent_id = ${agentId} AND status = 'running'`; + // A pause that landed during launch found no run to stop; stop it here instead. + if (yield* paused(orgId)) + yield* threads.interruptThread({ + projectId: project.id, + threadId, + runId: run.id, + commandId: CommandId.make(`kin-stop-launch:${wakeupId}`), + }); yield* wait(row, project.id); }).pipe( Effect.onExit((exit) => @@ -668,9 +686,7 @@ const make = Effect.gen(function* () { JOIN kin_agents a ON a.org_id = l.org_id AND a.agent_id = l.agent_id WHERE l.org_id = ${orgId} AND a.retired_at IS NOT NULL ORDER BY l.agent_id`; for (const row of rows) yield* retire(yield* lead(orgId, KinAgentId.make(row.agentId))); - const paused = - yield* sql`SELECT 1 FROM kin_org_state WHERE org_id = ${orgId} AND paused = 1`; - const next = paused.length === 0 ? yield* queue.nextReady(orgId) : undefined; + const next = (yield* paused(orgId)) ? undefined : yield* queue.nextReady(orgId); const now = yield* Clock.currentTimeMillis; if (next !== undefined && next.readyAt <= now) { yield* runOnce(orgId, next.agentId); diff --git a/apps/server/src/kin/lead/Workers.test.ts b/apps/server/src/kin/lead/Workers.test.ts index fc0a22e6b9fe..19008935319b 100644 --- a/apps/server/src/kin/lead/Workers.test.ts +++ b/apps/server/src/kin/lead/Workers.test.ts @@ -565,7 +565,7 @@ it.effect( ); it.effect( - "kin_delegate refuses at 0 worker units, below L2, foreign ids and non-lead callers", + "kin_delegate refuses at 0 worker units, while paused, below L2, foreign ids and non-lead callers", () => run(({ ctx, fakes, workers }) => Effect.gen(function* () { @@ -576,6 +576,13 @@ it.effect( assert.strictEqual(refused.reason, "limit"); assert.lengthOf(fakes.launches, 0); yield* ctx.sql`UPDATE kin_budget_limits SET limits_json = '{"leadUnits":10,"workerUnits":10}' WHERE org_id = ${ctx.orgId}`; + yield* ctx.sql`INSERT INTO kin_org_state (org_id, paused, paused_at, paused_by) SELECT ${ctx.orgId}, 1, 0, person_id FROM kin_people WHERE org_id = ${ctx.orgId} ORDER BY person_id LIMIT 1`; + assert.include(yield* Effect.flip(workers.delegate(scope, delegateInput)), { + reason: "not_allowed", + message: "Kin is paused by the owner. Nothing new starts until they resume.", + }); + assert.lengthOf(fakes.launches, 0); + yield* ctx.sql`DELETE FROM kin_org_state WHERE org_id = ${ctx.orgId}`; const foreign: ReadonlyArray> = [ workers.delegate(scope, { diff --git a/apps/server/src/kin/lead/Workers.ts b/apps/server/src/kin/lead/Workers.ts index 99e7a4f27e73..a41e35252c75 100644 --- a/apps/server/src/kin/lead/Workers.ts +++ b/apps/server/src/kin/lead/Workers.ts @@ -17,6 +17,7 @@ import { WorkerReport, type KinLevel, type OrchestrationV2Run, + type OrchestrationV2ThreadProjection, } from "@t3tools/contracts"; import { stableStringify } from "@t3tools/shared/relaySigning"; import { formatThreadLink } from "@t3tools/shared/threadLinks"; @@ -41,6 +42,7 @@ import { KinChatService } from "../chat/KinChatService.ts"; import { KinIdentity } from "../chat/KinIdentity.ts"; import * as Memory from "../memory/Memory.ts"; import { KinBudget } from "./KinBudget.ts"; +import { isOrgPaused } from "./orgState.ts"; import { resolveLead } from "./LeadMemory.ts"; import { quote } from "./LeadPrompt.ts"; import { buildBrief, Quoted, reportFormat } from "./workerBrief.ts"; @@ -87,6 +89,37 @@ export const WorkerLaunchGate = Context.Reference("t3/kin/lead/Workers/ defaultValue: () => workerLaunchReady, }); +const isLive = (run: OrchestrationV2Run) => + run.status === "queued" || ThreadManagement.isActiveRun(run); +const liveRun = (projection: OrchestrationV2ThreadProjection) => projection.runs.findLast(isLive); +/** Interrupts the thread's live run and stops its delegated tasks; true when a run was live. */ +export const stopThreadTree = Effect.fn("KinWorkers.stopThreadTree")(function* ( + threads: ThreadManagement.ThreadManagementService["Service"], + input: { + readonly projection: OrchestrationV2ThreadProjection; + readonly prefix: string; + readonly tag: string; + readonly reason: string; + }, +) { + const { projection, prefix, tag, reason } = input; + const active = liveRun(projection); + if (active !== undefined) + yield* threads.interruptThread({ + projectId: projection.thread.projectId, + threadId: projection.thread.id, + runId: active.id, + commandId: CommandId.make(`${prefix}:${tag}`), + reason, + }); + yield* threads.stopDelegatedTasks({ + threadId: projection.thread.id, + commandId: CommandId.make(`${prefix}-tasks:${tag}`), + reason, + }); + return active !== undefined; +}); + const supportedDrivers = new Set([ ProviderDriverKind.make("claudeAgent"), ProviderDriverKind.make("codex"), @@ -221,6 +254,8 @@ const make = Effect.gen(function* () { const admit = Effect.fn("KinWorkers.admit")(function* (orgId: KinOrgId) { if (!(yield* WorkerLaunchGate)) return yield* denied("Worker launches are disabled until T3 tool scoping lands."); + if (yield* isOrgPaused(orgId).pipe(Effect.provideService(SqlClient.SqlClient, sql))) + return yield* denied("Kin is paused by the owner. Nothing new starts until they resume."); if (!(yield* budget.admitWorker(orgId))) return yield* new KinChatError({ reason: "limit", @@ -243,8 +278,6 @@ const make = Effect.gen(function* () { return yield* denied("Kin workers run on an enabled Claude Code or Codex provider only."); return chosen; }); - const isLive = (run: OrchestrationV2Run) => - run.status === "queued" || ThreadManagement.isActiveRun(run); const threadLink = Effect.fn("KinWorkers.threadLink")(function* (threadId: ThreadId) { return formatThreadLink({ environmentId: yield* environment.getEnvironmentId, threadId }); }); @@ -507,22 +540,13 @@ const make = Effect.gen(function* () { const projection = yield* threads .getThreadProjection(record.threadId) .pipe(Effect.mapError(() => missing())); - const active = projection.runs.findLast(isLive); - const reason = input.reason ?? "Stopped by the Kin lead."; // A follow-up in the same wake-up starts a new run, which a second stop must still reach. - const tag = `${input.workerId}:${ctx.wakeupId}:${(active ?? projection.runs.at(-1))?.id ?? "idle"}`; - if (active !== undefined) - yield* threads.interruptThread({ - projectId: projection.thread.projectId, - threadId: record.threadId, - runId: active.id, - commandId: CommandId.make(`kin-worker-stop:${tag}`), - reason, - }); - yield* threads.stopDelegatedTasks({ - threadId: record.threadId, - commandId: CommandId.make(`kin-worker-stop-tasks:${tag}`), - reason, + const tag = `${input.workerId}:${ctx.wakeupId}:${(liveRun(projection) ?? projection.runs.at(-1))?.id ?? "idle"}`; + const stopped = yield* stopThreadTree(threads, { + projection, + prefix: "kin-worker-stop", + tag, + reason: input.reason ?? "Stopped by the Kin lead.", }); if (record.chatId !== null) yield* chat @@ -537,7 +561,7 @@ const make = Effect.gen(function* () { Effect.logWarning("Kin worker task card could not close", { cause }), ), ); - return { workerId: input.workerId, stopped: active !== undefined }; + return { workerId: input.workerId, stopped }; }, guarded("KinWorkers.stop")); return KinWorkers.of({ delegate, followUp, stop }); diff --git a/apps/server/src/kin/lead/layer.ts b/apps/server/src/kin/lead/layer.ts index 202fdfdf0fdf..48f595d41533 100644 --- a/apps/server/src/kin/lead/layer.ts +++ b/apps/server/src/kin/lead/layer.ts @@ -13,6 +13,7 @@ import * as SqlClient from "effect/sql/SqlClient"; import * as Option from "effect/Option"; import { KinChatHub } from "../chat/KinChatHub.ts"; import * as Config from "effect/Config"; +import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Schema from "effect/Schema"; @@ -31,6 +32,7 @@ import { handlers } from "./mcp/handlers.ts"; import * as KinLeadRegistry from "./KinLeadRegistry.ts"; import { KinLeadPrompt } from "./LeadPrompt.ts"; import * as WakeupRunner from "./WakeupRunner.ts"; +import * as KinStop from "./KinStop.ts"; import * as DreamRunner from "../dreaming/DreamRunner.ts"; import * as DreamSession from "../dreaming/DreamSession.ts"; import * as Workers from "./Workers.ts"; @@ -73,11 +75,15 @@ const dreamRuntime = DreamRunner.layer.pipe( const runner = Layer.effectDiscard( Effect.gen(function* () { const runner = yield* WakeupRunner.WakeupRunner; + const stop = yield* KinStop.KinStop; yield* Effect.gen(function* () { const context = yield* Layer.build(routineRuntime); yield* Layer.build(dreamRuntime.pipe(Layer.provide(Layer.succeedContext(context)))).pipe( Effect.catchCause((cause) => Effect.logError("Kin dreams unavailable", { cause })), ); + yield* stop.attach( + yield* KinStop.makeFanOut(Context.getOption(context, Routines.KinRoutines)), + ); yield* runner.start.pipe(Effect.provideContext(context)); }).pipe(forkParked); }), diff --git a/apps/server/src/kin/lead/orgState.ts b/apps/server/src/kin/lead/orgState.ts new file mode 100644 index 000000000000..00ee82bae2ed --- /dev/null +++ b/apps/server/src/kin/lead/orgState.ts @@ -0,0 +1,31 @@ +import type { KinOrgId, KinPersonId } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/sql/SqlClient"; +import { toPersistenceSqlError } from "../../persistence/Errors.ts"; + +/** The org owner owns one of its leads; the reserved bootstrap owner counts before any lead exists. */ +export const isOrgOwner = Effect.fn("KinOrgState.isOrgOwner")(function* ( + orgId: KinOrgId, + personId: KinPersonId, +) { + const sql = yield* SqlClient.SqlClient; + const rows = + yield* sql`SELECT 1 FROM kin_people p WHERE p.org_id = ${orgId} AND p.person_id = ${personId} + AND ((p.org_id = 'kin-default' AND p.person_id = 'kin-default-owner' + AND NOT EXISTS (SELECT 1 FROM kin_leads WHERE org_id = p.org_id)) OR EXISTS ( + SELECT 1 FROM kin_leads l JOIN kin_agents a ON a.org_id = l.org_id AND a.agent_id = l.agent_id + WHERE l.org_id = p.org_id AND a.owner_person_id = p.person_id)) + ORDER BY p.person_id`; + return rows.length > 0; +}); + +/** The stop switch. Paused orgs start no wake-up, worker, dream or routine. */ +export const isOrgPaused = Effect.fn("KinOrgState.isOrgPaused")( + function* (orgId: KinOrgId) { + const sql = yield* SqlClient.SqlClient; + const rows = + yield* sql`SELECT 1 FROM kin_org_state WHERE org_id = ${orgId} AND paused = 1 ORDER BY org_id`; + return rows.length > 0; + }, + Effect.mapError(toPersistenceSqlError("KinOrgState.isOrgPaused")), +); diff --git a/docs/kin/specs/lead.md b/docs/kin/specs/lead.md index 128f77585375..7be3f48ed4ab 100644 --- a/docs/kin/specs/lead.md +++ b/docs/kin/specs/lead.md @@ -133,7 +133,21 @@ export const KinEvent = Schema.Union([ pendingActionId: TrimmedNonEmptyString, outcome: Schema.Literals(["executed", "rejected", "expired", "failed"]), }), - Schema.TaggedStruct("resume", { ...EventBase, pausedAt: CanonicalUtcTimestamp }), + Schema.TaggedStruct("resume", { + ...EventBase, + pausedAt: CanonicalUtcTimestamp, + // The events expired at resume: the total and the first 50 by receipt. + expired: Schema.Struct({ + count: NonNegativeInt, + events: Schema.Array( + Schema.Struct({ + eventId: KinEventId, + tag: TrimmedNonEmptyString, + receivedAt: CanonicalUtcTimestamp, + }), + ), + }), + }), ]); ``` @@ -179,7 +193,7 @@ Coalescing: - Events that arrive while a wake-up runs wait for the next one. The runner never starts a second session for the same organization. This is the one-writer rule: memory is written only inside a wake-up, and wake-ups are serial. - Implementation: an organization permit is held for the whole wake-up, but never while waiting for that lead’s quiet window. `claim({ orgId, agentId, wakeupId, model })` waits interruptibly, takes the permit, rechecks readiness in the transaction and atomically claims up to 50 events, recording their ordered ids without incrementing `attempts`. A stale claim rolls back and waits again; an empty wake-up is never committed. The caller keeps its Effect scope open through the provider run and terminal database write; scope closure releases only the permit. LEAD-5 calls `finish(timed_out | failed)` for real failures and timeouts; graceful scope interruption uses `requeueUnfinished` without consuming an attempt. Organization signal entries stay available for idle runner subscriptions. Cross-process exclusion rests on SQLite’s running-wake-up unique index; a loser receives `KinEventQueueBusy`. -The runner owns stop/budget admission and provider execution; all event transitions go through the queue. `release({ orgId, wakeupId, eventIds?, notBefore?, countAttempt })` returns the selected events (all by default) to the queue. Pause and budget deferrals use `countAttempt: false`, which requires `notBefore`; LEAD-11 pause handling must supply `notBefore = now + pause poll interval`, so a paused organization waits instead of repeatedly claiming and releasing. Once all claimed events are released, the wake-up closes as `released` (or `failed` for a counted failure). A partial release leaves it `running` for the admitted events. `finish({ orgId, wakeupId, status })` closes the wake-up as `completed`, `failed` or `timed_out`. Completion marks the remaining events `done`; failure increments attempts and requeues them for five minutes, marking them `failed` at three attempts. `release` with `countAttempt: true` applies the same failure limit and defaults to the same five-minute delay. `release` returns `{ released, failedEventIds }` and `finish` returns `{ failedEventIds }`; the ids are the events that transitioned to terminal `failed` in that call, ordered by id. Unknown or foreign lead ids fail with `KinEventQueueUnknownAgent`, and reused wake-up ids fail with `KinEventQueueWakeupConflict`, not SQL constraint errors. Enqueue validates the lead and computes owner priority inside the insert transaction. Identical enqueue retries return `{ event, duplicate: true }` with the stored event; conflicting retries fail with `KinEventQueueConflict`. +The runner owns stop/budget admission and provider execution; all event transitions go through the queue. `release({ orgId, wakeupId, eventIds?, notBefore?, countAttempt })` returns the selected events (all by default) to the queue. Pause and budget deferrals use `countAttempt: false`, which requires `notBefore`; LEAD-11 pause handling must supply `notBefore = now + pause poll interval` (`PAUSE_POLL_INTERVAL_MS`, 5 s), so a paused organization waits instead of repeatedly claiming and releasing. Once all claimed events are released, the wake-up closes as `released` (or `failed` for a counted failure). A partial release leaves it `running` for the admitted events. `finish({ orgId, wakeupId, status })` closes the wake-up as `completed`, `failed` or `timed_out`. Completion marks the remaining events `done`; failure increments attempts and requeues them for five minutes, marking them `failed` at three attempts. `release` with `countAttempt: true` applies the same failure limit and defaults to the same five-minute delay. `release` returns `{ released, failedEventIds }` and `finish` returns `{ failedEventIds }`; the ids are the events that transitioned to terminal `failed` in that call, ordered by id. Unknown or foreign lead ids fail with `KinEventQueueUnknownAgent`, and reused wake-up ids fail with `KinEventQueueWakeupConflict`, not SQL constraint errors. Enqueue validates the lead and computes owner priority inside the insert transaction. Identical enqueue retries return `{ event, duplicate: true }` with the stored event; conflicting retries fail with `KinEventQueueConflict`. Ownership: T3's `serverOwnership.ts` takes `server-owner.sqlite` before opening the application database, guaranteeing one server per T3 home. Kin relies on that ownership and has no sidecar lock. @@ -490,16 +504,18 @@ Budget RPC payloads (wiring in LEAD-10 through `kinRpcs.ts`): `kin.budget.get` u `KinStop.pause(orgId, by)`: -1. Sets `paused` first, so nothing new starts. -2. Interrupts the running lead turn (`interruptThread`) and every `kin_workers` thread with an active run, plus `stopDelegatedTasks` on each. +1. Sets `paused` first, so nothing new starts: the runner claims nothing, `kin_delegate` and `kin_follow_up` refuse with `not_allowed`, and dreams wait. +2. Interrupts the active run on every lead thread and on every `kin_workers` thread whose `ended_at` is null, plus `stopDelegatedTasks` on each. A wake-up whose run ends this way returns its events with `release({ countAttempt: false, notBefore: now + pause poll interval })`, so no attempt is spent; a pause that lands while the runner is launching is caught by a recheck right after launch. 3. Tells the triggers spec (`KinRoutines.pauseAll(orgId)`) and the approvals spec (the executor checks `KinStop.isPaused` before every run and refuses). 4. Posts "Kin is paused. Nothing runs until you resume." to the owner's direct chat. -While paused, `enqueue` still records events, so nothing is lost. `KinStop.resume(orgId)` clears the flag and enqueues one `resume` event. That wake-up sees what arrived during the pause. Events older than 24 hours are marked `expired` and listed, not acted on. A deferral that came due during the pause is treated as `on_due: ask`, never `start`. +`KinStop` lives with the chat and budget services the WebSocket handlers use; those are built outside the orchestration runtime, so the lead runtime attaches steps 2 and 3 to it (`KinStop.attach`) and replays them for orgs already paused at startup. Pausing a paused org keeps the original `paused_at`, `paused_by` and reason and posts nothing, but runs steps 2 and 3 again, so a retry reaches work a failed fan-out missed. A fan-out failure returns `unavailable` with the org still paused. + +While paused, `enqueue` still records events, so nothing is lost. `KinStop.resume(orgId)` clears the flag, calls `KinRoutines.resumeAll(orgId)` and enqueues one `resume` event per active lead (dedupe key `resume::`). That wake-up sees what arrived during the pause. Events queued (`queued_at`, server receipt) more than 24 hours before the resume are marked `expired` in the same transaction and listed in the resume event, not acted on. Resuming an org that is not paused changes nothing. A deferral that came due during the pause is treated as `on_due: ask`, never `start`. -Payloads: `KinOrgPauseInput { orgId, reason? }`, `KinOrgResumeInput { orgId }`; both return `KinOrgState { orgId, paused, pausedAt, pausedBy, reason }`. +Payloads: `KinOrgGetInput { orgId }`, `KinOrgPauseInput { orgId, reason? }`, `KinOrgResumeInput { orgId }`; all return `KinOrgState { orgId, paused, pausedAt, pausedBy, reason }`. -RPC (**new**, through the chat spec's `kinRpcs.ts`): `kin.org.pause` and `kin.org.resume`, owner role only (checked in `RpcAuthorization`). Pausing needs no signature because it only stops things. Resuming needs an owner session. The mobile spec draws the switch in settings and as a lock screen and notification action that calls `kin.org.pause`. +RPC (**new**, through the chat spec's `kinRpcs.ts`): `kin.org.get` for any person in the organization (`orchestration:read`), and `kin.org.pause` and `kin.org.resume`, owner role only (`orchestration:operate`, then the same owner check as `kin.budget.set`). Pausing needs no signature because it only stops things. Resuming needs an owner session. No MCP tool pauses or resumes. The mobile spec draws the switch in settings and as a lock screen and notification action that calls `kin.org.pause`. ### Morning and evening routines: content @@ -559,14 +575,14 @@ Upstream files touched, one line each: ## Interfaces to other specs -| This spec provides | Used by | -| ------------------------------------------------------------- | ------------------------------------------------------------------ | -| `KinEventQueue.enqueue(event)` | chat, triggers, approvals | -| `KinStop.isPaused(orgId)`, `kin.org.pause` / `kin.org.resume` | approvals (executor), triggers (routines, dreams), mobile (switch) | -| `KinBudget.admit(orgId, priority)` | triggers (dreaming at P4) | -| `Memory.query`, cache change stream, per-org write semaphore | triggers (watchdogs, deferral scheduling, dreaming, secret scan) | -| `KinLevels.effectiveLevel(orgId, agentId, chatId?)` | approvals (refuse actions above level), chat (badge) | -| `kin_workers.provenance_json` | approvals (provenance ceiling) | +| This spec provides | Used by | +| ----------------------------------------------------------------------------- | ------------------------------------------------------------------ | +| `KinEventQueue.enqueue(event)` | chat, triggers, approvals | +| `KinStop.isPaused(orgId)`, `kin.org.get` / `kin.org.pause` / `kin.org.resume` | approvals (executor), triggers (routines, dreams), mobile (switch) | +| `KinBudget.admit(orgId, priority)` | triggers (dreaming at P4) | +| `Memory.query`, cache change stream, per-org write semaphore | triggers (watchdogs, deferral scheduling, dreaming, secret scan) | +| `KinLevels.effectiveLevel(orgId, agentId, chatId?)` | approvals (refuse actions above level), chat (badge) | +| `kin_workers.provenance_json` | approvals (provenance ceiling) | | This spec needs | From | | ------------------------------------------------------------------------------------------------------------------------ | --------- | diff --git a/packages/client-runtime/src/state/commandPermissions.test.ts b/packages/client-runtime/src/state/commandPermissions.test.ts index a1413d5d20ea..ede94e9d9afe 100644 --- a/packages/client-runtime/src/state/commandPermissions.test.ts +++ b/packages/client-runtime/src/state/commandPermissions.test.ts @@ -96,6 +96,24 @@ describe("command permissions", () => { }), ), ); + it.effect("guards the Kin stop switch with the destination operate grant", () => + Effect.scoped( + Effect.gen(function* () { + const registry = yield* setup; + for (const method of ["kin.org.pause", "kin.org.resume"] as const) { + const permissions = createCommandPermissions(runtime, method); + registry.set(sessions(env), AsyncResult.success(grant(false))); + expect(registry.get(permissions.permissionAtom(env))).toBe(false); + expect((yield* permissions.authorize(registry, env).pipe(Effect.flip))._tag).toBe( + "EnvironmentAuthorizationError", + ); + registry.set(sessions(env), AsyncResult.success(grant(true))); + expect(registry.get(permissions.permissionAtom(env))).toBe(true); + yield* permissions.authorize(registry, env); + } + }), + ), + ); it.effect("requires the destination read grant to refresh providers", () => Effect.scoped( Effect.gen(function* () { diff --git a/packages/contracts/src/kinLead.test.ts b/packages/contracts/src/kinLead.test.ts index 0a3af3b6c83b..9b1eef90e882 100644 --- a/packages/contracts/src/kinLead.test.ts +++ b/packages/contracts/src/kinLead.test.ts @@ -94,7 +94,12 @@ const eventSamples = { pendingActionId: "action-1", outcome: "executed", }, - resume: { ...event, _tag: "resume", pausedAt: at }, + resume: { + ...event, + _tag: "resume", + pausedAt: at, + expired: { count: 1, events: [{ eventId: "event-0", tag: "github", receivedAt: at }] }, + }, } satisfies Record; const samples = { @@ -167,6 +172,7 @@ const samples = { values: [{ orgId: "org-1" }, { orgId: "org-1", reason: "Stop" }], }, KinOrgResumeInput: { schema: Kin.KinOrgResumeInput, values: [{ orgId: "org-1" }] }, + KinOrgGetInput: { schema: Kin.KinOrgGetInput, values: [{ orgId: "org-1" }] }, } satisfies Record; values: ReadonlyArray }>; describe("Kin lead contracts", () => { diff --git a/packages/contracts/src/kinLead.ts b/packages/contracts/src/kinLead.ts index c3594ec11f9d..941bbb82c962 100644 --- a/packages/contracts/src/kinLead.ts +++ b/packages/contracts/src/kinLead.ts @@ -242,7 +242,21 @@ export const KinEvent = Schema.Union([ pendingActionId: TrimmedNonEmptyString, outcome: Schema.Literals(["executed", "rejected", "expired", "failed"]), }), - Schema.TaggedStruct("resume", { ...EventBase, pausedAt: CanonicalUtcTimestamp }), + Schema.TaggedStruct("resume", { + ...EventBase, + pausedAt: CanonicalUtcTimestamp, + // Events queued more than 24 hours before the resume are expired, not replayed. + expired: Schema.Struct({ + count: NonNegativeInt, + events: Schema.Array( + Schema.Struct({ + eventId: KinEventId, + tag: TrimmedNonEmptyString, + receivedAt: CanonicalUtcTimestamp, + }), + ).check(Schema.isMaxLength(50)), + }), + }), ]); export type KinEvent = typeof KinEvent.Type; @@ -298,3 +312,5 @@ export const KinOrgPauseInput = Schema.Struct({ export type KinOrgPauseInput = typeof KinOrgPauseInput.Type; export const KinOrgResumeInput = Schema.Struct({ orgId: KinOrgId }); export type KinOrgResumeInput = typeof KinOrgResumeInput.Type; +export const KinOrgGetInput = Schema.Struct({ orgId: KinOrgId }); +export type KinOrgGetInput = typeof KinOrgGetInput.Type; diff --git a/packages/contracts/src/kinRpcs.ts b/packages/contracts/src/kinRpcs.ts index f9aaf7228d15..d8277041ee14 100644 --- a/packages/contracts/src/kinRpcs.ts +++ b/packages/contracts/src/kinRpcs.ts @@ -1,36 +1,54 @@ import * as Rpc from "effect/rpc/Rpc"; import * as Schema from "effect/Schema"; -import { KinBudget, KinBudgetGetInput, KinBudgetSetInput } from "./kinLead.ts"; +import { + KinBudget, + KinBudgetGetInput, + KinBudgetSetInput, + KinOrgGetInput, + KinOrgPauseInput, + KinOrgResumeInput, + KinOrgState, +} from "./kinLead.ts"; import { KinChatError, KinChatRpcs, KIN_CHAT_RPC_SCOPES } from "./kinChat.ts"; import { EnvironmentAuthorizationError, + AuthOrchestrationOperateScope, AuthOrchestrationReadScope, AuthSettingsWriteScope, } from "./auth.ts"; +const KinError = Schema.Union([KinChatError, EnvironmentAuthorizationError]); const KinBudgetRpcs = [ - Rpc.make("kin.budget.get", { - payload: KinBudgetGetInput, - success: KinBudget, - error: Schema.Union([KinChatError, EnvironmentAuthorizationError]), - }), - Rpc.make("kin.budget.set", { - payload: KinBudgetSetInput, - success: KinBudget, - error: Schema.Union([KinChatError, EnvironmentAuthorizationError]), + Rpc.make("kin.budget.get", { payload: KinBudgetGetInput, success: KinBudget, error: KinError }), + Rpc.make("kin.budget.set", { payload: KinBudgetSetInput, success: KinBudget, error: KinError }), +] as const; +// The stop switch: pause and resume are owner-only; any member of the org may read the state. +const KinOrgRpcs = [ + Rpc.make("kin.org.get", { payload: KinOrgGetInput, success: KinOrgState, error: KinError }), + Rpc.make("kin.org.pause", { payload: KinOrgPauseInput, success: KinOrgState, error: KinError }), + Rpc.make("kin.org.resume", { + payload: KinOrgResumeInput, + success: KinOrgState, + error: KinError, }), ] as const; -export const KinChatAndBudgetRpcs = [...KinChatRpcs, ...KinBudgetRpcs] as const; -export const KinRpcs = [...KinChatAndBudgetRpcs] as const; +export const KinRpcs = [...KinChatRpcs, ...KinBudgetRpcs, ...KinOrgRpcs] as const; export const KIN_RPC_SCOPES = { ...KIN_CHAT_RPC_SCOPES, "kin.budget.get": AuthOrchestrationReadScope, "kin.budget.set": AuthSettingsWriteScope, + "kin.org.get": AuthOrchestrationReadScope, + // Stopping agent work is an orchestration operation, as force-stopping a thread is. + "kin.org.pause": AuthOrchestrationOperateScope, + "kin.org.resume": AuthOrchestrationOperateScope, } as const; export const KIN_RPC_AGGREGATES = { "kin.budget.get": "kin-budget", "kin.budget.set": "kin-budget", + "kin.org.get": "kin-org", + "kin.org.pause": "kin-org", + "kin.org.resume": "kin-org", "kin.orgs.list": "kin-chat", "kin.chats.subscribe": "kin-chat", "kin.chats.createDirect": "kin-chat", From a1105f5b4fc7d749e8c0002af6ade1afe41b165c Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Sun, 8 Nov 2026 22:28:12 +0530 Subject: [PATCH 2/3] test(kin): type the stop test's owner assertion Co-Authored-By: Claude Opus 5.5 --- apps/server/src/kin/lead/KinStop.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/server/src/kin/lead/KinStop.test.ts b/apps/server/src/kin/lead/KinStop.test.ts index cb37a8eae00e..6ca98bf98c30 100644 --- a/apps/server/src/kin/lead/KinStop.test.ts +++ b/apps/server/src/kin/lead/KinStop.test.ts @@ -97,7 +97,7 @@ describe("KinStop", () => { assert.include(ThreadManagement.ACTIVE_RUN_STATUSES, yield* live(workerThread)); const paused = yield* stop.pause(owner, { orgId, reason: "Testing" }); - assert.include(paused, { paused: true, pausedBy: "owner", reason: "Testing" }); + assert.include(paused, { paused: true, pausedBy: owner[0]!.personId, reason: "Testing" }); yield* worker.drain(); yield* Fiber.join(wakeup); From 1ca9f7b649c2bed9126752968921d01267c061a1 Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Mon, 9 Nov 2026 00:07:47 +0530 Subject: [PATCH 3/3] fix(kin): stop switch reaches dreams, survives races and tells the owner on resume - Fan-out also interrupts running dream threads; DreamRunner records the turn's thread before it starts and fails a dream that sees the pause after a turn or its push, so its branch is deleted, not merged. - Pause plus fan-out and resume plus resumeAll hold a per-org lock; attach calls resumeAll for resumed orgs, healing a detached or failed resume. - kin_delegate and kin_follow_up recheck the pause after launch or send and stop the new run. - Each fan-out thread stop runs under Effect.exit; pauseAll always runs and the fan-out fails at the end if anything failed. - Resume posts "Kin resumed." with the expired count to the owner chat. - Fan-out command ids carry a per-boot nonce: recovery keeps run ids. - Tests for each, including the runner's post-launch recheck. Co-Authored-By: Claude Opus 5.5 --- .../src/kin/dreaming/DreamRunner.test.ts | 20 +- apps/server/src/kin/dreaming/DreamRunner.ts | 15 +- apps/server/src/kin/dreaming/DreamSession.ts | 5 +- apps/server/src/kin/lead/KinStop.test.ts | 251 ++++++++++++++++-- apps/server/src/kin/lead/KinStop.ts | 162 +++++++---- apps/server/src/kin/lead/Workers.test.ts | 44 +++ apps/server/src/kin/lead/Workers.ts | 24 +- apps/server/src/kin/lead/orgState.ts | 2 + docs/kin/specs/lead.md | 10 +- 9 files changed, 442 insertions(+), 91 deletions(-) diff --git a/apps/server/src/kin/dreaming/DreamRunner.test.ts b/apps/server/src/kin/dreaming/DreamRunner.test.ts index 8eb10185ae0d..2093a3fdda3e 100644 --- a/apps/server/src/kin/dreaming/DreamRunner.test.ts +++ b/apps/server/src/kin/dreaming/DreamRunner.test.ts @@ -54,6 +54,8 @@ type Behavior = | "token" | "failed-turn" | "contradiction" + /** The owner presses stop while the extract turn runs; the turn itself completes. */ + | "paused" | { readonly hold: Deferred.Deferred }; const newClaim = (body = "Invented report ships Friday.\n") => @@ -91,6 +93,8 @@ const fixture = ( readonly root: string; readonly behave: (behavior: Behavior) => void; readonly turns: Array; + /** `kin_dream_runs.thread_id` as each turn saw it while running. */ + readonly turnThreads: Array; readonly notices: Array; readonly refuseBudget: () => void; readonly hooks: RunnerHooks; @@ -107,6 +111,7 @@ const fixture = ( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const vcs = yield* Vcs.VcsProcess; + const db = yield* SqlClient.SqlClient; const root = yield* fs.realPath(yield* fs.makeTempDirectoryScoped()); const source = `${root}/seed`; const remote = `${root}/remote.git`; @@ -158,13 +163,20 @@ const fixture = ( let budgetAllowed = true; const hooks: RunnerHooks = { beforeDelete: Effect.void, removed: () => Effect.void }; const turns: Array = []; + const turnThreads: Array = []; const notices: Array = []; const session = Layer.succeed(DreamSession.DreamSession, { run: (turn) => Effect.gen(function* () { turns.push(turn); const threadId = ThreadId.make(`kin-dream:${turn.dreamId}:${turn.turn}`); + const [recorded] = yield* db<{ + threadId: string | null; + }>`SELECT thread_id AS "threadId" FROM kin_dream_runs WHERE org_id = ${orgId} AND dream_id = ${turn.dreamId} ORDER BY dream_id`; + turnThreads.push(recorded?.threadId ?? null); const current = behavior; + if (current === "paused") + yield* db`INSERT INTO kin_org_state (org_id, paused, paused_at, paused_by) VALUES (${orgId}, 1, 0, 'owner')`; if (typeof current === "object") { yield* Deferred.succeed(current.hold, undefined); return yield* Effect.never; @@ -312,6 +324,7 @@ const fixture = ( behavior = next; }, turns, + turnThreads, notices, refuseBudget: () => { budgetAllowed = false; @@ -504,10 +517,11 @@ it.live.each([ { behavior: "secret", failure: "The secret scan found" }, { behavior: "token", failure: "a live T3 credential" }, { behavior: "failed-turn", failure: "The dream session failed" }, + { behavior: "paused", failure: "Kin was paused during the dream" }, ] as const)( "a $behavior dream fails, pushes nothing and tells the owner", ({ behavior, failure }) => - fixture(({ git, remote, root, main, behave, notices }) => + fixture(({ git, remote, root, main, behave, notices, turns, turnThreads }) => Effect.gen(function* () { behave(behavior); const runner = yield* DreamRunner.DreamRunner; @@ -523,6 +537,10 @@ it.live.each([ expect(notices).toHaveLength(1); expect(notices[0]).toContain("merged nothing"); expect(notices[0]).not.toContain("ghp_"); + // The stop switch's fan-out finds the running turn's thread. + expect(turnThreads).toEqual( + turns.map((turn) => DreamSession.dreamThreadId(turn.dreamId, turn.turn)), + ); }), ), ); diff --git a/apps/server/src/kin/dreaming/DreamRunner.ts b/apps/server/src/kin/dreaming/DreamRunner.ts index 90e713268c51..6fdfd8a24646 100644 --- a/apps/server/src/kin/dreaming/DreamRunner.ts +++ b/apps/server/src/kin/dreaming/DreamRunner.ts @@ -25,6 +25,7 @@ import { nextScheduledRunAt } from "../../scheduledTasks/Schedule.ts"; import * as Scheduler from "../../scheduling/Scheduler.ts"; import { VcsProcess } from "../../vcs/VcsProcess.ts"; import * as KinBudget from "../lead/KinBudget.ts"; +import { isOrgPaused } from "../lead/orgState.ts"; import * as Memory from "../memory/Memory.ts"; import * as MemoryCache from "../memory/MemoryCache.ts"; import * as Writer from "../memory/MemoryWriter.ts"; @@ -308,6 +309,8 @@ const make = Effect.gen(function* () { branch, failures, }); + const paused = isOrgPaused(orgId).pipe(Effect.provideService(SqlClient.SqlClient, sql)); + const pausedFailure = "Kin was paused during the dream"; if (Option.isNone(writer)) return failed(["The memory writer is not available"]); const base = yield* writer.value.dreamBase(orgId); yield* sql`UPDATE kin_dream_runs SET base_head = ${base.head} @@ -329,6 +332,10 @@ const make = Effect.gen(function* () { model: ModelSelection, text: string, ) { + if (yield* paused) return "paused"; + // Recorded before the turn starts, so the stop switch's fan-out can interrupt it. + yield* sql`UPDATE kin_dream_runs SET thread_id = ${DreamSession.dreamThreadId(dreamId, name)} + WHERE org_id = ${orgId} AND dream_id = ${dreamId} AND status = 'running' RETURNING dream_id`; const result = yield* session.run({ orgId, dreamId, @@ -337,15 +344,14 @@ const make = Effect.gen(function* () { model, text, }); - yield* sql`UPDATE kin_dream_runs SET thread_id = ${result.threadId} - WHERE org_id = ${orgId} AND dream_id = ${dreamId} AND status = 'running' RETURNING dream_id`; - return result.status; + return (yield* paused) ? "paused" : result.status; }); const extracted = yield* turn( "extract", claimed.models.cheap, extractPrompt({ orgId, head: base.head, full }), ); + if (extracted === "paused") return failed([pausedFailure]); if (extracted !== "completed") return failed([`The dream session ${extracted}`]); let checkout = yield* readCheckout(root); const draft = @@ -357,6 +363,7 @@ const make = Effect.gen(function* () { draft.value.contradictions.some((entry) => !entry.resolution && !entry.question) ) { const resolved = yield* turn("resolve", claimed.models.strong, resolvePrompt); + if (resolved === "paused") return failed([pausedFailure]); if (resolved !== "completed") return failed([`The contradiction session ${resolved}`]); checkout = yield* readCheckout(root); } @@ -401,6 +408,8 @@ const make = Effect.gen(function* () { files: checked.changes, message: `kin(dream): ${ownerDate}\n\nKin-Dream: ${dreamId}\nMaterial-Until: ${iso(claimed.until)}\n`, }); + // A pause during the checks or the push fails the dream, and settling deletes its branch. + if (yield* paused) return failed([pausedFailure]); return { _tag: "succeeded", dreamId, diff --git a/apps/server/src/kin/dreaming/DreamSession.ts b/apps/server/src/kin/dreaming/DreamSession.ts index 117baad50073..14d8fc717034 100644 --- a/apps/server/src/kin/dreaming/DreamSession.ts +++ b/apps/server/src/kin/dreaming/DreamSession.ts @@ -65,6 +65,9 @@ export class DreamSession extends Context.Service< } >()("t3/kin/dreaming/DreamSession") {} +export const dreamThreadId = (dreamId: string, turn: DreamTurn["turn"]) => + ThreadId.make(`${threadPrefix}${dreamId}:${turn}`); + const timeoutMs = 900_000; const isRefused = Schema.is(DreamRefused); @@ -100,7 +103,7 @@ export const layer = Layer.effect( title: `Kin dreams · ${turn.orgId}`, }); const key = `${turn.dreamId}:${turn.turn}`; - const threadId = ThreadId.make(`${threadPrefix}${key}`); + const threadId = dreamThreadId(turn.dreamId, turn.turn); const messageId = MessageId.make(`kin-dream-msg:${key}`); const sent = yield* launch.launch({ commandId: CommandId.make(`kin-dream:${key}`), diff --git a/apps/server/src/kin/lead/KinStop.test.ts b/apps/server/src/kin/lead/KinStop.test.ts index 6ca98bf98c30..1de70f64b88c 100644 --- a/apps/server/src/kin/lead/KinStop.test.ts +++ b/apps/server/src/kin/lead/KinStop.test.ts @@ -12,6 +12,7 @@ import * as Clock from "effect/Clock"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; +import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as DateTime from "effect/DateTime"; import * as SqlClient from "effect/sql/SqlClient"; @@ -21,6 +22,7 @@ import * as EffectWorker from "../../orchestration-v2/EffectWorker.ts"; import * as ThreadLaunch from "../../orchestration-v2/ThreadLaunchService.ts"; import * as ThreadManagement from "../../orchestration-v2/ThreadManagementService.ts"; import { provideDeterministicTestRuntime } from "../../orchestration-v2/testkit/DeterministicRuntime.ts"; +import * as Routines from "../triggers/routines.ts"; import * as KinEventQueue from "./KinEventQueue.ts"; import * as KinStop from "./KinStop.ts"; import * as WakeupRunner from "./WakeupRunner.ts"; @@ -28,6 +30,48 @@ import { agentId, event, harness, orgId, seed } from "./WakeupRunner.testkit.ts" const owner = [{ _tag: "person" as const, orgId, personId: KinPersonId.make("owner") }]; +/** Starts a fake-provider run on a new thread; under `untilInterrupted` it runs until stopped. */ +const launchThread = Effect.fn("KinStopTest.launchThread")(function* (threadId: ThreadId) { + const launch = yield* ThreadLaunch.ThreadLaunchService; + yield* launch.launch({ + commandId: CommandId.make(`launch:${threadId}`), + threadId, + projectId: ProjectId.make("kin:org"), + title: "Work", + modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5.1-codex" }, + runtimeMode: "full-access", + interactionMode: "default", + workspaceStrategy: { type: "root" }, + initialMessage: { messageId: MessageId.make(`msg:${threadId}`), text: "Work", attachments: [] }, + createdBy: "agent", + creationSource: "server", + }); +}); +const lastRunStatus = (threadId: ThreadId) => + Effect.flatMap(ThreadManagement.ThreadManagementService, (threads) => + threads + .getThreadProjection(threadId) + .pipe(Effect.map((projection) => projection.runs.at(-1)?.status)), + ); +/** Routines that record each pause and resume; `pauseAll` may be slowed down. */ +const recordingRoutines = ( + log: Array, + beforePause: Effect.Effect = Effect.void, + resumeAll: Routines.KinRoutines["Service"]["resumeAll"] = () => + Effect.sync(() => void log.push("resume")), +) => + Effect.asSome( + Routines.KinRoutines.pipe( + Effect.provide( + Layer.mock(Routines.KinRoutines)({ + pauseAll: () => + beforePause.pipe(Effect.andThen(Effect.sync(() => void log.push("pause")))), + resumeAll, + }), + ), + ), + ); + describe("KinStop", () => { it.effect( "stops a running wake-up and worker, holds their events and stays paused when pressed twice", @@ -41,7 +85,6 @@ describe("KinStop", () => { const runner = yield* WakeupRunner.WakeupRunner; const stop = yield* KinStop.KinStop; const threads = yield* ThreadManagement.ThreadManagementService; - const launch = yield* ThreadLaunch.ThreadLaunchService; const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; const delegated: Array = []; yield* stop.attach( @@ -67,34 +110,17 @@ describe("KinStop", () => { const leadThread = ThreadId.make(running!.threadId); const workerThread = ThreadId.make("kin-worker-stop-test"); - yield* launch.launch({ - commandId: CommandId.make("kin-worker:stop-test"), - threadId: workerThread, - projectId: ProjectId.make("kin:org"), - title: "Worker", - modelSelection: { - instanceId: ProviderInstanceId.make("codex"), - model: "gpt-5.1-codex", - }, - runtimeMode: "full-access", - interactionMode: "default", - workspaceStrategy: { type: "root" }, - initialMessage: { - messageId: MessageId.make("kin-worker-msg:stop-test"), - text: "Work", - attachments: [], - }, - createdBy: "agent", - creationSource: "server", - }); + yield* launchThread(workerThread); yield* sql`INSERT INTO kin_workers (worker_id, org_id, agent_id, wakeup_id, kind, thread_id, model_json, provenance_json, status, started_at) VALUES ('worker', ${orgId}, ${agentId}, ${running!.wakeupId}, 'build', ${workerThread}, '{}', '{"_tag":"system"}', 'running', 0)`; + const dreamThread = ThreadId.make("kin-dream:dream-stop-test:extract"); + yield* launchThread(dreamThread); + yield* sql`INSERT INTO kin_dream_runs (org_id, owner_date, dream_id, full, status, branch, material_from, material_until, started_at, thread_id) + VALUES (${orgId}, '2026-11-08', 'dream-stop-test', 0, 'running', 'dream/2026-11-08', 0, 0, 0, ${dreamThread})`; yield* worker.drain(); - const live = (threadId: ThreadId) => - threads - .getThreadProjection(threadId) - .pipe(Effect.map((projection) => projection.runs.at(-1)?.status)); + const live = lastRunStatus; assert.include(ThreadManagement.ACTIVE_RUN_STATUSES, yield* live(workerThread)); + assert.include(ThreadManagement.ACTIVE_RUN_STATUSES, yield* live(dreamThread)); const paused = yield* stop.pause(owner, { orgId, reason: "Testing" }); assert.include(paused, { paused: true, pausedBy: owner[0]!.personId, reason: "Testing" }); @@ -103,7 +129,8 @@ describe("KinStop", () => { assert.strictEqual(yield* live(leadThread), "interrupted"); assert.strictEqual(yield* live(workerThread), "interrupted"); - assert.sameMembers(delegated, [leadThread, workerThread]); + assert.strictEqual(yield* live(dreamThread), "interrupted"); + assert.sameMembers(delegated, [leadThread, workerThread, dreamThread]); assert.deepStrictEqual( yield* sql`SELECT status FROM kin_wakeups WHERE org_id = ${orgId} ORDER BY wakeup_id`, [{ status: "released" }], @@ -157,6 +184,13 @@ describe("KinStop", () => { }); assert.deepStrictEqual(yield* stop.resume(owner, { orgId }), resumed); assert.isFalse(yield* stop.isPaused(orgId)); + assert.deepStrictEqual( + yield* sql`SELECT author_kind, body FROM kin_messages WHERE org_id = ${orgId} ORDER BY seq`, + [ + { author_kind: "system", body: KinStop.PAUSED_NOTICE }, + { author_kind: "system", body: "Kin resumed. 1 queued event expired." }, + ], + ); assert.deepStrictEqual( yield* sql`SELECT event_id, state FROM kin_events WHERE org_id = ${orgId} AND tag != 'resume' ORDER BY event_id`, [ @@ -190,4 +224,169 @@ describe("KinStop", () => { ); }).pipe(Effect.provide(harness()), provideDeterministicTestRuntime), ); + it.effect( + "a pause that lands while the runner launches interrupts the new run and returns its events", + () => + Effect.gen(function* () { + const ready = yield* Deferred.make(); + // Pausing inside the wake-up, with no fan-out attached, leaves only the post-launch recheck. + const prepare = () => + Deferred.await(ready).pipe( + Effect.flatMap((stop) => stop.pause(owner, { orgId })), + Effect.orDie, + Effect.as({ head: "abc", indexAsOf: DateTime.makeUnsafe("2026-11-08T00:00:00.000Z") }), + ); + yield* Effect.gen(function* () { + yield* seed; + const sql = yield* SqlClient.SqlClient; + const queue = yield* KinEventQueue.KinEventQueue; + const runner = yield* WakeupRunner.WakeupRunner; + yield* Deferred.succeed(ready, yield* KinStop.KinStop); + yield* queue.enqueue(event("work")); + const wakeup = yield* runner.runOnce(orgId, agentId).pipe(Effect.forkChild); + yield* TestClock.adjust("20 seconds"); + yield* Fiber.join(wakeup); + const [row] = yield* sql<{ + status: string; + threadId: string; + }>`SELECT status, thread_id AS "threadId" FROM kin_wakeups WHERE org_id = ${orgId} ORDER BY wakeup_id`; + assert.strictEqual(row!.status, "released"); + assert.strictEqual(yield* lastRunStatus(ThreadId.make(row!.threadId)), "interrupted"); + const now = yield* Clock.currentTimeMillis; + assert.deepStrictEqual( + yield* sql`SELECT state, attempts, not_before FROM kin_events WHERE org_id = ${orgId} ORDER BY event_id`, + [{ state: "queued", attempts: 0, not_before: now + KinStop.PAUSE_POLL_INTERVAL_MS }], + ); + }).pipe(Effect.provide(harness({ untilInterrupted: true, prepare }))); + }).pipe(provideDeterministicTestRuntime), + ); + + it.effect("a thread that cannot be stopped still lets the others and routines pause", () => + Effect.gen(function* () { + yield* seed; + const sql = yield* SqlClient.SqlClient; + const stop = yield* KinStop.KinStop; + const threads = yield* ThreadManagement.ThreadManagementService; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const log: Array = []; + const broken = ThreadId.make("broken-lead-thread"); + yield* stop.attach( + yield* KinStop.makeFanOut(yield* recordingRoutines(log)).pipe( + Effect.provideService(ThreadManagement.ThreadManagementService, { + ...threads, + getThreadProjection: (threadId) => + threadId === broken + ? Effect.die("unreadable projection") + : threads.getThreadProjection(threadId), + }), + ), + ); + yield* sql`UPDATE kin_leads SET thread_id = ${broken} WHERE org_id = ${orgId} AND agent_id = ${agentId}`; + const workerThread = ThreadId.make("kin-worker-stop-test"); + yield* launchThread(workerThread); + yield* sql`INSERT INTO kin_wakeups (wakeup_id, org_id, agent_id, model, started_at, status, event_ids) VALUES ('wake', ${orgId}, ${agentId}, 'test', 0, 'completed', '[]')`; + yield* sql`INSERT INTO kin_workers (worker_id, org_id, agent_id, wakeup_id, kind, thread_id, model_json, provenance_json, status, started_at) + VALUES ('worker', ${orgId}, ${agentId}, 'wake', 'build', ${workerThread}, '{}', '{"_tag":"system"}', 'running', 0)`; + yield* worker.drain(); + + const failure = yield* Effect.flip(stop.pause(owner, { orgId })); + assert.include(failure, { _tag: "KinChatError", reason: "unavailable" }); + yield* worker.drain(); + assert.strictEqual(yield* lastRunStatus(workerThread), "interrupted"); + assert.deepStrictEqual(log, ["pause"]); + assert.isTrue(yield* stop.isPaused(orgId)); + }).pipe(Effect.provide(harness({ untilInterrupted: true })), provideDeterministicTestRuntime), + ); + + it.effect("a resume during a slow pause fan-out waits for it and leaves routines running", () => + Effect.gen(function* () { + yield* seed; + const stop = yield* KinStop.KinStop; + const entered = yield* Deferred.make(); + const gate = yield* Deferred.make(); + const log: Array = []; + yield* stop.attach( + yield* KinStop.makeFanOut( + yield* recordingRoutines( + log, + Deferred.succeed(entered, undefined).pipe(Effect.andThen(Deferred.await(gate))), + ), + ), + ); + const pausing = yield* stop.pause(owner, { orgId }).pipe(Effect.forkChild); + yield* Deferred.await(entered); + const resuming = yield* Effect.sleep("1 second").pipe( + Effect.andThen(stop.resume(owner, { orgId })), + Effect.forkChild, + ); + // Runs the resume until it finishes or blocks, with routines still pausing. + yield* TestClock.adjust("1 second"); + yield* Deferred.succeed(gate, undefined); + yield* Fiber.join(pausing); + yield* Fiber.join(resuming); + assert.deepStrictEqual(log, ["pause", "resume"]); + assert.isFalse(yield* stop.isPaused(orgId)); + }).pipe(Effect.provide(harness()), provideDeterministicTestRuntime), + ); + + it.effect("attach resumes routines a resume left paused while detached or failing", () => + Effect.gen(function* () { + yield* seed; + const stop = yield* KinStop.KinStop; + const log: Array = []; + const failing = yield* recordingRoutines(log, Effect.void, () => + Effect.fail(new Routines.KinRoutinesError({ orgId, cause: "unavailable" })), + ); + yield* Effect.scoped( + Effect.gen(function* () { + yield* stop.attach(yield* KinStop.makeFanOut(failing)); + yield* stop.pause(owner, { orgId }); + yield* stop.resume(owner, { orgId }); + }), + ); + // A restart: this pause and resume run before the lead runtime attaches. + yield* TestClock.adjust("1 second"); + yield* stop.pause(owner, { orgId }); + yield* stop.resume(owner, { orgId }); + assert.deepStrictEqual(log, ["pause"]); + + yield* stop.attach(yield* KinStop.makeFanOut(yield* recordingRoutines(log))); + assert.deepStrictEqual(log, ["pause", "resume"]); + assert.isFalse(yield* stop.isPaused(orgId)); + }).pipe(Effect.provide(harness()), provideDeterministicTestRuntime), + ); + + it.effect("a startup replay stops with new command ids; a repeat press reuses them", () => + Effect.gen(function* () { + yield* seed; + const sql = yield* SqlClient.SqlClient; + const stop = yield* KinStop.KinStop; + const threads = yield* ThreadManagement.ThreadManagementService; + const commandIds: Array = []; + const fanOut = KinStop.makeFanOut(Option.none()).pipe( + Effect.provideService(ThreadManagement.ThreadManagementService, { + ...threads, + stopDelegatedTasks: (input) => + Effect.sync(() => commandIds.push(input.commandId)).pipe( + Effect.andThen(threads.stopDelegatedTasks(input)), + ), + }), + ); + const leadThread = ThreadId.make("kin-lead-stop-test"); + yield* launchThread(leadThread); + yield* sql`UPDATE kin_leads SET thread_id = ${leadThread} WHERE org_id = ${orgId} AND agent_id = ${agentId}`; + yield* Effect.scoped( + Effect.gen(function* () { + yield* stop.attach(yield* fanOut); + yield* stop.pause(owner, { orgId }); + yield* stop.pause(owner, { orgId }); + }), + ); + // Recovery keeps the run id, so only the boot tells the replay from the first fan-out. + yield* stop.attach(yield* fanOut); + assert.lengthOf(commandIds, 3); + assert.strictEqual(commandIds[1], commandIds[0]); + assert.notStrictEqual(commandIds[2], commandIds[1]); + }).pipe(Effect.provide(harness()), provideDeterministicTestRuntime), + ); }); diff --git a/apps/server/src/kin/lead/KinStop.ts b/apps/server/src/kin/lead/KinStop.ts index 7a66e110d14a..5909efc79f6f 100644 --- a/apps/server/src/kin/lead/KinStop.ts +++ b/apps/server/src/kin/lead/KinStop.ts @@ -10,13 +10,17 @@ import { type KinOrgPauseInput, type KinOrgResumeInput, } from "@t3tools/contracts"; +import * as Cause from "effect/Cause"; import * as Clock from "effect/Clock"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; +import * as Random from "effect/Random"; import * as Schema from "effect/Schema"; +import * as Semaphore from "effect/Semaphore"; import type * as Scope from "effect/Scope"; import * as SqlClient from "effect/sql/SqlClient"; import { isSqlError } from "effect/sql/SqlError"; @@ -28,7 +32,7 @@ import type { PersonIdentity } from "../chat/KinIdentity.ts"; import * as Chat from "../chat/KinChatRepository.ts"; import type * as Routines from "../triggers/routines.ts"; import * as Queue from "./KinEventQueue.ts"; -import { isOrgOwner, isOrgPaused } from "./orgState.ts"; +import { isOrgOwner, isOrgPaused, STOP_REASON } from "./orgState.ts"; import { stopThreadTree } from "./Workers.ts"; /** A claim that races a pause returns its events for this long, so a paused org does not spin. */ @@ -36,7 +40,10 @@ export const PAUSE_POLL_INTERVAL_MS = 5_000; /** Resume expires events queued longer ago than this instead of replaying them. */ const PAUSE_EXPIRY_MS = 24 * 60 * 60 * 1_000; export const PAUSED_NOTICE = "Kin is paused. Nothing runs until you resume."; -const STOP_REASON = "Stopped by the Kin stop switch."; +const resumedNotice = (expired: number) => + expired === 0 + ? "Kin resumed." + : `Kin resumed. ${expired} queued ${expired === 1 ? "event" : "events"} expired.`; const EXPIRED_LISTED = 50; export class KinStopFanOutError extends Schema.TaggedError()( @@ -100,6 +107,17 @@ const make = Effect.gen(function* () { const chat = yield* Chat.KinChatRepository; const hub = yield* Hub.KinChatHub; let attached: KinStopFanOut | undefined; + // Pause plus its fan-out and resume plus resumeAll run one at a time per org, so the last + // routine write always matches the stop switch. + const locks = new Map(); + const withOrgLock = (orgId: KinOrgId, effect: Effect.Effect) => { + let lock = locks.get(orgId); + if (lock === undefined) { + lock = Semaphore.makeUnsafe(1); + locks.set(orgId, lock); + } + return lock.withPermits(1)(effect); + }; const state = Effect.fn("KinStop.state")(function* (orgId: KinOrgId) { const [row] = yield* sql<{ @@ -143,7 +161,12 @@ const make = Effect.gen(function* () { if (!allowed) return yield* notAllowed; return person; }); - const notice = Effect.fn("KinStop.notice")(function* (orgId: KinOrgId, now: number) { + const notice = Effect.fn("KinStop.notice")(function* ( + orgId: KinOrgId, + now: number, + kind: "pause" | "resume", + body: string, + ) { const chats = yield* sql<{ chatId: string; }>`SELECT DISTINCT owner_chat_id AS "chatId" FROM kin_leads WHERE org_id = ${orgId} ORDER BY owner_chat_id`; @@ -151,10 +174,10 @@ const make = Effect.gen(function* () { chat.insertMessage({ orgId, chatId: KinChatId.make(row.chatId), - messageId: KinMessageId.make(`kin-pause:${now}:${row.chatId}`), + messageId: KinMessageId.make(`kin-${kind}:${now}:${row.chatId}`), author: { _tag: "system" }, kind: "text", - body: PAUSED_NOTICE, + body, intent: null, mentions: [], replyTo: null, @@ -176,9 +199,22 @@ const make = Effect.gen(function* () { ); }); + const publish = Effect.fn("KinStop.publish")(function* ( + orgId: KinOrgId, + messages: ReadonlyArray, + ) { + for (const message of messages) yield* hub.publish({ type: "message.posted", orgId, message }); + }); + const pause = Effect.fn("KinStop.pause")(function* ( people: ReadonlyArray, input: KinOrgPauseInput, + ) { + return yield* withOrgLock(input.orgId, pauseLocked(people, input)); + }, Effect.mapError(mapError)); + const pauseLocked = Effect.fn("KinStop.pauseLocked")(function* ( + people: ReadonlyArray, + input: KinOrgPauseInput, ) { const result = yield* sql.withTransaction( Effect.gen(function* () { @@ -196,30 +232,35 @@ const make = Effect.gen(function* () { paused_by = ${person.personId}, reason = ${reason} WHERE org_id = ${input.orgId} AND paused = 0 RETURNING org_id`; const messages: ReadonlyArray = - changed.length === 0 ? [] : yield* notice(input.orgId, now); + changed.length === 0 ? [] : yield* notice(input.orgId, now, "pause", PAUSED_NOTICE); return { state: yield* state(input.orgId), messages }; }), ); - for (const message of result.messages) - yield* hub.publish({ type: "message.posted", orgId: input.orgId, message }); + yield* publish(input.orgId, result.messages); if (result.state.paused) yield* fanOut( input.orgId, DateTime.toEpochMillis(DateTime.makeUnsafe(result.state.pausedAt)), ); return result.state; - }, Effect.mapError(mapError)); + }); const resume = Effect.fn("KinStop.resume")(function* ( people: ReadonlyArray, input: KinOrgResumeInput, + ) { + return yield* withOrgLock(input.orgId, resumeLocked(people, input)); + }, Effect.mapError(mapError)); + const resumeLocked = Effect.fn("KinStop.resumeLocked")(function* ( + people: ReadonlyArray, + input: KinOrgResumeInput, ) { const resumed = yield* sql.withTransaction( Effect.gen(function* () { yield* owner(people, input.orgId); const [cleared] = yield* sql<{ pausedAt: number }>`SELECT paused_at AS "pausedAt" FROM kin_org_state WHERE org_id = ${input.orgId} AND paused = 1 ORDER BY org_id`; - if (cleared === undefined) return false; + if (cleared === undefined) return undefined; yield* sql`UPDATE kin_org_state SET paused = 0, paused_at = NULL, paused_by = NULL, reason = NULL WHERE org_id = ${input.orgId} AND paused = 1`; const now = yield* Clock.currentTimeMillis; @@ -266,12 +307,15 @@ const make = Effect.gen(function* () { }), ); } - return true; + return yield* notice(input.orgId, now, "resume", resumedNotice(expired.length)); }), ); - if (resumed && attached !== undefined) yield* attached.resume(input.orgId); + if (resumed !== undefined) { + yield* publish(input.orgId, resumed); + if (attached !== undefined) yield* attached.resume(input.orgId); + } return yield* state(input.orgId); - }, Effect.mapError(mapError)); + }); return KinStop.of({ isPaused: Effect.fn("KinStop.isPaused")((orgId) => @@ -293,64 +337,78 @@ const make = Effect.gen(function* () { if (attached === next) attached = undefined; }), ); - // A pause from before a restart still owes its fan-out to work recovered since. - const paused = yield* sql<{ + // A pause from before a restart still owes its fan-out to work recovered since, and a + // resume that ran detached or failed its resumeAll still owes the routines theirs. + const orgs = yield* sql<{ orgId: KinOrgId; - pausedAt: number; - }>`SELECT org_id AS "orgId", paused_at AS "pausedAt" FROM kin_org_state WHERE paused = 1 ORDER BY org_id`.pipe( + }>`SELECT org_id AS "orgId" FROM kin_org_state ORDER BY org_id`.pipe( Effect.catchCause((cause) => Effect.logError("Kin stop catch-up unavailable", { cause }).pipe(Effect.as([])), ), ); - for (const row of paused) yield* fanOut(row.orgId, row.pausedAt).pipe(Effect.ignore); + for (const { orgId } of orgs) + yield* withOrgLock( + orgId, + Effect.gen(function* () { + const current = yield* state(orgId); + if (current.pausedAt !== null) + yield* fanOut(orgId, DateTime.toEpochMillis(current.pausedAt)).pipe(Effect.ignore); + else yield* next.resume(orgId); + }), + ).pipe( + Effect.catchCause((cause) => + Effect.logError("Kin stop catch-up failed", { orgId, cause }), + ), + ); }), }); }); export const layer = Layer.effect(KinStop, make); -/** The lead runtime's fan-out: lead runs, worker runs and their delegated tasks, then routines. */ +/** The lead runtime's fan-out: lead, worker and dream runs and their delegated tasks, then routines. */ export const makeFanOut = Effect.fn("KinStop.makeFanOut")(function* ( routines: Option.Option, ) { const sql = yield* SqlClient.SqlClient; const threads = yield* ThreadManagement.ThreadManagementService; + // Recovery keeps run ids across a restart, so a replay must not dedupe on a pre-crash receipt. + const boot = (yield* Random.nextInt).toString(36); return { - pause: Effect.fn("KinStop.fanOut.pause")( - function* (orgId: KinOrgId, pausedAt: number) { - const rows = yield* sql<{ threadId: string }>` + pause: Effect.fn("KinStop.fanOut.pause")(function* (orgId: KinOrgId, pausedAt: number) { + const failed = (cause: unknown) => new KinStopFanOutError({ orgId, cause }); + const rows = yield* sql<{ threadId: string }>` SELECT thread_id AS "threadId" FROM kin_leads WHERE org_id = ${orgId} AND thread_id IS NOT NULL UNION SELECT thread_id FROM kin_workers WHERE org_id = ${orgId} AND ended_at IS NULL - ORDER BY 1`; - yield* Effect.forEach( - rows, - (row) => - Effect.gen(function* () { - const threadId = ThreadId.make(row.threadId); - const projection = yield* threads.getThreadProjection(threadId).pipe(Effect.option); - if (Option.isNone(projection)) return; - const run = projection.value.runs.at(-1)?.id ?? "idle"; - yield* stopThreadTree(threads, { - projection: projection.value, - prefix: "kin-stop", - tag: `${threadId}:${pausedAt}:${run}`, - reason: STOP_REASON, - }); - }), - { concurrency: "unbounded", discard: true }, - ); - if (Option.isSome(routines)) - yield* routines.value - .pauseAll(orgId) - .pipe( - Effect.catchCause((cause) => - Effect.logWarning("Kin routines could not pause", { orgId, cause }), - ), - ); - }, - (effect, orgId) => - effect.pipe(Effect.mapError((cause) => new KinStopFanOutError({ orgId, cause }))), - ), + UNION SELECT thread_id FROM kin_dream_runs + WHERE org_id = ${orgId} AND status = 'running' AND thread_id IS NOT NULL + ORDER BY 1`.pipe(Effect.mapError(failed)); + const exits = yield* Effect.forEach( + rows, + (row) => + Effect.gen(function* () { + const threadId = ThreadId.make(row.threadId); + const projection = yield* threads.getThreadProjection(threadId).pipe(Effect.option); + if (Option.isNone(projection)) return; + const run = projection.value.runs.at(-1)?.id ?? "idle"; + yield* stopThreadTree(threads, { + projection: projection.value, + prefix: "kin-stop", + tag: `${threadId}:${pausedAt}:${run}:${boot}`, + reason: STOP_REASON, + }); + }).pipe(Effect.exit), + { concurrency: "unbounded" }, + ); + const routineExit = Option.isSome(routines) + ? yield* Effect.exit(routines.value.pauseAll(orgId)) + : Exit.void; + const failures: Array> = []; + for (const exit of exits) if (Exit.isFailure(exit)) failures.push(exit.cause); + if (Exit.isFailure(routineExit)) failures.push(routineExit.cause); + if (failures.length > 0) + return yield* failed(failures.reduce((all, cause) => Cause.combine(all, cause))); + }), resume: Effect.fn("KinStop.fanOut.resume")(function* (orgId: KinOrgId) { if (Option.isSome(routines)) yield* routines.value diff --git a/apps/server/src/kin/lead/Workers.test.ts b/apps/server/src/kin/lead/Workers.test.ts index 19008935319b..186018a1fed4 100644 --- a/apps/server/src/kin/lead/Workers.test.ts +++ b/apps/server/src/kin/lead/Workers.test.ts @@ -195,6 +195,11 @@ const harness = () => { // The tracker follows these instead of the real stored-event stream. const stored: Array = []; const faults = { enqueue: 0, projection: new Set() }; + /** Run once a launch or send has started its run, before it returns. */ + const hooks: { + launched: Effect.Effect; + sent: Effect.Effect; + } = { launched: Effect.void, sent: Effect.void }; const settles = new Map>(); const drained = Deferred.makeUnsafe(); const orchestration = Layer.unwrap( @@ -212,6 +217,7 @@ const harness = () => { Effect.gen(function* () { launches.push(input); const threadId = yield* startThread(input); + yield* hooks.launched; return { threadId, projection: yield* projection(threadId), resumed: false }; }).pipe(Effect.provideService(OrchestratorV2, orchestrator), Effect.orDie), }), @@ -235,6 +241,7 @@ const harness = () => { createdBy: input.createdBy, creationSource: input.creationSource, }); + yield* hooks.sent; const sent = yield* projection(input.threadId); return { dispatch, @@ -313,6 +320,7 @@ const harness = () => { shells, stored, faults, + hooks, settles, drained, orchestration, @@ -894,6 +902,42 @@ it.effect("a parent run re-emitted after its delivery run settled reports that a ), ); +it.effect( + "a pause that lands while kin_delegate or kin_follow_up starts a run stops that run", + () => + run(({ ctx, fakes, workers }) => + Effect.gen(function* () { + const pause = + ctx.sql`INSERT INTO kin_org_state (org_id, paused, paused_at, paused_by) SELECT ${ctx.orgId}, 1, 0, person_id FROM kin_people WHERE org_id = ${ctx.orgId} ORDER BY person_id LIMIT 1`.pipe( + Effect.orDie, + Effect.asVoid, + ); + fakes.hooks.launched = pause; + const worker = yield* workers.delegate(scope, delegateInput); + assert.ok("workerId" in worker); + const threadId = ThreadId.make(`kin-worker-${worker.workerId}`); + assert.deepStrictEqual( + fakes.interrupts.map((input) => [input.threadId, input.commandId.split(":")[0]]), + [[threadId, "kin-stop-launch"]], + ); + assert.deepStrictEqual(fakes.stopped, [threadId]); + + yield* ctx.sql`DELETE FROM kin_org_state WHERE org_id = ${ctx.orgId}`; + fakes.hooks.launched = Effect.void; + fakes.hooks.sent = pause; + yield* workers.followUp(scope, { + workerId: worker.workerId, + brief: "Also cover the retry path.", + sourceEventId: "lead-event", + }); + assert.lengthOf(fakes.interrupts, 2); + assert.strictEqual(fakes.interrupts[1]!.threadId, threadId); + assert.notStrictEqual(fakes.interrupts[1]!.runId, fakes.interrupts[0]!.runId); + assert.deepStrictEqual(fakes.stopped, [threadId, threadId]); + }), + ), +); + it.effect("stopping a worker closes its card and a follow-up reopens the same card", () => run(({ ctx, fakes, workers }) => Effect.gen(function* () { diff --git a/apps/server/src/kin/lead/Workers.ts b/apps/server/src/kin/lead/Workers.ts index a41e35252c75..94e71921332b 100644 --- a/apps/server/src/kin/lead/Workers.ts +++ b/apps/server/src/kin/lead/Workers.ts @@ -42,7 +42,7 @@ import { KinChatService } from "../chat/KinChatService.ts"; import { KinIdentity } from "../chat/KinIdentity.ts"; import * as Memory from "../memory/Memory.ts"; import { KinBudget } from "./KinBudget.ts"; -import { isOrgPaused } from "./orgState.ts"; +import { isOrgPaused, STOP_REASON } from "./orgState.ts"; import { resolveLead } from "./LeadMemory.ts"; import { quote } from "./LeadPrompt.ts"; import { buildBrief, Quoted, reportFormat } from "./workerBrief.ts"; @@ -91,12 +91,13 @@ export const WorkerLaunchGate = Context.Reference("t3/kin/lead/Workers/ const isLive = (run: OrchestrationV2Run) => run.status === "queued" || ThreadManagement.isActiveRun(run); -const liveRun = (projection: OrchestrationV2ThreadProjection) => projection.runs.findLast(isLive); +type ThreadRuns = Pick; +const liveRun = (projection: ThreadRuns) => projection.runs.findLast(isLive); /** Interrupts the thread's live run and stops its delegated tasks; true when a run was live. */ export const stopThreadTree = Effect.fn("KinWorkers.stopThreadTree")(function* ( threads: ThreadManagement.ThreadManagementService["Service"], input: { - readonly projection: OrchestrationV2ThreadProjection; + readonly projection: ThreadRuns; readonly prefix: string; readonly tag: string; readonly reason: string; @@ -278,6 +279,19 @@ const make = Effect.gen(function* () { return yield* denied("Kin workers run on an enabled Claude Code or Codex provider only."); return chosen; }); + /** The stop fan-out misses a run that starts during its pass; a pause seen after the launch or send stops it. */ + const stopIfPaused = Effect.fn("KinWorkers.stopIfPaused")(function* ( + orgId: KinOrgId, + projection: ThreadRuns, + ) { + if (!(yield* isOrgPaused(orgId).pipe(Effect.provideService(SqlClient.SqlClient, sql)))) return; + yield* stopThreadTree(threads, { + projection, + prefix: "kin-stop-launch", + tag: `${projection.thread.id}:${liveRun(projection)?.id ?? "idle"}`, + reason: STOP_REASON, + }); + }); const threadLink = Effect.fn("KinWorkers.threadLink")(function* (threadId: ThreadId) { return formatThreadLink({ environmentId: yield* environment.getEnvironmentId, threadId }); }); @@ -442,6 +456,7 @@ const make = Effect.gen(function* () { }), ), ); + yield* stopIfPaused(ctx.orgId, launched.projection); const live = launched.projection.runs.some(isLive); // A resumed launch can start nothing; the row then takes the thread's real outcome. if (!live) { @@ -490,7 +505,7 @@ const make = Effect.gen(function* () { const projection = yield* threads .getThreadProjection(plan.worker.threadId) .pipe(Effect.mapError(() => missing())); - yield* threads + const sent = yield* threads .sendToThread({ projectId: projection.thread.projectId, commandId: CommandId.make(`kin-follow-up:${ctx.wakeupId}:${key}`), @@ -518,6 +533,7 @@ const make = Effect.gen(function* () { ); yield* sql`UPDATE kin_workers SET status = 'queued', ended_at = NULL WHERE org_id = ${ctx.orgId} AND worker_id = ${input.workerId}`; + yield* stopIfPaused(ctx.orgId, sent.projection); if (plan.worker.chatId !== null) yield* showCard({ orgId: ctx.orgId, diff --git a/apps/server/src/kin/lead/orgState.ts b/apps/server/src/kin/lead/orgState.ts index 00ee82bae2ed..30dacd2c4e69 100644 --- a/apps/server/src/kin/lead/orgState.ts +++ b/apps/server/src/kin/lead/orgState.ts @@ -19,6 +19,8 @@ export const isOrgOwner = Effect.fn("KinOrgState.isOrgOwner")(function* ( return rows.length > 0; }); +export const STOP_REASON = "Stopped by the Kin stop switch."; + /** The stop switch. Paused orgs start no wake-up, worker, dream or routine. */ export const isOrgPaused = Effect.fn("KinOrgState.isOrgPaused")( function* (orgId: KinOrgId) { diff --git a/docs/kin/specs/lead.md b/docs/kin/specs/lead.md index 7be3f48ed4ab..2642babc35d1 100644 --- a/docs/kin/specs/lead.md +++ b/docs/kin/specs/lead.md @@ -145,7 +145,7 @@ export const KinEvent = Schema.Union([ tag: TrimmedNonEmptyString, receivedAt: CanonicalUtcTimestamp, }), - ), + ).check(Schema.isMaxLength(50)), }), }), ]); @@ -505,13 +505,15 @@ Budget RPC payloads (wiring in LEAD-10 through `kinRpcs.ts`): `kin.budget.get` u `KinStop.pause(orgId, by)`: 1. Sets `paused` first, so nothing new starts: the runner claims nothing, `kin_delegate` and `kin_follow_up` refuse with `not_allowed`, and dreams wait. -2. Interrupts the active run on every lead thread and on every `kin_workers` thread whose `ended_at` is null, plus `stopDelegatedTasks` on each. A wake-up whose run ends this way returns its events with `release({ countAttempt: false, notBefore: now + pause poll interval })`, so no attempt is spent; a pause that lands while the runner is launching is caught by a recheck right after launch. +2. Interrupts the active run on every lead thread, on every `kin_workers` thread whose `ended_at` is null and on the current thread of every running dream, plus `stopDelegatedTasks` on each. Each thread is stopped independently; one failure does not skip the others or step 3, and the fan-out fails once all have run. A wake-up whose run ends this way returns its events with `release({ countAttempt: false, notBefore: now + pause poll interval })`, so no attempt is spent. A pause that lands while the runner, `kin_delegate` or `kin_follow_up` is launching is caught by a recheck right after the launch or send. A dream that sees the pause after any turn or after its push fails and deletes its branch, so nothing it wrote is merged. 3. Tells the triggers spec (`KinRoutines.pauseAll(orgId)`) and the approvals spec (the executor checks `KinStop.isPaused` before every run and refuses). 4. Posts "Kin is paused. Nothing runs until you resume." to the owner's direct chat. -`KinStop` lives with the chat and budget services the WebSocket handlers use; those are built outside the orchestration runtime, so the lead runtime attaches steps 2 and 3 to it (`KinStop.attach`) and replays them for orgs already paused at startup. Pausing a paused org keeps the original `paused_at`, `paused_by` and reason and posts nothing, but runs steps 2 and 3 again, so a retry reaches work a failed fan-out missed. A fan-out failure returns `unavailable` with the org still paused. +Pause with steps 2 and 3, and resume with `resumeAll`, hold a per-org lock, so a resume during a slow fan-out runs after it and routines end unpaused. + +`KinStop` lives with the chat and budget services the WebSocket handlers use; those are built outside the orchestration runtime, so the lead runtime attaches steps 2 and 3 to it (`KinStop.attach`). At startup it replays them for orgs still paused and calls `resumeAll` for orgs resumed since, which heals a resume that ran before the attach or whose `resumeAll` failed. Replayed interrupts carry a per-boot nonce in their command ids: recovery keeps run ids, and a pre-crash receipt must not swallow the replay. Pausing a paused org keeps the original `paused_at`, `paused_by` and reason and posts nothing, but runs steps 2 and 3 again, so a retry reaches work a failed fan-out missed. A fan-out failure returns `unavailable` with the org still paused. -While paused, `enqueue` still records events, so nothing is lost. `KinStop.resume(orgId)` clears the flag, calls `KinRoutines.resumeAll(orgId)` and enqueues one `resume` event per active lead (dedupe key `resume::`). That wake-up sees what arrived during the pause. Events queued (`queued_at`, server receipt) more than 24 hours before the resume are marked `expired` in the same transaction and listed in the resume event, not acted on. Resuming an org that is not paused changes nothing. A deferral that came due during the pause is treated as `on_due: ask`, never `start`. +While paused, `enqueue` still records events, so nothing is lost. `KinStop.resume(orgId)` clears the flag, calls `KinRoutines.resumeAll(orgId)` and enqueues one `resume` event per active lead (dedupe key `resume::`). It posts "Kin resumed." to the owner's direct chat, adding how many queued events expired. That wake-up sees what arrived during the pause. Events queued (`queued_at`, server receipt) more than 24 hours before the resume are marked `expired` in the same transaction and listed in the resume event, not acted on. Resuming an org that is not paused changes nothing. A deferral that came due during the pause is treated as `on_due: ask`, never `start`. Payloads: `KinOrgGetInput { orgId }`, `KinOrgPauseInput { orgId, reason? }`, `KinOrgResumeInput { orgId }`; all return `KinOrgState { orgId, paused, pausedAt, pausedBy, reason }`.