diff --git a/apps/server/src/kin/KinServer.ts b/apps/server/src/kin/KinServer.ts index 33bc02a3f5fef..d61dcc1cc5031 100644 --- a/apps/server/src/kin/KinServer.ts +++ b/apps/server/src/kin/KinServer.ts @@ -8,7 +8,7 @@ import { KinChatHub } from "./chat/KinChatHub.ts"; import { KinBudget } from "./lead/KinBudget.ts"; import { KinEventQueue } from "./lead/KinEventQueue.ts"; -export type Services = KinIdentity | KinChatService | KinBudget | KinEventQueue; +export type Services = KinIdentity | KinChatService | KinBudget | KinEventQueue | KinChatHub; export class KinServer extends Context.Service< KinServer, diff --git a/apps/server/src/kin/chat/KinChatService.ts b/apps/server/src/kin/chat/KinChatService.ts index 1661671409ab4..9ddf482ff10a7 100644 --- a/apps/server/src/kin/chat/KinChatService.ts +++ b/apps/server/src/kin/chat/KinChatService.ts @@ -10,6 +10,7 @@ import { KinMessageId, KinWaitingId, KinWidgetId, + KinOwnerCheckCard, KinMessagesSendInput, KinWidgetsActInput, KinAgentIntent, @@ -179,6 +180,7 @@ const decodeWidgetId = Schema.decodeEffect(KinWidgetId); const decodeWaitingId = Schema.decodeEffect(KinWaitingId); const decodeSend = Schema.decodeEffect(KinMessagesSendInput); const decodeIntent = Schema.decodeEffect(KinAgentIntent); +const decodeOwnerCheckCard = Schema.decodeUnknownEffect(KinOwnerCheckCard); const decodeAct = Schema.decodeEffect(KinWidgetsActInput); const encodeAction = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown)); const invalidState = new KinChatError({ reason: "invalid_state", message: "Invalid chat input." }); @@ -947,7 +949,8 @@ const make = Effect.gen(function* () { reason: "use_approvals", message: "Use the signed approvals service.", }); - if (input.type === "task" && input.widgetId === undefined) return yield* notAllowed; + if (input.type === "owner_check" || (input.type === "task" && input.widgetId === undefined)) + return yield* notAllowed; let state = yield* validateState(input.type, input.state); let widget: KinWidget; if (input.widgetId === undefined) { @@ -1074,6 +1077,17 @@ const make = Effect.gen(function* () { reason: "use_approvals", message: "Use the signed approvals service.", }); + if (widget.type === "owner_check") { + if (member.role !== "owner") return yield* notAllowed; + const card = yield* decodeOwnerCheckCard(widget.state).pipe( + Effect.mapError(() => invalidState), + ); + if ( + !["keep", "update", "drop"].includes(request.actionId) || + !card.items.some((item) => item.itemId === request.value) + ) + return yield* invalidState; + } const body = yield* encodeAction({ widgetId: request.widgetId, revision: request.revision, diff --git a/apps/server/src/kin/lead/KinEventQueue.test.ts b/apps/server/src/kin/lead/KinEventQueue.test.ts index e20021ce4a87d..65ec358124f4c 100644 --- a/apps/server/src/kin/lead/KinEventQueue.test.ts +++ b/apps/server/src/kin/lead/KinEventQueue.test.ts @@ -186,11 +186,12 @@ describe("KinEventQueue", () => { "report", "resolved", "github", - "routine", "noisy", + "routine", "watch", ], ); + assert.equal(events.find((entry) => entry.eventId === "routine")?.priority, "P3"); yield* finish("priority"); yield* sql`UPDATE kin_chat_members SET role = 'member'`; yield* queue.enqueue(yield* event("member", "chat.message")); diff --git a/apps/server/src/kin/lead/KinEventQueue.ts b/apps/server/src/kin/lead/KinEventQueue.ts index 8fae9a32d440f..58b11a39fa91d 100644 --- a/apps/server/src/kin/lead/KinEventQueue.ts +++ b/apps/server/src/kin/lead/KinEventQueue.ts @@ -228,6 +228,9 @@ const make = Effect.gen(function* () { case "worker.report": priority = "P1"; break; + case "routine": + if (event.routine === "owner_check") priority = "P3"; + break; case "watchdog": priority = "P3"; break; diff --git a/apps/server/src/kin/lead/WakeupRunner.test.ts b/apps/server/src/kin/lead/WakeupRunner.test.ts index ef5df2302c96f..ff326c900c6dc 100644 --- a/apps/server/src/kin/lead/WakeupRunner.test.ts +++ b/apps/server/src/kin/lead/WakeupRunner.test.ts @@ -22,6 +22,9 @@ import { ProjectId, ProviderInstanceId, } from "@t3tools/contracts"; +import * as OwnerCheck from "../triggers/ownerCheck.ts"; +import * as ChatService from "../chat/KinChatService.ts"; +import * as ChatHub from "../chat/KinChatHub.ts"; import * as Context from "effect/Context"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; @@ -86,6 +89,34 @@ const run = Effect.fn("test.run")(function* () { }); describe("WakeupRunner", () => { + it.effect("handles the owner check in code without starting a provider turn", () => + Effect.gen(function* () { + yield* seed; + const queue = yield* KinEventQueue.KinEventQueue; + const sql = yield* SqlClient.SqlClient; + yield* queue.enqueue(event("owner-check")); + yield* Fiber.join(yield* run()); + assert.deepEqual( + yield* sql`SELECT state FROM kin_events WHERE org_id = ${orgId} ORDER BY event_id`, + [{ state: "done" }], + ); + assert.deepEqual( + yield* sql`SELECT thread_id, status FROM kin_wakeups WHERE org_id = ${orgId} ORDER BY wakeup_id`, + [{ thread_id: null, status: "completed" }], + ); + assert.lengthOf( + yield* sql`SELECT thread_id FROM kin_agent_threads WHERE org_id = ${orgId} ORDER BY thread_id`, + 0, + ); + }).pipe( + Effect.provide( + harness().pipe( + Layer.provide(Layer.mock(OwnerCheck.OwnerCheck)({ post: () => Effect.succeed([]) })), + ), + ), + provideDeterministicTestRuntime, + ), + ); it.effect("uses the cheap model for exhausted P0 even when a routine is in the batch", () => Effect.gen(function* () { yield* seed; @@ -962,6 +993,8 @@ describe("WakeupRunner", () => { Layer.provide(Layer.mock(KinIdentity.KinIdentity)({ isAgentThread: () => false })), Layer.provide( Layer.mergeAll( + Layer.mock(ChatService.KinChatService)({}), + Layer.mock(ChatHub.KinChatHub)({}), Layer.mock(Budget.KinBudget)({}), Layer.mock(KinEventQueue.KinEventQueue)({}), Layer.mock(ThreadLaunch.ThreadLaunchService)({}), diff --git a/apps/server/src/kin/lead/WakeupRunner.ts b/apps/server/src/kin/lead/WakeupRunner.ts index cac394b78d9c1..9d84ce6df8316 100644 --- a/apps/server/src/kin/lead/WakeupRunner.ts +++ b/apps/server/src/kin/lead/WakeupRunner.ts @@ -30,11 +30,15 @@ import { toPersistenceSqlError } from "../../persistence/Errors.ts"; import * as ProjectService from "../../project/ProjectService.ts"; import * as KinIdentity from "../chat/KinIdentity.ts"; import * as Memory from "../memory/Memory.ts"; +import * as Routines from "../triggers/routines.ts"; +import * as OwnerCheck from "../triggers/ownerCheck.ts"; import * as KinEventQueue from "./KinEventQueue.ts"; import * as KinBudget from "./KinBudget.ts"; import * as KinLeadRegistry from "./KinLeadRegistry.ts"; import { KinLeadPrompt } from "./LeadPrompt.ts"; +const ownerCheckDispatcherReady = false; + export class KinWakeupError extends Schema.TaggedError()("KinWakeupError", { orgId: KinOrgId, agentId: KinAgentId, @@ -93,6 +97,7 @@ const ulid = Effect.gen(function* () { }); const make = Effect.gen(function* () { + const ownerCheck = yield* Effect.serviceOption(OwnerCheck.OwnerCheck); const sql = yield* SqlClient.SqlClient; const queue = yield* KinEventQueue.KinEventQueue; const budget = yield* KinBudget.KinBudget; @@ -349,7 +354,17 @@ const make = Effect.gen(function* () { events = events.filter((event) => !deferred.has(event.eventId)); } } - if (events.length === 0) return; + const routines = yield* Effect.serviceOption(Routines.KinRoutines); + if (Option.isSome(routines)) + events = yield* Effect.filter(events, (event) => + event._tag === "routine" + ? routines.value.isEnabled(orgId, event.routine) + : Effect.succeed(true), + ); + if (events.length === 0) { + yield* queue.finish({ orgId, wakeupId, status: "completed" }); + return; + } const budgetContext = yield* budget.promptContext(orgId); const pinned = yield* memory.prepareWakeup(orgId, wakeupId).pipe( Effect.tapError((error) => @@ -365,6 +380,21 @@ const make = Effect.gen(function* () { : Effect.void, ), ); + if (Option.isSome(ownerCheck)) { + for (const event of events) { + if (event._tag === "routine" && event.routine === "owner_check") + // Keep cards dark until the verified action dispatcher and clients land. + if (ownerCheckDispatcherReady) + yield* ownerCheck.value.post(orgId, event.eventId, event.ownerCheckItems ?? []); + } + events = events.filter( + (event) => event._tag !== "routine" || event.routine !== "owner_check", + ); + if (events.length === 0) { + yield* queue.finish({ orgId, wakeupId, status: "completed" }); + return; + } + } const paths = yield* memory.ensureCheckouts(orgId); const { project } = yield* projects.bootstrap({ commandId: CommandId.make(`kin-project:${orgId}`), diff --git a/apps/server/src/kin/lead/layer.test.ts b/apps/server/src/kin/lead/layer.test.ts index c2d16b667511a..4cf8917612a81 100644 --- a/apps/server/src/kin/lead/layer.test.ts +++ b/apps/server/src/kin/lead/layer.test.ts @@ -14,11 +14,15 @@ import * as ServerConfig from "../../config.ts"; import * as EffectWorker from "../../orchestration-v2/EffectWorker.ts"; import * as ThreadManagement from "../../orchestration-v2/ThreadManagementService.ts"; import { provideDeterministicTestRuntime } from "../../orchestration-v2/testkit/DeterministicRuntime.ts"; +import * as ChatService from "../chat/KinChatService.ts"; +import * as ChatHub from "../chat/KinChatHub.ts"; import * as KinEventQueue from "./KinEventQueue.ts"; import * as KinLead from "./layer.ts"; import { event, harness, seed, orgId } from "./WakeupRunner.testkit.ts"; const external = Layer.mergeAll( + Layer.mock(ChatService.KinChatService)({}), + Layer.mock(ChatHub.KinChatHub)({}), ServerConfig.layerTest(process.cwd(), { prefix: "kin-lead-layer-" }), Layer.mock(VcsProcess.VcsProcess)({}), ).pipe(Layer.provideMerge(NodeServices.layer)); @@ -52,7 +56,7 @@ vi.mock("../memory/Memory.ts", async (importOriginal) => { : Effect.die("Prompt and runner must share pinned memory"), readIndex: () => Effect.die("unused"), readRules: () => Effect.die("unused"), - query: () => Effect.die("unused"), + query: () => Effect.succeed([]), readItem: () => Effect.die("unused"), readItemQuietly: () => Effect.die("unused"), rebuildIndex: () => Effect.die("unused"), @@ -69,7 +73,11 @@ it.effect("starts configured Kin with the real prompt and the runner's pinned me const hold = yield* Deferred.make(); yield* Effect.gen(function* () { yield* seed; - yield* (yield* KinEventQueue.KinEventQueue).enqueue(event("real-prompt")); + yield* (yield* KinEventQueue.KinEventQueue).enqueue({ + ...event("real-prompt"), + _tag: "routine", + routine: "morning_brief", + }); yield* Layer.build( Layer.fresh(KinLead.layerServer).pipe( Layer.provide(external), @@ -114,5 +122,5 @@ it.effect("leaves unconfigured Kin silent and inert without building its depende ), ); assert.deepStrictEqual(entries, []); - }).pipe(Effect.provide(harness()), provideDeterministicTestRuntime), + }).pipe(Effect.provide(Layer.mergeAll(harness(), external)), provideDeterministicTestRuntime), ); diff --git a/apps/server/src/kin/lead/layer.ts b/apps/server/src/kin/lead/layer.ts index 56cdbaeb98469..7d566aa80d4d2 100644 --- a/apps/server/src/kin/lead/layer.ts +++ b/apps/server/src/kin/lead/layer.ts @@ -1,8 +1,14 @@ import { forkParked } from "../../serverActivation.ts"; import { KinOrgId } from "@t3tools/contracts"; +import * as Scheduler from "../../scheduling/Scheduler.ts"; import * as Config from "effect/Config"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Routines from "../triggers/routines.ts"; +import * as OwnerCheck from "../triggers/ownerCheck.ts"; +import * as TriggerScheduler from "../triggers/KinTriggerScheduler.ts"; +import * as TriggerStore from "../triggers/KinTriggerStore.ts"; +import * as TriggerInbox from "../triggers/KinTriggerInbox.ts"; import * as Schema from "effect/Schema"; import * as KinChatRepository from "../chat/KinChatRepository.ts"; @@ -35,13 +41,24 @@ export function containStartup(layer: Layer.Layer) { ); } +const routineRuntime = Routines.layer.pipe( + Layer.provideMerge(TriggerScheduler.layer), + Layer.provide(Layer.mergeAll(TriggerStore.layer, TriggerInbox.layer)), + // Kin builds in its contained startup scope; orchestration runtime does not expose its scheduler. + Layer.provide(Scheduler.layer), +); + export const layer = Layer.effectDiscard( Effect.gen(function* () { const runner = yield* WakeupRunner.WakeupRunner; - yield* runner.start.pipe(forkParked); + yield* Effect.gen(function* () { + const context = yield* Layer.build(routineRuntime); + yield* runner.start.pipe(Effect.provideContext(context)); + }).pipe(forkParked); }), ).pipe( Layer.provide(WakeupRunner.layer), + Layer.provide(OwnerCheck.layer), Layer.provide( Layer.mergeAll(KinLeadRegistry.layer, KinLeadPrompt.layer).pipe( Layer.provideMerge( diff --git a/apps/server/src/kin/memory/Memory.test.ts b/apps/server/src/kin/memory/Memory.test.ts index 402e17e08e711..942b89ffbba43 100644 --- a/apps/server/src/kin/memory/Memory.test.ts +++ b/apps/server/src/kin/memory/Memory.test.ts @@ -858,6 +858,10 @@ it.live("uses filesystem identity for a case-mismatched base without recloning", it.live.each([ "diverged", + "secret merge", + "scan failure merge", + "remote secret merge", + "large remote merge", "conflict", "network down", "dirty writer", @@ -932,7 +936,11 @@ it.live.each([ } else yield* fs.writeFileString( `${writer}/rules.md`, - scenario === "H2-merge-swap" ? h2Local : "Local invented rules\n", + scenario === "H2-merge-swap" + ? h2Local + : scenario === "secret merge" + ? ["gh", "p_", "a".repeat(36)].join("") + : "Local invented rules\n", ); let local = yield* commit(writer); if (scenario === "push retry") { @@ -952,13 +960,20 @@ it.live.each([ } yield* memory.record(orgA, "main", initial.head, local); yield* fs.writeFileString( - `${source}/${scenario === "push retry" || scenario === "diverged" || scenario === "network down" || scenario === "dirty writer" || scenario === "H2-merge-swap" ? "remote-only" : "rules.md"}`, + `${source}/${scenario === "push retry" || scenario === "diverged" || scenario === "scan failure merge" || scenario === "secret merge" || scenario === "remote secret merge" || scenario === "large remote merge" || scenario === "network down" || scenario === "dirty writer" || scenario === "H2-merge-swap" ? "remote-only" : "rules.md"}`, "Remote invented edit\n", ); if (scenario === "journal conflict" || scenario === "push retry") { yield* fs.writeFileString(`${source}/journal/day.md`, "remote entry\n"); yield* fs.writeFileString(`${source}/INDEX.md`, "remote generated index\n"); } + if (scenario === "remote secret merge") + yield* fs.writeFileString(`${source}/remote-only`, ["gh", "p_", "a".repeat(36)].join("")); + if (scenario === "large remote merge") { + yield* fs.makeDirectory(`${source}/archive`); + for (let i = 0; i < 1030; i++) + yield* fs.writeFileString(`${source}/archive/${i}.txt`, "clean"); + } if (scenario === "H2-merge-swap") yield* fs.writeFileString(`${source}/rules.md`, h2Base.replace("base tail", "remote tail")); yield* commit(source); @@ -1004,6 +1019,10 @@ it.live.each([ yield* memory.record(orgA, "main", local, replacement); local = replacement; } + if (scenario === "scan failure merge") { + const sql = yield* SqlClient.SqlClient; + yield* sql`DROP TABLE kin_secret_allow`; + } const result = yield* memory.prepareWakeup(orgA, "fixture-wakeup").pipe(Effect.result); if (scenario === "unrelated" || scenario === "criss-cross") { expect(result._tag).toBe("Failure"); @@ -1038,6 +1057,22 @@ it.live.each([ expect( (yield* memory.prepareWakeup(orgA, "fixture-wakeup-3").pipe(Effect.result))._tag, ).toBe("Success"); + } else if (scenario === "scan failure merge") { + expect(result._tag).toBe("Failure"); + expect( + messages.some((message) => message.includes("Secret scan failed: Allowlist unavailable")), + ).toBe(true); + expect(yield* memory.anchor(orgA, "main")).toBe(local); + } else if (scenario === "secret merge") { + expect(result._tag).toBe("Failure"); + if (result._tag === "Failure") { + expect(result.failure._tag).toBe("MemoryError"); + expect(result.failure.operation).toBe("secretScan"); + } + expect(yield* memory.anchor(orgA, "main")).toBe(local); + expect(yield* git(root, [`--git-dir=${remote}`, "rev-parse", "main"])).toBe( + yield* git(source, ["rev-parse", "HEAD"]), + ); } else if (scenario === "push retry") { expect(result._tag).toBe("Success"); if (result._tag === "Success") { @@ -1063,6 +1098,8 @@ it.live.each([ } else if ( scenario === "network down" || scenario === "diverged" || + scenario === "remote secret merge" || + scenario === "large remote merge" || scenario === "dirty writer" ) { expect(result._tag).toBe("Success"); @@ -1072,7 +1109,9 @@ it.live.each([ expect(result.success.head).not.toBe(local); expect(yield* memory.readRules(orgA)).toBe("Local invented rules\n"); expect(yield* git(writer, ["show", `${result.success.head}:remote-only`])).toBe( - "Remote invented edit", + scenario === "remote secret merge" + ? ["gh", "p_", "a".repeat(36)].join("") + : "Remote invented edit", ); } } diff --git a/apps/server/src/kin/memory/Memory.ts b/apps/server/src/kin/memory/Memory.ts index 4ecc9ded5fd74..d8aafb19a722a 100644 --- a/apps/server/src/kin/memory/Memory.ts +++ b/apps/server/src/kin/memory/Memory.ts @@ -1,3 +1,4 @@ +import * as SecretScan from "./secretScan.ts"; import { readTree, readObjects, @@ -203,6 +204,7 @@ interface BlobCache { } const make = Effect.gen(function* () { + const scanner = yield* SecretScan.KinSecretScan; const config = yield* MemoryConfig; const fs = yield* FileSystem.FileSystem; const { notify } = yield* MemoryRecoveryHooks; @@ -1079,6 +1081,7 @@ const make = Effect.gen(function* () { }); } else { const { files: mergedFiles, modes } = merged; + const remoteOriginal = new Map(remoteFiles); remoteFiles.clear(); for (const [file, bytes] of mergedFiles) remoteFiles.set(file, bytes); const heads = []; @@ -1098,6 +1101,36 @@ const make = Effect.gen(function* () { "INDEX.md", Buffer.from(buildIndex(heads, reads, yield* Clock.currentTimeMillis, zone)), ); + const findings = yield* scanner + .scan({ + orgId, + files: [...remoteFiles] + .filter( + ([path, bytes]) => + path === "INDEX.md" || !remoteOriginal.get(path)?.equals(bytes), + ) + .map(([path, content]) => ({ path, content })), + }) + .pipe( + Effect.tapError((cause) => + notify({ orgId, branch, reason: `Secret scan failed: ${cause.reason}` }), + ), + Effect.mapError( + (cause) => new MemoryError({ orgId, operation: "secretScan", cause }), + ), + ); + if (findings.length) { + yield* notify({ + orgId, + branch, + reason: `Secret scan blocked merge: ${findings[0]!.ruleId} in ${findings[0]!.pathHit ? "[redacted]" : (findings[0]!.itemPath ?? findings[0]!.path)}`, + }); + return yield* new MemoryError({ + orgId, + operation: "secretScan", + reason: `Secret scan blocked merge: ${findings[0]!.ruleId} in ${findings[0]!.pathHit ? "[redacted]" : (findings[0]!.itemPath ?? findings[0]!.path)}`, + }); + } tree = yield* Effect.scoped( Effect.gen(function* () { const temp = yield* io(orgId, "journalMerge", fs.makeTempDirectoryScoped()); @@ -1286,4 +1319,7 @@ const make = Effect.gen(function* () { }); }); -export const layer = Layer.effect(Memory, make).pipe(Layer.provide(MemoryCache.layer)); +export const layer = Layer.effect(Memory, make).pipe( + Layer.provide(MemoryCache.layer), + Layer.provide(SecretScan.layer), +); diff --git a/apps/server/src/kin/memory/MemoryWriter.test.ts b/apps/server/src/kin/memory/MemoryWriter.test.ts index 5dc139d5b196c..b2c814d055865 100644 --- a/apps/server/src/kin/memory/MemoryWriter.test.ts +++ b/apps/server/src/kin/memory/MemoryWriter.test.ts @@ -1,3 +1,4 @@ +import * as SqlClient from "effect/sql/SqlClient"; import { memoryGit } from "./git.ts"; import * as NodeZlib from "node:zlib"; import * as NodeServices from "@effect/platform-node/NodeServices"; @@ -23,6 +24,8 @@ it.live.each([ "completed", "failed", "blocked", + "scan unavailable", + "scan limit", "forged", "conflict", "B1-shadow", @@ -42,6 +45,11 @@ it.live.each([ "P9-shadow", "P10-lost-git-dir", "sha256", + "owner-keep", + "owner-other-item-keep", + "owner-drop-keep", + "owner-chat-keep", + "agent-keep", "review-default", "review-null", "credentials", @@ -83,7 +91,12 @@ it.live.each([ ]); yield* fs.writeFileString(`${source}/rules.md`, "Invented rules\n"); yield* fs.writeFileString(`${source}/INDEX.md`, "As of 2026-11-08T00:00:00.000Z\n"); - if (scenario.startsWith("B6") || scenario === "P3-object" || scenario === "P6-malformed") { + if ( + scenario.startsWith("B6") || + scenario.endsWith("-keep") || + scenario === "P3-object" || + scenario === "P6-malformed" + ) { yield* fs.makeDirectory(`${source}/claims`); yield* fs.writeFileString( `${source}/claims/existing.md`, @@ -92,7 +105,7 @@ it.live.each([ : fixtureItem({ kind: "claim", status: scenario === "B6-owner-open" ? "open" : "done", - trust: scenario === "B6-agent-own" ? "agent" : "owner", + trust: scenario === "B6-agent-own" || scenario === "agent-keep" ? "agent" : "owner", said_by: scenario === "B6-agent-own" ? agentId : "fixture-owner", }), ); @@ -206,10 +219,28 @@ it.live.each([ timeZone: "Asia/Kolkata", startedAt: "2026-11-08T00:00:00.000Z", eventIds: [], - sources: new Map([ + sources: new Map< + string, + Writer.WakeupContext["sources"] extends ReadonlyMap ? S : never + >([ [ "owner", - { saidBy: "fixture-owner", trust: "owner", source: "fixture-owner-message" }, + scenario === "owner-keep" || + scenario === "owner-other-item-keep" || + scenario === "owner-drop-keep" + ? { + saidBy: "fixture-owner", + trust: "owner", + source: "fixture-owner-message", + kind: "owner_check", + itemId: scenario === "owner-other-item-keep" ? "other-item" : "def-001", + action: scenario === "owner-drop-keep" ? "drop" : "keep", + } + : { + saidBy: "fixture-owner", + trust: "owner", + source: "fixture-owner-message", + }, ], [ "member", @@ -223,10 +254,7 @@ it.live.each([ }), ), }); - const scan = - scenario === "blocked" - ? Scan.testLayer([{ path: "claims/new.md", ruleId: "fixture", line: 1 }]) - : Scan.noFindingsStubLayer; + const scan = Scan.layer; const layer = Writer.layer.pipe( Layer.provideMerge(memoryLayer), Layer.provide(Layer.succeed(Memory.MemoryRecoveryHooks, { notify: () => Effect.void })), @@ -502,6 +530,31 @@ it.live.each([ ).toBe("Success"); return; } + if (scenario.endsWith("-keep")) { + const content = fixtureItem({ + kind: "claim", + status: "done", + trust: scenario === "agent-keep" ? "agent" : "owner", + said_by: "fixture-owner", + review_by: "2026-12-13", + }); + const result = yield* writer + .stage(orgId, "wake1", { + path: "claims/existing.md", + content, + ...(scenario !== "agent-keep" ? { sourceEventId: "owner" } : {}), + }) + .pipe(Effect.result); + expect(result._tag).toBe(scenario === "owner-keep" ? "Success" : "Failure"); + if (result._tag === "Failure") expect(result.failure._tag).toBe("MemoryConflict"); + if (result._tag === "Success") { + const committed = yield* writer.commitWakeup(orgId, "wake1", "completed"); + expect( + yield* git(paths.writer, ["show", `${committed.commit}:claims/existing.md`]), + ).toContain("review_by: 2026-12-13"); + } + return; + } if (scenario.startsWith("B6")) { const result = yield* writer .stage(orgId, "wake1", { @@ -600,10 +653,52 @@ it.live.each([ } if (scenario === "tree-substitution") substituteTree = yield* git(paths.writer, ["rev-parse", `${original}^{tree}`]); + if (scenario === "blocked") { + const tokenPath = "claims/" + ["gh", "p_", "a".repeat(36)].join("") + ".md"; + const malformed = yield* writer + .stage(orgId, "wake1", { + path: tokenPath, + content: "---\nid: [\n" + tokenPath + ": [\n---\nbody", + }) + .pipe(Effect.result); + expect(malformed._tag).toBe("Failure"); + if (malformed._tag === "Failure") { + expect(malformed.failure._tag).toBe("MemoryItemError"); + expect(JSON.stringify(malformed.failure).includes(tokenPath.slice(7, -3))).toBe(false); + if (malformed.failure._tag === "MemoryItemError") + expect(malformed.failure.path).toBe("[redacted]"); + } + const inaccessible = yield* writer + .stage(orgId, "wake1", { + path: tokenPath.slice(0, -3) + "_".repeat(260) + ".md", + content: "missing front matter", + }) + .pipe(Effect.result); + expect(inaccessible._tag).toBe("Failure"); + if (inaccessible._tag === "Failure") + expect(JSON.stringify(inaccessible.failure).includes(tokenPath.slice(7, -3))).toBe(false); + } + if (scenario === "blocked") + yield* writer.appendJournal(orgId, "wake1", [ + ["gh", "p_"].join("") + "aB3dE6gH9jK2mN5pQ8sT1vW4yZ7cF0iL3oR6", + ]); + if (scenario === "scan unavailable") { + const sql = yield* SqlClient.SqlClient; + yield* sql`DROP TABLE kin_secret_allow`; + } + if (scenario === "scan limit") + yield* writer.appendJournal(orgId, "wake1", ["a".repeat(1024 * 1024)]); const result = yield* writer .commitWakeup(orgId, "wake1", scenario === "failed" ? "failed" : "completed") .pipe(Effect.result); if (result._tag === "Failure") yield* Effect.logError(result.failure); + if (scenario === "scan unavailable" || scenario === "scan limit") { + expect(result._tag).toBe("Failure"); + expect(notices).toHaveLength(1); + expect(notices[0]!.error._tag).toBe("KinSecretScanError"); + expect(yield* git(paths.writer, ["rev-parse", "HEAD"])).toBe(original); + return; + } if (scenario === "tree-substitution") { expect(result._tag).toBe("Failure"); if (result._tag === "Failure") expect(result.failure._tag).toBe("MemoryCheckoutInvalid"); @@ -618,6 +713,19 @@ it.live.each([ ); expect(notices).toHaveLength(1); + if (scenario === "blocked") { + expect(JSON.stringify(notices)).not.toContain("aB3dE6gH9jK2mN5pQ8sT1vW4yZ7cF0iL3oR6"); + expect(yield* git(paths.writer, ["rev-parse", "HEAD"])).toBe(original); + expect( + (yield* git(root, [ + `--git-dir=${remote}`, + "show-ref", + "--verify", + `refs/heads/kin/shadow/${agentId}`, + ]).pipe(Effect.result))._tag, + ).toBe("Failure"); + expect(yield* fs.exists(`${paths.writer}/claims/new.md`)).toBe(false); + } expect(yield* fs.exists(`${paths.writer}/.git/rebase-merge`)).toBe(false); } else { expect(result._tag).toBe("Success"); diff --git a/apps/server/src/kin/memory/MemoryWriter.ts b/apps/server/src/kin/memory/MemoryWriter.ts index 113280e8906c5..1f602817bf729 100644 --- a/apps/server/src/kin/memory/MemoryWriter.ts +++ b/apps/server/src/kin/memory/MemoryWriter.ts @@ -34,7 +34,14 @@ export interface WakeupContext extends JournalContext { readonly level: "L0" | "L1" | "L2" | "L3"; readonly sources: ReadonlyMap< string, - { readonly saidBy: string; readonly trust: MemoryTrust; readonly source: string } + { readonly saidBy: string; readonly trust: MemoryTrust; readonly source: string } & ( + | { readonly kind?: undefined } + | { + readonly kind: "owner_check"; + readonly itemId: string; + readonly action: "keep" | "update" | "drop"; + } + ) >; } // The runner resolves this context from same-org records, never tool input. @@ -61,14 +68,24 @@ export class MemorySecretBlocked extends Schema.TaggedError { orgId: Schema.String, findings: Schema.Array( - Schema.Struct({ path: Schema.String, ruleId: Schema.String, line: Schema.Number }), + Schema.Struct({ + path: Schema.String, + ruleId: Schema.String, + line: Schema.Number, + pathHit: Schema.optional(Schema.Boolean), + itemPath: Schema.optional(Schema.String), + }), ), }, ) {} export type WriterNotice = { readonly orgId: KinOrgId; readonly wakeupId: string; - readonly error: MemoryCheckoutInvalid | MemorySyncConflict | MemorySecretBlocked; + readonly error: + | MemoryCheckoutInvalid + | MemorySyncConflict + | MemorySecretBlocked + | KinSecretScanError; }; export class MemoryWriterHooks extends Context.Service< MemoryWriterHooks, @@ -288,7 +305,11 @@ const make = Effect.gen(function* () { const result = yield* fs.lstat(current).pipe(Effect.result); if (result._tag === "Failure") { if (result.failure.reason._tag !== "NotFound") - return yield* new MemoryError({ orgId, operation: "path", cause: result.failure }); + return yield* new MemoryError({ + orgId, + operation: "path", + cause: "Unable to inspect memory path", + }); } else if ( result.success.type === "SymbolicLink" || (result.success.type === "File" && @@ -306,10 +327,25 @@ const make = Effect.gen(function* () { }); const report = (orgId: KinOrgId, wakeupId: string, effect: Effect.Effect) => effect.pipe( + Effect.catchTags({ + MemoryItemError: (error) => + scanner.scan({ orgId, files: [{ path: error.path, content: new Uint8Array() }] }).pipe( + Effect.flatMap((findings) => + Effect.fail( + new MemoryItemError({ + orgId: error.orgId, + path: findings.length ? "[redacted]" : error.path, + reason: error.reason, + }), + ), + ), + ), + }), Effect.tapError((error) => error._tag === "MemoryCheckoutInvalid" || error._tag === "MemorySyncConflict" || - error._tag === "MemorySecretBlocked" + error._tag === "MemorySecretBlocked" || + error._tag === "KinSecretScanError" ? hooks .notify({ orgId, wakeupId, error }) .pipe( @@ -338,7 +374,12 @@ const make = Effect.gen(function* () { const parsed = yield* parseFrontMatter(orgId, change.path, change.content, true); const source = change.sourceEventId === undefined - ? { saidBy: ctx.agentId, trust: "agent", source: `wakeup:${wakeupId}` } + ? { + saidBy: ctx.agentId, + trust: "agent", + source: `wakeup:${wakeupId}`, + kind: undefined, + } : ctx.sources.get(change.sourceEventId); if (!source) return yield* new MemoryWriteRejected({ @@ -443,6 +484,12 @@ const make = Effect.gen(function* () { "said_by", "source", ]); + if ( + source.kind === "owner_check" && + source.itemId === item.id && + source.action === "keep" + ) + mutable.add("review_by"); if ( Object.entries(item).some( ([key, value]) => diff --git a/apps/server/src/kin/memory/frontMatter.ts b/apps/server/src/kin/memory/frontMatter.ts index 692502849fca7..49b7bb49b70b7 100644 --- a/apps/server/src/kin/memory/frontMatter.ts +++ b/apps/server/src/kin/memory/frontMatter.ts @@ -89,8 +89,12 @@ export const parseFrontMatter = Effect.fnUntraced(function* ( new MemoryItemError({ orgId, path, - reason: cause instanceof Error ? cause.message : "Malformed YAML", - cause, + reason: staged + ? "Malformed YAML front matter" + : cause instanceof Error + ? cause.message + : "Malformed YAML", + ...(staged ? {} : { cause }), }), }); const decoded = yield* (staged ? decodeStaged(value) : decodeFrontMatter(value)).pipe( diff --git a/apps/server/src/kin/memory/secretScan.test.ts b/apps/server/src/kin/memory/secretScan.test.ts new file mode 100644 index 0000000000000..319b4fc2aff03 --- /dev/null +++ b/apps/server/src/kin/memory/secretScan.test.ts @@ -0,0 +1,387 @@ +// @effect-diagnostics-next-line nodeBuiltinImport:off -- Bounded synchronous fingerprint hashing shares the scan loop. +import * as NodeCrypto from "node:crypto"; +import { vi } from "vite-plus/test"; +import { expect, it } from "@effect/vitest"; +import { KinOrgId } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as SqlClient from "effect/sql/SqlClient"; +import * as Sqlite from "../../persistence/Sqlite.ts"; +import * as Persistence from "../persistence/KinPersistence.ts"; +import * as Scan from "./secretScan.ts"; + +const orgId = KinOrgId.make("fixture-org"); +const database = Layer.provideMerge(Persistence.layer, Sqlite.layerMemory); +const layer = Scan.layer.pipe(Layer.provideMerge(database)); +const body = "aB3dE6gH9jK2mN5pQ8sT1vW4yZ7cF0iL3oR6"; +const github = ["gh", "p_"].join("") + body; +const fixtures = [ + ...[ + ["stripe-key", ["sk_", "live_"].join("") + body + "_old"], + ["slack-token", ["xo", "xb-"].join("") + body + "_old"], + ["google-key", ["AI", "za"].join("") + "X".repeat(35) + "_old"], + ["google-key", "key_" + ["AI", "za"].join("") + "X".repeat(35)], + ["anthropic-key", "prod-" + ["sk-", "ant-"].join("") + body.repeat(2)], + ...[ + Buffer.from(JSON.stringify({ alg: "RS256", x5c: ["a".repeat(499)] })).toString("base64url"), + ["ey", "J"].join("") + "a".repeat(2048), + ].map((header) => [ + "jwt", + header + "." + ["ey", "J"].join("") + "b".repeat(2600) + ".signature", + ]), + ].map(([rule, hit]) => [rule, hit, "task-follow-up-with-design-team"]), + + ...["X-Api-Key: ", "client-secret: ", '{"password": "', '"api_key":"'].map((prefix) => [ + "secret-assignment", + prefix + body + '"', + prefix + "short", + ]), + ...["_", "token_"].map((prefix) => [ + "github-token", + prefix + github + "_rotated", + "task-follow-up-with-design-team", + ]), + ["aws-access-key", ["AK", "IA"].join("") + "Q".repeat(16) + "_old", "ASIAPACIFICREGION2026"], + [ + "jwt", + ["ey", "J"].join("") + "a".repeat(100) + ".eyJ" + "b".repeat(1800) + "." + "c".repeat(100), + "eyJshort.two.parts", + ], + + ...["p", "o", "u", "s", "r"].map((kind) => [ + "github-token", + "gh" + kind + "_" + body, + "gh" + kind + "_short", + ]), + [ + "github-token", + ["github_", "pat_"].join("") + "a".repeat(82), + ["github_", "pat_short"].join(""), + ], + ["anthropic-key", ["sk-", "ant-"].join("") + body.repeat(2), ["sk-", "ant-short"].join("")], + ["openai-key", "sk-" + body.slice(0, 20) + "T3BlbkFJ" + body.slice(0, 20), "sk-short"], + ...["proj", "svcacct", "admin"].map((kind) => [ + "openai-key", + ["sk-", kind, "-", body.repeat(3)].join(""), + ["sk-", kind, "-", "a".repeat(40)].join(""), + ]), + ...["AK", "AS"].map((kind) => [ + "aws-access-key", + kind + "IA" + "X".repeat(16), + kind + "IA" + "X".repeat(15), + ]), + ["slack-token", ["xo", "xb-"].join("") + body, ["xo", "xb-short"].join("")], + ["stripe-key", ["sk_", "live_"].join("") + body, "sk_live_short"], + ["google-key", ["AI", "za"].join("") + "X".repeat(35), ["AI", "za_short"].join("")], + ["pem-private-key", "-----BEGIN RSA PRIVATE KEY-----", "-----BEGIN PUBLIC KEY-----"], + [ + "jwt", + ["ey", "J"].join("") + "a".repeat(10) + ".eyJ" + "b".repeat(12) + "." + "c".repeat(20), + "eyJshort.two.parts", + ], + [ + "url-password", + "https://fixture:fake-password@example.invalid", + "https://fixture@example.invalid", + ], + ["secret-assignment", "password=" + body, "password=" + "a".repeat(40)], + ["secret-assignment", "export API_KEY='" + body + "'", "export API_KEY='short'"], + ["secret-assignment", "AWS_SECRET_ACCESS_KEY=" + body + "aB3d", "AWS_SECRET_ACCESS_KEY=example"], +]; +it.effect.each(fixtures.map(([rule, hit, miss], i) => ({ rule, hit, miss, id: i })))( + "rule fixture $id: $rule", + ({ rule, hit, miss }) => + Effect.gen(function* () { + const scanner = yield* Scan.KinSecretScan; + const run = (content: string) => + scanner.scan({ orgId, files: [{ path: "claims/test.md", content: Buffer.from(content) }] }); + // Assert only metadata so a failed assertion never prints the synthetic matched token. + expect((yield* run(hit!)).some((finding) => finding.ruleId === rule)).toBe(true); + expect((yield* run(miss!)).length).toBe(0); + }).pipe(Effect.provide(layer)), +); +it.effect("blocks encoded, wrapped and confusable tokens with safe findings", () => + Effect.gen(function* () { + const scanner = yield* Scan.KinSecretScan; + const values = [ + github, + "The deploy key is " + github.slice(0, 12) + "\n" + github.slice(12), + github.slice(0, 12) + "\n" + github.slice(12) + " and more", + github.slice(0, 12) + "\n" + github.slice(12) + "\nThanks", + "The deploy key is " + + Buffer.from(github).toString("base64").slice(0, 16) + + "\n" + + Buffer.from(github).toString("base64").slice(16) + + " and more", + "%67" + github.slice(1), + ...["/", "-", "_", "+", "word"].map( + (prefix) => prefix + Buffer.from(github).toString("base64"), + ), + Buffer.from("é" + github + "é").toString("base64"), + [...Buffer.from(github.slice(0, 5) + "\u034f" + github.slice(5))] + .map((byte) => `%${byte.toString(16).padStart(2, "0")}`) + .join(""), + "%FF" + [...github].map((c) => `%${c.charCodeAt(0).toString(16)}`).join(""), + ...["\u00ad", "\u034f", "\ufe0f", "\u{e0061}"].map( + (separator) => github.slice(0, 5) + separator + github.slice(5), + ), + github.replace("g", "ɡ"), + github.replace("h", "һ"), + ["gh", "ѕ_", body].join(""), + ["gh", "ο_", body].join(""), + " ".repeat(8190) + github, + Buffer.from(github).toString("base64"), + Buffer.from(Buffer.from(github).toString("base64")).toString("base64"), + [...github].map((c) => `%${c.charCodeAt(0).toString(16)}`).join(""), + github.slice(0, 12) + "\n" + github.slice(12), + ...["next word", " next word"].map( + (suffix) => github.slice(0, 12) + "\n" + github.slice(12) + "\n" + suffix, + ), + github.slice(0, 12) + "\\\n " + github.slice(12), + github.replace("p", "р"), + [...github].map((c) => String.fromCharCode(c.charCodeAt(0) + 0xfee0)).join(""), + github.slice(0, 5) + "\u200b" + github.slice(5), + ]; + for (const content of values) { + const result = yield* scanner.scan({ + orgId, + files: [{ path: "journal/test.md", content: Buffer.from(content) }], + }); + expect(result.some((f) => f.ruleId === "github-token")).toBe(true); + expect(JSON.stringify(result).includes(body)).toBe(false); + expect(Object.keys(result[0]!).sort()).toEqual(["line", "path", "ruleId"]); + } + }).pipe(Effect.provide(layer)), +); +it.effect("reports source lines and blocks AWS credential pairs without leaking values", () => + Effect.gen(function* () { + const scanner = yield* Scan.KinSecretScan; + for (const content of [ + "safe\n\n" + github, + "safe\n\n" + github.slice(0, 12) + "\n" + github.slice(12), + ]) { + const result = yield* scanner.scan({ + orgId, + files: [{ path: "claims/test.md", content: Buffer.from(content) }], + }); + expect(result.every((f) => f.line === 3)).toBe(true); + expect(result.length).toBeGreaterThan(0); + } + const pair = [ + "AWS_ACCESS_KEY_ID=" + ["AK", "IA", "X".repeat(16)].join(""), + "AWS_SECRET_ACCESS_KEY=" + body + "aB3d", + ].join("\n"); + const findings = yield* scanner.scan({ + orgId, + files: [{ path: ".env", content: Buffer.from(pair) }], + }); + expect(findings.some((f) => f.ruleId === "aws-access-key")).toBe(true); + expect(findings.some((f) => f.ruleId === "secret-assignment")).toBe(true); + }).pipe(Effect.provide(layer)), +); +it.effect("allowlist is exact, org scoped, and read only to scanner callers", () => + Effect.gen(function* () { + const scanner = yield* Scan.KinSecretScan; + const sql = yield* SqlClient.SqlClient; + const hash = NodeCrypto.createHash("sha256") + .update("github-token\0") + .update(github) + .digest("hex"); + // Fixture stands in for the future APPR owner-signed executor; scanner has no write method. + yield* sql`INSERT INTO kin_secret_allow VALUES (${orgId}, 'claims/test.md', 'github-token', ${hash}, 'signed-fixture', 0)`; + const run = (org: KinOrgId, path: string, value = github) => + scanner.scan({ orgId: org, files: [{ path, content: Buffer.from(value) }] }); + expect((yield* run(orgId, "claims/test.md")).length).toBe(0); + expect((yield* run(KinOrgId.make("other"), "claims/test.md")).length).toBeGreaterThan(0); + expect((yield* run(orgId, "claims/other.md")).length).toBeGreaterThan(0); + expect((yield* run(orgId, "claims/test.md", github.replace("a", "z"))).length).toBeGreaterThan( + 0, + ); + expect(Object.keys(scanner)).toEqual(["scan"]); + yield* sql`DROP TABLE kin_secret_allow`; + expect((yield* run(orgId, "claims/test.md").pipe(Effect.result))._tag).toBe("Failure"); + }).pipe(Effect.provide(layer)), +); +it.effect("fails closed when decoded findings or the allowlist exceed their bounds", () => + Effect.gen(function* () { + const scanner = yield* Scan.KinSecretScan; + const content = Buffer.from( + Array.from({ length: 1100 }, () => Buffer.from(github).toString("base64")).join("\n"), + ); + expect( + (yield* scanner + .scan({ orgId, files: [{ path: "claims/many.md", content }] }) + .pipe(Effect.result))._tag, + ).toBe("Failure"); + const exhausted = yield* scanner + .scan({ + orgId, + files: Array.from({ length: 8 }, (_, i) => ({ + path: `claims/${i}.md`, + content: Buffer.from("a".repeat(1024 * 1024 - 100)), + })), + }) + .pipe(Effect.result); + expect(exhausted._tag).toBe("Failure"); + if (exhausted._tag === "Failure") + expect(exhausted.failure.reason).toBe("Scan work limit exceeded"); + const sql = yield* SqlClient.SqlClient; + yield* sql`WITH RECURSIVE entries(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM entries WHERE n < 4097) + INSERT INTO kin_secret_allow SELECT ${orgId}, 'claims/' || n, 'github-token', ${"0".repeat(64)}, 'signed-fixture', 0 FROM entries`; + expect((yield* scanner.scan({ orgId, files: [] }).pipe(Effect.result))._tag).toBe("Failure"); + }).pipe(Effect.provide(layer)), +); +it.effect("fails closed at file, batch and count limits", () => + Effect.gen(function* () { + const scanner = yield* Scan.KinSecretScan; + const scan = (files: { path: string; content: Uint8Array }[]) => scanner.scan({ orgId, files }); + for (const files of [ + [{ path: "huge", content: Buffer.alloc(1024 * 1024 + 1) }], + Array.from({ length: 9 }, () => ({ path: "batch", content: Buffer.alloc(1024 * 1024) })), + Array.from({ length: 1025 }, () => ({ path: "count", content: Buffer.alloc(0) })), + ]) + expect((yield* scan(files).pipe(Effect.result))._tag).toBe("Failure"); + const content = Buffer.from( + "a".repeat(1024 * 1024 - github.length - 1 - "large".length) + "\n" + github, + ); + expect( + (yield* scan([{ path: "large", content }])).some((f) => f.ruleId === "github-token"), + ).toBe(true); + }).pipe(Effect.provide(layer)), +); + +it.effect("near misses and private key headers", () => + Effect.gen(function* () { + const scanner = yield* Scan.KinSecretScan; + for (const content of [ + "task-follow-up-with-design-team-on-onboarding", + "risk-follow-up-with-design-team-on-onboarding", + "desk-follow-up-with-design-team-on-onboarding", + "token: 0123456789abcdef0123456789abcdef01234567", + "Password: https://vault.example.invalid/credentials/team", + "secret: projects/example-project/secrets/database/versions/latest", + "github_token_scopes: repo,workflow,admin:org,read:user", + "postgres://user:${DB_PASSWORD}@example.invalid/db", + "ASIAPACIFICREGION2026", + ["sk-", "proj-2026-11-follow-up-with-design-team-on-onboarding"].join(""), + ["sk-", "admin-rotation-plan-for-2026-q3-and-next-year"].join(""), + "z".repeat(4096) + ["AK", "IA"].join("") + "Q".repeat(16), + " ".repeat(8173) + ["AK", "IA"].join("") + "Q".repeat(16) + "z", + ]) + expect( + (yield* scanner.scan({ + orgId, + files: [{ path: "claims/test.md", content: Buffer.from(content) }], + })).length, + ).toBe(0); + for (const content of [ + "-----BEGIN PGP PRIVATE KEY BLOCK-----", + "-----BEGIN CUSTOM PRIVATE KEY-----", + ]) + expect( + (yield* scanner.scan({ + orgId, + files: [{ path: "claims/test.md", content: Buffer.from(content) }], + })).some((f) => f.ruleId === "pem-private-key"), + ).toBe(true); + }).pipe(Effect.provide(layer)), +); +it.effect("scans paths without exposing them", () => + Effect.gen(function* () { + const scanner = yield* Scan.KinSecretScan; + const findings = yield* scanner.scan({ + orgId, + files: [{ path: `claims/${github}.md`, content: Buffer.from("clean") }], + }); + expect(findings.some((f) => f.ruleId === "github-token")).toBe(true); + expect(JSON.stringify(findings).includes(github)).toBe(false); + const indexFindings = yield* scanner.scan({ + orgId, + files: [ + { path: "claims/item.md", content: Buffer.from("clean") }, + { + path: "INDEX.md", + content: Buffer.from("- [claim (claims/wrong.md)] " + github + " (claims/item.md)"), + }, + ], + }); + expect(indexFindings.some((f) => f.itemPath === "claims/item.md")).toBe(true); + }).pipe(Effect.provide(layer)), +); +it.effect("bounds adversarial rule inputs structurally", () => + Effect.gen(function* () { + const scanner = yield* Scan.KinSecretScan; + for (const prefix of ["eyJ", "sk-", "AKIA"]) { + const content = prefix.repeat(Math.floor((1024 * 1024 - 100) / prefix.length)); + let examined = 0; + for (const window of Scan.ruleWindows(content)) { + expect(window.text.length).toBeLessThanOrEqual(8194); + examined += window.text.length; + } + expect(examined).toBeLessThanOrEqual( + content.length * 2 + Math.ceil(content.length / 4096) * 2, + ); + expect( + (yield* scanner.scan({ + orgId, + files: [{ path: "claims/test.md", content: Buffer.from(content) }], + })).length, + ).toBe(0); + } + }).pipe(Effect.provide(layer)), +); + +it.effect.each([ + { name: "256 KiB journal", count: 1, bytes: 256 * 1024 }, + { name: "30 x 60 KiB batch", count: 30, bytes: 60 * 1024 }, +])("scans realistic $name without exhausting work", ({ count, bytes }) => + Effect.gen(function* () { + const scanner = yield* Scan.KinSecretScan; + const journal = + [ + "Discussed the next steps with the team. Follow up tomorrow.", + "Commit " + "0123456789abcdef".repeat(2) + "01234567 reviewed.", + "Updated apps/server/src/kin/memory/secretScan.test.ts and docs/operations/development.md", + "Trace 12345678-1234-5678-9abc-123456789abc completed successfully.", + ].join("\n") + "\n"; + const files = Array.from({ length: count }, (_, i) => ({ + path: `journal/${i}.md`, + content: Buffer.from(journal.repeat(Math.ceil(bytes / journal.length)).slice(0, bytes)), + })); + expect((yield* scanner.scan({ orgId, files })).length).toBe(0); + }).pipe(Effect.provide(layer)), +); + +it.effect("bounds rule executions for random runs, word lines and multi-file inputs", () => + Effect.gen(function* () { + const scanner = yield* Scan.KinSecretScan; + let seed = 42; + const random = (length: number) => + Array.from({ length }, () => { + seed = (Math.imul(seed, 1664525) + 1013904223) >>> 0; + return "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"[seed % 62]; + }).join(""); + const runs = Array.from({ length: Math.floor((1024 * 1024 - 100) / 25) }, () => + random(24), + ).join(" "); + const words = (Array.from({ length: 10 }, () => random(7)).join(" ") + "\n").repeat(13000); + for (const files of [ + [{ path: "runs", content: Buffer.from(runs) }], + [{ path: "words", content: Buffer.from(words) }], + Array.from({ length: 8 }, (_, i) => ({ path: `runs-${i}`, content: Buffer.from(runs) })), + ]) { + let executions = 0; + const exec = RegExp.prototype.exec; + using _spy = vi.spyOn(RegExp.prototype, "exec").mockImplementation(function ( + this: RegExp, + value: string, + ) { + if (this.source.startsWith("(?()( "KinSecretScanError", { reason: Schema.String, + cause: Schema.optional(Schema.Defect()), }, ) {} export interface SecretFinding { readonly path: string; readonly ruleId: string; readonly line: number; + readonly pathHit?: boolean; + readonly itemPath?: string; } export class KinSecretScan extends Context.Service< KinSecretScan, @@ -25,14 +32,263 @@ export class KinSecretScan extends Context.Service< } >()("t3/kin/memory/secretScan/KinSecretScan") {} -export const noFindingsStubLayer = Layer.effect( - KinSecretScan, - Effect.gen(function* () { - yield* Effect.logWarning("SECURITY: Kin secret scanning is disabled (no-findings stub)"); - return KinSecretScan.of({ scan: Effect.fn("KinSecretScan.stub")(() => Effect.succeed([])) }); - }), -); -export const testLayer = (findings: ReadonlyArray) => - Layer.succeed(KinSecretScan, { - scan: Effect.fn("KinSecretScan.fixture")(() => Effect.succeed(findings)), +const rules = [ + ["github-token", /(?:gh[pousr]_[A-Za-z0-9]{36}|github_pat_[A-Za-z0-9_]{82})/], + ["anthropic-key", /sk-ant-[A-Za-z0-9_-]{40,200}/], + [ + "openai-key", + /sk-(?:(?:proj|svcacct|admin)-[A-Za-z0-9_-]{32,200}|[A-Za-z0-9]{20}T3BlbkFJ[A-Za-z0-9]{20})/, + ], + ["aws-access-key", /(?:AKIA|ASIA)[A-Z0-9]{16}/], + ["slack-token", /xox[abposr]-[A-Za-z0-9-]{20,200}/], + ["stripe-key", /[rs]k_live_[A-Za-z0-9]{24,200}/], + ["google-key", /AIza[A-Za-z0-9_-]{35}/], + ["pem-private-key", /-----BEGIN (?:[A-Z0-9]+ ){0,8}PRIVATE KEY(?: BLOCK)?-----/], + ["jwt", /eyJ[A-Za-z0-9_-]{5,2048}\.eyJ[A-Za-z0-9_-]{8,1024}/], + ["url-password", /\b[a-z][a-z0-9+.-]{0,31}:\/\/[^\s/:@]{1,512}:([^\s/@]{1,512})@/i], + [ + "secret-assignment", + /\b[A-Za-z0-9_]{0,64}(?:password|secret|token|api[_-]?key|access[_-]?key)[A-Za-z0-9_]{0,64}["']?\s{0,32}[:=]\s{0,32}["']?([^\s"';#]{20,512})/i, + ], +] as const; +const compiledRules = rules.map(([id, pattern]) => { + const alphabet = + id === "github-token" || id === "aws-access-key" || id === "stripe-key" || id === "google-key" + ? "A-Za-z0-9" + : id === "slack-token" + ? "A-Za-z0-9-" + : "A-Za-z0-9_-"; + const right = ["url-password", "secret-assignment", "jwt"].includes(id) + ? "" + : `(?![${alphabet}])`; + return [ + id, + new RegExp(`(? 0 ? 1 : 0, + rightContext: offset + 8192 < value.length ? 1 : 0, + }; +}; +const entropy = (value: string) => { + const counts = new Map(); + for (const char of value) counts.set(char, (counts.get(char) ?? 0) + 1); + let bits = 0; + for (const count of counts.values()) { + const p = count / value.length; + bits -= p * Math.log2(p); + } + return bits; +}; +// NFKC covers fullwidth forms; common Cyrillic/Greek confusables are normalized as well. +const lookalikes: Readonly> = { + ѕ: "s", + һ: "h", + ο: "o", + ɡ: "g", + а: "a", + е: "e", + о: "o", + р: "p", + с: "c", + х: "x", + у: "y", + і: "i", + ј: "j", + А: "A", + В: "B", + Е: "E", + К: "K", + М: "M", + Н: "H", + О: "O", + Р: "P", + С: "C", + Т: "T", + Х: "X", + Α: "A", + Β: "B", + Ε: "E", + Ι: "I", + Κ: "K", + Μ: "M", + Ν: "N", + Ο: "O", + Ρ: "P", + Τ: "T", + Χ: "X", +}; +const normalize = (text: string) => + text + .normalize("NFKC") + .replace(/\p{Default_Ignorable_Code_Point}/gu, "") + .replace(/[ѕһοɡаеорсхуіјАВЕКМНОРСТХΑΒΕΙΚΜΝΟΡΤΧ]/g, (c) => lookalikes[c] ?? c); +const fingerprint = (ruleId: string, value: string) => + NodeCrypto.createHash("sha256").update(ruleId).update("\0").update(value).digest("hex"); +const make = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const scan = Effect.fn("KinSecretScan.scan")(function* (input: { + orgId: KinOrgId; + files: ReadonlyArray<{ path: string; content: Uint8Array }>; + }) { + if ( + input.files.length > 1024 || + input.files.some((f) => f.content.byteLength + Buffer.byteLength(f.path) > 1024 * 1024) || + input.files.reduce((sum, f) => sum + f.content.byteLength + Buffer.byteLength(f.path), 0) > + 8 * 1024 * 1024 + ) + return yield* new KinSecretScanError({ reason: "Scan size limit exceeded" }); + // No mutation API: the owner-signed action executor owns allowlist insertion. + const allowed = yield* sql<{ path: string; rule_id: string; fingerprint: string }>` + SELECT path, rule_id, fingerprint FROM kin_secret_allow WHERE org_id = ${input.orgId} LIMIT 4097 + `.pipe( + Effect.mapError(toPersistenceSqlError("KinSecretScan.allowlist")), + Effect.mapError( + (cause) => new KinSecretScanError({ reason: "Allowlist unavailable", cause }), + ), + ); + if (allowed.length > 4096) + return yield* new KinSecretScanError({ reason: "Allowlist size limit exceeded" }); + const allow = new Set(allowed.map((r) => JSON.stringify([r.path, r.rule_id, r.fingerprint]))); + const findings: SecretFinding[] = []; + const seen = new Set(); + let budget = 64 * 1024 * 1024; + for (const file of input.files) { + const text = normalize(Buffer.from(file.content).toString("utf8")); + const lines = text.split("\n"); + const pathFindingStart = findings.length; + let pathHit = true; + let redactPath = false; + const inspect = function* (value: string, baseLine: number, depth: number): Generator { + budget -= value.length; + if (budget < 0 || findings.length > 1024) return; + for (const window of ruleWindows(value)) { + budget -= window.text.length; + if (budget < 0) return; + yield; + if (!anchors.test(window.text)) continue; + for (const [ruleId, bounded] of compiledRules) { + bounded.lastIndex = 0; + let match: RegExpExecArray | null; + while ((match = bounded.exec(window.text)) !== null) { + budget -= 64; + if (budget < 0) return; + if ( + match.index < window.leftContext || + match.index + match[0].length > window.text.length - window.rightContext + ) + continue; + const index = window.offset + match.index; + const secret = match[1] ?? match[0]; + if (ruleId === "openai-key" && !/[A-Z]/.test(secret)) continue; + if (ruleId === "url-password" && /^\$\{[^}]+\}$/.test(secret)) continue; + if ( + ruleId === "secret-assignment" && + (entropy(secret) < 3.5 || + /^[a-f0-9]{40}(?:[a-f0-9]{24})?$/i.test(secret) || + /^https?:\/\//i.test(secret) || + /^projects\/[^/]+\/(?:locations\/[^/]+\/)?secrets\//.test(secret) || + /^[^:=]*token_scopes\s*[:=]/i.test(match[0])) + ) + continue; + if (allow.has(JSON.stringify([file.path, ruleId, fingerprint(ruleId, secret)]))) + continue; + const line = baseLine + (value.slice(0, index).match(/\n/g)?.length ?? 0); + const key = JSON.stringify([file.path, ruleId, line, pathHit]); + if (!seen.has(key)) { + const indexLine = file.path === "INDEX.md" ? lines[line - 1] : undefined; + const itemPath = Array.from(indexLine?.matchAll(/\(([^()]+\.md)\)/g) ?? []).at( + -1, + )?.[1]; + seen.add(key); + findings.push({ + path: pathHit || redactPath ? "[redacted]" : file.path, + ruleId, + line, + ...(pathHit ? { pathHit: true } : {}), + ...(itemPath && input.files.some((f) => f.path === itemPath) ? { itemPath } : {}), + }); + } + break; + } + } + } + if (depth === 2) return; + let cursor = 0; + let decodedLine = baseLine; + for (const match of value.matchAll(/(?:%[0-9a-fA-F]{2}|[A-Za-z0-9_+/-]){20,}={0,2}/g)) { + const encoded = match[0]; + // Mixed percent/plain encodings can represent 20 bytes in fewer than 60 characters. + if (!encoded.includes("%") && encoded.length < 27) continue; + decodedLine += value.slice(cursor, match.index).match(/\n/g)?.length ?? 0; + cursor = match.index; + yield; + budget -= encoded.length; + if (budget < 0) return; + const candidates = encoded.includes("%") + ? [ + Buffer.from( + encoded.replace(/%([0-9a-f]{2})/gi, (_, hex: string) => + String.fromCharCode(Number.parseInt(hex, 16)), + ), + "latin1", + ).toString("utf8"), + ] + : Array.from({ length: 4 }, (_, alignment) => + Buffer.from(encoded.slice(alignment), "base64url").toString("utf8"), + ); + for (const candidate of candidates) { + if (budget < 0 || findings.length > 1024) return; + const decoded = normalize(candidate).replace(/[^\x09\x0a\x0d\x20-\x7e]/g, " "); + if (decoded !== encoded) yield* inspect(decoded, decodedLine, depth + 1); + } + } + }; + for (const _ of inspect(normalize(file.path), 1, 0)) yield* Effect.yieldNow; + redactPath = findings.length > pathFindingStart; + pathHit = false; + for (const _ of inspect(text, 1, 0)) yield* Effect.yieldNow; + let run: { text: string; line: number }[] = []; + const append = function* (text: string, line: number): Generator { + run.push({ text, line }); + while (run.length > 8 || run.reduce((sum, part) => sum + part.text.length, 0) > 4096) + run.shift(); + if (run.length < 2) return; + const joined = run.map((part) => part.text).join(""); + // No rule matches without an anchor and no decoder starts below 20 characters. + if (joined.length >= 20 || anchors.test(joined)) yield* inspect(joined, run[0]!.line, 0); + }; + for (const [index, raw] of lines.entries()) { + if (index % 1024 === 1023) yield* Effect.yieldNow; + const words = raw + .replace(/\\?\r$/, "") + .replace(/\\$/, "") + .trim() + .split(/\s+/); + const first = words[0]!; + if (!first) { + run = []; + continue; + } + for (const _ of append(first, index + 1)) yield* Effect.yieldNow; + if (words.length > 1) run = [{ text: words.at(-1)!, line: index + 1 }]; + } + } + if (budget < 0 || findings.length > 1024) + return yield* new KinSecretScanError({ reason: "Scan work limit exceeded" }); + const unsafePaths = findings.some((f) => f.pathHit); + return findings.map(({ itemPath, ...finding }) => ({ + ...finding, + ...(itemPath && !unsafePaths ? { itemPath } : {}), + })); }); + return KinSecretScan.of({ scan }); +}); +export const layer = Layer.effect(KinSecretScan, make); diff --git a/apps/server/src/kin/persistence/KinMigrations.test.ts b/apps/server/src/kin/persistence/KinMigrations.test.ts index 4b855164bf64d..8a3537cb5d44a 100644 --- a/apps/server/src/kin/persistence/KinMigrations.test.ts +++ b/apps/server/src/kin/persistence/KinMigrations.test.ts @@ -36,6 +36,9 @@ const tableNames = [ "kin_ping_items", "kin_ping_ledger", "kin_read_state", + "kin_routine_folds", + "kin_routine_preferences", + "kin_secret_allow", "kin_session_bindings", "kin_shadow_entries", "kin_trigger_fires", @@ -98,6 +101,8 @@ describe("Kin migrations", () => { { migration_id: 9, name: "JudgeLog" }, { migration_id: 10, name: "ChatChanges" }, { migration_id: 11, name: "MemoryHeads" }, + { migration_id: 12, name: "SecretAllow" }, + { migration_id: 13, name: "Routines" }, ]); assert.deepStrictEqual( upstreamBefore.map((row) => [row.migration_id, row.name] as const), @@ -151,6 +156,8 @@ describe("Kin migrations", () => { { migration_id: 9, name: "JudgeLog" }, { migration_id: 10, name: "ChatChanges" }, { migration_id: 11, name: "MemoryHeads" }, + { migration_id: 12, name: "SecretAllow" }, + { migration_id: 13, name: "Routines" }, ]); assert.deepStrictEqual(yield* upstreamHistory, upstreamBefore); assert.deepStrictEqual(yield* sql`SELECT * FROM kin_orgs`, [ @@ -160,6 +167,10 @@ describe("Kin migrations", () => { yield* sql`SELECT name FROM pragma_index_info('idx_kin_triggers_due') ORDER BY seqno`, ["next_run_at", "org_id", "trigger_id"].map((name) => ({ name })), ); + assert.deepStrictEqual( + yield* sql`SELECT name FROM pragma_index_info('kin_workers_owner_review') ORDER BY seqno`, + ["org_id", "ended_at", "worker_id"].map((name) => ({ name })), + ); const indexes = yield* sql<{ name: string }>` SELECT name FROM sqlite_master WHERE type = 'index' AND name GLOB 'idx_kin_*' ORDER BY name `; @@ -229,6 +240,8 @@ describe("Kin migrations", () => { [9, "JudgeLog"], [10, "ChatChanges"], [11, "MemoryHeads"], + [12, "SecretAllow"], + [13, "Routines"], ]); const before = yield* kinSchema; assert.deepStrictEqual( @@ -266,6 +279,10 @@ describe("Kin migrations", () => { yield* sql`DROP TABLE kin_ping_items`; yield* sql`DROP INDEX kin_chat_members_one_owner`; yield* sql`DROP TABLE kin_memory_heads`; + yield* sql`DROP TABLE kin_secret_allow`; + yield* sql`DROP INDEX kin_workers_owner_review`; + yield* sql`DROP TABLE kin_routine_folds`; + yield* sql`DROP TABLE kin_routine_preferences`; yield* sql`DROP TABLE kin_budget_charges`; yield* sql`ALTER TABLE kin_events DROP COLUMN budget_deferred`; yield* sql`UPDATE kin_chats SET last_seq = 1 WHERE org_id = 'org' AND chat_id = 'existing-chat'`; @@ -280,6 +297,8 @@ describe("Kin migrations", () => { [9, "JudgeLog"], [10, "ChatChanges"], [11, "MemoryHeads"], + [12, "SecretAllow"], + [13, "Routines"], ]); assert.deepStrictEqual( yield* sql`SELECT member_id FROM kin_chat_members WHERE org_id = 'org' ORDER BY member_id`, @@ -316,6 +335,8 @@ describe("Kin migrations", () => { { migration_id: 9, name: "JudgeLog" }, { migration_id: 10, name: "ChatChanges" }, { migration_id: 11, name: "MemoryHeads" }, + { migration_id: 12, name: "SecretAllow" }, + { migration_id: 13, name: "Routines" }, ]); assert.deepStrictEqual(yield* runKinMigrations(), []); assert.deepStrictEqual(yield* runMigrations(), []); diff --git a/apps/server/src/kin/persistence/KinMigrations.ts b/apps/server/src/kin/persistence/KinMigrations.ts index b7bba32965691..64fe950fca142 100644 --- a/apps/server/src/kin/persistence/KinMigrations.ts +++ b/apps/server/src/kin/persistence/KinMigrations.ts @@ -14,6 +14,8 @@ import JudgeLog from "./migrations/K009_JudgeLog.ts"; import ChatChanges from "./migrations/K010_ChatChanges.ts"; import MemoryHeads from "./migrations/K011_MemoryHeads.ts"; +import SecretAllow from "./migrations/K012_SecretAllow.ts"; +import Routines from "./migrations/K013_Routines.ts"; const run = Migrator.make({}); const loader = Migrator.fromRecord({ @@ -28,6 +30,8 @@ const loader = Migrator.fromRecord({ "9_JudgeLog": JudgeLog, "10_ChatChanges": ChatChanges, "11_MemoryHeads": MemoryHeads, + "12_SecretAllow": SecretAllow, + "13_Routines": Routines, }); export const runKinMigrations = Effect.fn("KinMigrations.run")(function* () { diff --git a/apps/server/src/kin/persistence/migrations/K012_SecretAllow.ts b/apps/server/src/kin/persistence/migrations/K012_SecretAllow.ts new file mode 100644 index 0000000000000..c5563c34c8717 --- /dev/null +++ b/apps/server/src/kin/persistence/migrations/K012_SecretAllow.ts @@ -0,0 +1,15 @@ +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/sql/SqlClient"; + +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql`CREATE TABLE kin_secret_allow ( + org_id TEXT NOT NULL, + path TEXT NOT NULL, + rule_id TEXT NOT NULL, + fingerprint TEXT NOT NULL CHECK(length(fingerprint) = 64), + signed_action_id TEXT NOT NULL, + created_at INTEGER NOT NULL, + PRIMARY KEY (org_id, path, rule_id, fingerprint) + )`; +}); diff --git a/apps/server/src/kin/persistence/migrations/K013_Routines.ts b/apps/server/src/kin/persistence/migrations/K013_Routines.ts new file mode 100644 index 0000000000000..cacbc61e1bb18 --- /dev/null +++ b/apps/server/src/kin/persistence/migrations/K013_Routines.ts @@ -0,0 +1,22 @@ +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/sql/SqlClient"; + +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql`CREATE INDEX kin_workers_owner_review ON kin_workers (org_id, ended_at, worker_id) WHERE report_json IS NOT NULL`; + yield* sql`CREATE TABLE kin_routine_preferences ( + org_id TEXT NOT NULL REFERENCES kin_orgs(org_id), + routine TEXT NOT NULL CHECK (routine IN ('morning_brief', 'evening_summary', 'owner_check')), + enabled INTEGER NOT NULL DEFAULT 1 CHECK (enabled IN (0, 1)), + paused INTEGER NOT NULL DEFAULT 0 CHECK (paused IN (0, 1)), + PRIMARY KEY (org_id, routine) + )`; + yield* sql`CREATE TABLE kin_routine_folds ( + org_id TEXT NOT NULL REFERENCES kin_orgs(org_id), + fire_id TEXT NOT NULL, + routine TEXT NOT NULL, + due_at INTEGER NOT NULL CHECK (typeof(due_at) = 'integer'), + fold_into TEXT NOT NULL CHECK (fold_into IN ('morning_brief', 'evening_summary')), + PRIMARY KEY (org_id, fire_id) + )`; +}); diff --git a/apps/server/src/kin/triggers/KinTriggerInbox.ts b/apps/server/src/kin/triggers/KinTriggerInbox.ts index b21e6f0e325f2..d6e2cfa90e0fb 100644 --- a/apps/server/src/kin/triggers/KinTriggerInbox.ts +++ b/apps/server/src/kin/triggers/KinTriggerInbox.ts @@ -1,4 +1,10 @@ -import { KinOrgId, KinTriggerFireId, KinTriggerFired, NonNegativeInt } from "@t3tools/contracts"; +import { + KinOrgId, + KinRoutine, + KinTriggerFireId, + KinTriggerFired, + NonNegativeInt, +} from "@t3tools/contracts"; import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; @@ -36,6 +42,7 @@ const isInboxConflict = Schema.is(KinTriggerInboxConflict); const PendingRequest = Schema.Struct({ orgId: KinOrgId, limit: Schema.Int.check(Schema.isGreaterThan(0)), + routines: Schema.optional(Schema.Array(KinRoutine)), }); const InboxRow = Schema.Struct({ payload: Schema.fromJsonString(KinTriggerFired) }); const decodeInboxRow = Schema.decodeUnknownEffect(InboxRow); @@ -50,6 +57,7 @@ export class KinTriggerInbox extends Context.Service< readonly listPending: ( orgId: KinOrgId, limit?: number, + routines?: ReadonlyArray, ) => Effect.Effect, InboxError>; readonly markConsumed: ( orgId: KinOrgId, @@ -73,9 +81,14 @@ const make = Effect.gen(function* () { const pending = SqlSchema.findAll({ Request: PendingRequest, Result: InboxRow, - execute: ({ orgId, limit }) => sql` + execute: ({ orgId, limit, routines }) => sql` SELECT payload_json AS payload FROM kin_trigger_inbox WHERE org_id = ${orgId} AND consumed_at IS NULL + ${ + routines === undefined + ? sql`` + : sql`AND json_extract(payload_json, '$.payload.subject.routine') IN (${sql.join(",", false)(routines.map((routine) => sql`${routine}`))})` + } ORDER BY received_at, fire_id LIMIT ${limit} `, }); @@ -144,10 +157,12 @@ const make = Effect.gen(function* () { Effect.mapError(toInboxError("KinTriggerInbox.put")), ), listPending: Effect.fn("KinTriggerInbox.listPending")( - function* (orgId, limit = 100) { - return (yield* pending({ orgId, limit }).pipe(NodeSqliteClient.readOnly)).map( - (row) => row.payload, - ); + function* (orgId, limit = 100, routines) { + return (yield* pending({ + orgId, + limit, + ...(routines === undefined ? {} : { routines }), + }).pipe(NodeSqliteClient.readOnly)).map((row) => row.payload); }, Effect.mapError(toInboxError("KinTriggerInbox.listPending")), ), diff --git a/apps/server/src/kin/triggers/ownerCheck.test.ts b/apps/server/src/kin/triggers/ownerCheck.test.ts new file mode 100644 index 0000000000000..64048b929c60c --- /dev/null +++ b/apps/server/src/kin/triggers/ownerCheck.test.ts @@ -0,0 +1,276 @@ +import { assert, it } from "@effect/vitest"; +import { + KinEvent, + KinEventId, + KinChatId, + KinWidgetId, + KinPersonId, + KinWakeupId, + KinAgentId, + KinOwnerCheckCard, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; +import * as SqlClient from "effect/sql/SqlClient"; +import * as TestClock from "effect/testing/TestClock"; +import * as MemoryWriter from "../memory/MemoryWriter.ts"; +import * as Service from "../chat/KinChatService.ts"; +import * as Queue from "../lead/KinEventQueue.ts"; +import * as OwnerCheck from "./ownerCheck.ts"; +import { fixture, seed, orgId, otherOrg, item } from "./routines.testkit.ts"; + +const encodeJson = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown)); +const decodeCard = Schema.decodeSync(Schema.fromJsonString(KinOwnerCheckCard)); +const decodeEvent = Schema.decodeSync(KinEvent); +const now = Date.parse("2026-11-09T09:00:00.000Z"); +const memoryItems = [ + item("old-review", { review_by: "2026-11-01" }), + item("misleading"), + item("low-trust", { trust: "external" }), + item("no-evidence", { links: [] }), + item("last-review", { review_by: "2026-11-02" }), + item("sixth", { links: [] }), + item("invalid", { invalid_at: "2026-11-01", review_by: "2026-10-01" }), + item("other", { orgId: otherOrg, review_by: "2026-10-01" }), +]; +it.effect("picks five distinct, valid same-org items in the specified category order", () => + Effect.gen(function* () { + yield* seed(); + const sql = yield* SqlClient.SqlClient; + const report = { + result: "done", + summary: "Done", + evidence: [], + openQuestions: [], + memorySuggestions: [], + misleadingMemory: [{ itemId: "misleading", why: "Wrong evidence" }], + parsed: true, + }; + const json = yield* encodeJson(report); + yield* sql`INSERT INTO kin_wakeups (org_id, wakeup_id, agent_id, model, started_at, status, event_ids) VALUES (${orgId}, 'worker-wakeup', 'lead-a', 'fixture', 0, 'completed', '[]')`; + yield* sql`INSERT INTO kin_workers (worker_id, org_id, agent_id, wakeup_id, thread_id, kind, model_json, provenance_json, status, report_json, started_at, ended_at) VALUES ('worker', ${orgId}, 'lead-a', 'worker-wakeup', 'thread', 'research', '{}', '{}', 'done', ${json}, 0, ${now})`; + yield* sql`INSERT INTO kin_workers (worker_id, org_id, agent_id, wakeup_id, thread_id, kind, model_json, provenance_json, status, report_json, started_at, ended_at) VALUES ('ancient-worker', ${orgId}, 'lead-a', 'worker-wakeup', 'thread-old', 'research', '{}', '{}', 'done', 'invalid ancient report', 0, ${now - 91 * 86_400_000})`; + yield* TestClock.setTime(now); + const service = yield* OwnerCheck.OwnerCheck; + const selected = yield* service.select(orgId); + assert.deepEqual( + selected.map((item) => [item.itemId, item.reason]), + [ + ["old-review", "past_review"], + ["last-review", "past_review"], + ["misleading", "misleading"], + ["low-trust", "old_low_trust"], + ["no-evidence", "no_evidence"], + ], + ); + assert.deepEqual( + OwnerCheck.pickOwnerCheckItems(orgId, memoryItems.toReversed(), new Set(["misleading"]), now), + selected, + ); + }).pipe(Effect.provide(fixture(memoryItems))), +); + +it.effect( + "posts one native card per fire and routes owner taps to the writer without accepting model authority", + () => + Effect.gen(function* () { + yield* seed(); + yield* TestClock.setTime(now); + const service = yield* OwnerCheck.OwnerCheck; + const chat = yield* Service.KinChatService; + const queue = yield* Queue.KinEventQueue; + const sql = yield* SqlClient.SqlClient; + yield* service.post(orgId, "weekly", yield* service.select(orgId)); + yield* service.post(orgId, "weekly", yield* service.select(orgId)); + const [widget] = yield* sql<{ + state_json: string; + }>`SELECT state_json FROM kin_widgets WHERE org_id = ${orgId} ORDER BY widget_id`; + const card = decodeCard(widget!.state_json); + assert.equal(card.items[0]?.itemId, "old-review"); + assert.lengthOf( + yield* sql`SELECT widget_id FROM kin_widgets WHERE org_id = ${orgId} ORDER BY widget_id`, + 1, + ); + yield* chat + .upsertWidget( + { _tag: "person", orgId, personId: KinPersonId.make("owner-a") }, + { chatId: KinChatId.make("chat-a"), type: "owner_check", state: card }, + ) + .pipe(Effect.flip); + yield* chat + .act( + { _tag: "person", orgId, personId: KinPersonId.make("owner-a") }, + { + widgetId: KinWidgetId.make("owner-check:a:weekly"), + revision: 1, + actionId: "drop", + value: "foreign-item", + clientActionId: "forged-target", + }, + ) + .pipe(Effect.flip); + const staged: MemoryWriter.MemoryChange[] = []; + yield* Effect.scoped( + Effect.gen(function* () { + const writer = Layer.mock(MemoryWriter.MemoryWriter)({ + stage: (_org, _wakeup, change) => + Effect.sync(() => { + staged.push(change); + }), + }); + for (const [index, action] of (["keep", "update", "drop"] as const).entries()) { + const message = yield* chat.act( + { _tag: "person", orgId, personId: KinPersonId.make("owner-a") }, + { + widgetId: KinWidgetId.make("owner-check:a:weekly"), + revision: index + 1, + actionId: action, + value: card.items[index]!.itemId, + clientActionId: action, + }, + ); + yield* queue.enqueue( + decodeEvent({ + _tag: "widget.action", + orgId, + eventId: action, + agentId: "lead-a", + receivedAt: "2026-11-09T09:00:00.000Z", + dedupeKey: action, + provenance: { _tag: "human", personId: "owner-a", memberId: "owner-member-a" }, + chatId: "chat-a", + messageId: message.messageId, + actionId: action, + }), + ); + yield* TestClock.adjust("20 seconds"); + const wakeupId = KinWakeupId.make(`review-${action}`); + yield* Effect.scoped( + Effect.gen(function* () { + yield* queue.claim({ + orgId, + agentId: KinAgentId.make("lead-a"), + wakeupId, + model: "fixture", + }); + const result = yield* service + .act(orgId, wakeupId, KinEventId.make(action)) + .pipe(Effect.provide(writer)); + assert.equal(result.action, action); + const [updated] = yield* sql<{ + revision: number; + closed_at: string | null; + state_json: string; + }>`SELECT revision, closed_at, state_json FROM kin_widgets WHERE org_id = ${orgId} ORDER BY widget_id`; + assert.equal(updated!.revision, index + 2); + assert.equal(decodeCard(updated!.state_json).items[index]!.handled, action); + assert.equal(updated!.closed_at !== null, index === 2); + yield* queue.finish({ orgId, wakeupId, status: "completed" }); + }), + ); + } + assert.lengthOf(staged, 2); + assert.equal(staged[0]?.sourceEventId, "keep"); + assert.include(staged[0]!.content, "2026-12-09T09:00:20.000Z"); + assert.include(staged[1]!.content, "2026-11-09T09:01:00.000Z"); + for (const [org, eventId, wakeup] of [ + [otherOrg, "keep", "review-keep"], + [orgId, "keep", "foreign-wakeup"], + ] as const) { + const result = yield* service + .act(org, wakeup, KinEventId.make(eventId)) + .pipe(Effect.provide(writer), Effect.flip); + assert.equal(result._tag, "OwnerCheckError"); + } + yield* sql`UPDATE kin_chat_members SET role = 'member' WHERE org_id = ${orgId} AND member_id = 'owner-member-a'`; + yield* service + .act(orgId, "review-keep", KinEventId.make("keep")) + .pipe(Effect.provide(writer), Effect.flip); + assert.lengthOf(staged, 2); + yield* chat + .act( + { _tag: "person", orgId, personId: KinPersonId.make("owner-a") }, + { + widgetId: KinWidgetId.make("owner-check:a:weekly"), + revision: 1, + actionId: "keep", + value: "old-review", + clientActionId: "revoked-owner", + }, + ) + .pipe(Effect.flip); + }), + ); + }).pipe( + Effect.provide( + fixture( + memoryItems + .slice(0, 3) + .map((item, index) => ({ ...item, review_by: `2026-11-0${index + 1}` })), + ), + ), + ), +); + +it.effect.each(["keep", "drop"] as const)("discards staged %s when the card changes", (action) => + Effect.gen(function* () { + yield* seed(); + yield* TestClock.setTime(now); + const service = yield* OwnerCheck.OwnerCheck; + const chat = yield* Service.KinChatService; + const queue = yield* Queue.KinEventQueue; + const sql = yield* SqlClient.SqlClient; + yield* service.post(orgId, "race", yield* service.select(orgId)); + const widgetId = KinWidgetId.make("owner-check:a:race"); + const message = yield* chat.act( + { _tag: "person", orgId, personId: KinPersonId.make("owner-a") }, + { widgetId, revision: 1, actionId: action, value: "old-review", clientActionId: action }, + ); + yield* queue.enqueue( + decodeEvent({ + _tag: "widget.action", + orgId, + eventId: action, + agentId: "lead-a", + receivedAt: "2026-11-09T09:00:00.000Z", + dedupeKey: action, + provenance: { _tag: "human", personId: "owner-a", memberId: "owner-member-a" }, + chatId: "chat-a", + messageId: message.messageId, + actionId: action, + }), + ); + yield* TestClock.adjust("20 seconds"); + const wakeupId = KinWakeupId.make("racing-review"); + yield* queue.claim({ orgId, agentId: KinAgentId.make("lead-a"), wakeupId, model: "fixture" }); + let staged = false; + let discarded = false; + const writer = Layer.mock(MemoryWriter.MemoryWriter)({ + stage: () => + Effect.gen(function* () { + staged = true; + yield* sql`UPDATE kin_widgets SET revision = revision + 1 WHERE org_id = ${orgId} AND widget_id = ${widgetId}`.pipe( + Effect.orDie, + ); + }), + discard: (org, wakeup) => + Effect.sync(() => { + assert.equal(org, orgId); + assert.equal(wakeup, wakeupId); + discarded = true; + staged = false; + }), + }); + const failure = yield* service + .act(orgId, wakeupId, KinEventId.make(action)) + .pipe(Effect.provide(writer), Effect.flip); + assert.equal(failure._tag, "OwnerCheckError"); + assert.isTrue(discarded); + assert.isFalse(staged); + const [widget] = yield* sql<{ + state_json: string; + }>`SELECT state_json FROM kin_widgets WHERE org_id = ${orgId} AND widget_id = ${widgetId}`; + assert.isUndefined(decodeCard(widget!.state_json).items[0]?.handled); + }).pipe(Effect.provide(fixture(memoryItems)), Effect.scoped), +); diff --git a/apps/server/src/kin/triggers/ownerCheck.ts b/apps/server/src/kin/triggers/ownerCheck.ts new file mode 100644 index 0000000000000..f31102d5990c0 --- /dev/null +++ b/apps/server/src/kin/triggers/ownerCheck.ts @@ -0,0 +1,354 @@ +import { + KinEvent, + KinEventId, + KinMemoryItemId, + KinMessageId, + KinOrgId, + KinWidgetId, + KinPersonId, + type MemoryItemHead, + WorkerReport, + type KinWidget, + KinChatId, + KinOwnerCheckCard, +} from "@t3tools/contracts"; +import * as Clock from "effect/Clock"; +import * as Context from "effect/Context"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; +import * as SqlClient from "effect/sql/SqlClient"; +import { isSqlError } from "effect/sql/SqlError"; +import { stringify } from "yaml"; +import { toPersistenceSqlError } from "../../persistence/Errors.ts"; +import * as Memory from "../memory/Memory.ts"; +import * as MemoryWriter from "../memory/MemoryWriter.ts"; +import { memoryDateMillis } from "../memory/frontMatter.ts"; +import * as Chat from "../chat/KinChatService.ts"; +import * as Repository from "../chat/KinChatRepository.ts"; +import * as Hub from "../chat/KinChatHub.ts"; + +const day = 86_400_000; +const reasons = ["past_review", "misleading", "old_low_trust", "no_evidence"] as const; +const Selection = Schema.Struct({ itemId: KinMemoryItemId, reason: Schema.Literals(reasons) }); +type Selection = typeof Selection.Type; +const decodeCard = Schema.decodeUnknownEffect(KinOwnerCheckCard); +const decodeEvent = Schema.decodeUnknownEffect(Schema.fromJsonString(KinEvent)); +const decodeAction = Schema.decodeUnknownEffect( + Schema.fromJsonString( + Schema.Struct({ + widgetId: KinWidgetId, + revision: Schema.Int, + actionId: Schema.Literals(["keep", "update", "drop"]), + value: KinMemoryItemId, + }), + ), +); +const decodeReport = Schema.decodeUnknownEffect(Schema.fromJsonString(WorkerReport)); +const ordinal = (a: string, b: string) => (a < b ? -1 : a > b ? 1 : 0); + +export function pickOwnerCheckItems( + orgId: KinOrgId, + items: ReadonlyArray, + misleading: ReadonlySet, + now: number, +): ReadonlyArray { + return items + .filter( + (item) => + item.orgId === orgId && + !item.duplicate && + item.invalid_at === null && + item.superseded_by === null, + ) + .map((item) => { + const review = memoryDateMillis(item.review_by); + const rank = + review < now + ? 0 + : misleading.has(item.id) + ? 1 + : item.kind === "claim" && + item.trust !== "owner" && + memoryDateMillis(item.recorded_at) < now - 30 * day + ? 2 + : item.kind === "claim" && item.links.length === 0 + ? 3 + : -1; + return { item, review, rank }; + }) + .filter(({ rank }) => rank >= 0) + .toSorted( + (a, b) => + a.rank - b.rank || + (Number.isFinite(a.review) ? a.review : Infinity) - + (Number.isFinite(b.review) ? b.review : Infinity) || + ordinal(a.item.id, b.item.id) || + ordinal(a.item.path, b.item.path), + ) + .filter( + (row, index, rows) => rows.findIndex((other) => other.item.id === row.item.id) === index, + ) + .slice(0, 5) + .map(({ item, rank }) => ({ itemId: item.id, reason: reasons[rank] ?? "no_evidence" })); +} + +export class OwnerCheckError extends Schema.TaggedError()("OwnerCheckError", { + orgId: KinOrgId, + cause: Schema.Defect(), +}) {} +export class OwnerCheck extends Context.Service< + OwnerCheck, + { + readonly select: (orgId: KinOrgId) => Effect.Effect, OwnerCheckError>; + readonly post: ( + orgId: KinOrgId, + fireId: string, + selections: ReadonlyArray, + ) => Effect.Effect, OwnerCheckError>; + readonly act: ( + orgId: KinOrgId, + wakeupId: string, + eventId: KinEventId, + ) => Effect.Effect< + { + readonly chatId: KinChatId; + readonly itemId: KinMemoryItemId; + readonly action: "keep" | "update" | "drop"; + }, + OwnerCheckError, + MemoryWriter.MemoryWriter + >; + } +>()("t3/kin/triggers/ownerCheck") {} + +const make = Effect.gen(function* () { + const memory = yield* Memory.Memory; + const sql = yield* SqlClient.SqlClient; + const chat = yield* Chat.KinChatService; + const repo = yield* Repository.KinChatRepository; + const hub = yield* Hub.KinChatHub; + const wrap = (orgId: KinOrgId, effect: Effect.Effect) => + effect.pipe(Effect.mapError((cause) => new OwnerCheckError({ orgId, cause }))); + const select = Effect.fn("OwnerCheck.select")((orgId: KinOrgId) => + wrap( + orgId, + Effect.gen(function* () { + const now = yield* Clock.currentTimeMillis; + const items = yield* memory.query(orgId, { excludeInvalid: true }); + const reports = yield* sql<{ + report_json: string; + }>`SELECT report_json FROM kin_workers WHERE org_id = ${orgId} AND report_json IS NOT NULL AND ended_at >= ${now - 90 * day} ORDER BY ended_at, worker_id`.pipe( + Effect.mapError((cause) => + isSqlError(cause) ? toPersistenceSqlError("OwnerCheck.select")(cause) : cause, + ), + ); + const misleading = new Set(); + for (const row of reports) { + const report = yield* decodeReport(row.report_json); + for (const finding of report.misleadingMemory) misleading.add(finding.itemId); + } + return pickOwnerCheckItems(orgId, items, misleading, now); + }), + ), + ); + const post = Effect.fn("OwnerCheck.post")( + (orgId: KinOrgId, fireId: string, selections: ReadonlyArray) => + wrap( + orgId, + Effect.gen(function* () { + const widgetId = KinWidgetId.make(`owner-check:${orgId}:${fireId}`); + const previous = yield* repo.getWidget({ orgId, widgetId }); + if (Option.isSome(previous)) + return (yield* decodeCard(previous.value.state)).items.map(({ itemId, reason }) => ({ + itemId, + reason, + })); + const heads = yield* Effect.forEach(selections, (selection) => + memory.readItemQuietly(orgId, selection.itemId), + ); + return yield* sql + .withTransaction( + Effect.gen(function* () { + const existing = yield* repo.getWidget({ orgId, widgetId }); + if (Option.isSome(existing)) + return (yield* decodeCard(existing.value.state)).items.map( + ({ itemId, reason }) => ({ itemId, reason }), + ); + if (selections.length === 0) return selections; + const [lead] = yield* sql<{ + owner_chat_id: string; + owner_person_id: string; + }>`SELECT l.owner_chat_id, a.owner_person_id FROM kin_leads l JOIN kin_agents a ON a.org_id = l.org_id AND a.agent_id = l.agent_id WHERE l.org_id = ${orgId} AND a.retired_at IS NULL ORDER BY l.agent_id LIMIT 1`; + if (lead === undefined) return yield* Effect.fail("missing_lead"); + const chatId = KinChatId.make(lead.owner_chat_id); + const { message } = yield* chat.notice({ + orgId, + chatId, + messageId: KinMessageId.make(widgetId), + body: "Memory to review", + }); + const now = DateTime.formatIso(yield* DateTime.now); + const state = { + items: selections.map((selection, index) => ({ + ...selection, + title: heads[index]?.head.title ?? "", + path: heads[index]?.head.path ?? "", + })), + }; + const widget: KinWidget = { + orgId, + chatId, + messageId: message.messageId, + widgetId, + widgetClass: "native", + type: "owner_check", + revision: 1, + state, + source: null, + feed: null, + awaiting: [KinPersonId.make(lead.owner_person_id)], + pinnedAt: null, + closedAt: null, + updatedAt: now, + }; + yield* repo.insertWidget(widget); + yield* sql.afterCommit(hub.publish({ type: "widget.updated", orgId, widget })); + return selections; + }), + ) + .pipe( + Effect.mapError((cause) => + isSqlError(cause) ? toPersistenceSqlError("OwnerCheck.post")(cause) : cause, + ), + ); + }), + ), + ); + const act = Effect.fn("OwnerCheck.act")( + (orgId: KinOrgId, wakeupId: string, eventId: KinEventId) => + wrap( + orgId, + Effect.gen(function* () { + const writer = yield* MemoryWriter.MemoryWriter; + const verified = yield* sql + .withTransaction( + Effect.gen(function* () { + const [row] = yield* sql<{ + json: string; + }>`SELECT e.json FROM kin_events e JOIN kin_wakeups w ON w.org_id = e.org_id AND w.wakeup_id = e.wakeup_id WHERE e.org_id = ${orgId} AND e.event_id = ${eventId} AND e.wakeup_id = ${wakeupId} AND e.state = 'claimed' AND w.status = 'running' ORDER BY e.event_id`; + if (row === undefined) return yield* Effect.fail("unclaimed_action"); + const event = yield* decodeEvent(row.json); + if ( + event.orgId !== orgId || + event.eventId !== eventId || + event._tag !== "widget.action" || + event.provenance._tag !== "human" + ) + return yield* Effect.fail("not_owner"); + const message = yield* repo.getMessage({ orgId, messageId: event.messageId }); + if ( + Option.isNone(message) || + message.value.chatId !== event.chatId || + message.value.kind !== "widget_action" || + message.value.authorRole !== "owner" || + message.value.author._tag !== "human" || + message.value.author.personId !== event.provenance.personId || + message.value.author.memberId !== event.provenance.memberId + ) + return yield* Effect.fail("not_owner"); + const owners = + yield* sql`SELECT m.member_id FROM kin_chat_members m JOIN kin_agents a ON a.org_id = m.org_id AND a.owner_person_id = m.person_id WHERE m.org_id = ${orgId} AND m.member_id = ${event.provenance.memberId} AND m.chat_id = ${event.chatId} AND m.person_id = ${event.provenance.personId} AND m.role = 'owner' AND m.left_at IS NULL AND a.agent_id = ${event.agentId} AND a.retired_at IS NULL ORDER BY m.member_id`; + if (owners.length === 0) return yield* Effect.fail("not_owner"); + const action = yield* decodeAction(message.value.body); + const widget = yield* repo.getWidget({ orgId, widgetId: action.widgetId }); + if ( + Option.isNone(widget) || + widget.value.type !== "owner_check" || + widget.value.chatId !== event.chatId || + message.value.widgetId !== action.widgetId || + widget.value.closedAt !== null || + widget.value.revision !== action.revision || + event.actionId !== action.actionId + ) + return yield* Effect.fail("invalid_card"); + const card = yield* decodeCard(widget.value.state); + if ( + !card.items.some( + (item) => item.itemId === action.value && item.handled === undefined, + ) + ) + return yield* Effect.fail("unknown_item"); + return { action, chatId: event.chatId, widget: widget.value, card }; + }), + ) + .pipe( + Effect.mapError((cause) => + isSqlError(cause) ? toPersistenceSqlError("OwnerCheck.act")(cause) : cause, + ), + ); + const item = yield* memory.readItemQuietly(orgId, verified.action.value); + if (verified.action.actionId !== "update") { + const { orgId: _org, path, ...frontMatter } = item.head; + const now = yield* Clock.currentTimeMillis; + const period = (frontMatter.kind === "person" ? 90 : 30) * day; + const updated = + verified.action.actionId === "keep" + ? { + ...frontMatter, + review_by: DateTime.formatIso(DateTime.makeUnsafe(now + period)), + } + : { ...frontMatter, invalid_at: DateTime.formatIso(DateTime.makeUnsafe(now)) }; + yield* writer.stage(orgId, wakeupId, { + path, + content: `---\n${stringify(updated)}---\n${item.body}`, + sourceEventId: eventId, + }); + } + const at = DateTime.formatIso(yield* DateTime.now); + const items = verified.card.items.map((item) => + item.itemId === verified.action.value + ? { ...item, handled: verified.action.actionId } + : item, + ); + yield* sql + .withTransaction( + Effect.gen(function* () { + const widget = yield* repo.updateWidget({ + ...verified.widget, + expectedRevision: verified.widget.revision, + state: { items }, + awaiting: items.every((item) => item.handled !== undefined) + ? [] + : verified.widget.awaiting, + closedAt: items.every((item) => item.handled !== undefined) ? at : null, + updatedAt: at, + }); + yield* sql.afterCommit(hub.publish({ type: "widget.updated", orgId, widget })); + }), + ) + .pipe( + Effect.mapError((cause) => + isSqlError(cause) ? toPersistenceSqlError("OwnerCheck.act")(cause) : cause, + ), + Effect.tapError((cause) => + cause._tag === "KinChatError" && + cause.reason === "widget_changed" && + verified.action.actionId !== "update" + ? writer.discard(orgId, wakeupId) + : Effect.void, + ), + ); + return { + chatId: verified.chatId, + itemId: verified.action.value, + action: verified.action.actionId, + }; + }), + ), + ); + return OwnerCheck.of({ select, post, act }); +}); +export const layer = Layer.effect(OwnerCheck, make); diff --git a/apps/server/src/kin/triggers/routines.test.ts b/apps/server/src/kin/triggers/routines.test.ts new file mode 100644 index 0000000000000..ed48e02633d73 --- /dev/null +++ b/apps/server/src/kin/triggers/routines.test.ts @@ -0,0 +1,397 @@ +import { assert, it } from "@effect/vitest"; +import { KinOrgId, KinTriggerId, KinEvent, type KinTriggerSettings } from "@t3tools/contracts"; +import * as Deferred from "effect/Deferred"; +import * as Fiber from "effect/Fiber"; +import * as Option from "effect/Option"; +import * as Semaphore from "effect/Semaphore"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; +import * as SqlClient from "effect/sql/SqlClient"; +import * as TestClock from "effect/testing/TestClock"; +import * as Memory from "../memory/Memory.ts"; +import * as Routines from "./routines.ts"; +import * as TriggerScheduler from "./KinTriggerScheduler.ts"; +import * as TriggerStore from "./KinTriggerStore.ts"; +import { fixture, seed, orgId as orgA, otherOrg } from "./routines.testkit.ts"; +import { routineDisposition } from "./routines.ts"; + +const decodeEventJson = Schema.decodeSync(Schema.fromJsonString(KinEvent)); +const settings: KinTriggerSettings = { + orgId: KinOrgId.make("a"), + timeZone: "Asia/Kolkata", + morningBrief: "08:30", + eveningSummary: "18:30", + dream: "03:00", + ownerCheck: "09:00", + ownerCheckWeekday: 1, + fullRebuildWeekday: 0, + quietHours: { from: "21:00", to: "08:00" }, +}; +it.effect("folds late briefs and summaries at their owner-local boundaries", () => + Effect.sync(() => { + for (const [routine, due, now, expected] of [ + ["morning_brief", "2026-11-09T03:00:00.000Z", "2026-11-09T12:59:59.999Z", "post"], + ["morning_brief", "2026-11-09T03:00:00.000Z", "2026-11-09T13:00:00.000Z", "fold"], + ["evening_summary", "2026-11-09T13:00:00.000Z", "2026-11-09T21:29:59.999Z", "post"], + ["evening_summary", "2026-11-09T13:00:00.000Z", "2026-11-09T21:30:00.000Z", "fold"], + ["owner_check", "2026-11-09T03:30:00.000Z", "2026-11-12T13:00:00.000Z", "post"], + ] as const) + assert.equal(routineDisposition(routine, due, now, settings), expected); + }), +); + +it.effect( + "emits one routine per org at owner time, survives replay, and folds expired slots durably", + () => + Effect.gen(function* () { + yield* seed(); + const routines = yield* Routines.KinRoutines; + const scheduler = yield* TriggerScheduler.KinTriggerScheduler; + const sql = yield* SqlClient.SqlClient; + yield* TestClock.setTime(Date.parse("2026-11-09T02:59:59.999Z")); + for (const orgId of [orgA, otherOrg]) yield* routines.configure({ ...settings, orgId }); + yield* scheduler.runDue(); + yield* routines.drain(orgA); + assert.lengthOf( + yield* sql`SELECT event_id FROM kin_events WHERE org_id = ${orgA} ORDER BY event_id`, + 0, + ); + yield* TestClock.adjust(1); + yield* scheduler.runDue(); + for (const orgId of [orgA, otherOrg]) yield* routines.drain(orgId); + const rows = yield* sql<{ + dedupe_key: string; + json: string; + }>`SELECT dedupe_key, json FROM kin_events WHERE org_id IN (${orgA}, ${otherOrg}) ORDER BY org_id, event_id`; + assert.lengthOf(rows, 2); + assert.equal(new Set(rows.map((row) => row.dedupe_key)).size, 2); + assert.equal(decodeEventJson(rows[0]!.json)._tag, "routine"); + yield* TestClock.setTime(Date.parse("2026-11-09T13:00:00.000Z")); + yield* scheduler.runDue(); + yield* routines.drain(orgA); + assert.lengthOf( + yield* sql`SELECT event_id FROM kin_events WHERE org_id = ${orgA} ORDER BY event_id`, + 3, + ); + // Replay an already consumed inbox row, as after a crash before the consume marker. + yield* sql`UPDATE kin_trigger_inbox SET consumed_at = NULL WHERE org_id = ${orgA}`; + yield* routines.drain(orgA); + assert.lengthOf( + yield* sql`SELECT event_id FROM kin_events WHERE org_id = ${orgA} ORDER BY event_id`, + 3, + ); + }).pipe(Effect.provide(fixture())), +); + +it.effect( + "folds a brief after summary time and a summary after the next dream without duplicate catch-up", + () => + Effect.gen(function* () { + yield* seed(); + const routines = yield* Routines.KinRoutines; + const scheduler = yield* TriggerScheduler.KinTriggerScheduler; + const sql = yield* SqlClient.SqlClient; + yield* TestClock.setTime(Date.parse("2026-11-09T02:00:00.000Z")); + yield* routines.configure(settings); + yield* TestClock.setTime(Date.parse("2026-11-09T13:00:00.000Z")); + yield* scheduler.runDue(); + yield* routines.drain(orgA); + const rows = yield* sql<{ + json: string; + }>`SELECT json FROM kin_events WHERE org_id = ${orgA} AND tag = 'routine' ORDER BY event_id`; + const events = rows.map((row) => decodeEventJson(row.json)); + assert.isFalse( + events.some((event) => event._tag === "routine" && event.routine === "morning_brief"), + ); + const summary = events.find( + (event) => event._tag === "routine" && event.routine === "evening_summary", + ); + assert.equal(summary?._tag, "routine"); + if (summary?._tag === "routine") + assert.equal(summary.foldedRoutines?.[0]?.routine, "morning_brief"); + yield* TestClock.setTime(Date.parse("2026-11-10T03:00:00.000Z")); + yield* scheduler.runDue(); + yield* routines.drain(orgA); + yield* TestClock.setTime(Date.parse("2026-11-11T03:00:00.000Z")); + yield* scheduler.runDue(); + yield* routines.drain(orgA); + const [latest] = yield* sql<{ + json: string; + }>`SELECT json FROM kin_events WHERE org_id = ${orgA} AND json_extract(json, '$.routine') = 'morning_brief' ORDER BY received_at DESC LIMIT 1`; + const brief = decodeEventJson(latest!.json); + assert.equal(brief._tag, "routine"); + if (brief._tag === "routine") + assert.equal(brief.foldedRoutines?.[0]?.routine, "evening_summary"); + assert.lengthOf( + yield* sql`SELECT fire_id FROM kin_routine_folds WHERE org_id = ${orgA} ORDER BY fire_id`, + 0, + ); + }).pipe(Effect.provide(fixture())), +); + +it.effect( + "disarms and re-arms individual routines and preserves opt-outs across pause, restart and timezone changes", + () => + Effect.gen(function* () { + yield* seed(); + const routines = yield* Routines.KinRoutines; + const store = yield* TriggerStore.KinTriggerStore; + const scheduler = yield* TriggerScheduler.KinTriggerScheduler; + yield* TestClock.setTime(Date.parse("2026-03-07T12:00:00.000Z")); + yield* routines.configure({ ...settings, timeZone: "America/New_York" }); + const id = KinTriggerId.make("routine:morning_brief"); + assert.equal((yield* store.get(orgA, id))?.nextRunAt, "2026-03-07T13:30:00.000Z"); + yield* TestClock.setTime(Date.parse("2026-03-07T13:30:00.000Z")); + yield* scheduler.runDue(); + assert.equal((yield* store.get(orgA, id))?.nextRunAt, "2026-03-08T12:30:00.000Z"); + yield* routines.drain(orgA); + const sql = yield* SqlClient.SqlClient; + yield* routines.setEnabled(orgA, "morning_brief", false); + assert.deepEqual( + yield* sql`SELECT state FROM kin_events WHERE org_id = ${orgA} AND json_extract(json, '$.routine') = 'morning_brief' ORDER BY event_id`, + [{ state: "expired" }], + ); + assert.isNull((yield* store.get(orgA, id))?.nextRunAt); + yield* routines.configure(settings).pipe(Effect.provide(Layer.fresh(Routines.layer))); + assert.isFalse((yield* store.get(orgA, id))?.enabled); + yield* routines.pauseAll(orgA); + assert.isTrue((yield* store.list(orgA)).every((trigger) => !trigger.enabled)); + yield* routines.resumeAll(orgA); + assert.isFalse((yield* store.get(orgA, id))?.enabled); + yield* routines.setEnabled(orgA, "morning_brief", true); + assert.equal((yield* store.get(orgA, id))?.nextRunAt, "2026-03-08T03:00:00.000Z"); + yield* TestClock.setTime(Date.parse("2026-03-08T03:01:00.000Z")); + yield* routines.configure({ ...settings, timeZone: "America/New_York" }); + assert.equal((yield* store.get(orgA, id))?.nextRunAt, "2026-03-08T03:00:00.000Z"); + yield* scheduler.runDue(); + assert.equal((yield* store.get(orgA, id))?.nextRunAt, "2026-03-08T12:30:00.000Z"); + yield* TestClock.setTime(Date.parse("2026-10-31T12:00:00.000Z")); + yield* scheduler.runDue(); + yield* TestClock.setTime(Date.parse("2026-10-31T12:30:00.000Z")); + yield* scheduler.runDue(); + assert.equal((yield* store.get(orgA, id))?.nextRunAt, "2026-11-01T13:30:00.000Z"); + yield* sql`UPDATE kin_agents SET retired_at = '2026-11-01T13:00:00.000Z' WHERE org_id = ${orgA} AND agent_id = 'lead-a'`; + yield* routines.drain(orgA); + yield* routines.configure(settings); + assert.isTrue((yield* store.list(orgA)).every((trigger) => !trigger.enabled)); + assert.lengthOf( + yield* sql`SELECT fire_id FROM kin_trigger_inbox WHERE org_id = ${orgA} AND consumed_at IS NULL ORDER BY fire_id`, + 0, + ); + }).pipe(Effect.provide(fixture())), +); + +it.effect.each(["transient", "decode", "expired"] as const)( + "handles a %s failure without blocking the following good fire", + (failure) => + Effect.gen(function* () { + yield* seed(); + const routines = yield* Routines.KinRoutines; + const scheduler = yield* TriggerScheduler.KinTriggerScheduler; + const sql = yield* SqlClient.SqlClient; + yield* TestClock.setTime(Date.parse("2026-11-09T02:00:00.000Z")); + yield* routines.configure(settings); + yield* TestClock.setTime(Date.parse("2026-11-09T13:00:00.000Z")); + yield* scheduler.runDue(); + if (failure === "decode") + yield* sql`INSERT INTO kin_routine_folds VALUES (${orgA}, 'bad-fold', 'invalid-routine', 1, 'evening_summary')`; + else + yield* sql`CREATE TRIGGER reject_fold BEFORE INSERT ON kin_routine_folds BEGIN SELECT RAISE(ABORT, 'fixture failure'); END`; + if (failure === "expired") { + yield* routines.drain(orgA); + assert.lengthOf( + yield* sql`SELECT fire_id FROM kin_trigger_inbox WHERE org_id = ${orgA} AND consumed_at IS NULL`, + 1, + ); + yield* TestClock.adjust("25 hours"); + } + yield* routines.drain(orgA); + const rows = yield* sql<{ + json: string; + }>`SELECT json FROM kin_events WHERE org_id = ${orgA} AND tag = 'routine' ORDER BY event_id`; + assert.equal( + rows.some((row) => { + const event = decodeEventJson(row.json); + return ( + event._tag === "routine" && event.routine === "evening_summary" && event.lateByMs === 0 + ); + }), + failure !== "decode", + ); + assert.lengthOf( + yield* sql`SELECT fire_id FROM kin_trigger_inbox WHERE org_id = ${orgA} AND consumed_at IS NULL ORDER BY fire_id`, + failure === "transient" ? 1 : 0, + ); + if (failure === "transient") { + yield* sql`DROP TRIGGER reject_fold`; + yield* routines.drain(orgA); + assert.lengthOf( + yield* sql`SELECT fire_id FROM kin_trigger_inbox WHERE org_id = ${orgA} AND consumed_at IS NULL`, + 0, + ); + } + }).pipe(Effect.provide(fixture())), +); + +it.effect.each([false, true])( + "repoints stranded folds, or drops them when both routines are off: %s", + (bothOff) => + Effect.gen(function* () { + yield* seed(); + const routines = yield* Routines.KinRoutines; + const sql = yield* SqlClient.SqlClient; + yield* TestClock.setTime(Date.parse("2026-11-09T02:00:00.000Z")); + yield* routines.configure(settings); + yield* sql`INSERT INTO kin_routine_folds VALUES (${orgA}, 'old-fire', 'morning_brief', 1, 'evening_summary')`; + yield* routines.setEnabled(orgA, "evening_summary", false); + const folds = yield* sql<{ + fold_into: string; + }>`SELECT fold_into FROM kin_routine_folds WHERE org_id = ${orgA} ORDER BY fire_id`; + assert.deepEqual(folds, [{ fold_into: "morning_brief" }]); + if (bothOff) { + yield* routines.setEnabled(orgA, "morning_brief", false); + assert.lengthOf( + yield* sql`SELECT fire_id FROM kin_routine_folds WHERE org_id = ${orgA} ORDER BY fire_id`, + 0, + ); + } else { + const scheduler = yield* TriggerScheduler.KinTriggerScheduler; + yield* TestClock.setTime(Date.parse("2026-11-09T03:00:00.000Z")); + yield* scheduler.runDue(); + yield* routines.drain(orgA); + const [row] = yield* sql<{ + json: string; + }>`SELECT json FROM kin_events WHERE org_id = ${orgA} ORDER BY event_id LIMIT 1`; + const event = decodeEventJson(row!.json); + assert.equal(event._tag, "routine"); + if (event._tag === "routine") + assert.equal(event.foldedRoutines?.[0]?.routine, "morning_brief"); + } + }).pipe(Effect.provide(fixture())), +); + +it.effect("waits for Memory's org lock without holding SQLite's writer", () => + Effect.gen(function* () { + const lock = yield* Semaphore.make(1); + const held = yield* Deferred.make(); + const selecting = yield* Deferred.make(); + const write = yield* Deferred.make(); + yield* Effect.gen(function* () { + yield* seed(); + const sql = yield* SqlClient.SqlClient; + const routines = yield* Routines.KinRoutines; + const scheduler = yield* TriggerScheduler.KinTriggerScheduler; + yield* TestClock.setTime(Date.parse("2026-11-09T03:29:59.000Z")); + yield* routines.configure(settings); + yield* TestClock.adjust("1 second"); + yield* scheduler.runDue(); + const holder = yield* lock + .withPermits(1)( + Effect.gen(function* () { + yield* Deferred.succeed(held, undefined); + const canWrite = yield* Deferred.await(write); + if (canWrite) + yield* sql.withTransaction( + sql`UPDATE kin_orgs SET name = 'Memory anchor' WHERE org_id = ${orgA}`, + ); + }), + ) + .pipe(Effect.forkScoped); + yield* Deferred.await(held); + const drain = yield* routines.drain(orgA).pipe(Effect.forkScoped); + const inTransaction = yield* Deferred.await(selecting); + yield* Deferred.succeed(write, !inTransaction); + yield* Fiber.join(holder); + yield* Fiber.join(drain); + assert.isFalse(inTransaction); + assert.lengthOf( + yield* sql`SELECT event_id FROM kin_events WHERE org_id = ${orgA} AND json_extract(json, '$.routine') = 'owner_check'`, + 1, + ); + }).pipe( + Effect.provide( + fixture([], { + query: (sql) => () => + Effect.gen(function* () { + const transaction = yield* Effect.serviceOption(sql.transactionService); + yield* Deferred.succeed(selecting, Option.isSome(transaction)); + return yield* lock.withPermits(1)(Effect.succeed([])); + }), + }), + ), + ); + }).pipe(Effect.scoped), +); + +const decodeNoItems = Schema.decodeUnknownEffect(Schema.Array(Schema.Never)); + +it.effect("consumes an owner check whose selection fails to decode", () => + Effect.gen(function* () { + yield* seed(); + const sql = yield* SqlClient.SqlClient; + const routines = yield* Routines.KinRoutines; + const scheduler = yield* TriggerScheduler.KinTriggerScheduler; + yield* TestClock.setTime(Date.parse("2026-11-09T03:29:59.000Z")); + yield* routines.configure(settings); + yield* TestClock.adjust("1 second"); + yield* scheduler.runDue(); + yield* routines.drain(orgA); + assert.lengthOf( + yield* sql`SELECT fire_id FROM kin_trigger_inbox WHERE org_id = ${orgA} AND consumed_at IS NULL ORDER BY fire_id`, + 0, + ); + assert.lengthOf( + yield* sql`SELECT event_id FROM kin_events WHERE org_id = ${orgA} AND json_extract(json, '$.routine') = 'owner_check' ORDER BY event_id`, + 0, + ); + }).pipe( + Effect.provide( + fixture([], { + query: () => () => + decodeNoItems(["bad"]).pipe( + Effect.mapError( + (cause) => new Memory.MemoryError({ orgId: orgA, operation: "query", cause }), + ), + ), + }), + ), + ), +); + +it.effect("reconciles at fifteen minutes, not intervening five-second ticks", () => + Effect.gen(function* () { + let tick = Effect.void; + yield* Effect.gen(function* () { + yield* seed(); + const sql = yield* SqlClient.SqlClient; + yield* tick; + yield* sql`DELETE FROM kin_triggers WHERE org_id = ${orgA} AND trigger_id = 'routine:owner_check'`; + for (let index = 1; index < 180; index++) { + yield* TestClock.adjust("5 seconds"); + yield* tick; + assert.lengthOf( + yield* sql`SELECT trigger_id FROM kin_triggers WHERE org_id = ${orgA} AND trigger_id = 'routine:owner_check'`, + 0, + ); + } + yield* TestClock.adjust("5 seconds"); + yield* tick; + assert.lengthOf( + yield* sql`SELECT trigger_id FROM kin_triggers WHERE org_id = ${orgA} AND trigger_id = 'routine:owner_check'`, + 1, + ); + }).pipe( + Effect.provide( + fixture([], { + register: Effect.fn(function* (name: string, work: Effect.Effect) { + if (name === "kin-routines") { + const context = yield* Effect.context(); + tick = work.pipe(Effect.provideContext(context), Effect.orDie); + } + }), + }), + ), + ); + }), +); diff --git a/apps/server/src/kin/triggers/routines.testkit.ts b/apps/server/src/kin/triggers/routines.testkit.ts new file mode 100644 index 0000000000000..a1c47e4ae09af --- /dev/null +++ b/apps/server/src/kin/triggers/routines.testkit.ts @@ -0,0 +1,112 @@ +import { KinOrgId, KinMemoryItemId, MemoryItemHead } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; +import * as SqlClient from "effect/sql/SqlClient"; +import * as Sqlite from "../../persistence/Sqlite.ts"; +import * as Scheduler from "../../scheduling/Scheduler.ts"; +import * as ThreadManagement from "../../orchestration-v2/ThreadManagementService.ts"; +import * as Memory from "../memory/Memory.ts"; +import * as Chat from "../chat/KinChatService.ts"; +import * as JudgeShadow from "../floor/JudgeShadow.ts"; +import * as Repository from "../chat/KinChatRepository.ts"; +import * as Hub from "../chat/KinChatHub.ts"; +import * as Persistence from "../persistence/KinPersistence.ts"; +import * as EventQueue from "../lead/KinEventQueue.ts"; +import * as TriggerStore from "./KinTriggerStore.ts"; +import * as TriggerInbox from "./KinTriggerInbox.ts"; +import * as TriggerScheduler from "./KinTriggerScheduler.ts"; +import * as Routines from "./routines.ts"; +import * as OwnerCheck from "./ownerCheck.ts"; + +export const orgId = KinOrgId.make("a"); +export const otherOrg = KinOrgId.make("b"); +const timestamp = "2026-11-07T00:00:00.000Z"; +const decodeHead = Schema.decodeUnknownSync(MemoryItemHead); +export const item = (id: string, overrides: Partial = {}) => + decodeHead({ + orgId, + id: KinMemoryItemId.make(id), + path: `claims/${id}.md`, + kind: "claim", + title: id, + status: "open", + due: null, + recorded_at: "2026-09-01", + valid_from: "2026-09-01", + invalid_at: null, + superseded_by: null, + review_by: "2026-12-01", + source: "owner-chat", + said_by: "owner", + trust: "owner", + links: ["https://example.com/evidence"], + ...overrides, + }); +export function fixture( + items: ReadonlyArray = [], + options: { + query?: (sql: SqlClient.SqlClient) => Memory.Memory["Service"]["query"]; + register?: Scheduler.Scheduler["Service"]["register"]; + } = {}, +) { + const database = Persistence.layer.pipe(Layer.provideMerge(Sqlite.layerMemory)); + const repositories = Layer.mergeAll( + Repository.layer, + TriggerStore.layer, + TriggerInbox.layer, + EventQueue.layer, + ).pipe( + Layer.provideMerge(database), + Layer.provide(Layer.mock(ThreadManagement.ThreadManagementService)({})), + ); + const dependencies = Layer.mergeAll( + Layer.unwrap( + Effect.map(SqlClient.SqlClient, (sql) => + Layer.mock(Memory.Memory)({ + query: + options.query?.(sql) ?? + ((orgId) => + Effect.succeed( + items + .filter((item) => item.orgId === orgId) + .map((item) => ({ ...item, duplicate: false })), + )), + readItemQuietly: (orgId, id) => { + const head = items.find((item) => item.orgId === orgId && item.id === id); + return head === undefined + ? Effect.fail(new Memory.MemoryNotFound({ orgId, id })) + : Effect.succeed({ head, body: "Body\n" }); + }, + }), + ), + ), + Chat.layer.pipe(Layer.provide(JudgeShadow.layerOff), Layer.provideMerge(Hub.layer)), + TriggerScheduler.layer, + ).pipe( + Layer.provideMerge(repositories), + Layer.provide( + Layer.succeed(Scheduler.Scheduler, { register: options.register ?? (() => Effect.void) }), + ), + ); + const owner = OwnerCheck.layer.pipe(Layer.provideMerge(dependencies)); + return Routines.layer.pipe( + Layer.provideMerge(owner), + Layer.provideMerge( + Layer.succeed(Scheduler.Scheduler, { register: options.register ?? (() => Effect.void) }), + ), + ); +} +export const seed = Effect.fn("test.routinesSeed")(function* () { + const sql = yield* SqlClient.SqlClient; + for (const org of [orgId, otherOrg]) { + const agent = `lead-${org}`; + yield* sql`INSERT INTO kin_orgs VALUES (${org}, 'Kin', ${timestamp})`; + yield* sql`INSERT INTO kin_people (person_id, org_id, handle, display_name, created_at) VALUES (${`owner-${org}`}, ${org}, 'owner', 'Owner', ${timestamp})`; + yield* sql`INSERT INTO kin_agents (agent_id, org_id, display_name, role, owner_person_id, created_at) VALUES (${agent}, ${org}, 'Lead', 'lead', ${`owner-${org}`}, ${timestamp})`; + yield* sql`INSERT INTO kin_chats (chat_id, org_id, kind, direct_key, last_seq, created_at) VALUES (${`chat-${org}`}, ${org}, 'direct', ${`direct-${org}`}, 0, ${timestamp})`; + yield* sql`INSERT INTO kin_chat_members (member_id, org_id, chat_id, principal_kind, person_id, role, joined_at) VALUES (${`owner-member-${org}`}, ${org}, ${`chat-${org}`}, 'person', ${`owner-${org}`}, 'owner', ${timestamp})`; + yield* sql`INSERT INTO kin_chat_members (member_id, org_id, chat_id, principal_kind, agent_id, role, joined_at) VALUES (${`lead-member-${org}`}, ${org}, ${`chat-${org}`}, 'agent', ${agent}, 'member', ${timestamp})`; + yield* sql`INSERT INTO kin_leads (agent_id, org_id, name, level_since, models_json, worker_projects_json, owner_chat_id) VALUES (${agent}, ${org}, 'Lead', 0, '{}', '[]', ${`chat-${org}`})`; + } +}); diff --git a/apps/server/src/kin/triggers/routines.ts b/apps/server/src/kin/triggers/routines.ts new file mode 100644 index 0000000000000..691851fbf2693 --- /dev/null +++ b/apps/server/src/kin/triggers/routines.ts @@ -0,0 +1,455 @@ +import { + KinAgentId, + KinEvent, + KinEventId, + KinLeadRoutine, + KinOrgId, + KinTriggerId, + KinTriggerSettings, + type KinTrigger, +} from "@t3tools/contracts"; +import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; +import * as Clock from "effect/Clock"; +import * as Context from "effect/Context"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; +import * as SqlClient from "effect/sql/SqlClient"; +import { isSqlError } from "effect/sql/SqlError"; +import { PersistenceDecodeError, toPersistenceSqlError } from "../../persistence/Errors.ts"; +import { nextScheduledRunAt } from "../../scheduledTasks/Schedule.ts"; +import * as Scheduler from "../../scheduling/Scheduler.ts"; +import * as KinEventQueue from "../lead/KinEventQueue.ts"; +import * as KinTriggerInbox from "./KinTriggerInbox.ts"; +import * as KinTriggerStore from "./KinTriggerStore.ts"; +import * as KinTriggerScheduler from "./KinTriggerScheduler.ts"; +import * as OwnerCheck from "./ownerCheck.ts"; + +const routines = ["morning_brief", "evening_summary", "owner_check"] as const; +const iso = (now: number) => DateTime.formatIso(DateTime.makeUnsafe(now)); +const decodeSettings = Schema.decodeUnknownEffect(KinTriggerSettings); +const decodeEvent = Schema.decodeUnknownEffect(KinEvent); +const decodeQuietHours = Schema.decodeUnknownEffect( + Schema.fromJsonString(KinTriggerSettings.fields.quietHours), +); +const encodeQuietHours = Schema.encodeEffect( + Schema.fromJsonString(KinTriggerSettings.fields.quietHours), +); +const decodeRoutine = Schema.decodeUnknownEffect(KinLeadRoutine); + +export function routineDisposition( + routine: KinLeadRoutine, + dueAt: string, + now: string, + settings: KinTriggerSettings, +) { + if (routine === "owner_check") return "post"; + const due = DateTime.makeZonedUnsafe(dueAt, { timeZone: settings.timeZone }); + const [hour = 0, minute = 0] = ( + routine === "morning_brief" ? settings.eveningSummary : settings.dream + ) + .split(":") + .map(Number); + let boundary = DateTime.setParts(due, { hour, minute, second: 0, millisecond: 0 }); + if ( + routine === "evening_summary" && + DateTime.toEpochMillis(boundary) <= DateTime.toEpochMillis(due) + ) + boundary = DateTime.add(boundary, { days: 1 }); + return Date.parse(now) < DateTime.toEpochMillis(boundary) ? "post" : "fold"; +} + +function schedule(routine: KinLeadRoutine, settings: KinTriggerSettings): KinTrigger["schedule"] { + switch (routine) { + case "morning_brief": + return { type: "fixed_time", timeOfDay: settings.morningBrief }; + case "evening_summary": + return { type: "fixed_time", timeOfDay: settings.eveningSummary, weekdays: [1, 2, 3, 4, 5] }; + case "owner_check": + return { + type: "fixed_time", + timeOfDay: settings.ownerCheck, + weekdays: [settings.ownerCheckWeekday], + }; + } +} + +export class KinRoutinesError extends Schema.TaggedError()("KinRoutinesError", { + orgId: KinOrgId, + cause: Schema.Defect(), +}) {} +export class KinRoutines extends Context.Service< + KinRoutines, + { + readonly isEnabled: ( + orgId: KinOrgId, + routine: KinLeadRoutine, + ) => Effect.Effect; + readonly configure: (settings: KinTriggerSettings) => Effect.Effect; + readonly setEnabled: ( + orgId: KinOrgId, + routine: KinLeadRoutine, + enabled: boolean, + ) => Effect.Effect; + readonly pauseAll: (orgId: KinOrgId) => Effect.Effect; + readonly resumeAll: (orgId: KinOrgId) => Effect.Effect; + readonly drain: (orgId: KinOrgId) => Effect.Effect; + } +>()("t3/kin/triggers/routines/KinRoutines") {} + +const isDecodeError = Schema.is(PersistenceDecodeError); +const isPermanent = (error: unknown) => { + let current = error; + for (let depth = 0; depth < 8 && typeof current === "object" && current !== null; depth++) { + if (Schema.isSchemaError(current) || isDecodeError(current)) return true; + current = "cause" in current ? current.cause : undefined; + } + return false; +}; + +const make = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const store = yield* KinTriggerStore.KinTriggerStore; + const scheduler = yield* KinTriggerScheduler.KinTriggerScheduler; + const inbox = yield* KinTriggerInbox.KinTriggerInbox; + const queue = yield* KinEventQueue.KinEventQueue; + const ownerCheck = yield* OwnerCheck.OwnerCheck; + const ticks = yield* Scheduler.Scheduler; + const settingsFor = Effect.fn("KinRoutines.settingsFor")(function* (orgId: KinOrgId) { + const [row] = yield* sql<{ + time_zone: string; + morning_brief: string; + evening_summary: string; + dream: string; + full_rebuild_weekday: number; + owner_check_weekday: number; + owner_check: string; + quiet_hours_json: string; + }>`SELECT * FROM kin_trigger_settings WHERE org_id = ${orgId} ORDER BY org_id`; + return yield* decodeSettings( + row === undefined + ? { + orgId, + timeZone: "UTC", + morningBrief: "08:30", + eveningSummary: "18:30", + dream: "03:00", + fullRebuildWeekday: 0, + ownerCheckWeekday: 1, + ownerCheck: "09:00", + quietHours: { from: "21:00", to: "08:00" }, + } + : { + orgId, + timeZone: row.time_zone, + morningBrief: row.morning_brief, + eveningSummary: row.evening_summary, + dream: row.dream, + fullRebuildWeekday: row.full_rebuild_weekday, + ownerCheckWeekday: row.owner_check_weekday, + ownerCheck: row.owner_check, + quietHours: yield* decodeQuietHours(row.quiet_hours_json), + }, + ); + }); + const nextFoldTarget = Effect.fn("KinRoutines.nextFoldTarget")(function* ( + orgId: KinOrgId, + excludeFireId: string, + ) { + const [next] = yield* sql<{ + routine: string; + }>`SELECT json_extract(t.subject_json, '$.routine') AS routine + FROM kin_triggers t WHERE t.org_id = ${orgId} AND t.enabled = 1 + AND t.trigger_id IN ('routine:morning_brief', 'routine:evening_summary') + ORDER BY MIN(t.next_run_at, COALESCE(( + SELECT MIN(CAST(unixepoch(json_extract(i.payload_json, '$.dueAt')) * 1000 AS INTEGER)) + FROM kin_trigger_inbox i WHERE i.org_id = t.org_id AND i.consumed_at IS NULL + AND i.fire_id <> ${excludeFireId} + AND json_extract(i.payload_json, '$.payload.subject.routine') = json_extract(t.subject_json, '$.routine') + ), t.next_run_at)), t.trigger_id LIMIT 1`; + return next?.routine; + }); + const reconcile = Effect.fn("KinRoutines.reconcile")(function* (settings: KinTriggerSettings) { + const now = yield* Clock.currentTimeMillis; + const activeLead = + yield* sql`SELECT l.agent_id FROM kin_leads l JOIN kin_agents a ON a.org_id = l.org_id AND a.agent_id = l.agent_id WHERE l.org_id = ${settings.orgId} AND a.retired_at IS NULL ORDER BY l.agent_id LIMIT 1`; + for (const routine of routines) { + const id = KinTriggerId.make(`routine:${routine}`); + const existing = yield* store.get(settings.orgId, id); + yield* sql`INSERT INTO kin_routine_preferences (org_id, routine, enabled) VALUES (${settings.orgId}, ${routine}, ${existing?.enabled === false ? 0 : 1}) ON CONFLICT DO NOTHING RETURNING routine`; + const [preference] = yield* sql<{ + enabled: number; + paused: number; + }>`SELECT enabled, paused FROM kin_routine_preferences WHERE org_id = ${settings.orgId} AND routine = ${routine} ORDER BY routine`; + const enabled = activeLead.length > 0 && preference?.enabled === 1 && preference.paused === 0; + if (existing !== undefined && existing.enabled === enabled) continue; + const timing = schedule(routine, settings); + const next = enabled + ? nextScheduledRunAt(timing, DateTime.makeZonedUnsafe(now, { timeZone: settings.timeZone })) + : null; + yield* store.put({ + id, + orgId: settings.orgId, + subject: { type: "routine", routine }, + schedule: timing, + timeZone: settings.timeZone, + enabled, + nextRunAt: next === null ? null : DateTime.formatIso(next), + createdAt: existing?.createdAt ?? iso(now), + updatedAt: iso(Math.max(now, Date.parse(existing?.updatedAt ?? iso(now)))), + }); + } + yield* scheduler.recompute(settings); + const target = yield* nextFoldTarget(settings.orgId, ""); + if (target !== undefined) { + yield* sql`UPDATE kin_routine_folds SET fold_into = ${target} WHERE org_id = ${settings.orgId}`; + } else { + const dropped = + yield* sql`DELETE FROM kin_routine_folds WHERE org_id = ${settings.orgId} RETURNING fire_id`; + if (dropped.length > 0) + yield* Effect.logWarning("Kin routine folds dropped: no enabled destination", { + orgId: settings.orgId, + count: dropped.length, + }); + } + }); + const wrap = (orgId: KinOrgId, effect: Effect.Effect) => + effect.pipe(Effect.mapError((cause) => new KinRoutinesError({ orgId, cause }))); + const configure = Effect.fn("KinRoutines.configure")((input: KinTriggerSettings) => + wrap( + input.orgId, + sql + .withTransaction( + Effect.gen(function* () { + const settings = yield* decodeSettings(input); + yield* sql`INSERT INTO kin_trigger_settings (org_id, time_zone, morning_brief, evening_summary, dream, full_rebuild_weekday, owner_check_weekday, owner_check, quiet_hours_json) + VALUES (${settings.orgId}, ${settings.timeZone}, ${settings.morningBrief}, ${settings.eveningSummary}, ${settings.dream}, ${settings.fullRebuildWeekday}, ${settings.ownerCheckWeekday}, ${settings.ownerCheck}, ${yield* encodeQuietHours(settings.quietHours)}) + ON CONFLICT (org_id) DO UPDATE SET time_zone = excluded.time_zone, morning_brief = excluded.morning_brief, evening_summary = excluded.evening_summary, dream = excluded.dream, full_rebuild_weekday = excluded.full_rebuild_weekday, owner_check_weekday = excluded.owner_check_weekday, owner_check = excluded.owner_check, quiet_hours_json = excluded.quiet_hours_json RETURNING org_id`; + yield* reconcile(settings); + }), + ) + .pipe( + Effect.mapError((cause) => + isSqlError(cause) ? toPersistenceSqlError("KinRoutines.configure")(cause) : cause, + ), + ), + ), + ); + const toggle = Effect.fn("KinRoutines.toggle")( + (orgId: KinOrgId, routine: KinLeadRoutine | null, value: boolean) => + wrap( + orgId, + sql + .withTransaction( + Effect.gen(function* () { + const settings = yield* settingsFor(orgId); + yield* reconcile(settings); + if (routine === null) + yield* sql`UPDATE kin_routine_preferences SET paused = ${value ? 1 : 0} WHERE org_id = ${orgId}`; + else { + yield* decodeRoutine(routine); + yield* sql`UPDATE kin_routine_preferences SET enabled = ${value ? 1 : 0} WHERE org_id = ${orgId} AND routine = ${routine}`; + } + yield* reconcile(settings); + if ((routine === null && value) || (routine !== null && !value)) { + yield* sql`UPDATE kin_events SET state = 'expired' WHERE org_id = ${orgId} AND tag = 'routine' AND state = 'queued' AND (${routine} IS NULL OR json_extract(json, '$.routine') = ${routine})`; + } + }), + ) + .pipe( + Effect.mapError((cause) => + isSqlError(cause) ? toPersistenceSqlError("KinRoutines.toggle")(cause) : cause, + ), + ), + ), + ); + const drain = Effect.fn("KinRoutines.drain")((orgId: KinOrgId) => + wrap( + orgId, + Effect.gen(function* () { + const now = yield* Clock.currentTimeMillis; + const pending = yield* inbox.listPending(orgId, 100, routines); + for (const fire of pending.toSorted( + (a, b) => + Date.parse(a.dueAt) - Date.parse(b.dueAt) || + (a.fireId < b.fireId ? -1 : a.fireId > b.fireId ? 1 : 0), + )) { + if (fire.payload.kind !== "time" || fire.payload.subject.type !== "routine") continue; + const name = fire.payload.subject.routine; + if (name !== "morning_brief" && name !== "evening_summary" && name !== "owner_check") + continue; + + const eventId = KinEventId.make( + `routine_${Array.from(new TextEncoder().encode(`${orgId.length}:${orgId}${fire.fireId}`), (byte) => byte.toString(16).padStart(2, "0")).join("")}`, + ); + yield* Effect.gen(function* () { + const [pending] = yield* sql<{ handled: number }>`SELECT EXISTS( + SELECT 1 FROM kin_events WHERE org_id = ${orgId} AND event_id = ${eventId} + ) AS handled FROM kin_trigger_inbox WHERE org_id = ${orgId} + AND fire_id = ${fire.fireId} AND consumed_at IS NULL`.pipe(NodeSqliteClient.readOnly); + if (pending === undefined) return; + if (pending.handled === 1) { + yield* inbox.markConsumed(orgId, [fire.fireId]); + return; + } + // Memory takes an org lock and may anchor a checkout in SQLite. + const items = name === "owner_check" ? yield* ownerCheck.select(orgId) : undefined; + yield* sql.withTransaction( + Effect.gen(function* () { + const settings = yield* settingsFor(orgId); + const unconsumed = + yield* sql`SELECT fire_id FROM kin_trigger_inbox WHERE org_id = ${orgId} AND fire_id = ${fire.fireId} AND consumed_at IS NULL ORDER BY fire_id`; + if (unconsumed.length === 0) return; + const [lead] = yield* sql<{ + agent_id: string; + }>`SELECT l.agent_id FROM kin_leads l JOIN kin_agents a ON a.org_id = l.org_id AND a.agent_id = l.agent_id WHERE l.org_id = ${orgId} AND a.retired_at IS NULL ORDER BY l.agent_id LIMIT 1`; + if (lead === undefined) { + yield* inbox.markConsumed(orgId, [fire.fireId]); + yield* sql`UPDATE kin_triggers SET enabled = 0, next_run_at = NULL WHERE org_id = ${orgId} AND trigger_id IN ('routine:morning_brief', 'routine:evening_summary', 'routine:owner_check')`; + yield* Effect.logWarning("Kin routine consumed: lead retired", { + orgId, + fireId: fire.fireId, + }); + return; + } + const previous = + yield* sql`SELECT event_id FROM kin_events WHERE org_id = ${orgId} AND event_id = ${eventId} ORDER BY event_id`; + if (previous.length > 0) { + yield* inbox.markConsumed(orgId, [fire.fireId]); + return; + } + const [preference] = yield* sql<{ + enabled: number; + paused: number; + }>`SELECT enabled, paused FROM kin_routine_preferences WHERE org_id = ${orgId} AND routine = ${name} ORDER BY routine`; + if (preference?.enabled !== 1 || preference.paused === 1) { + yield* inbox.markConsumed(orgId, [fire.fireId]); + return; + } + if (routineDisposition(name, fire.dueAt, iso(now), settings) === "fold") { + const target = yield* nextFoldTarget(orgId, fire.fireId); + if (target === undefined) { + yield* Effect.logWarning("Kin routine fold dropped: no enabled destination", { + orgId, + fireId: fire.fireId, + }); + yield* inbox.markConsumed(orgId, [fire.fireId]); + return; + } + yield* sql`UPDATE kin_routine_folds SET fold_into = ${target} WHERE org_id = ${orgId} AND fold_into = ${name}`; + yield* sql`INSERT INTO kin_routine_folds (org_id, fire_id, routine, due_at, fold_into) VALUES (${orgId}, ${fire.fireId}, ${name}, ${Date.parse(fire.dueAt)}, ${target}) ON CONFLICT DO NOTHING RETURNING fire_id`; + } else { + const folds = yield* sql<{ + routine: string; + due_at: number; + }>`SELECT routine, due_at FROM kin_routine_folds WHERE org_id = ${orgId} AND fold_into = ${name} ORDER BY due_at, fire_id`; + const event = yield* decodeEvent({ + _tag: "routine", + eventId, + orgId, + agentId: KinAgentId.make(lead.agent_id), + receivedAt: iso(now), + provenance: { _tag: "system" }, + dedupeKey: `routine:${orgId}:${fire.fireId}`, + routine: name, + dueAt: fire.dueAt, + lateByMs: Math.max(0, now - Date.parse(fire.dueAt)), + foldedRoutines: folds.map((fold) => ({ + routine: fold.routine, + dueAt: iso(fold.due_at), + })), + ...(items === undefined ? {} : { ownerCheckItems: items }), + }); + yield* queue.enqueue(event); + yield* sql`DELETE FROM kin_routine_folds WHERE org_id = ${orgId} AND fold_into = ${name}`; + } + yield* inbox.markConsumed(orgId, [fire.fireId]); + }), + ); + }).pipe( + Effect.mapError((cause) => + isSqlError(cause) ? toPersistenceSqlError("KinRoutines.drain")(cause) : cause, + ), + Effect.catch((cause) => + Effect.gen(function* () { + const consume = + isPermanent(cause) || now - Date.parse(fire.dueAt) > 24 * 60 * 60 * 1000; + // Retries recur every tick for up to 24 h, so only the final drop warns. + yield* (consume ? Effect.logWarning : Effect.logDebug)( + "Kin routine fire processing failed", + { + orgId, + fireId: fire.fireId, + errorTag: cause._tag, + message: cause.message, + disposition: consume ? "consumed" : "retry", + }, + ); + if (consume) yield* inbox.markConsumed(orgId, [fire.fireId]); + }), + ), + ); + } + }), + ), + ); + let nextReconcileAt = 0; + yield* ticks.register( + "kin-routines", + Effect.gen(function* () { + const now = yield* Clock.currentTimeMillis; + const reconcileDue = now >= nextReconcileAt; + // Scheduler maintenance deliberately scans organizations; all work below remains org-scoped. + const orgs = yield* sql<{ + org_id: string; + }>`SELECT org_id FROM kin_leads WHERE org_id IN (SELECT org_id FROM kin_orgs) ORDER BY org_id`; + for (const row of orgs) { + const orgId = KinOrgId.make(row.org_id); + yield* Effect.gen(function* () { + if (reconcileDue) + yield* sql.withTransaction( + Effect.gen(function* () { + yield* reconcile(yield* settingsFor(orgId)); + }), + ); + yield* drain(orgId); + }).pipe( + Effect.catch((cause) => Effect.logWarning("Kin routines tick failed", { orgId, cause })), + ); + } + if (reconcileDue) nextReconcileAt = now + 15 * 60 * 1000; + }).pipe( + Effect.catch((cause) => + Effect.logWarning("Kin routines scan failed", { + errorTag: cause._tag, + message: cause.message, + }), + ), + ), + ); + return KinRoutines.of({ + isEnabled: Effect.fn("KinRoutines.isEnabled")((orgId, routine) => + wrap( + orgId, + Effect.gen(function* () { + const [row] = yield* sql<{ + enabled: number; + paused: number; + }>`SELECT enabled, paused FROM kin_routine_preferences WHERE org_id = ${orgId} AND routine = ${routine} ORDER BY routine`.pipe( + Effect.mapError(toPersistenceSqlError("KinRoutines.isEnabled")), + ); + return row === undefined || (row.enabled === 1 && row.paused === 0); + }), + ), + ), + configure, + setEnabled: Effect.fn("KinRoutines.setEnabled")((orgId, routine, enabled) => + toggle(orgId, routine, enabled), + ), + pauseAll: Effect.fn("KinRoutines.pauseAll")((orgId) => toggle(orgId, null, true)), + resumeAll: Effect.fn("KinRoutines.resumeAll")((orgId) => toggle(orgId, null, false)), + drain, + }); +}); +export const layer = Layer.effect(KinRoutines, make); diff --git a/apps/server/src/kin/widgets/registry.ts b/apps/server/src/kin/widgets/registry.ts index 59c189ee84d71..31b038ab429bf 100644 --- a/apps/server/src/kin/widgets/registry.ts +++ b/apps/server/src/kin/widgets/registry.ts @@ -1,8 +1,11 @@ -import { KinChatError, KinTaskCardState } from "@t3tools/contracts"; +import { KinChatError, KinTaskCardState, KinOwnerCheckCard } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; import * as Schema from "effect/Schema"; -const schemas = new Map>([["task", KinTaskCardState]]); +const schemas = new Map>([ + ["task", KinTaskCardState], + ["owner_check", KinOwnerCheckCard], +]); export const validateState = Effect.fn("KinWidgetRegistry.validateState")(function* ( type: string, diff --git a/docs/kin/specs/lead.md b/docs/kin/specs/lead.md index b6d23aaafa0b2..afd27a41e8b93 100644 --- a/docs/kin/specs/lead.md +++ b/docs/kin/specs/lead.md @@ -494,7 +494,7 @@ RPC (**new**, through the chat spec's `kinRpcs.ts`): `kin.org.pause` and `kin.or ### Morning and evening routines: content -Scheduling is the triggers spec's. Each fires a `routine` event, and the lead writes the message. `routines.ts` holds these templates. Each is one message of at most 12 lines, plus widgets, posted to the owner's direct chat (shadow summary at L0). +Scheduling is the triggers spec's. Each fires a `routine` event, and the lead writes the message. A request carries its original due time, lateness and any folded routine dates; content must acknowledge those dates rather than imply an on-time run. The owner check selects memory-review items in code and posts its native card after budget admission; its taps use owner-authored writer sources, not model authority. `routines.ts` holds these templates. Each is one message of at most 12 lines, plus widgets, posted to the owner's direct chat (shadow summary at L0). Morning brief, from the dream's prepared brief plus a live recheck: @@ -600,7 +600,7 @@ LEAD-1. **Contracts and tables.** `kinLead.ts` schemas; Kin migrations for all l LEAD-2. **Memory read side.** `ensureCheckouts`, `prepareWakeup`, front matter parser, cache, `query`, `readItem`, `rebuildIndex` (200-line cap, "As of" line, fading after 8 weeks unread). Files: `apps/server/src/kin/memory/{Memory,MemoryCache,frontMatter}.ts`. Depends on: LEAD-1. Done when: tests create a temporary git repo from invented fixtures matching the kin-memory README; kind/date/review queries, malformed-item isolation, organization isolation, cache invalidation, index cap/As-of and TestClock fading all pass. Never use real organization memory as a fixture. Owner: backend. -LEAD-3. **Memory git writer.** `stage` with every writer rule, `commitWakeup`, push and rebase, shadow branch, the secret scan hook (stub until the triggers spec lands). Files: `MemoryWriter.ts`, `journal.ts`. Depends on: LEAD-2, TRIG-16 (stub allowed). Done when: tests show provenance override, `rules.md` refusal, conflict error, one commit per wake-up, discard on failure. Owner: backend. +LEAD-3. **Memory git writer.** `stage` with every writer rule, `commitWakeup`, push and rebase, shadow branch, the secret scan hook (TRIG-16 supplies the scanner layer). Files: `MemoryWriter.ts`, `journal.ts`. Depends on: LEAD-2, TRIG-16 (stub allowed). Done when: tests show provenance override, `rules.md` refusal, conflict error, one commit per wake-up, discard on failure. Owner: backend. LEAD-4. **Event queue and coalescer.** `enqueue`, dedupe, priorities, quiet window, claim, restart requeue, per-org semaphore. Files: `KinEventQueue.ts`. Depends on: LEAD-1. Done when: coalescing and one-writer tests pass with `TestClock`. Owner: backend. @@ -647,7 +647,7 @@ LEAD-14. **End-to-end shadow week dry run.** Script that replays a day of record `MemoryWriterAuthority.resolve` is runner-only: resolve organization, wake-up, active lead, level, timestamps, timezone and event provenance from same-organization rows inside the runner's transaction. Never expose this context as tool input. Owner-confirmed sources must be verified there; absent a source, provenance is the lead's agent id. `MemoryWriterHooks.notify` consumes checkout, conflict and secret-block notices without chat transport coupling. -`KinSecretScan` is deliberately supplied separately. `noFindingsStubLayer` returns no findings until TRIG-16 replaces it; it is not a production secret detector. Any finding or scanner failure blocks the whole commit. TRIG-16 must preserve this fail-closed interface: `scan({ orgId, files: ReadonlyArray<{ path: string; content: Uint8Array }> })` returns `{ path, ruleId, line }[]`, with typed `KinSecretScanError`. +`KinSecretScan` is deliberately supplied separately. `secretScan.layer` supplies the TRIG-16 scanner and reads the owner-signed allowlist; there is no no-findings production stub. Any finding or scanner failure blocks the whole commit. TRIG-16 must preserve this fail-closed interface: `scan({ orgId, files: ReadonlyArray<{ path: string; content: Uint8Array }> })` returns `{ path, ruleId, line }[]`, with typed `KinSecretScanError`. `commitWakeup` builds TRIG-12's sectioned `INDEX.md` from the hash-verified recorded tree plus staged files, read history, the server clock and the owner's time zone, in the same wake-up commit. The read-side rebuild is display-only; no read-checkout text is trusted. K011 adds the SQLite branch anchors required by the writer. The runner serializes prepare/stage/commit and dreaming per organization. Identical commit retries find the existing wake-up commit; failed pushes retain local commits, and failures discard uncommitted files. Journal additions use `appendJournal`, never replacement through `stage`. diff --git a/docs/kin/specs/triggers.md b/docs/kin/specs/triggers.md index 1890e5a05a9df..9b29e137dafb5 100644 --- a/docs/kin/specs/triggers.md +++ b/docs/kin/specs/triggers.md @@ -250,6 +250,31 @@ The dream consumer suppresses a `dream` fire whose `dueAt`, in owner time, falls | `dream_full` | The same pipeline over all items, not just new material. | | `owner_check` | Code picks 5 items: past `review_by`, flagged misleading, low-trust claims older than 30 days, and evidence-free status claims, in that order. It posts a native card with Keep (push `review_by` out one period), Update (opens the direct chat on the item) and Drop (set `invalid_at` now). Taps are owner-authored memory writes. | +Routine delivery is a lead-queue request, not a second posting path: TRIG-9 enqueues `routine`; LEAD-13 owns the brief/summary message and prepared-brief content. Late boundaries are measured from the scheduled owner's calendar date (a summary expires at the first dream after its due time). Folds retain the original routine and due instant, survive restart, and move with an expired summary into the next brief. Same-tick fires are handled by due time, not inbox receipt order. Disabling a routine expires its queued requests; enabling it schedules the next slot without backfill. Pause/resume preserves individual opt-outs. + +Routine processing consumes schema/decode failures immediately. Other failures (including SQLite contention, enqueue failure and unavailable memory checkouts) stay pending for the next tick; a failing fire more than 24 hours past its due instant is consumed to bound poison retries. Successful late fires still follow the normal fold/post policy. Owner-check selection runs before the enqueue transaction, which rechecks consumption and event identity. + +Owner-check ties use review date (missing last), then item id and path. Invalid, superseded and duplicate-id items are excluded. Low trust means any trust below owner; with no status-claim subtype in memory, evidence-free means a claim with no evidence links. Keep sets the next review to now +90 days for people or +30 days otherwise, so repeated Keeps do not grow the review interval. Update returns the owner direct-chat/item target; Keep and Drop stage through the writer with the verified, claimed owner action as their source, never model-supplied provenance. Native clients own presenting the three actions; the server owns their authorization. + +**Owner-check release gate (TRIG-9):** selection and routine enqueue are live, but +`WakeupRunner` keeps native card posting explicitly disabled until the dispatcher +lands. Before enabling it: + +- chat act must enqueue `widget.action`; +- `WakeupRunner` must intercept `owner_check` actions, run `OwnerCheck.act`, + then `MemoryWriter.commitWakeup`, then queue `finish`, without exposing these + actions to the model prompt; +- production `MemoryWriterAuthority.resolve` must grant `kind: "owner_check"` + only after verifying the claimed owner tap, binding its item id and action. + An owner chat citation is not review-date authority; +- web and mobile must render the native card, its handled items, and closed state; +- a revision conflict after Keep or Drop is staged must unstage only that item's path. Today it discards the whole wake-up session, which would drop the lead's other staged writes. + +Each accepted action advances the card revision and marks that item handled; +the last handled item closes it. Misleading-memory selection reads reports +ended in the last 90 days. K013_Routines is provisional and must be renumbered at +merge if TRIG-16 or LEAD-7 takes the slot first. + ### Watchdogs (`interval`) All are plain code. They write memory only through the LEAD queue as code-only events, which keeps one writer per org (gap 1). @@ -324,12 +349,12 @@ GitHub then blocks a bad auto-merge even if Kin's classifier is wrong. This forc **Rules:** - GitHub tokens (`ghp_`, `gho_`, `ghu_`, `ghs_`, `ghr_`, `github_pat_`); -- Anthropic `sk-ant-`, OpenAI `sk-`/`sk-proj-`; +- Anthropic `sk-ant-`; OpenAI legacy `sk-` with the `T3BlbkFJ` marker and `sk-proj-`/`sk-svcacct-`/`sk-admin-` containing an uppercase letter; - AWS `AKIA…`/`ASIA…`; - Slack `xox[abposr]-`; - Stripe `sk_live_`/`rk_live_`; - Google `AIza`; -- PEM `-----BEGIN … PRIVATE KEY-----`; +- PEM `-----BEGIN … PRIVATE KEY-----`, including `PGP PRIVATE KEY BLOCK`; - JWTs (`eyJ…`, three base64url parts); - URLs with a password in them (`scheme://user:pass@`); - `password|secret|token|api_key = `. @@ -342,7 +367,11 @@ GitHub then blocks a bad auto-merge even if Kin's classifier is wrong. This forc **On a finding**, the writer blocks the entire commit, discards staged text and emits a typed owner-notification hook. The runner journals the refusal without secret content and tells the owner to review and rotate the affected credential. Automatic redaction is not part of the writer contract. -**False positives:** the owner adds the fingerprint to `kin_secret_allow` through a signed action. A model can never add to it. +**False positives:** the owner adds the fingerprint to `kin_secret_allow` through a signed action. A model can never add to it. Entries are scoped to organization, exact path, rule and SHA-256 of `ruleId + "\0" + normalized matched value`, with `created_at` in integer milliseconds and `signed_action_id` reserved for the future APPR executor id. The scanner is read-only; the APPR executor must verify the owner grant before inserting. Until that executor lands, there is no agent or MCP allowlist-write path. Migration K012 is provisional and will be renumbered at merge. + +The scanner checks paths as well as content, redacts paths that contain a finding, normalizes NFKC, common Cyrillic/Greek/Latin lookalikes and Default Ignorable code points, checks newline-wrapped tokens, and decodes URL/base64 content up to two levels. Base64 decoding tries all four alignments; malformed percent bytes and non-ASCII decoded bytes become separators. Prefix rules use alphanumeric left boundaries and body-specific right boundaries (fixed-width Google keys also tolerate underscore metadata suffixes); each precompiled regex sees at most 8192 characters plus one character of boundary context on each side, with 4096-character overlap. JWT detection matches a bounded prefix: an eyJ header followed by 5–2048 base64url characters, then an eyJ payload prefix followed by 8–1024 characters, without requiring the signature or the payload end. Its maximum match is 3080 characters, within the overlap. Wrapped tokens are inspected after each appended fragment once the joined run has 20 characters or an anchor, retaining up to eight fragments and 4 KiB. A prose line closes the prior run with its first word and opens a new run with its last word, so adjacent prose and trailing words do not hide a wrapped token. Merge scanning covers only bytes changed from the fetched remote plus the rebuilt `INDEX.md`, not pre-existing remote content. Scanner failures notify through the writer hook, just like findings. Staged YAML parser errors omit source-bearing diagnostics and filesystem path errors omit the underlying path-bearing cause. Merge blocks use the recovery hook; production owner delivery still waits on a notifier. It fails closed above 1 MiB per file, 8 MiB or 1024 files per call, 4096 allowlist entries per organization, or 64 MiB of weighted work (inspected characters, encoded candidate characters, window characters, and 64 units per regex match). A cheap anchor pre-filter skips rule execution on irrelevant windows, including decoded candidates; recursive decoding remains active to find double-encoded tokens. Scanning yields between windows and decoder candidates so multi-file batches do not monopolize the event loop. Base64 runs shorter than 27 characters and fully percent-encoded runs shorter than 60 characters are not decoded; mixed percent/plain runs retain a 20-byte minimum to prevent partial-encoding bypasses. Wrapped runs beyond eight fragments or 4 KiB, splits by a space, `\r`, U+2028, across journal bullets, markdown or HTML comments, `\u` escapes, and hex encoding remain out of scope. These are conservative detectors, not a guarantee against arbitrary encryption or steganography. + +Known gaps, accepted for phase 1: JWE (five-part) tokens and JWT headers longer than 2051 characters; multi-part secrets (JWT, PEM header, URL password, `KEY=value`) with only some bytes percent-encoded; lookalikes outside the folding table (for example Greek `ρ`, Cyrillic `І`/`Ј`/`к`, small caps, U+2800, combining marks inside an anchor). A file of about 1 MiB made only of long whitespace-free lines (such as one path per line) exhausts the work budget after about 15 s and fails closed, blocking that commit; realistic journals up to 1 MiB pass. ### Owner-signed purge (gap 5) @@ -475,7 +504,7 @@ Kin runs on one always-on machine, the **Kin host**. The laptop and phone are cl - **Sink:** a signed GitHub `pull_request` request through the real `ScheduledTaskService` gives exactly one inbox row and no thread launch. A replay gives `duplicate` and no new row. A bot-sender request gives `filtered`. A non-Kin task still launches normally. - **Deferral:** unchanged recheck, then heads-up, then silence, then `started` at due. A linked issue closed before due leads to `asked`, and nothing starts. With the stop switch on, it asks. [Monday] re-arms to next Monday 09:00. - **Dream:** a dream touching `rules.md` fails, and `main` is unchanged. A dream with one journal fold and one `decisions/` change opens two PRs, and only the auto one merges. -- **Secret scan:** a journal line holding `ghp_` plus 36 characters is committed as `[redacted:github-token]`, and the owner gets one message. +- **Secret scan:** a journal line holding `ghp_` plus 36 characters blocks the entire commit, and the owner gets one typed notification without the value. - **Purge:** on a fixture repo, the file is gone from `git log --all -p`, a tombstone exists, and an audit row exists. - **Contract tests:** `vp test run kin-contract` passes on today's upstream. Renaming `scheduledTaskId` in a scratch branch makes it fail. diff --git a/packages/contracts/src/kinLead.ts b/packages/contracts/src/kinLead.ts index b7bd25ccadac7..9c71c77dfd31b 100644 --- a/packages/contracts/src/kinLead.ts +++ b/packages/contracts/src/kinLead.ts @@ -147,6 +147,18 @@ const EventBase = { dedupeKey: Schema.NullOr(TrimmedNonEmptyString), provenance: KinAuthor, }; +export const KinOwnerCheckCard = Schema.Struct({ + items: Schema.Array( + Schema.Struct({ + itemId: KinMemoryItemId, + reason: Schema.Literals(["past_review", "misleading", "old_low_trust", "no_evidence"]), + handled: Schema.optional(Schema.Literals(["keep", "update", "drop"])), + title: Schema.String, + path: Schema.String, + }), + ), +}); + export const KinEvent = Schema.Union([ Schema.TaggedStruct("chat.message", { ...EventBase, chatId: KinChatId, messageId: KinMessageId }), Schema.TaggedStruct("widget.action", { @@ -184,6 +196,24 @@ export const KinEvent = Schema.Union([ Schema.TaggedStruct("routine", { ...EventBase, routine: KinLeadRoutine, + dueAt: Schema.optional(CanonicalUtcTimestamp), + lateByMs: Schema.optional(NonNegativeInt), + foldedRoutines: Schema.optional( + Schema.Array( + Schema.Struct({ + routine: KinLeadRoutine, + dueAt: CanonicalUtcTimestamp, + }), + ), + ), + ownerCheckItems: Schema.optional( + Schema.Array( + Schema.Struct({ + itemId: KinMemoryItemId, + reason: Schema.Literals(["past_review", "misleading", "old_low_trust", "no_evidence"]), + }), + ), + ), }), Schema.TaggedStruct("watchdog", { ...EventBase,