From 0fa77307592d058559ec9121616d8fa409c11df3 Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Mon, 9 Nov 2026 01:22:23 +0530 Subject: [PATCH 1/4] feat(kin): add watchdogs that compare memory with GitHub and reconcile triggers Co-Authored-By: Claude Opus 5.5 --- apps/server/src/kin/lead/layer.ts | 2 + apps/server/src/kin/memory/Memory.ts | 13 +- apps/server/src/kin/memory/buildIndex.ts | 8 + .../server/src/kin/triggers/KinTriggerSink.ts | 3 +- .../src/kin/triggers/KinTriggerStore.ts | 47 ++ apps/server/src/kin/triggers/githubEvents.ts | 2 +- .../server/src/kin/triggers/watchdogs.test.ts | 633 +++++++++++++++++ apps/server/src/kin/triggers/watchdogs.ts | 635 ++++++++++++++++++ .../src/kin/triggers/webhooks.testkit.ts | 3 +- 9 files changed, 1332 insertions(+), 14 deletions(-) create mode 100644 apps/server/src/kin/triggers/watchdogs.test.ts create mode 100644 apps/server/src/kin/triggers/watchdogs.ts diff --git a/apps/server/src/kin/lead/layer.ts b/apps/server/src/kin/lead/layer.ts index 55200e7f68fbf..9ce1a207b9116 100644 --- a/apps/server/src/kin/lead/layer.ts +++ b/apps/server/src/kin/lead/layer.ts @@ -2,6 +2,7 @@ import * as Scheduler from "../../scheduling/Scheduler.ts"; import * as Routines from "../triggers/routines.ts"; import * as DeferralFlow from "../triggers/deferralFlow.ts"; import * as GithubEvents from "../triggers/githubEvents.ts"; +import * as Watchdogs from "../triggers/watchdogs.ts"; import * as OwnerCheck from "../triggers/ownerCheck.ts"; import * as TriggerScheduler from "../triggers/KinTriggerScheduler.ts"; import * as TriggerStore from "../triggers/KinTriggerStore.ts"; @@ -64,6 +65,7 @@ const routineRuntime = Layer.mergeAll( Routines.layer, DeferralFlow.layer, GithubEvents.consumerLayer, + Watchdogs.runtimeLayer, ).pipe( Layer.provideMerge(TriggerScheduler.layer), Layer.provideMerge(Layer.mergeAll(TriggerStore.layer, TriggerInbox.layer)), diff --git a/apps/server/src/kin/memory/Memory.ts b/apps/server/src/kin/memory/Memory.ts index 037fc25fc2784..c2968bbce9c99 100644 --- a/apps/server/src/kin/memory/Memory.ts +++ b/apps/server/src/kin/memory/Memory.ts @@ -31,7 +31,7 @@ import { memoryGit as runMemoryGit } from "./git.ts"; import { VcsProcess } from "../../vcs/VcsProcess.ts"; import * as MemoryCache from "./MemoryCache.ts"; import { mergeMemoryFiles, uniqueMergeBase } from "./merge.ts"; -import { buildIndex } from "./buildIndex.ts"; +import { buildIndex, indexIsStale } from "./buildIndex.ts"; import { MemoryConfig } from "./MemoryConfig.ts"; import { MemoryItemError, @@ -1319,16 +1319,7 @@ const make = Effect.gen(function* () { const index = yield* readFile(orgId, "INDEX.md").pipe( Effect.catchTags({ MemoryError: () => Effect.succeed("") }), ); - const asOf = /^As of ([^\r\n]+)(?:\r?\n|$)/.exec(index)?.[1]; - const now = yield* Clock.currentTimeMillis; - if ( - !asOf || - !Number.isFinite(dateMillis(asOf)) || - !/(?:Z|[+-]\d{2}:\d{2})$/.test(asOf) || - now - dateMillis(asOf) >= 36 * 60 * 60 * 1000 || - dateMillis(asOf) > now - ) - yield* rebuildIndex(orgId); + if (indexIsStale(index, yield* Clock.currentTimeMillis)) yield* rebuildIndex(orgId); const rebuilt = yield* readFile(orgId, "INDEX.md"); const indexAsOf = yield* Effect.fromOption( DateTime.make(/^As of ([^\r\n]+)(?:\r?\n|$)/.exec(rebuilt)?.[1] ?? ""), diff --git a/apps/server/src/kin/memory/buildIndex.ts b/apps/server/src/kin/memory/buildIndex.ts index c2518402f0b7a..5fd8b5f08f69a 100644 --- a/apps/server/src/kin/memory/buildIndex.ts +++ b/apps/server/src/kin/memory/buildIndex.ts @@ -13,6 +13,14 @@ const dateOrder = (value: string | null) => { type IndexItem = MemoryItemHead & { readonly misleading?: boolean }; +/** Needs a rebuild: no zoned "As of" first line, one in the future, or one 36 h old. */ +export function indexIsStale(index: string, now: number): boolean { + const asOf = /^As of ([^\r\n]+)(?:\r?\n|$)/.exec(index)?.[1]; + if (asOf === undefined || !/(?:Z|[+-]\d{2}:\d{2})$/.test(asOf)) return true; + const at = memoryDateMillis(asOf); + return !Number.isFinite(at) || at > now || now - at >= 36 * 60 * 60 * 1000; +} + export function buildIndex( items: ReadonlyArray, readHistory: Readonly>, diff --git a/apps/server/src/kin/triggers/KinTriggerSink.ts b/apps/server/src/kin/triggers/KinTriggerSink.ts index d605a60a16f20..8e1b5167ba56f 100644 --- a/apps/server/src/kin/triggers/KinTriggerSink.ts +++ b/apps/server/src/kin/triggers/KinTriggerSink.ts @@ -98,7 +98,8 @@ const make = Effect.gen(function* () { return; } const prefix = `scheduled-task:${taskId}:webhook:`; - if (!input.commandId.startsWith(prefix) || source.status !== "active") + // `status` is what trigger_reconcile last saw; the task shape and secret below are the gate. + if (!input.commandId.startsWith(prefix)) return yield* new KinTriggerInbox.KinTriggerInboxConflict({ orgId: source.orgId, reason: "unknown_source", diff --git a/apps/server/src/kin/triggers/KinTriggerStore.ts b/apps/server/src/kin/triggers/KinTriggerStore.ts index 87e11cf076114..363a2d83b4737 100644 --- a/apps/server/src/kin/triggers/KinTriggerStore.ts +++ b/apps/server/src/kin/triggers/KinTriggerStore.ts @@ -161,6 +161,21 @@ export class KinTriggerStore extends Context.Service< readonly sourceByTaskId: ( id: ScheduledTaskId, ) => Effect.Effect; + readonly listSources: ( + orgId: KinOrgId, + ) => Effect.Effect, StoreError>; + /** Records what `trigger_reconcile` saw; false when the status was no longer `from`. */ + readonly setSourceStatus: ( + orgId: KinOrgId, + id: KinWebhookSourceId, + from: KinWebhookSource["status"], + to: KinWebhookSource["status"], + ) => Effect.Effect; + /** True when the task's current secret is the one bound to the source. */ + readonly secretMatchesTask: ( + orgId: KinOrgId, + id: KinWebhookSourceId, + ) => Effect.Effect; readonly put: (trigger: Omit) => Effect.Effect; readonly get: ( orgId: KinOrgId, @@ -395,6 +410,38 @@ const make = Effect.gen(function* () { }, Effect.mapError(toPersistenceOrDecodeError("KinTriggerStore.sourceByTaskId")), ), + listSources: Effect.fn("KinTriggerStore.listSources")( + function* (orgId) { + const rows = yield* sql`SELECT ${sql.literal(sourceColumns)} FROM kin_webhook_sources + WHERE org_id = ${orgId} ORDER BY source_id`.pipe(NodeSqliteClient.readOnly); + return yield* Effect.forEach(rows, (row) => decodeSource(row)); + }, + Effect.mapError(toPersistenceOrDecodeError("KinTriggerStore.listSources")), + ), + setSourceStatus: Effect.fn("KinTriggerStore.setSourceStatus")( + function* (orgId, id, from, to) { + const rows = yield* sql`UPDATE kin_webhook_sources SET status = ${to} + WHERE org_id = ${orgId} AND source_id = ${id} AND status = ${from} + RETURNING source_id`; + return rows.length === 1; + }, + Effect.mapError(toPersistenceSqlError("KinTriggerStore.setSourceStatus")), + ), + secretMatchesTask: Effect.fn("KinTriggerStore.secretMatchesTask")( + function* (orgId, id) { + const [row] = yield* sql<{ + readonly hash: string | null; + readonly secret: string | null; + }>`SELECT s.secret_sha256 AS hash, t.webhook_secret AS secret FROM kin_webhook_sources s + JOIN scheduled_tasks t ON t.task_id = s.scheduled_task_id + WHERE s.org_id = ${orgId} AND s.source_id = ${id}`.pipe(NodeSqliteClient.readOnly); + if (row?.hash == null || row.secret == null) return false; + const bound = Buffer.from(row.hash, "hex"); + const given = secretDigest(Redacted.make(row.secret)); + return bound.length === given.length && NodeCrypto.timingSafeEqual(bound, given); + }, + Effect.mapError(toPersistenceSqlError("KinTriggerStore.secretMatchesTask")), + ), put: Effect.fn("KinTriggerStore.put")( function* (trigger) { const value = yield* decodePutTrigger({ ...trigger, lastFireId: null }); diff --git a/apps/server/src/kin/triggers/githubEvents.ts b/apps/server/src/kin/triggers/githubEvents.ts index b75576072e113..88305b68af7cc 100644 --- a/apps/server/src/kin/triggers/githubEvents.ts +++ b/apps/server/src/kin/triggers/githubEvents.ts @@ -101,7 +101,7 @@ export const repoScopeLayer = Layer.effect( ), ); -const inRepoScope = (repos: ReadonlyArray, repo: string) => +export const inRepoScope = (repos: ReadonlyArray, repo: string) => repos.some((scoped) => scoped.toLowerCase() === repo.toLowerCase()); const header = (headers: Readonly>, name: string) => diff --git a/apps/server/src/kin/triggers/watchdogs.test.ts b/apps/server/src/kin/triggers/watchdogs.test.ts new file mode 100644 index 0000000000000..943e48db3bb6b --- /dev/null +++ b/apps/server/src/kin/triggers/watchdogs.test.ts @@ -0,0 +1,633 @@ +import { assert, describe, it } from "@effect/vitest"; +import { + KIN_GITHUB_APP_SOURCE, + KinEvent, + KinOrgId, + KinWebhookSource, + ManagedPolicyDocument, + MemoryItemHead, +} from "@t3tools/contracts"; +import * as ConfigProvider from "effect/ConfigProvider"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Redacted from "effect/Redacted"; +import * as Ref from "effect/Ref"; +import * as Schema from "effect/Schema"; +import * as SqlClient from "effect/sql/SqlClient"; +import * as TestClock from "effect/testing/TestClock"; + +import * as ThreadManagementService from "../../orchestration-v2/ThreadManagementService.ts"; +import * as Sqlite from "../../persistence/Sqlite.ts"; +import type * as ScheduledTaskService from "../../scheduledTasks/ScheduledTaskService.ts"; +import * as Scheduler from "../../scheduling/Scheduler.ts"; +import * as Hub from "../chat/KinChatHub.ts"; +import * as Repository from "../chat/KinChatRepository.ts"; +import * as Chat from "../chat/KinChatService.ts"; +import * as JudgeShadow from "../floor/JudgeShadow.ts"; +import * as KinEventQueue from "../lead/KinEventQueue.ts"; +import * as Memory from "../memory/Memory.ts"; +import * as KinPersistence from "../persistence/KinPersistence.ts"; +import * as GithubEvents from "./githubEvents.ts"; +import * as KinTriggerInbox from "./KinTriggerInbox.ts"; +import * as KinTriggerStore from "./KinTriggerStore.ts"; +import * as Watchdogs from "./watchdogs.ts"; +import { + projection, + signedRequest, + timestamp, + waitForDispatch, + webhookTask, + withScheduledTasks, +} from "./webhooks.testkit.ts"; + +const NOW = 1794009600000; +const HOUR = 60 * 60 * 1000; +const DAY = 24 * HOUR; +const iso = (ms: number) => new Date(ms).toISOString(); +const orgId = KinOrgId.make("org-a"); +const decodeHead = Schema.decodeUnknownSync(MemoryItemHead); +const decodeEvent = Schema.decodeSync(Schema.fromJsonString(KinEvent)); +const decodePolicy = Schema.decodeEffect(ManagedPolicyDocument); +const source = Schema.decodeSync(KinWebhookSource)({ + id: KIN_GITHUB_APP_SOURCE, + orgId, + repo: KIN_GITHUB_APP_SOURCE, + scheduledTaskId: `task-${orgId}`, + inboxThreadId: `inbox-${orgId}`, + githubHookId: null, + botLogin: "kin[bot]", + status: "active", +}); + +const item = (id: string, overrides: Record = {}) => + decodeHead({ + orgId, + id, + path: `commitments/${id}.md`, + kind: "commitment", + title: id, + status: "open", + due: null, + chat: "team", + recorded_at: "2026-09-01", + valid_from: "2026-09-01", + invalid_at: null, + superseded_by: null, + review_by: "2026-12-01", + source: "owner-chat", + said_by: "owner", + trust: "owner", + links: [], + ...overrides, + }); + +type State = Watchdogs.GithubItemState; +const issue = (number: number, state: State["state"]): State => ({ + number, + kind: "issue", + state, + reviewRequests: [], +}); +const pull = ( + number: number, + state: State["state"], + reviewRequests: State["reviewRequests"] = [], +): State => ({ number, kind: "pull", state, reviewRequests }); + +const seed = Effect.gen(function* () { + yield* TestClock.setTime(NOW); + const sql = yield* SqlClient.SqlClient; + yield* sql`INSERT INTO kin_orgs VALUES (${orgId}, 'Kin', ${timestamp})`; + yield* sql`INSERT INTO kin_people VALUES ('owner', ${orgId}, 'owner', 'Owner', 'owner-gh', ${timestamp})`; + yield* sql`INSERT INTO kin_people VALUES ('alice', ${orgId}, 'alice', 'Alice', 'Alice-GH', ${timestamp})`; + yield* sql`INSERT INTO kin_agents VALUES ('lead', ${orgId}, 'Lead', 'lead', 'owner', ${timestamp}, NULL)`; + for (const chat of ["chat", "team"]) + yield* sql`INSERT INTO kin_chats (chat_id, org_id, kind, direct_key, last_seq, created_at) + VALUES (${chat}, ${orgId}, ${chat === "chat" ? "direct" : "team_group"}, ${chat === "chat" ? "direct" : null}, 0, ${timestamp})`; + for (const [member, chat, person] of [ + ["m-owner", "chat", "owner"], + ["m-owner-team", "team", "owner"], + ["m-alice", "team", "alice"], + ] as const) + yield* sql`INSERT INTO kin_chat_members (member_id, org_id, chat_id, principal_kind, person_id, role, joined_at) + VALUES (${member}, ${orgId}, ${chat}, 'person', ${person}, ${person === "owner" ? "owner" : "member"}, ${timestamp})`; + yield* sql`INSERT INTO kin_leads (agent_id, org_id, name, level_since, models_json, worker_projects_json, owner_chat_id) + VALUES ('lead', ${orgId}, 'Lead', 0, '{}', '[]', 'chat')`; + yield* sql`INSERT INTO orchestration_v2_projection_threads + (thread_id, project_id, title, default_provider, runtime_mode, interaction_mode, created_at, updated_at, payload_json) + VALUES (${`inbox-${orgId}`}, ${`project-${orgId}`}, 'Inbox', 'codex', 'approval-required', 'plan', ${timestamp}, ${timestamp}, '{}')`; +}); + +/** A `github_recheck` request as TRIG-6 leaves it; the sweep only reads and consumes it. */ +const putRecheck = (delivery: string) => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const fireId = `gh:${orgId}:recheck:delivery:${delivery}`; + yield* sql`INSERT INTO kin_trigger_inbox (fire_id, org_id, payload_json, received_at) + VALUES (${fireId}, ${orgId}, ${JSON.stringify({ + fireId, + orgId, + dueAt: iso(NOW), + firedAt: iso(NOW), + lateByMs: 0, + payload: { + kind: "github_recheck", + repo: null, + reason: "delivery_missing", + sourceFireId: `gh:${orgId}:delivery:${delivery}`, + }, + })}, ${NOW})`; + return fireId; + }); + +const pendingRechecks = Effect.gen(function* () { + const inbox = yield* KinTriggerInbox.KinTriggerInbox; + return (yield* inbox.listPending(orgId, 100, undefined, ["github_recheck"])).length; +}); + +const watchdogEvents = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const rows = yield* sql<{ json: string }>`SELECT json FROM kin_events + WHERE org_id = ${orgId} AND tag = 'watchdog' ORDER BY dedupe_key`; + return rows.flatMap((row) => { + const event = decodeEvent(row.json); + return event._tag === "watchdog" ? [{ finding: event.finding, itemIds: event.itemIds }] : []; + }); +}); + +const pingsFor = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + return yield* sql<{ person: string; itemId: string }>`SELECT person, item_id AS "itemId" + FROM kin_ping_items WHERE org_id = ${orgId} ORDER BY person, item_id`; +}); + +const notices = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const rows = yield* sql<{ body: string }>`SELECT body FROM kin_messages + WHERE org_id = ${orgId} AND chat_id = 'chat' AND author_kind = 'system' ORDER BY seq`; + return rows.map((row) => row.body); +}); + +interface Harness { + readonly items: Ref.Ref>; + readonly index: Ref.Ref; + readonly rebuilds: Ref.Ref; + readonly ticks: Map>; +} + +function services(harness: Harness) { + const memory = Layer.unwrap( + Effect.map(SqlClient.SqlClient, (sql) => { + // Memory takes an org lock that a memory load holds while it waits for the SQLite writer. + const outside = (name: string, effect: Effect.Effect) => + Effect.serviceOption(sql.transactionService).pipe( + Effect.flatMap((open) => + Option.isSome(open) ? Effect.die(`Memory.${name} ran inside a transaction`) : effect, + ), + ); + return Layer.mock(Memory.Memory)({ + query: (org, query) => + outside( + "query", + Ref.get(harness.items).pipe( + Effect.map((items) => + items + .filter((head) => head.orgId === org) + .filter((head) => query.status === undefined || head.status === query.status) + .map((head) => ({ ...head, duplicate: false })), + ), + ), + ), + readIndex: () => outside("readIndex", Ref.get(harness.index)), + rebuildIndex: () => + outside( + "rebuildIndex", + Ref.update(harness.rebuilds, (n) => n + 1), + ), + }); + }), + ); + return Layer.mergeAll( + memory, + Chat.layer.pipe(Layer.provide(JudgeShadow.layerOff), Layer.provideMerge(Hub.layer)), + Layer.succeed(GithubEvents.KinGithubRepoScope, { repos: ["Owner/Repo"] }), + Layer.succeed(Scheduler.Scheduler, { + register: Effect.fn("KinWatchdogsTest.register")(function* ( + name: string, + work: Effect.Effect, + ) { + const context = yield* Effect.context(); + harness.ticks.set(name, work.pipe(Effect.provideContext(context), Effect.orDie)); + }), + }), + ).pipe( + Layer.provideMerge( + Layer.mergeAll( + Repository.layer, + KinTriggerStore.layer, + KinTriggerInbox.layer, + KinEventQueue.layer, + ), + ), + Layer.provide( + Layer.mock(ThreadManagementService.ThreadManagementService)({ + getThreadProjection: () => Effect.succeed(projection(orgId)), + }), + ), + Layer.provideMerge( + KinPersistence.layer.pipe( + Layer.provideMerge(Sqlite.layerMemory), + Layer.provideMerge(ConfigProvider.layer(ConfigProvider.fromUnknown({ KIN_ENABLED: true }))), + ), + ), + ); +} + +/** Builds the service in the caller's context, so a fake reader or T3 tasks there reach its tick. */ +const build = Effect.provide(Effect.service(Watchdogs.KinWatchdogs), Watchdogs.layer); + +/** Seeded state with these memory items; GitHub is always a fake passed per call. */ +const run = ( + items: ReadonlyArray, + body: (harness: Harness) => Effect.Effect, +) => + Effect.gen(function* () { + const harness: Harness = { + items: yield* Ref.make(items), + index: yield* Ref.make(`As of ${iso(NOW)}\n`), + rebuilds: yield* Ref.make(0), + ticks: new Map(), + }; + return yield* Effect.gen(function* () { + yield* seed; + return yield* body(harness); + }).pipe(Effect.provide(services(harness))); + }); + +/** A fake GitHub that records each batched read: `repo:numbers`. */ +const fakeGithub = (states: ReadonlyArray, reads: Array, fail = false) => ({ + readRepo: (org: KinOrgId, repo: string, numbers: ReadonlyArray) => { + reads.push(`${repo}:${numbers.join(",")}`); + return fail + ? Effect.fail( + new Watchdogs.KinWatchdogGithubError({ orgId: org, repo, reason: "unavailable" }), + ) + : Effect.succeed(states.filter((state) => numbers.includes(state.number))); + }, +}); +const withGithub = (reader: Watchdogs.KinWatchdogGithubReaderShape | null) => + Effect.provideService(Watchdogs.KinWatchdogGithubReader, reader); + +describe("github_recheck", () => { + const closesIssue = item("ship-login", { links: ["https://github.com/owner/repo/issues/7"] }); + const mergesPr = item("rate-limit", { + kind: "deferral", + path: "deferrals/rate-limit.md", + links: ["owner/repo#9"], + }); + const halfDone = item("two-links", { + links: ["https://github.com/owner/repo/issues/7", "https://github.com/owner/repo/issues/8"], + }); + const outOfScope = item("elsewhere", { links: ["https://github.com/other/repo/issues/7"] }); + const condition = item("when-212-merges", { + kind: "deferral", + path: "deferrals/when-212-merges.md", + on: { event: "pull_request", action: "closed", merged: true, number: 212 }, + }); + const states = [issue(7, "closed"), issue(8, "open"), pull(9, "merged"), pull(212, "merged")]; + + it.effect("marks a commitment done when its linked issue closes, with the evidence link", () => + run([closesIssue, mergesPr, halfDone, outOfScope, condition], () => + Effect.gen(function* () { + const service = yield* build; + yield* putRecheck("1"); + const reads: Array = []; + yield* service.run("github_recheck", orgId).pipe(withGithub(fakeGithub(states, reads))); + + // One batched read for the scoped repo; the out-of-scope link is never read. + assert.deepEqual(reads, ["owner/repo:7,8,9,212"]); + assert.deepEqual(yield* watchdogEvents, [ + { + finding: + "GitHub state shows condition met (read from GitHub by code): https://github.com/Owner/Repo/pull/212 is merged.", + itemIds: ["when-212-merges"], + }, + { + finding: + "Verified on GitHub by code: https://github.com/owner/repo/pull/9 is merged. Mark deferral rate-limit done and record that link as its evidence.", + itemIds: ["rate-limit"], + }, + { + finding: + "Verified on GitHub by code: https://github.com/owner/repo/issues/7 is closed. Mark commitment ship-login done and record that link as its evidence.", + itemIds: ["ship-login"], + }, + ]); + assert.equal(yield* pendingRechecks, 0); + + // A later sweep reads again but queues nothing new for the same findings. + yield* putRecheck("2"); + yield* service.run("github_recheck", orgId).pipe(withGithub(fakeGithub(states, reads))); + assert.equal(reads.length, 2); + assert.equal((yield* watchdogEvents).length, 3); + assert.equal(yield* pendingRechecks, 0); + }), + ), + ); + + it.effect("leaves recheck requests pending until a reader reads every repo", () => + run([closesIssue], () => + Effect.gen(function* () { + const service = yield* build; + yield* putRecheck("1"); + // The default reader is unavailable: nothing is read, queued or consumed. + yield* service.run("github_recheck", orgId); + assert.equal(yield* pendingRechecks, 1); + + const reads: Array = []; + yield* service + .run("github_recheck", orgId) + .pipe(withGithub(fakeGithub([issue(7, "closed")], reads, true))); + assert.equal(yield* pendingRechecks, 1); + assert.deepEqual(yield* watchdogEvents, []); + + // A read that never answers times out instead of holding the tick. + const hung = yield* service + .run("github_recheck", orgId) + .pipe(withGithub({ readRepo: () => Effect.never }), Effect.forkChild); + yield* TestClock.adjust("30 seconds"); + yield* Fiber.join(hung); + assert.equal(yield* pendingRechecks, 1); + + yield* service + .run("github_recheck", orgId) + .pipe(withGithub(fakeGithub([issue(7, "closed")], reads))); + assert.equal(yield* pendingRechecks, 0); + assert.equal((yield* watchdogEvents).length, 1); + }), + ), + ); +}); + +describe("chases", () => { + it.effect("promise_overdue chases whoever promised an overdue commitment, once", () => + run( + [ + item("overdue", { owner: "alice", due: iso(NOW - HOUR) }), + item("later", { owner: "alice", due: iso(NOW + DAY) }), + item("done", { owner: "alice", due: iso(NOW - DAY), status: "done" }), + ], + () => + Effect.gen(function* () { + const service = yield* build; + yield* service.run("promise_overdue", orgId); + yield* service.run("promise_overdue", orgId); + assert.deepEqual(yield* pingsFor, [{ person: "alice", itemId: "overdue" }]); + }), + ), + ); + + it.effect("stale_review chases a reviewer requested over 24 hours ago", () => + run([item("review-me", { links: ["https://github.com/owner/repo/pull/5"] })], () => + Effect.gen(function* () { + const service = yield* build; + const states = [ + pull(5, "open", [ + { login: "alice-gh", requestedAt: NOW - 25 * HOUR }, + { login: "owner-gh", requestedAt: NOW - 2 * HOUR }, + ]), + ]; + yield* service.run("stale_review", orgId); + assert.deepEqual(yield* pingsFor, []); + yield* service.run("stale_review", orgId).pipe(withGithub(fakeGithub(states, []))); + assert.deepEqual(yield* pingsFor, [{ person: "alice", itemId: "review-me" }]); + }), + ), + ); + + it.effect("objective_at_risk flags an objective due within 3 days with open issues", () => + run( + [ + item("launch", { + kind: "objective", + path: "objectives/launch.md", + due: iso(NOW + 2 * DAY), + links: ["owner/repo#3", "owner/repo#4"], + }), + item("someday", { + kind: "objective", + path: "objectives/someday.md", + due: iso(NOW + 5 * DAY), + links: ["owner/repo#3"], + }), + ], + () => + Effect.gen(function* () { + const service = yield* build; + const github = withGithub(fakeGithub([issue(3, "open"), issue(4, "closed")], [])); + yield* service.run("objective_at_risk", orgId).pipe(github); + yield* service.run("objective_at_risk", orgId).pipe(github); + assert.deepEqual(yield* notices, [ + `launch is due ${iso(NOW + 2 * DAY)} and still has open GitHub work: https://github.com/owner/repo/issues/3.`, + ]); + }), + ), + ); +}); + +it.effect("index_freshness rebuilds INDEX.md once it is 36 hours old", () => + run([], ({ index, rebuilds }) => + Effect.gen(function* () { + const service = yield* build; + yield* Ref.set(index, `As of ${iso(NOW - 35 * HOUR)}\n`); + yield* service.run("index_freshness", orgId); + assert.equal(yield* Ref.get(rebuilds), 0); + yield* Ref.set(index, `As of ${iso(NOW - 36 * HOUR)}\n`); + yield* service.run("index_freshness", orgId); + assert.equal(yield* Ref.get(rebuilds), 1); + }), + ), +); + +describe("trigger_reconcile", () => { + const sourceStatus = Effect.gen(function* () { + const store = yield* KinTriggerStore.KinTriggerStore; + return (yield* store.getSource(orgId, source.id))?.status; + }); + const registered = ( + body: ( + tasks: ScheduledTaskService.ScheduledTaskService["Service"], + reconcile: Effect.Effect, + taskId: string, + ) => Effect.Effect, + ) => + run([], () => + withScheduledTasks((tasks) => + Effect.gen(function* () { + const { task } = yield* tasks.upsert( + yield* webhookTask(source.scheduledTaskId, `project-${orgId}`), + ); + const store = yield* KinTriggerStore.KinTriggerStore; + yield* store.registerSource(source); + yield* store.bindSecret(orgId, source.id, Redacted.make("test-secret")); + yield* TestClock.adjust("1 second"); + const service = yield* build; + yield* body(tasks, service.run("trigger_reconcile", orgId).pipe(Effect.orDie), task.id); + }), + ), + ); + + it.effect("reports a paused task without re-enabling it, then sees it resume", () => + registered((tasks, reconcile, taskId) => + Effect.gen(function* () { + yield* reconcile; + assert.equal(yield* sourceStatus, "active"); + assert.deepEqual(yield* notices, []); + + yield* tasks.setEnabled({ id: source.scheduledTaskId, enabled: false }); + yield* reconcile; + yield* reconcile; + assert.equal(yield* sourceStatus, "paused_in_t3"); + const listed = (yield* tasks.list()).tasks.find((task) => task.id === taskId); + assert.equal(listed?.enabled, false); + assert.lengthOf(yield* notices, 1); + assert.include((yield* notices)[0], "Kin won't turn it back on"); + + // The owner turns it on in T3; the sink takes deliveries before the next reconcile. + const { task } = yield* tasks.setEnabled({ id: source.scheduledTaskId, enabled: true }); + yield* waitForDispatch( + tasks.triggerWebhook( + signedRequest(task, "gh-1", { + action: "opened", + repository: { full_name: "owner/repo" }, + sender: { login: "alice" }, + pull_request: { number: 1 }, + }), + ), + ); + const inbox = yield* KinTriggerInbox.KinTriggerInbox; + assert.lengthOf(yield* inbox.listPending(orgId, 10, undefined, ["github"]), 1); + yield* reconcile; + assert.equal(yield* sourceStatus, "active"); + assert.deepEqual((yield* notices).at(-1), "GitHub events reach Kin again."); + }), + ), + ); + + it.effect("reports a swapped secret once per binding and a deleted task as missing", () => + registered((tasks, reconcile) => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql`UPDATE scheduled_tasks SET webhook_secret = 'agent-chosen-secret' + WHERE task_id = ${source.scheduledTaskId}`; + yield* reconcile; + yield* reconcile; + assert.equal(yield* sourceStatus, "active"); + assert.lengthOf(yield* notices, 1); + assert.include((yield* notices)[0], "a secret Kin didn't bind"); + const bound = yield* sql<{ hash: string }>`SELECT secret_sha256 AS hash + FROM kin_webhook_sources WHERE org_id = ${orgId} AND source_id = ${source.id}`; + assert.isString(bound[0]?.hash); + + yield* tasks.delete({ id: source.scheduledTaskId }); + yield* reconcile; + assert.equal(yield* sourceStatus, "missing"); + assert.include((yield* notices).at(-1), "was deleted"); + }), + ), + ); + + it.effect("reports a task blocked by managed policy", () => + run([], () => + Effect.gen(function* () { + yield* withScheduledTasks((tasks) => + Effect.gen(function* () { + yield* tasks.upsert(yield* webhookTask(source.scheduledTaskId, `project-${orgId}`)); + const store = yield* KinTriggerStore.KinTriggerStore; + yield* store.registerSource(source); + yield* store.bindSecret(orgId, source.id, Redacted.make("test-secret")); + }), + ); + const policy = yield* decodePolicy({ + schemaVersion: 1, + organization: "Kin", + limits: { webhookTasks: false }, + }); + yield* withScheduledTasks( + () => + Effect.gen(function* () { + const service = yield* build; + yield* service.run("trigger_reconcile", orgId); + }), + undefined, + policy, + ); + assert.equal(yield* sourceStatus, "blocked_by_policy"); + assert.include((yield* notices)[0], "managed policy"); + }), + ), + ); + + it.effect("re-arms open deferrals from memory", () => + run( + [ + item("ping-ops", { + kind: "deferral", + path: "deferrals/ping-ops.md", + due: iso(NOW + DAY), + on_due: "remind", + }), + ], + () => + Effect.gen(function* () { + const service = yield* build; + yield* service.run("trigger_reconcile", orgId); + const store = yield* KinTriggerStore.KinTriggerStore; + const armed = (yield* store.list(orgId)).filter((trigger) => trigger.enabled); + assert.deepEqual( + armed.map((trigger) => [trigger.id, trigger.nextRunAt]), + [["deferral:ping-ops:due", iso(NOW + DAY)]], + ); + }), + ), + ); +}); + +it.effect("the tick runs every watchdog at startup, then each on its own interval", () => + run([item("ship-login", { links: ["owner/repo#7"] })], ({ ticks }) => + Effect.gen(function* () { + const reads: Array = []; + yield* build.pipe(withGithub(fakeGithub([issue(7, "open")], reads))); + const sql = yield* SqlClient.SqlClient; + // Retention runs with trigger_reconcile: consumed rows go after 7 days, pending ones stay. + for (const [fireId, consumedAt] of [ + ["old", NOW - 8 * DAY], + ["recent", NOW - DAY], + ] as const) + yield* sql`INSERT INTO kin_trigger_inbox (fire_id, org_id, payload_json, received_at, consumed_at) + VALUES (${fireId}, ${orgId}, '{}', ${consumedAt}, ${consumedAt})`; + const tick = ticks.get("kin-watchdogs") ?? Effect.die("tick not registered"); + yield* tick; + // github_recheck and stale_review each read once. + assert.lengthOf(reads, 2); + const left = yield* sql<{ fireId: string }>`SELECT fire_id AS "fireId" FROM kin_trigger_inbox + WHERE org_id = ${orgId} ORDER BY fire_id`; + assert.deepEqual( + left.map((row) => row.fireId), + ["recent"], + ); + + yield* TestClock.adjust("5 minutes"); + yield* tick; + assert.lengthOf(reads, 2); + yield* TestClock.adjust("5 minutes"); + yield* tick; + assert.lengthOf(reads, 3); + yield* TestClock.adjust("20 minutes"); + yield* tick; + assert.lengthOf(reads, 5); + }), + ), +); diff --git a/apps/server/src/kin/triggers/watchdogs.ts b/apps/server/src/kin/triggers/watchdogs.ts new file mode 100644 index 0000000000000..d8ba787f247c7 --- /dev/null +++ b/apps/server/src/kin/triggers/watchdogs.ts @@ -0,0 +1,635 @@ +import { + KinAgentId, + KinChatId, + KinEvent, + KinEventId, + KinMessageId, + KinOrgId, + KinPersonId, + KinWatchdog, + type KinWebhookSource, + type MemoryItemHead, + type ScheduledTask, +} from "@t3tools/contracts"; +import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; +import * as Clock from "effect/Clock"; +import * as Context from "effect/Context"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Result from "effect/Result"; +import * as Schema from "effect/Schema"; +import * as SqlClient from "effect/sql/SqlClient"; +import { isSqlError } from "effect/sql/SqlError"; + +import { toPersistenceSqlError } from "../../persistence/Errors.ts"; +import * as ScheduledTaskService from "../../scheduledTasks/ScheduledTaskService.ts"; +import * as Scheduler from "../../scheduling/Scheduler.ts"; +import * as Chat from "../chat/KinChatService.ts"; +import * as KinEventQueue from "../lead/KinEventQueue.ts"; +import { indexIsStale } from "../memory/buildIndex.ts"; +import { memoryDateMillis } from "../memory/frontMatter.ts"; +import * as Memory from "../memory/Memory.ts"; +import * as MemoryCache from "../memory/MemoryCache.ts"; +import { githubRef } from "./deferralGithub.ts"; +import { + type GithubConditionFacts, + inRepoScope, + KinGithubRepoScope, + matchConditions, + repoScopeLayer, +} from "./githubEvents.ts"; +import { KinGithubWebhookLock } from "./githubWebhooks.ts"; +import * as KinTriggerInbox from "./KinTriggerInbox.ts"; +import * as KinTriggerService from "./KinTriggerService.ts"; +import * as KinTriggerStore from "./KinTriggerStore.ts"; +import * as Pings from "./pings.ts"; + +const MINUTE = 60_000; +const HOUR = 60 * MINUTE; +const DAY = 24 * HOUR; +const WATCHDOG_EVERY_MS: Readonly> = { + github_recheck: 10 * MINUTE, + stale_review: 30 * MINUTE, + promise_overdue: 30 * MINUTE, + objective_at_risk: 6 * HOUR, + index_freshness: HOUR, + trigger_reconcile: 15 * MINUTE, +}; +const READ_TIMEOUT = "30 seconds"; +const PAGE = 100; +const decodeEvent = Schema.decodeUnknownEffect(KinEvent); +const decodePing = Schema.decodeOption( + Schema.fromJsonString(Schema.Struct({ nextStep: Schema.String })), +); + +/** One issue or pull request as GitHub has it now. */ +export interface GithubItemState { + readonly number: number; + readonly kind: "issue" | "pull"; + readonly state: "open" | "closed" | "merged"; + /** Pending review requests on a pull request, with when each was made (epoch ms). */ + readonly reviewRequests: ReadonlyArray<{ readonly login: string; readonly requestedAt: number }>; +} + +export class KinWatchdogGithubError extends Schema.TaggedError()( + "KinWatchdogGithubError", + { orgId: KinOrgId, repo: Schema.String, reason: Schema.Literals(["unavailable", "timeout"]) }, +) {} + +export interface KinWatchdogGithubReaderShape { + /** Read-only and batched: one call per repo for all of its numbers. Unknown numbers are left out. */ + readonly readRepo: ( + orgId: KinOrgId, + repo: string, + numbers: ReadonlyArray, + ) => Effect.Effect, KinWatchdogGithubError>; +} + +// No credentialed GitHub client exists yet (APPR-8), so the GitHub watchdogs stay off and +// `github_recheck` requests wait in the inbox until a reader backs this. +export const KinWatchdogGithubReader = Context.Reference( + "t3/kin/triggers/watchdogs/KinWatchdogGithubReader", + { defaultValue: () => null }, +); + +export class KinWatchdogsError extends Schema.TaggedError()( + "KinWatchdogsError", + { orgId: KinOrgId, watchdog: Schema.String, cause: Schema.Defect() }, +) {} + +export class KinWatchdogs extends Context.Service< + KinWatchdogs, + { + readonly run: ( + watchdog: KinWatchdog, + orgId: KinOrgId, + ) => Effect.Effect; + } +>()("t3/kin/triggers/watchdogs/KinWatchdogs") {} + +interface Link { + readonly repo: string; + readonly number: number; +} +type Item = MemoryItemHead & { readonly duplicate: boolean }; +const refKey = (repo: string, number: number) => `${repo.toLowerCase()}#${number}`; +const githubUrl = (repo: string, state: GithubItemState) => + `https://github.com/${repo}/${state.kind === "issue" ? "issues" : "pull"}/${state.number}`; +const isDone = (state: GithubItemState) => + state.state === (state.kind === "issue" ? "closed" : "merged"); +const clip = (text: string, max: number) => + text.length <= max ? text : `${text.slice(0, max - 1)}…`; +const hex = (text: string) => + Array.from(new TextEncoder().encode(text), (byte) => byte.toString(16).padStart(2, "0")).join(""); + +/** What a condition can test, from an issue or PR's current state rather than an event. */ +function conditionFactsFromState(repo: string, state: GithubItemState): GithubConditionFacts { + return { + event: state.kind === "pull" ? "pull_request" : "issues", + action: state.state === "open" ? "opened" : "closed", + repo, + number: state.number, + merged: state.kind === "pull" ? state.state === "merged" : null, + }; +} + +/** The org's scoped GitHub links on an item, deduplicated. */ +function linksOf(item: MemoryItemHead, repos: ReadonlyArray): ReadonlyArray { + const seen = new Map(); + for (const link of item.links) { + const ref = githubRef(link); + if (ref !== undefined && inRepoScope(repos, ref.repo)) + seen.set(refKey(ref.repo, ref.number), ref); + } + return [...seen.values()]; +} + +/** The repos a condition can be met in: its own, or every scoped repo. */ +function conditionLinks(item: MemoryItemHead, repos: ReadonlyArray): ReadonlyArray { + if (item.on === undefined) return []; + const { repo, number } = item.on; + if (repo !== undefined) return inRepoScope(repos, repo) ? [{ repo, number }] : []; + return repos.map((scoped) => ({ repo: scoped, number })); +} + +/** Which way, if any, a webhook source has stopped delivering. Only the owner turns it back on. */ +function observedSourceStatus( + task: Pick | undefined, +): KinWebhookSource["status"] { + if (task === undefined) return "missing"; + if (task.blockedByPolicy !== undefined) return "blocked_by_policy"; + return task.enabled ? "active" : "paused_in_t3"; +} + +const STATUS_NOTICE: Readonly> = { + active: "GitHub events reach Kin again.", + paused_in_t3: + "GitHub events are paused: Kin's GitHub webhook task is turned off in T3's scheduled tasks. Kin won't turn it back on. Turn it on there, or run GitHub setup again.", + blocked_by_policy: + "GitHub events are blocked: this host's managed policy turns off webhook tasks. Kin will catch up from GitHub when a reader is available.", + missing: + "GitHub events stopped: Kin's GitHub webhook task was deleted. Run GitHub setup again to recreate it.", +}; +const SECRET_NOTICE = + "GitHub events are refused: Kin's GitHub webhook task has a secret Kin didn't bind. Send the GitHub App's webhook secret to Kin again, or run GitHub setup again."; + +interface Lead { + readonly agentId: KinAgentId; + readonly owner: KinPersonId; + readonly chatId: KinChatId; +} + +const make = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const inbox = yield* KinTriggerInbox.KinTriggerInbox; + const store = yield* KinTriggerStore.KinTriggerStore; + const triggers = yield* KinTriggerService.KinTriggerService; + const queue = yield* KinEventQueue.KinEventQueue; + const memory = yield* Memory.Memory; + const chat = yield* Chat.KinChatService; + const pings = yield* Pings.KinPings; + const scope = yield* KinGithubRepoScope; + const ticks = yield* Scheduler.Scheduler; + // Optional, as in githubEvents: lead-only test runtimes have no T3 scheduling. + const tasks = yield* Effect.serviceOption(ScheduledTaskService.ScheduledTaskService); + // The RPC handlers own this lock; without it a re-key can briefly read as a foreign secret. + const lock = yield* Effect.serviceOption(KinGithubWebhookLock); + + const leadOf = Effect.fn("KinWatchdogs.lead")(function* (orgId: KinOrgId) { + const [row] = yield* sql<{ agentId: string; owner: string; chatId: string }>`SELECT + l.agent_id AS "agentId", a.owner_person_id AS owner, l.owner_chat_id AS "chatId" + FROM kin_leads l JOIN kin_agents a ON a.org_id = l.org_id AND a.agent_id = l.agent_id + WHERE l.org_id = ${orgId} AND a.retired_at IS NULL ORDER BY l.agent_id LIMIT 1`.pipe( + NodeSqliteClient.readOnly, + ); + return row === undefined + ? undefined + : ({ + agentId: KinAgentId.make(row.agentId), + owner: KinPersonId.make(row.owner), + chatId: KinChatId.make(row.chatId), + } satisfies Lead); + }); + + const openItems = (orgId: KinOrgId) => + memory + .query(orgId, { status: "open", excludeInvalid: true }) + .pipe( + Effect.map((items) => + items.filter((item) => !item.duplicate && item.superseded_by === null), + ), + ); + + /** One bounded read per repo; a repo that fails or times out is reported, not retried here. */ + const readStates = Effect.fn("KinWatchdogs.readStates")(function* ( + orgId: KinOrgId, + reader: KinWatchdogGithubReaderShape, + links: ReadonlyArray, + ) { + const byRepo = new Map }>(); + for (const link of links) { + const key = link.repo.toLowerCase(); + const entry = byRepo.get(key) ?? { repo: link.repo, numbers: new Set() }; + entry.numbers.add(link.number); + byRepo.set(key, entry); + } + const states = new Map(); + const failed = new Set(); + for (const [key, { repo, numbers }] of [...byRepo].toSorted(([a], [b]) => + a < b ? -1 : a > b ? 1 : 0, + )) { + const read = yield* reader + .readRepo( + orgId, + repo, + [...numbers].toSorted((a, b) => a - b), + ) + .pipe( + Effect.timeoutOrElse({ + duration: READ_TIMEOUT, + orElse: () => + Effect.fail(new KinWatchdogGithubError({ orgId, repo, reason: "timeout" })), + }), + Effect.result, + ); + if (Result.isFailure(read)) { + failed.add(key); + yield* Effect.logWarning("Kin watchdog GitHub read failed", { + orgId, + repo, + reason: read.failure.reason, + }); + continue; + } + for (const state of read.success) + if (numbers.has(state.number)) states.set(refKey(repo, state.number), state); + } + return { + failed: (link: Link) => failed.has(link.repo.toLowerCase()), + state: (link: Link) => states.get(refKey(link.repo, link.number)), + }; + }); + + type Finding = { readonly dedupeKey: string; readonly finding: string; readonly item: Item }; + /** Enqueues code-built findings for the lead, skipping ones already queued under their key. */ + const enqueueFindings = Effect.fn("KinWatchdogs.enqueueFindings")(function* ( + orgId: KinOrgId, + lead: Lead, + findings: ReadonlyArray, + now: number, + ) { + for (const { dedupeKey, finding, item } of findings) { + const [seen] = yield* sql`SELECT event_id FROM kin_events + WHERE org_id = ${orgId} AND dedupe_key = ${dedupeKey} ORDER BY event_id LIMIT 1`; + if (seen !== undefined) continue; + yield* queue.enqueue( + yield* decodeEvent({ + _tag: "watchdog", + orgId, + eventId: KinEventId.make( + `watchdog_${hex(`${orgId.length}:${orgId}${dedupeKey}`)}_${now}`, + ), + agentId: lead.agentId, + receivedAt: DateTime.formatIso(DateTime.makeUnsafe(now)), + dedupeKey, + // Server-written text; GitHub only supplied states, never prose. + provenance: { _tag: "system" }, + finding, + itemIds: [item.id], + }), + ); + } + }); + + const githubRecheck = Effect.fn("KinWatchdogs.githubRecheck")(function* (orgId: KinOrgId) { + const reader = yield* KinWatchdogGithubReader; + // Without a reader nothing is compared, so the requests wait rather than being consumed. + if (reader === null) return; + const now = yield* Clock.currentTimeMillis; + const requests = yield* inbox.listPending(orgId, PAGE, undefined, ["github_recheck"]); + const items = yield* openItems(orgId); + const tracked = items.filter((item) => item.kind === "commitment" || item.kind === "deferral"); + const conditions = items.filter((item) => item.on !== undefined); + const wanted = [ + ...tracked.flatMap((item) => linksOf(item, scope.repos)), + ...conditions.flatMap((item) => conditionLinks(item, scope.repos)), + ]; + const github = yield* readStates(orgId, reader, wanted); + + const findings: Array = []; + for (const item of tracked) { + const links = linksOf(item, scope.repos); + const states = links.map(github.state); + if (links.length === 0 || links.some(github.failed)) continue; + const done = states.filter((state) => state !== undefined && isDone(state)); + if (done.length !== links.length) continue; + const evidence = links.map((link, index) => githubUrl(link.repo, states[index]!)); + findings.push({ + item, + dedupeKey: `watchdog:github_recheck:done:${item.id}:${evidence.join(",")}`, + finding: `Verified on GitHub by code: ${evidence.join(", ")} ${evidence.length === 1 ? "is" : "are"} ${item.kind === "commitment" ? "closed" : "merged"}. Mark ${item.kind} ${item.id} done and record ${evidence.length === 1 ? "that link" : "those links"} as its evidence.`, + }); + } + for (const item of conditions) + for (const link of conditionLinks(item, scope.repos)) { + const state = github.state(link); + if (state === undefined) continue; + if (matchConditions(conditionFactsFromState(link.repo, state), [item]).length === 0) + continue; + const url = githubUrl(link.repo, state); + findings.push({ + item, + dedupeKey: `watchdog:github_recheck:condition:${item.id}:${refKey(link.repo, link.number)}:${state.state}`, + finding: `GitHub state shows condition met (read from GitHub by code): ${url} is ${state.state}.`, + }); + } + + const complete = wanted.every((link) => !github.failed(link)); + yield* sql.withTransaction( + Effect.gen(function* () { + const lead = yield* leadOf(orgId); + if (lead === undefined && findings.length > 0) + yield* Effect.logWarning("Kin GitHub recheck findings dropped: no active lead", { + orgId, + count: findings.length, + }); + if (lead !== undefined) yield* enqueueFindings(orgId, lead, findings, now); + // Only a sweep that read every repo answers a request; otherwise it waits for the next. + if (complete) + yield* inbox.markConsumed( + orgId, + requests.map((fire) => fire.fireId), + ); + }), + ); + }); + + /** Puts a chase unless this person already has one for the item with the same next step. */ + const chase = Effect.fn("KinWatchdogs.chase")(function* ( + orgId: KinOrgId, + lead: Lead, + item: MemoryItemHead, + person: KinPersonId, + nextStep: string, + ) { + if (item.chat === undefined) + return yield* Effect.logDebug("Kin watchdog chase skipped: item has no chat", { + orgId, + itemId: item.id, + }); + const [existing] = yield* sql<{ request: string }>`SELECT request_json AS request + FROM kin_ping_items WHERE org_id = ${orgId} AND person = ${person} AND item_id = ${item.id} + ORDER BY item_id`.pipe(NodeSqliteClient.readOnly); + if (existing !== undefined) { + const stored = Option.getOrUndefined(decodePing(existing.request)); + if (stored === undefined || stored.nextStep === nextStep) return; + } + yield* pings + .put( + { + orgId, + person, + itemId: item.id, + chatId: item.chat, + owner: lead.owner, + ownerChatId: lead.chatId, + nextStep, + }, + item, + ) + .pipe( + Effect.catch((cause) => + Effect.logWarning("Kin watchdog chase failed", { + orgId, + itemId: item.id, + errorTag: cause._tag, + }), + ), + ); + }); + + const promiseOverdue = Effect.fn("KinWatchdogs.promiseOverdue")(function* (orgId: KinOrgId) { + const lead = yield* leadOf(orgId); + if (lead === undefined) return; + const now = yield* Clock.currentTimeMillis; + for (const item of yield* openItems(orgId)) { + if (item.kind !== "commitment" || !(memoryDateMillis(item.due) < now)) continue; + const handle = item.owner ?? item.said_by; + const [person] = yield* sql<{ personId: string }>`SELECT person_id AS "personId" + FROM kin_people WHERE org_id = ${orgId} AND handle = ${handle} ORDER BY person_id`.pipe( + NodeSqliteClient.readOnly, + ); + if (person === undefined) { + yield* Effect.logDebug("Kin overdue chase skipped: promiser unknown", { + orgId, + itemId: item.id, + }); + continue; + } + yield* chase( + orgId, + lead, + item, + KinPersonId.make(person.personId), + `${clip(item.title, 400)} was due ${item.due}. What's the status?`, + ); + } + }); + + const staleReview = Effect.fn("KinWatchdogs.staleReview")(function* (orgId: KinOrgId) { + const reader = yield* KinWatchdogGithubReader; + if (reader === null) return; + const lead = yield* leadOf(orgId); + if (lead === undefined) return; + const now = yield* Clock.currentTimeMillis; + const items = (yield* openItems(orgId)).filter((item) => linksOf(item, scope.repos).length > 0); + const github = yield* readStates( + orgId, + reader, + items.flatMap((item) => linksOf(item, scope.repos)), + ); + for (const item of items) + for (const link of linksOf(item, scope.repos)) { + const state = github.state(link); + if (state?.kind !== "pull" || state.state !== "open") continue; + for (const request of state.reviewRequests) { + if (now - request.requestedAt < DAY) continue; + const [person] = yield* sql<{ personId: string }>`SELECT person_id AS "personId" + FROM kin_people WHERE org_id = ${orgId} AND lower(github_login) = lower(${request.login}) + ORDER BY person_id`.pipe(NodeSqliteClient.readOnly); + if (person === undefined) continue; + yield* chase( + orgId, + lead, + item, + KinPersonId.make(person.personId), + `Your review is requested on ${githubUrl(link.repo, state)} for ${clip(item.title, 400)}. Can you review it?`, + ); + } + } + }); + + const objectiveAtRisk = Effect.fn("KinWatchdogs.objectiveAtRisk")(function* (orgId: KinOrgId) { + const reader = yield* KinWatchdogGithubReader; + if (reader === null) return; + const lead = yield* leadOf(orgId); + if (lead === undefined) return; + const now = yield* Clock.currentTimeMillis; + const soon = (yield* openItems(orgId)).filter((item) => { + const due = memoryDateMillis(item.due); + return ( + item.kind === "objective" && + due > now && + due <= now + 3 * DAY && + linksOf(item, scope.repos).length > 0 + ); + }); + if (soon.length === 0) return; + const github = yield* readStates( + orgId, + reader, + soon.flatMap((item) => linksOf(item, scope.repos)), + ); + for (const item of soon) { + const links = linksOf(item, scope.repos); + if (links.some(github.failed)) continue; + const open = links.flatMap((link) => { + const state = github.state(link); + return state?.state === "open" ? [githubUrl(link.repo, state)] : []; + }); + if (open.length === 0) continue; + const messageId = KinMessageId.make( + `kin-watchdog:objective_at_risk:${orgId}:${item.id}:${memoryDateMillis(item.due)}`, + ); + // One flag per objective and due date, whatever its open work looks like on later runs. + const [posted] = yield* sql`SELECT message_id FROM kin_messages + WHERE org_id = ${orgId} AND message_id = ${messageId} ORDER BY message_id`; + if (posted !== undefined) continue; + yield* chat.notice({ + orgId, + chatId: lead.chatId, + messageId, + body: `${clip(item.title, 400)} is due ${item.due} and still has open GitHub work: ${open.slice(0, 20).join(", ")}${open.length > 20 ? ` and ${open.length - 20} more` : ""}.`, + }); + } + }); + + const indexFreshness = Effect.fn("KinWatchdogs.indexFreshness")(function* (orgId: KinOrgId) { + const index = yield* memory + .readIndex(orgId) + .pipe(Effect.catchTags({ MemoryError: () => Effect.succeed("") })); + if (indexIsStale(index, yield* Clock.currentTimeMillis)) yield* memory.rebuildIndex(orgId); + }); + + const reconcileSources = Effect.fn("KinWatchdogs.reconcileSources")(function* (orgId: KinOrgId) { + const sources = yield* store.listSources(orgId); + if (sources.length === 0 || Option.isNone(tasks)) return; + const all = (yield* tasks.value.list()).tasks; + const lead = yield* leadOf(orgId); + const notify = (messageId: string, body: string) => + lead === undefined + ? Effect.logWarning("Kin webhook source needs the owner", { orgId, body }) + : chat + .notice({ orgId, chatId: lead.chatId, messageId: KinMessageId.make(messageId), body }) + .pipe(Effect.asVoid); + for (const source of sources) { + const task = all.find((candidate) => candidate.id === source.scheduledTaskId); + const status = observedSourceStatus(task); + // Reports only: re-enabling, re-keying or rebinding is the owner's call. + if ( + status !== source.status && + (yield* store.setSourceStatus(orgId, source.id, source.status, status)) + ) + yield* notify( + `kin-reconcile:${orgId}:${source.id}:${status}:${yield* Clock.currentTimeMillis}`, + STATUS_NOTICE[status], + ); + if (task === undefined || (yield* store.secretMatchesTask(orgId, source.id))) continue; + const [bound] = yield* sql<{ boundAt: number | null }>`SELECT secret_bound_at AS "boundAt" + FROM kin_webhook_sources WHERE org_id = ${orgId} AND source_id = ${source.id} + ORDER BY source_id`.pipe(NodeSqliteClient.readOnly); + // Once per binding: a fresh bind is the only way out, and it gets its own notice if it fails. + yield* notify( + `kin-reconcile:${orgId}:${source.id}:secret:${bound?.boundAt ?? "unbound"}`, + SECRET_NOTICE, + ); + } + }); + + const triggerReconcile = Effect.fn("KinWatchdogs.triggerReconcile")(function* (orgId: KinOrgId) { + yield* Option.isSome(lock) + ? lock.value.withPermits(1)(reconcileSources(orgId)) + : reconcileSources(orgId); + // Re-arms open deferrals the cache hook missed; a failed read must not disarm everything. + const items = yield* memory.query(orgId, {}); + yield* triggers.syncOrg(orgId, items); + }); + + const run = Effect.fn("KinWatchdogs.run")( + function* (watchdog: KinWatchdog, orgId: KinOrgId) { + switch (watchdog) { + case "github_recheck": + return yield* githubRecheck(orgId); + case "stale_review": + return yield* staleReview(orgId); + case "promise_overdue": + return yield* promiseOverdue(orgId); + case "objective_at_risk": + return yield* objectiveAtRisk(orgId); + case "index_freshness": + return yield* indexFreshness(orgId); + case "trigger_reconcile": + return yield* triggerReconcile(orgId); + } + }, + (effect, watchdog, orgId) => + effect.pipe( + Effect.mapError((cause) => + isSqlError(cause) ? toPersistenceSqlError(`KinWatchdogs.${watchdog}`)(cause) : cause, + ), + Effect.mapError((cause) => new KinWatchdogsError({ orgId, watchdog, cause })), + ), + ); + + const prune = Effect.gen(function* () { + const now = yield* Clock.currentTimeMillis; + yield* inbox.pruneConsumed(now); + yield* store.pruneFires(now); + }).pipe(Effect.withSpan("KinWatchdogs.prune")); + + // Per process: no backfill, so each watchdog runs at startup and then every interval. + const nextAt = new Map(); + yield* ticks.register( + "kin-watchdogs", + Effect.gen(function* () { + const now = yield* Clock.currentTimeMillis; + const due = KinWatchdog.literals.filter((watchdog) => (nextAt.get(watchdog) ?? 0) <= now); + if (due.length === 0) return; + for (const watchdog of due) nextAt.set(watchdog, now + WATCHDOG_EVERY_MS[watchdog]); + if (due.includes("trigger_reconcile")) + yield* prune.pipe( + Effect.catch((cause) => Effect.logWarning("Kin trigger retention failed", { cause })), + ); + // Scheduler maintenance scans organizations; every watchdog below is org-scoped. + const orgs = yield* sql<{ orgId: string }>`SELECT org_id AS "orgId" FROM kin_orgs + ORDER BY org_id`; + for (const { orgId } of orgs) + for (const watchdog of due) + yield* run(watchdog, KinOrgId.make(orgId)).pipe( + Effect.catch((cause) => + Effect.logWarning("Kin watchdog failed", { orgId, watchdog, cause }), + ), + ); + }).pipe(Effect.catch((cause) => Effect.logWarning("Kin watchdog scan failed", { cause }))), + ); + + return KinWatchdogs.of({ run }); +}); + +export const layer = Layer.effect(KinWatchdogs, make).pipe( + Layer.provide(Layer.mergeAll(Pings.layer, KinTriggerService.layer)), + Layer.provide(MemoryCache.layer), +); +export const runtimeLayer = layer.pipe(Layer.provide(repoScopeLayer)); diff --git a/apps/server/src/kin/triggers/webhooks.testkit.ts b/apps/server/src/kin/triggers/webhooks.testkit.ts index 6a691fbdb6834..9d08259c9d949 100644 --- a/apps/server/src/kin/triggers/webhooks.testkit.ts +++ b/apps/server/src/kin/triggers/webhooks.testkit.ts @@ -130,6 +130,7 @@ export const withScheduledTasks = ( sends: ThreadManagementService.ThreadManagementSendInput[], ) => Effect.Effect, workers?: Map>, + policy?: Parameters[0], ) => Effect.gen(function* () { const launches: ThreadLaunchService.ThreadLaunchInput[] = []; @@ -169,7 +170,7 @@ export const withScheduledTasks = ( workers.set(name, work.pipe(Effect.provideContext(context), Effect.orDie)); }), }), - ManagedPolicy.layerTest(), + ManagedPolicy.layerTest(policy), Layer.mock(SecretRequests.SecretRequests)({}), Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({ namedProjectsRoot: "/projects", From e6b7115aebb27bff815734caba3c47a989f8a786 Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Mon, 9 Nov 2026 01:24:30 +0530 Subject: [PATCH 2/4] fix(kin): name each link's GitHub state in watchdog findings Co-Authored-By: Claude Opus 5.5 --- apps/server/src/kin/triggers/watchdogs.test.ts | 7 +++++-- apps/server/src/kin/triggers/watchdogs.ts | 14 +++++++++----- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/apps/server/src/kin/triggers/watchdogs.test.ts b/apps/server/src/kin/triggers/watchdogs.test.ts index 943e48db3bb6b..e484ecf1a3b66 100644 --- a/apps/server/src/kin/triggers/watchdogs.test.ts +++ b/apps/server/src/kin/triggers/watchdogs.test.ts @@ -8,6 +8,7 @@ import { MemoryItemHead, } from "@t3tools/contracts"; import * as ConfigProvider from "effect/ConfigProvider"; +import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; @@ -45,7 +46,7 @@ import { const NOW = 1794009600000; const HOUR = 60 * 60 * 1000; const DAY = 24 * HOUR; -const iso = (ms: number) => new Date(ms).toISOString(); +const iso = (ms: number) => DateTime.formatIso(DateTime.makeUnsafe(ms)); const orgId = KinOrgId.make("org-a"); const decodeHead = Schema.decodeUnknownSync(MemoryItemHead); const decodeEvent = Schema.decodeSync(Schema.fromJsonString(KinEvent)); @@ -153,7 +154,9 @@ const watchdogEvents = Effect.gen(function* () { WHERE org_id = ${orgId} AND tag = 'watchdog' ORDER BY dedupe_key`; return rows.flatMap((row) => { const event = decodeEvent(row.json); - return event._tag === "watchdog" ? [{ finding: event.finding, itemIds: event.itemIds }] : []; + return event._tag === "watchdog" + ? [{ finding: event.finding, itemIds: event.itemIds.map((id): string => id) }] + : []; }); }); diff --git a/apps/server/src/kin/triggers/watchdogs.ts b/apps/server/src/kin/triggers/watchdogs.ts index d8ba787f247c7..9407392041369 100644 --- a/apps/server/src/kin/triggers/watchdogs.ts +++ b/apps/server/src/kin/triggers/watchdogs.ts @@ -321,15 +321,19 @@ const make = Effect.gen(function* () { const findings: Array = []; for (const item of tracked) { const links = linksOf(item, scope.repos); - const states = links.map(github.state); if (links.length === 0 || links.some(github.failed)) continue; - const done = states.filter((state) => state !== undefined && isDone(state)); + const done = links.flatMap((link) => { + const state = github.state(link); + return state !== undefined && isDone(state) + ? [{ url: githubUrl(link.repo, state), state: state.state }] + : []; + }); if (done.length !== links.length) continue; - const evidence = links.map((link, index) => githubUrl(link.repo, states[index]!)); + const evidence = done.map(({ url }) => url); findings.push({ item, dedupeKey: `watchdog:github_recheck:done:${item.id}:${evidence.join(",")}`, - finding: `Verified on GitHub by code: ${evidence.join(", ")} ${evidence.length === 1 ? "is" : "are"} ${item.kind === "commitment" ? "closed" : "merged"}. Mark ${item.kind} ${item.id} done and record ${evidence.length === 1 ? "that link" : "those links"} as its evidence.`, + finding: `Verified on GitHub by code: ${done.map(({ url, state }) => `${url} is ${state}`).join(", ")}. Mark ${item.kind} ${item.id} done and record ${evidence.length === 1 ? "that link" : "those links"} as its evidence.`, }); } for (const item of conditions) @@ -401,7 +405,7 @@ const make = Effect.gen(function* () { ) .pipe( Effect.catch((cause) => - Effect.logWarning("Kin watchdog chase failed", { + Effect.logDebug("Kin watchdog chase failed", { orgId, itemId: item.id, errorTag: cause._tag, From 87919bc1ed7767517d0d90fe81b0bd92a9c876e7 Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Mon, 9 Nov 2026 01:24:40 +0530 Subject: [PATCH 3/4] docs(kin): describe watchdog scheduling, the GitHub reader gate and advisory source status Co-Authored-By: Claude Opus 5.5 --- docs/kin/specs/triggers.md | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/docs/kin/specs/triggers.md b/docs/kin/specs/triggers.md index 9c9f0a30d6b10..d81ab5afc90a8 100644 --- a/docs/kin/specs/triggers.md +++ b/docs/kin/specs/triggers.md @@ -178,7 +178,7 @@ receive: (input) => { return undefined; // not Kin's: unchanged } // From here every failure is a ThreadLaunchError("dispatch-message"): no real launch. - require(input.commandId starts with `scheduled-task:${taskId}:webhook:` && source.status === "active"); + require(input.commandId starts with `scheduled-task:${taskId}:webhook:`); // status is advisory, below const deliveryId = input.commandId.slice(prefix.length); const delivery = findLaunchDelivery(sql, { taskId, deliveryId, projectId: input.projectId, botLogin: source.botLogin }); @@ -198,7 +198,7 @@ A registered Kin task never falls through to a real launch. The wrapper is local **Secret binding.** Every Kin save that sets a secret (create, the re-key with the gate on, `setSecret`) is followed under the lock by `bindSecret`, which stores the secret's SHA-256 in `kin_webhook_sources.secret_sha256` and the bind time from `Clock` in `secret_bound_at` (migration K019). The sink hashes the task's current `webhook_secret` and compares the two with `timingSafeEqual`, and it requires the delivery's `received_at` to be later than `secret_bound_at`. T3 verifies a signature when the request arrives, but the sink reads the secret at launch, and a queued delivery can launch after a re-key; the time check ties each delivery to the secret it was verified against. `received_at` is the server clock, or the relay's receive time only with its signed proof (`RelayDeliveryProof`), capped at now, so a caller can backdate it only with the relay's signature. A NULL hash (a source made before K019), a mismatch (an agent swapped the secret, or a re-key is between its save and its bind, when the old hash is still bound) or an early delivery refuses the same way a weakened task does: each fails closed. The way out is `kin.webhooks.github.setSecret`, which re-saves and re-binds. A legitimate delivery the relay held while the host was offline and received before a re-key also refuses; `github_recheck` picks up what it carried. So an agent that re-saves the task with a secret it chose can get forged deliveries accepted upstream, but none reaches the inbox, even if the owner re-keys while one is queued. Neither value is logged, traced or put in an error. The sink also requires Kin's exact dedupe object (`{"replayKey":"{{headers.x-github-delivery}}"}`, no coalesce key or window) and event list, so an edit can't switch off dedupe to replay a delivery or hold one in a coalesce window. -What stays open: an agent can still delete, disable or re-key the task. That stops deliveries (denial of service) but cannot author an inbox row. `trigger_reconcile` (TRIG-10) must report a missing or disabled task, or a hash mismatch, to the owner. Fencing the MCP tools off Kin's task is still an open question. +What stays open: an agent can still delete, disable or re-key the task. That stops deliveries (denial of service) but cannot author an inbox row. `trigger_reconcile` (TRIG-10) reports a missing, disabled or policy-blocked task, or a hash mismatch, to the owner. Fencing the MCP tools off Kin's task is still an open question. `KinTriggerStore.registerSource` requires an existing org and an unbound `new_thread` webhook task. It reads the inbox projection before opening its transaction, then checks that the thread is not deleted and its project matches the task's. Identical registration returns the stored source; conflicting identity fails. Sources have no upstream org column: the `kin_webhook_sources` row establishes ownership; registration rejects task or inbox IDs already owned by another org. @@ -218,7 +218,7 @@ The sink verifies the delivery-log row belongs to the scheduled task. `dueAt` is **Missed while off.** The Kin host should be on, so outages are short: - **Hold on (recommended):** GitHub's request reaches the relay, the relay holds it up to 24 h, and `HeldHooksWaker` delivers it on reconnect. Replay keys stop doubles. -- **Hold off, or longer than 24 h:** GitHub gets a 530 or 502 and does not retry by itself. `github_recheck` runs at startup and every 10 minutes, compares memory with GitHub state, and finds what changed. +- **Hold off, or longer than 24 h:** GitHub gets a 530 or 502 and does not retry by itself. `github_recheck` runs at startup and every 10 minutes, compares memory with GitHub state, and finds what changed. It needs a GitHub reader, which arrives with APPR-8; until then its requests wait in the inbox. Webhooks are a fast path; the sweep is the guarantee. @@ -235,7 +235,7 @@ Arming comes from memory. When the front-matter cache (LEAD) sees a `deferrals/* Item edits move or disarm the triggers: a new `due` re-arms; `status` other than `open` or a set `invalid_at` disables them. A due time already past at save fires at once, marked late; no "pick a future time" error. A missing `on_due` means `remind`, so silence never starts work. -The cache hook is `MemorySyncHooks` in [`Memory.ts`](../../../apps/server/src/kin/memory/Memory.ts): each new head goes once to `KinTriggerService.syncOrg`, which also disarms armed deferrals missing from the snapshot or sharing a duplicate id. A failing item is logged without blocking the others, and the head retries on the next two loads, then waits for the next head. The deferral `trigger_reconcile` described below is not built yet, so until it is, a new head is the only recovery. A due time whose fires would land before 1970 never arms; `kin_defer` accepts only years 2000–2099. A commit to the default branch moves the cache to the new head and reloads it, so a `kin_defer` item arms when its wake-up commits, not at the next wake-up; shadow (L0) commits never arm. Trigger ids are `deferral::heads_up|due`. Re-syncing an unchanged `due` writes nothing, so a fired `once` is never re-armed. The run row in `kin_deferral_runs` (migration K016) holds the snapshot and the flow state (below). The snapshot holds the item half (`recorded_at`, `superseded_by`, links) and the GitHub half (state, assignees and linked PRs of each `github.com///issues|pull/` or `/#` link), read at arm time through `KinDeferralGithubReader` in [`deferralGithub.ts`](../../../apps/server/src/kin/triggers/deferralGithub.ts). The GitHub half is `null` when any link couldn't be read. That reader is a narrow read-only seam (one issue or PR per call) for #44's REST client to back; until then its default fails with `unavailable`. +The cache hook is `MemorySyncHooks` in [`Memory.ts`](../../../apps/server/src/kin/memory/Memory.ts): each new head goes once to `KinTriggerService.syncOrg`, which also disarms armed deferrals missing from the snapshot or sharing a duplicate id. A failing item is logged without blocking the others, and the head retries on the next two loads, then waits for the next head. `trigger_reconcile` (below) also re-syncs the whole org every 15 minutes and at startup, so a head that gave up is retried there. A due time whose fires would land before 1970 never arms; `kin_defer` accepts only years 2000–2099. A commit to the default branch moves the cache to the new head and reloads it, so a `kin_defer` item arms when its wake-up commits, not at the next wake-up; shadow (L0) commits never arm. Trigger ids are `deferral::heads_up|due`. Re-syncing an unchanged `due` writes nothing, so a fired `once` is never re-armed. The run row in `kin_deferral_runs` (migration K016) holds the snapshot and the flow state (below). The snapshot holds the item half (`recorded_at`, `superseded_by`, links) and the GitHub half (state, assignees and linked PRs of each `github.com///issues|pull/` or `/#` link), read at arm time through `KinDeferralGithubReader` in [`deferralGithub.ts`](../../../apps/server/src/kin/triggers/deferralGithub.ts). The GitHub half is `null` when any link couldn't be read. That reader is a narrow read-only seam (one issue or PR per call) for #44's REST client to back; until then its default fails with `unavailable`. **Confirm.** When the lead calls `kin_defer` (LEAD), Kin writes the item, arms the triggers, and the lead posts a `KinDeferralCard` widget (CHAT envelope): "Deferred to Fri 13 Nov, 9:00. I'll start then." [Change] [Just remind me]. [Just remind me] sets `on_due: remind`, and the triggers re-arm. [Change] writes nothing: it hands the item to the lead, which asks the owner what to change, marks the old item `cancelled` and calls `kin_defer` for the new one. @@ -320,16 +320,21 @@ ended in the last 90 days. ### Watchdogs (`interval`) -All are plain code. They write memory only through the LEAD queue as code-only events, which keeps one writer per org (gap 1). - -| Watchdog | Every | Does | -| ------------------- | ---------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `github_recheck` | 10 min, and at startup | For open items with GitHub links: a closed issue marks its commitment done, a merged PR closes its deferral, and the evidence link is recorded. Matches conditions. Batches GraphQL reads per repo. | -| `stale_review` | 30 min | PRs with a review requested over 24 h ago: chase the reviewer (ping rules). | -| `promise_overdue` | 30 min | Open commitments past `due`: chase whoever promised. | -| `objective_at_risk` | 6 h | Milestone due within 3 days with open linked issues: flag to the owner. | -| `index_freshness` | 1 h | `INDEX.md` "As of" older than 36 h: rebuild it (code, below). | -| `trigger_reconcile` | 15 min, and at startup | Every open deferral has armed triggers; every routine exists. Every `KinWebhookSource` task exists, is enabled, isn't `blockedByPolicy`, and its secret matches the bound hash; if not, set `status` and tell the owner. It never re-enables a task the owner paused in T3. Clears consumed inbox rows older than 7 days. | +All are plain code in [`watchdogs.ts`](../../../apps/server/src/kin/triggers/watchdogs.ts). They write memory only through the LEAD queue as code-only events, which keeps one writer per org (gap 1). They run on the shared 5-second tick (`kin-watchdogs`) with in-process intervals, not `kin_triggers` rows: a watchdog has no slot to keep, and the late policy is "run once now; no backfill". So every watchdog runs once at startup, then on its interval. Memory is read outside any SQLite transaction. + +| Watchdog | Every | Does | +| ------------------- | ------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `github_recheck` | 10 min | For open commitments and deferrals whose scoped GitHub links are all done (issue closed, PR merged), queues a `watchdog` event naming each evidence link and asking the lead to mark the item done. Matches conditions against current state (below). Then consumes the org's pending `github_recheck` rows, but only after a sweep that read every repo it needed. | +| `stale_review` | 30 min | Open items linking an open PR with a review requested over 24 h ago: chase the reviewer, matched by `kin_people.github_login`, in the item's chat (ping rules). | +| `promise_overdue` | 30 min | Open commitments past `due`: chase whoever promised (`owner`, else `said_by`) in the item's chat (ping rules). | +| `objective_at_risk` | 6 h | Open objectives due within 3 days with an open linked issue or PR: one owner notice per objective and due date. The objective's `due` stands in for its milestone. | +| `index_freshness` | 1 h | `INDEX.md` "As of" missing, in the future or 36 h old: `Memory.rebuildIndex` (code, below), the same check wake-up preparation runs. | +| `trigger_reconcile` | 15 min | Re-syncs deferral triggers from the whole memory snapshot (`KinTriggerService.syncOrg`). Checks every `KinWebhookSource` task exists, is enabled and isn't `blockedByPolicy`, records what it saw in `status` and tells the owner when it changes; a task whose secret doesn't match the bound hash gets one notice per binding. It never re-enables a task, re-keys or rebinds. Clears consumed inbox rows older than 7 days and fire-ledger rows older than 30 days. | + +- **GitHub reads.** Through `KinWatchdogGithubReader` only: one read-only, batched call per repo for all the numbers the sweep needs, each bounded to 30 s. Only repos in `KIN_GITHUB_REPOS` are read. Its default is `null` because no credentialed GitHub client exists until APPR-8, so `github_recheck`, `stale_review` and `objective_at_risk` do nothing and `github_recheck` rows stay pending; nothing is lost while it waits. A repo that fails or times out skips the items that need it and keeps the rows pending. +- **Conditions from state.** A PR gives `pull_request` with action `opened` or `closed` and `merged`; an issue gives `issues` with `opened` or `closed`. `matchConditions` then applies as for events, so a condition on another event or action matches only through webhooks. A match queues a `watchdog` event whose finding starts "GitHub state shows condition met (read from GitHub by code)". +- **Repeats.** Findings dedupe on `kin_events.dedupe_key` (item, links and state), so a later sweep queues nothing new. A chase is put only when the person has no ping for the item with the same next step, so an answered chase stays answered until the due date moves. +- **Source status is advisory.** The sink does not refuse on `status`: the task's shape and bound secret are the gate, and a delivery after the owner turns the task back on must not wait 15 minutes for reconcile. A hash mismatch has no status value; the sink already refuses it, and `setSecret` or setup is the way out. Routine rows are kept by the routines tick's own 15-minute reconcile and dream rows by `DreamRunner`, so `trigger_reconcile` doesn't repeat them. ### Staying fresh (section 4) From 89de99944bc335f54750bb3205d911c7557bf4ec Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Mon, 9 Nov 2026 01:51:45 +0530 Subject: [PATCH 4/4] fix(kin): keep answered watchdog chases answered until their prompt moves A chase now skips any active ping for the person and item, and any ping stopped after its due date or review request last changed, so alternating next steps and lead-written pings no longer re-arm. Watchdog event ids hash the dedupe key, memory titles in pings and notices are quoted, the two trigger_reconcile halves fail independently, and a runtime without T3 scheduled tasks warns at startup. The TRIG-10 acceptance criterion now matches the lead-queue design. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/kin/triggers/pings.ts | 2 +- .../server/src/kin/triggers/watchdogs.test.ts | 123 ++++++++++++------ apps/server/src/kin/triggers/watchdogs.ts | 61 +++++---- docs/kin/specs/triggers.md | 46 +++---- 4 files changed, 146 insertions(+), 86 deletions(-) diff --git a/apps/server/src/kin/triggers/pings.ts b/apps/server/src/kin/triggers/pings.ts index 30e220ae4b5d0..5b495a24ede4a 100644 --- a/apps/server/src/kin/triggers/pings.ts +++ b/apps/server/src/kin/triggers/pings.ts @@ -103,7 +103,7 @@ function isQuiet(now: number, zone: string, hours: { readonly from: string; read : hours.from > hours.to && (time >= hours.from || time < hours.to); } -function quoted(value: string) { +export function quoted(value: string) { const content = Array.from(value.replace(/[\p{Cc}\p{Cf}\p{Cs}\p{Zl}\p{Zp}]/gu, " ")) .slice(0, 500) .join(""); diff --git a/apps/server/src/kin/triggers/watchdogs.test.ts b/apps/server/src/kin/triggers/watchdogs.test.ts index e484ecf1a3b66..46611b398c84c 100644 --- a/apps/server/src/kin/triggers/watchdogs.test.ts +++ b/apps/server/src/kin/triggers/watchdogs.test.ts @@ -301,43 +301,45 @@ describe("github_recheck", () => { }); const states = [issue(7, "closed"), issue(8, "open"), pull(9, "merged"), pull(212, "merged")]; - it.effect("marks a commitment done when its linked issue closes, with the evidence link", () => - run([closesIssue, mergesPr, halfDone, outOfScope, condition], () => - Effect.gen(function* () { - const service = yield* build; - yield* putRecheck("1"); - const reads: Array = []; - yield* service.run("github_recheck", orgId).pipe(withGithub(fakeGithub(states, reads))); - - // One batched read for the scoped repo; the out-of-scope link is never read. - assert.deepEqual(reads, ["owner/repo:7,8,9,212"]); - assert.deepEqual(yield* watchdogEvents, [ - { - finding: - "GitHub state shows condition met (read from GitHub by code): https://github.com/Owner/Repo/pull/212 is merged.", - itemIds: ["when-212-merges"], - }, - { - finding: - "Verified on GitHub by code: https://github.com/owner/repo/pull/9 is merged. Mark deferral rate-limit done and record that link as its evidence.", - itemIds: ["rate-limit"], - }, - { - finding: - "Verified on GitHub by code: https://github.com/owner/repo/issues/7 is closed. Mark commitment ship-login done and record that link as its evidence.", - itemIds: ["ship-login"], - }, - ]); - assert.equal(yield* pendingRechecks, 0); - - // A later sweep reads again but queues nothing new for the same findings. - yield* putRecheck("2"); - yield* service.run("github_recheck", orgId).pipe(withGithub(fakeGithub(states, reads))); - assert.equal(reads.length, 2); - assert.equal((yield* watchdogEvents).length, 3); - assert.equal(yield* pendingRechecks, 0); - }), - ), + it.effect( + "queues a code-only watchdog event naming the commitment and the closed issue as evidence", + () => + run([closesIssue, mergesPr, halfDone, outOfScope, condition], () => + Effect.gen(function* () { + const service = yield* build; + yield* putRecheck("1"); + const reads: Array = []; + yield* service.run("github_recheck", orgId).pipe(withGithub(fakeGithub(states, reads))); + + // One batched read for the scoped repo; the out-of-scope link is never read. + assert.deepEqual(reads, ["owner/repo:7,8,9,212"]); + assert.deepEqual(yield* watchdogEvents, [ + { + finding: + "GitHub state shows condition met (read from GitHub by code): https://github.com/Owner/Repo/pull/212 is merged.", + itemIds: ["when-212-merges"], + }, + { + finding: + "Verified on GitHub by code: https://github.com/owner/repo/pull/9 is merged. Mark deferral rate-limit done and record that link as its evidence.", + itemIds: ["rate-limit"], + }, + { + finding: + "Verified on GitHub by code: https://github.com/owner/repo/issues/7 is closed. Mark commitment ship-login done and record that link as its evidence.", + itemIds: ["ship-login"], + }, + ]); + assert.equal(yield* pendingRechecks, 0); + + // A later sweep reads again but queues nothing new for the same findings. + yield* putRecheck("2"); + yield* service.run("github_recheck", orgId).pipe(withGithub(fakeGithub(states, reads))); + assert.equal(reads.length, 2); + assert.equal((yield* watchdogEvents).length, 3); + assert.equal(yield* pendingRechecks, 0); + }), + ), ); it.effect("leaves recheck requests pending until a reader reads every repo", () => @@ -410,6 +412,51 @@ describe("chases", () => { ), ); + it.effect("an answered chase stays answered until its due date moves", () => + run( + [ + item("both", { + owner: "alice", + due: iso(NOW - HOUR), + links: ["owner/repo#5", "owner/repo#6"], + }), + ], + ({ items }) => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const service = yield* build; + const stale = { login: "alice-gh", requestedAt: NOW - 25 * HOUR }; + const github = withGithub( + fakeGithub([pull(5, "open", [stale]), pull(6, "open", [stale])], []), + ); + const sweep = Effect.gen(function* () { + yield* service.run("promise_overdue", orgId); + yield* service.run("stale_review", orgId).pipe(github); + }); + const ping = Effect.gen(function* () { + const [row] = yield* sql<{ request: string; stoppedAt: number | null }>`SELECT + request_json AS request, stopped_at AS "stoppedAt" FROM kin_ping_items + WHERE org_id = ${orgId} AND person = 'alice' AND item_id = 'both' ORDER BY item_id`; + return row; + }); + yield* sweep; + const armed = yield* ping; + assert.isNull(armed?.stoppedAt); + + // Answered: neither the overdue chase nor either stale review re-arms it. + yield* sql`UPDATE kin_ping_items SET stopped_at = ${NOW} WHERE org_id = ${orgId}`; + yield* sweep; + yield* sweep; + assert.deepEqual(yield* ping, { request: armed?.request ?? "", stoppedAt: NOW }); + + yield* Ref.set(items, [item("both", { owner: "alice", due: iso(NOW + HOUR) })]); + yield* TestClock.adjust(2 * HOUR); + yield* service.run("promise_overdue", orgId); + assert.isNull((yield* ping)?.stoppedAt); + }), + ), + ); + it.effect("objective_at_risk flags an objective due within 3 days with open issues", () => run( [ @@ -433,7 +480,7 @@ describe("chases", () => { yield* service.run("objective_at_risk", orgId).pipe(github); yield* service.run("objective_at_risk", orgId).pipe(github); assert.deepEqual(yield* notices, [ - `launch is due ${iso(NOW + 2 * DAY)} and still has open GitHub work: https://github.com/owner/repo/issues/3.`, + `An objective due ${iso(NOW + 2 * DAY)} still has open GitHub work: https://github.com/owner/repo/issues/3.\n> \` launch \``, ]); }), ), diff --git a/apps/server/src/kin/triggers/watchdogs.ts b/apps/server/src/kin/triggers/watchdogs.ts index 9407392041369..11e641e85edc1 100644 --- a/apps/server/src/kin/triggers/watchdogs.ts +++ b/apps/server/src/kin/triggers/watchdogs.ts @@ -22,6 +22,8 @@ import * as Result from "effect/Result"; import * as Schema from "effect/Schema"; import * as SqlClient from "effect/sql/SqlClient"; import { isSqlError } from "effect/sql/SqlError"; +// @effect-diagnostics-next-line nodeBuiltinImport:off -- a pure, synchronous id hash; Digest would add a service. +import * as NodeCrypto from "node:crypto"; import { toPersistenceSqlError } from "../../persistence/Errors.ts"; import * as ScheduledTaskService from "../../scheduledTasks/ScheduledTaskService.ts"; @@ -60,9 +62,6 @@ const WATCHDOG_EVERY_MS: Readonly> = { const READ_TIMEOUT = "30 seconds"; const PAGE = 100; const decodeEvent = Schema.decodeUnknownEffect(KinEvent); -const decodePing = Schema.decodeOption( - Schema.fromJsonString(Schema.Struct({ nextStep: Schema.String })), -); /** One issue or pull request as GitHub has it now. */ export interface GithubItemState { @@ -119,10 +118,8 @@ const githubUrl = (repo: string, state: GithubItemState) => `https://github.com/${repo}/${state.kind === "issue" ? "issues" : "pull"}/${state.number}`; const isDone = (state: GithubItemState) => state.state === (state.kind === "issue" ? "closed" : "merged"); -const clip = (text: string, max: number) => - text.length <= max ? text : `${text.slice(0, max - 1)}…`; -const hex = (text: string) => - Array.from(new TextEncoder().encode(text), (byte) => byte.toString(16).padStart(2, "0")).join(""); +const shortHash = (text: string) => + NodeCrypto.createHash("sha256").update(text).digest("hex").slice(0, 32); /** What a condition can test, from an issue or PR's current state rather than an event. */ function conditionFactsFromState(repo: string, state: GithubItemState): GithubConditionFacts { @@ -194,6 +191,10 @@ const make = Effect.gen(function* () { const ticks = yield* Scheduler.Scheduler; // Optional, as in githubEvents: lead-only test runtimes have no T3 scheduling. const tasks = yield* Effect.serviceOption(ScheduledTaskService.ScheduledTaskService); + if (Option.isNone(tasks)) + yield* Effect.logWarning( + "Kin watchdogs started without T3 scheduled tasks: webhook sources go unreconciled", + ); // The RPC handlers own this lock; without it a re-key can briefly read as a foreign secret. const lock = yield* Effect.serviceOption(KinGithubWebhookLock); @@ -288,9 +289,7 @@ const make = Effect.gen(function* () { yield* decodeEvent({ _tag: "watchdog", orgId, - eventId: KinEventId.make( - `watchdog_${hex(`${orgId.length}:${orgId}${dedupeKey}`)}_${now}`, - ), + eventId: KinEventId.make(`watchdog-${shortHash(`${orgId}\0${dedupeKey}`)}`), agentId: lead.agentId, receivedAt: DateTime.formatIso(DateTime.makeUnsafe(now)), dedupeKey, @@ -370,26 +369,29 @@ const make = Effect.gen(function* () { ); }); - /** Puts a chase unless this person already has one for the item with the same next step. */ + /** Puts a chase unless this person has one for the item that is active or was stopped after `changedAt`. */ const chase = Effect.fn("KinWatchdogs.chase")(function* ( orgId: KinOrgId, lead: Lead, item: MemoryItemHead, person: KinPersonId, nextStep: string, + changedAt: number, ) { if (item.chat === undefined) return yield* Effect.logDebug("Kin watchdog chase skipped: item has no chat", { orgId, itemId: item.id, }); - const [existing] = yield* sql<{ request: string }>`SELECT request_json AS request + const [existing] = yield* sql<{ stoppedAt: number | null }>`SELECT stopped_at AS "stoppedAt" FROM kin_ping_items WHERE org_id = ${orgId} AND person = ${person} AND item_id = ${item.id} ORDER BY item_id`.pipe(NodeSqliteClient.readOnly); - if (existing !== undefined) { - const stored = Option.getOrUndefined(decodePing(existing.request)); - if (stored === undefined || stored.nextStep === nextStep) return; - } + // An answered chase stays answered until what prompted it (a due date, a review request) moves. + if ( + existing !== undefined && + (existing.stoppedAt === null || !(changedAt > existing.stoppedAt)) + ) + return; yield* pings .put( { @@ -437,7 +439,8 @@ const make = Effect.gen(function* () { lead, item, KinPersonId.make(person.personId), - `${clip(item.title, 400)} was due ${item.due}. What's the status?`, + `Was due ${item.due}. What's the status? ${Pings.quoted(item.title)}`, + memoryDateMillis(item.due), ); } }); @@ -469,7 +472,8 @@ const make = Effect.gen(function* () { lead, item, KinPersonId.make(person.personId), - `Your review is requested on ${githubUrl(link.repo, state)} for ${clip(item.title, 400)}. Can you review it?`, + `Your review is requested on ${githubUrl(link.repo, state)}. Can you review it? ${Pings.quoted(item.title)}`, + request.requestedAt, ); } } @@ -515,7 +519,7 @@ const make = Effect.gen(function* () { orgId, chatId: lead.chatId, messageId, - body: `${clip(item.title, 400)} is due ${item.due} and still has open GitHub work: ${open.slice(0, 20).join(", ")}${open.length > 20 ? ` and ${open.length - 20} more` : ""}.`, + body: `An objective due ${item.due} still has open GitHub work: ${open.slice(0, 20).join(", ")}${open.length > 20 ? ` and ${open.length - 20} more` : ""}.\n${Pings.quoted(item.title)}`, }); } }); @@ -563,12 +567,21 @@ const make = Effect.gen(function* () { }); const triggerReconcile = Effect.fn("KinWatchdogs.triggerReconcile")(function* (orgId: KinOrgId) { - yield* Option.isSome(lock) - ? lock.value.withPermits(1)(reconcileSources(orgId)) - : reconcileSources(orgId); + // Independent halves: a failure in one is logged and the other still runs. + yield* ( + Option.isSome(lock) + ? lock.value.withPermits(1)(reconcileSources(orgId)) + : reconcileSources(orgId) + ).pipe( + Effect.catch((cause) => + Effect.logWarning("Kin webhook source reconcile failed", { orgId, cause }), + ), + ); // Re-arms open deferrals the cache hook missed; a failed read must not disarm everything. - const items = yield* memory.query(orgId, {}); - yield* triggers.syncOrg(orgId, items); + yield* memory.query(orgId, {}).pipe( + Effect.flatMap((items) => triggers.syncOrg(orgId, items)), + Effect.catch((cause) => Effect.logWarning("Kin trigger re-arm failed", { orgId, cause })), + ); }); const run = Effect.fn("KinWatchdogs.run")( diff --git a/docs/kin/specs/triggers.md b/docs/kin/specs/triggers.md index d81ab5afc90a8..862a81305949e 100644 --- a/docs/kin/specs/triggers.md +++ b/docs/kin/specs/triggers.md @@ -333,7 +333,7 @@ All are plain code in [`watchdogs.ts`](../../../apps/server/src/kin/triggers/wat - **GitHub reads.** Through `KinWatchdogGithubReader` only: one read-only, batched call per repo for all the numbers the sweep needs, each bounded to 30 s. Only repos in `KIN_GITHUB_REPOS` are read. Its default is `null` because no credentialed GitHub client exists until APPR-8, so `github_recheck`, `stale_review` and `objective_at_risk` do nothing and `github_recheck` rows stay pending; nothing is lost while it waits. A repo that fails or times out skips the items that need it and keeps the rows pending. - **Conditions from state.** A PR gives `pull_request` with action `opened` or `closed` and `merged`; an issue gives `issues` with `opened` or `closed`. `matchConditions` then applies as for events, so a condition on another event or action matches only through webhooks. A match queues a `watchdog` event whose finding starts "GitHub state shows condition met (read from GitHub by code)". -- **Repeats.** Findings dedupe on `kin_events.dedupe_key` (item, links and state), so a later sweep queues nothing new. A chase is put only when the person has no ping for the item with the same next step, so an answered chase stays answered until the due date moves. +- **Repeats.** Findings dedupe on `kin_events.dedupe_key` (item, links and state), so a later sweep queues nothing new. A chase is skipped while the person's ping for the item is active, or when it was stopped after what prompted the chase last changed (the commitment's due date, or the review request's time), so an answered chase stays answered until the due date moves. - **Source status is advisory.** The sink does not refuse on `status`: the task's shape and bound secret are the gate, and a delivery after the owner turns the task back on must not wait 15 minutes for reconcile. A hash mismatch has no status value; the sink already refuses it, and `setSecret` or setup is the way out. Routine rows are kept by the routines tick's own 15-minute reconcile and dream rows by `DreamRunner`, so `trigger_reconcile` doesn't repeat them. ### Staying fresh (section 4) @@ -579,28 +579,28 @@ Kin runs on one always-on machine, the **Kin host**. The laptop and phone are cl **TRIG-4 constraints.** A Kin source task bound to a thread fails each delivery with `OrchestratorCommandRejectedError`; unbind it to resume. TRIG-4 is where Kin persistence first joins boot; CHAT-6/7's boot containment must cover it. -| Id | What | Files | Depends on | Done when | Owner | -| ------- | -------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------- | -| TRIG-1 | Trigger contracts | `kinTriggers.ts` | CHAT-1 | `vp run typecheck` passes; schema round-trip test | backend | -| TRIG-2 | Trigger tables and store | `KinTriggerStore.ts`, `KinTriggerInbox.ts`, Kin migrations | TRIG-1, CHAT-2 (Kin migration list) | store tests: idempotent `put`, `orgId` on every row | backend | -| TRIG-3 | `KinTriggerScheduler`: owner zone, firing, lateness | `KinTriggerScheduler.ts` | TRIG-2 | scheduler and timezone tests above pass | backend | -| TRIG-4 | `KinTriggerSink` and the runtime-layer hook | `KinTriggerSink.ts`, `runtimeLayer.ts` (1 line) | TRIG-2 | sink tests pass; non-Kin tasks unchanged (existing `ScheduledTaskService*.test.ts` green) | backend | -| TRIG-5 | GitHub App webhook source: setup, secret, rotate, remove, inbox thread; App hook via executor (gated) | `githubWebhooks.ts`, `webhookRpcHandlers.ts` | TRIG-4; APPR-9 opens the gate | test against a fake GitHub App creates a task with signature, replay key and filters; a null URL posts the T3 Connect message | backend | -| TRIG-6 | GitHub event consumer and conditions | `githubEvents.ts` | TRIG-5, LEAD-4 | delivery turns into a `KinGithubEvent`; a trimmed delivery falls back to `github_recheck`; a condition on #212 matches a merge | backend | -| TRIG-7 | Deferral arming from memory (including items written by `kin_defer`), and the confirm card | `KinTriggerService.ts`, `deferrals.ts` | TRIG-3, LEAD-2, LEAD-7 (`kin_defer`), CHAT-7 | editing `due` in an item re-arms; `status: done` disarms | backend | -| TRIG-8 | Deferral due flow: recheck, heads-up, grace, start/ask/remind/chase, late | `deferrals.ts` | TRIG-7, TRIG-11, LEAD-8, LEAD-9, LEAD-11 | the deferral tests above pass | backend | -| TRIG-9 | Routines: brief, summary, owner check | `routines.ts`, `ownerCheck.ts` | TRIG-3, TRIG-12 | at 08:30 owner time a brief is posted; late after 18:30 it folds into the summary; the owner check picks 5 by the stated order | backend | -| TRIG-10 | Watchdogs | `watchdogs.ts` | TRIG-3, TRIG-6, TRIG-11 | closing a linked issue on a fake GitHub marks the commitment done with an evidence link; a paused T3 task sets `paused_in_t3` and isn't re-enabled | backend | -| TRIG-11 | Ping ledger, batching, quiet hours, escalation | `pings.ts` | TRIG-2, CHAT-7 | two pings for one item on one day send one; a ping in quiet hours waits; day 2 escalates once | backend | -| TRIG-12 | Freshness defaults and code-built `INDEX.md` | `freshness.ts`, `buildIndex.ts` | LEAD-2 | golden-file test: ≤ 200 lines, As-of line, overdue flagged, 8-week-unread left out | backend | -| TRIG-13 | Dream runner: material, branch, dream session, checks | `DreamRunner.ts`, `DreamSession.ts`, `material.ts`, `checkDream.ts` | TRIG-12, TRIG-16, LEAD-5, LEAD-6 | fixture dream produces a branch and `dream.json`; `rules.md` edit fails it | backend | -| TRIG-14 | Dream split, auto-merge under delegation, review card, rejections; full weekly variant | `classifyDream.ts`, `DreamRunner.ts` | TRIG-13, APPR-4; auto-merge waits for APPR-12 (phase 3), and until then the owner approves both PRs | the two-PR test passes; a rejected item is not proposed again | backend | -| TRIG-15 | Memory repo guard rails | `kin-memory`: `CODEOWNERS`, `memory-check.yml`, ruleset | APPR-11 | a PR touching `decisions/` can't merge without owner review; a planted token fails the check | backend | -| TRIG-16 | Secret scanner, writer hook, allowlist | `secretScan.ts` | LEAD-3 | rule fixtures (one hit and one near-miss per rule) pass; commit refusal and typed owner notification tested | backend | -| TRIG-17 | Owner-signed purge | `purge.ts` | TRIG-16, APPR-9, APPR-10, LEAD-11 | purge test passes on a fixture repo; `git-filter-repo` provenance noted in the PR | backend | -| TRIG-18 | Server contract tests: triggers, relay, `delegate_task`, managed policy | `apps/server/src/kin/contract/*` | TRIG-4 | `cd apps/server && vp test run kin-contract` passes | backend | -| TRIG-19 | `McpAppWebView` contract test | `apps/mobile/src/features/chats/contract/*` | — | `cd apps/mobile && vp test run kin-contract` passes | UI | -| TRIG-20 | Kin host setup: systemd unit, `kin` user, `setup.sh`, `backup.sh`, verify source-build `up --connect` and hold | `ops/kin/host/*` | — | after a reboot, the server is reachable from the phone over T3 Connect; a GitHub redelivery while it's stopped arrives on start; a restore drill from backup works | backend | +| Id | What | Files | Depends on | Done when | Owner | +| ------- | -------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------- | +| TRIG-1 | Trigger contracts | `kinTriggers.ts` | CHAT-1 | `vp run typecheck` passes; schema round-trip test | backend | +| TRIG-2 | Trigger tables and store | `KinTriggerStore.ts`, `KinTriggerInbox.ts`, Kin migrations | TRIG-1, CHAT-2 (Kin migration list) | store tests: idempotent `put`, `orgId` on every row | backend | +| TRIG-3 | `KinTriggerScheduler`: owner zone, firing, lateness | `KinTriggerScheduler.ts` | TRIG-2 | scheduler and timezone tests above pass | backend | +| TRIG-4 | `KinTriggerSink` and the runtime-layer hook | `KinTriggerSink.ts`, `runtimeLayer.ts` (1 line) | TRIG-2 | sink tests pass; non-Kin tasks unchanged (existing `ScheduledTaskService*.test.ts` green) | backend | +| TRIG-5 | GitHub App webhook source: setup, secret, rotate, remove, inbox thread; App hook via executor (gated) | `githubWebhooks.ts`, `webhookRpcHandlers.ts` | TRIG-4; APPR-9 opens the gate | test against a fake GitHub App creates a task with signature, replay key and filters; a null URL posts the T3 Connect message | backend | +| TRIG-6 | GitHub event consumer and conditions | `githubEvents.ts` | TRIG-5, LEAD-4 | delivery turns into a `KinGithubEvent`; a trimmed delivery falls back to `github_recheck`; a condition on #212 matches a merge | backend | +| TRIG-7 | Deferral arming from memory (including items written by `kin_defer`), and the confirm card | `KinTriggerService.ts`, `deferrals.ts` | TRIG-3, LEAD-2, LEAD-7 (`kin_defer`), CHAT-7 | editing `due` in an item re-arms; `status: done` disarms | backend | +| TRIG-8 | Deferral due flow: recheck, heads-up, grace, start/ask/remind/chase, late | `deferrals.ts` | TRIG-7, TRIG-11, LEAD-8, LEAD-9, LEAD-11 | the deferral tests above pass | backend | +| TRIG-9 | Routines: brief, summary, owner check | `routines.ts`, `ownerCheck.ts` | TRIG-3, TRIG-12 | at 08:30 owner time a brief is posted; late after 18:30 it folds into the summary; the owner check picks 5 by the stated order | backend | +| TRIG-10 | Watchdogs | `watchdogs.ts` | TRIG-3, TRIG-6, TRIG-11 | closing a linked issue on a fake GitHub queues a code-only `watchdog` event naming the commitment and the closed issue as evidence, so the lead marks it done; a paused T3 task sets `paused_in_t3` and isn't re-enabled | backend | +| TRIG-11 | Ping ledger, batching, quiet hours, escalation | `pings.ts` | TRIG-2, CHAT-7 | two pings for one item on one day send one; a ping in quiet hours waits; day 2 escalates once | backend | +| TRIG-12 | Freshness defaults and code-built `INDEX.md` | `freshness.ts`, `buildIndex.ts` | LEAD-2 | golden-file test: ≤ 200 lines, As-of line, overdue flagged, 8-week-unread left out | backend | +| TRIG-13 | Dream runner: material, branch, dream session, checks | `DreamRunner.ts`, `DreamSession.ts`, `material.ts`, `checkDream.ts` | TRIG-12, TRIG-16, LEAD-5, LEAD-6 | fixture dream produces a branch and `dream.json`; `rules.md` edit fails it | backend | +| TRIG-14 | Dream split, auto-merge under delegation, review card, rejections; full weekly variant | `classifyDream.ts`, `DreamRunner.ts` | TRIG-13, APPR-4; auto-merge waits for APPR-12 (phase 3), and until then the owner approves both PRs | the two-PR test passes; a rejected item is not proposed again | backend | +| TRIG-15 | Memory repo guard rails | `kin-memory`: `CODEOWNERS`, `memory-check.yml`, ruleset | APPR-11 | a PR touching `decisions/` can't merge without owner review; a planted token fails the check | backend | +| TRIG-16 | Secret scanner, writer hook, allowlist | `secretScan.ts` | LEAD-3 | rule fixtures (one hit and one near-miss per rule) pass; commit refusal and typed owner notification tested | backend | +| TRIG-17 | Owner-signed purge | `purge.ts` | TRIG-16, APPR-9, APPR-10, LEAD-11 | purge test passes on a fixture repo; `git-filter-repo` provenance noted in the PR | backend | +| TRIG-18 | Server contract tests: triggers, relay, `delegate_task`, managed policy | `apps/server/src/kin/contract/*` | TRIG-4 | `cd apps/server && vp test run kin-contract` passes | backend | +| TRIG-19 | `McpAppWebView` contract test | `apps/mobile/src/features/chats/contract/*` | — | `cd apps/mobile && vp test run kin-contract` passes | UI | +| TRIG-20 | Kin host setup: systemd unit, `kin` user, `setup.sh`, `backup.sh`, verify source-build `up --connect` and hold | `ops/kin/host/*` | — | after a reboot, the server is reachable from the phone over T3 Connect; a GitHub redelivery while it's stopped arrives on start; a restore drill from backup works | backend | ## Design corrections