From c8630fbc76e6222175ddb679b9f7d42eef81cfcd Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Sun, 8 Nov 2026 21:39:46 +0530 Subject: [PATCH 1/7] feat(kin): deferral due flow with recheck, heads-up, grace and gated start Co-Authored-By: Claude Opus 5.5 --- apps/server/src/kin/lead/layer.ts | 3 +- .../src/kin/memory/MemoryWriter.test.ts | 50 +- apps/server/src/kin/memory/MemoryWriter.ts | 15 + .../src/kin/triggers/KinTriggerInbox.ts | 9 +- .../src/kin/triggers/KinTriggerService.ts | 88 +-- apps/server/src/kin/triggers/deferralFlow.ts | 387 +++++++++++++ .../server/src/kin/triggers/deferralGithub.ts | 73 +++ .../server/src/kin/triggers/deferrals.test.ts | 515 +++++++++++++++--- apps/server/src/kin/triggers/deferrals.ts | 336 +++++++++++- apps/server/src/kin/triggers/ownerCheck.ts | 2 +- apps/server/src/kin/widgets/deferralCard.ts | 2 +- docs/kin/specs/triggers.md | 20 +- packages/contracts/src/kinLead.ts | 4 + 13 files changed, 1307 insertions(+), 197 deletions(-) create mode 100644 apps/server/src/kin/triggers/deferralFlow.ts create mode 100644 apps/server/src/kin/triggers/deferralGithub.ts diff --git a/apps/server/src/kin/lead/layer.ts b/apps/server/src/kin/lead/layer.ts index 202fdfdf0fdfe..ff3efb2791552 100644 --- a/apps/server/src/kin/lead/layer.ts +++ b/apps/server/src/kin/lead/layer.ts @@ -1,5 +1,6 @@ import * as Scheduler from "../../scheduling/Scheduler.ts"; import * as Routines from "../triggers/routines.ts"; +import * as DeferralFlow from "../triggers/deferralFlow.ts"; import * as OwnerCheck from "../triggers/ownerCheck.ts"; import * as TriggerScheduler from "../triggers/KinTriggerScheduler.ts"; import * as TriggerStore from "../triggers/KinTriggerStore.ts"; @@ -58,7 +59,7 @@ export function containStartup(layer: Layer.Layer) { ); } -const routineRuntime = Routines.layer.pipe( +const routineRuntime = Layer.merge(Routines.layer, DeferralFlow.layer).pipe( Layer.provideMerge(TriggerScheduler.layer), Layer.provideMerge(Layer.mergeAll(TriggerStore.layer, TriggerInbox.layer)), // Kin builds in its contained startup scope; orchestration runtime does not expose its scheduler. diff --git a/apps/server/src/kin/memory/MemoryWriter.test.ts b/apps/server/src/kin/memory/MemoryWriter.test.ts index 13f18b2810bd3..3970678e0aff4 100644 --- a/apps/server/src/kin/memory/MemoryWriter.test.ts +++ b/apps/server/src/kin/memory/MemoryWriter.test.ts @@ -64,6 +64,7 @@ it.live.each([ "credentials", "read-during-record", "deferral-card-remind", + "deferral-card-monday", "deferral-plain-remind", "deferral-card-other-item", "H3-packed-ref", @@ -280,22 +281,32 @@ it.live.each([ itemId: scenario === "owner-other-item-keep" ? "other-item" : "def-001", action: scenario === "owner-drop-keep" ? "drop" : "keep", } - : scenario === "deferral-card-remind" || - scenario === "deferral-card-other-item" + : scenario === "deferral-card-monday" ? { saidBy: "fixture-owner", trust: "owner", source: "fixture-owner-message", kind: "deferral_card", - itemId: - scenario === "deferral-card-other-item" ? "other-item" : "def-001", - action: "remind", + itemId: "def-001", + action: "monday", + due: "2026-11-16T09:00:00.000+05:30", } - : { - saidBy: "fixture-owner", - trust: "owner", - source: "fixture-owner-message", - }, + : scenario === "deferral-card-remind" || + scenario === "deferral-card-other-item" + ? { + saidBy: "fixture-owner", + trust: "owner", + source: "fixture-owner-message", + kind: "deferral_card", + itemId: + scenario === "deferral-card-other-item" ? "other-item" : "def-001", + action: "remind", + } + : { + saidBy: "fixture-owner", + trust: "owner", + source: "fixture-owner-message", + }, ], [ "member", @@ -653,6 +664,25 @@ it.live.each([ expect(result._tag === "Failure" && result.failure._tag).toBe("MemoryConflict"); return; } + if (scenario === "deferral-card-monday") { + const stage = (fields: Record) => + writer + .stage(orgId, "wake1", { + path: "deferrals/def-001.md", + content: fixtureItem(fields), + sourceEventId: "owner", + }) + .pipe(Effect.result); + for (const forged of [ + { due: "2026-11-17T09:00:00.000+05:30" }, + { due: "2026-11-16T09:00:00.000+05:30", on_due: "remind" }, + ]) { + const result = yield* stage(forged); + expect(result._tag === "Failure" && result.failure._tag).toBe("MemoryConflict"); + } + expect((yield* stage({ due: "2026-11-16T09:00:00.000+05:30" }))._tag).toBe("Success"); + return; + } if (scenario === "deferral-card-remind") { const stage = (onDue: string) => writer diff --git a/apps/server/src/kin/memory/MemoryWriter.ts b/apps/server/src/kin/memory/MemoryWriter.ts index b08c73d835c50..86a40b2ead561 100644 --- a/apps/server/src/kin/memory/MemoryWriter.ts +++ b/apps/server/src/kin/memory/MemoryWriter.ts @@ -48,6 +48,12 @@ export interface WakeupContext extends JournalContext { readonly action: "keep" | "update" | "drop"; } | { readonly kind: "deferral_card"; readonly itemId: string; readonly action: "remind" } + | { + readonly kind: "deferral_card"; + readonly itemId: string; + readonly action: "monday"; + readonly due: string; + } ) >; } @@ -644,11 +650,20 @@ const make = Effect.gen(function* () { mutable.add("review_by"); if ( source.kind === "deferral_card" && + source.action === "remind" && source.itemId === item.id && item.kind === "deferral" && fm.on_due === "remind" ) mutable.add("on_due"); + if ( + source.kind === "deferral_card" && + source.action === "monday" && + source.itemId === item.id && + item.kind === "deferral" && + fm.due === source.due + ) + mutable.add("due"); const existing = new Map(Object.entries(item)); if ( [...new Set([...existing.keys(), ...proposed.keys()])].some( diff --git a/apps/server/src/kin/triggers/KinTriggerInbox.ts b/apps/server/src/kin/triggers/KinTriggerInbox.ts index d6e2cfa90e0fb..dc6b507e3b26a 100644 --- a/apps/server/src/kin/triggers/KinTriggerInbox.ts +++ b/apps/server/src/kin/triggers/KinTriggerInbox.ts @@ -42,7 +42,7 @@ const isInboxConflict = Schema.is(KinTriggerInboxConflict); const PendingRequest = Schema.Struct({ orgId: KinOrgId, limit: Schema.Int.check(Schema.isGreaterThan(0)), - routines: Schema.optional(Schema.Array(KinRoutine)), + routines: Schema.optional(Schema.Union([Schema.Array(KinRoutine), Schema.Literal("deferral")])), }); const InboxRow = Schema.Struct({ payload: Schema.fromJsonString(KinTriggerFired) }); const decodeInboxRow = Schema.decodeUnknownEffect(InboxRow); @@ -54,10 +54,11 @@ export class KinTriggerInbox extends Context.Service< event: KinTriggerFired, receivedAt: number, ) => Effect.Effect; + /** `subjects` narrows to those routines, or to deferral fires. */ readonly listPending: ( orgId: KinOrgId, limit?: number, - routines?: ReadonlyArray, + subjects?: ReadonlyArray | "deferral", ) => Effect.Effect, InboxError>; readonly markConsumed: ( orgId: KinOrgId, @@ -87,7 +88,9 @@ const make = Effect.gen(function* () { ${ routines === undefined ? sql`` - : sql`AND json_extract(payload_json, '$.payload.subject.routine') IN (${sql.join(",", false)(routines.map((routine) => sql`${routine}`))})` + : routines === "deferral" + ? sql`AND json_extract(payload_json, '$.payload.subject.type') = 'deferral'` + : sql`AND json_extract(payload_json, '$.payload.subject.routine') IN (${sql.join(",", false)(routines.map((routine) => sql`${routine}`))})` } ORDER BY received_at, fire_id LIMIT ${limit} `, diff --git a/apps/server/src/kin/triggers/KinTriggerService.ts b/apps/server/src/kin/triggers/KinTriggerService.ts index 0a2c2949591ca..db11142450c4b 100644 --- a/apps/server/src/kin/triggers/KinTriggerService.ts +++ b/apps/server/src/kin/triggers/KinTriggerService.ts @@ -2,7 +2,6 @@ import { KinChatId, type KinDeferralCard, KinMemoryItemId, - KinMessageId, KinOrgId, KinTriggerSchedule, KinTriggerSubject, @@ -31,9 +30,11 @@ import { deferralTriggerId, deferralWidgetId, planDeferral, + makeDeferralCards, type DeferralPhase, type DeferralPlan, } from "./deferrals.ts"; +import { type GithubLinkState, readGithubLinks } from "./deferralGithub.ts"; export class KinTriggerSyncError extends Schema.TaggedError()( "KinTriggerSyncError", @@ -101,6 +102,7 @@ const make = Effect.gen(function* () { const hub = yield* Effect.serviceOption(KinChatHub); const publish = (event: Parameters[0]) => Option.isSome(hub) ? sql.afterCommit(hub.value.publish(event)) : Effect.void; + const cards = yield* makeDeferralCards; // Trigger rows are only written in the same transaction as their run row. const readStored = Effect.fn("KinTriggerService.readStored")(function* ( @@ -187,11 +189,11 @@ const make = Effect.gen(function* () { timeZone: string, now: number, ) { - const widgetId = KinWidgetId.make(deferralWidgetId(orgId, itemId)); - const existing = yield* repo.getWidget({ orgId, widgetId }); - const at = iso(now); if (!plan.armed || plan.dueAt === null || item === undefined) { + const widgetId = KinWidgetId.make(deferralWidgetId(orgId, itemId)); + const existing = yield* repo.getWidget({ orgId, widgetId }); if (Option.isNone(existing) || existing.value.closedAt !== null) return; + const at = iso(now); const widget = yield* repo.updateWidget({ ...existing.value, expectedRevision: existing.value.revision, @@ -207,62 +209,16 @@ const make = Effect.gen(function* () { onDue: plan.onDue, text: confirmText(plan.dueAt, plan.onDue, timeZone), }; - if (Option.isSome(existing)) { - const widget = yield* repo.updateWidget({ - ...existing.value, - expectedRevision: existing.value.revision, - state, - closedAt: null, - updatedAt: at, - }); - return yield* publish({ type: "widget.updated", orgId, widget }); - } // Only kin_defer items name a chat; other deferrals arm without a card. - if (item.chat === undefined) return; - const chatId = KinChatId.make(item.chat); - const [chat] = yield* sql`SELECT 1 FROM kin_chats - WHERE org_id = ${orgId} AND chat_id = ${chatId} ORDER BY chat_id`; - if (chat === undefined) - return yield* Effect.logWarning("Deferral chat is not in its organization", { - orgId, - itemId, - }); - const message = yield* repo.insertMessage({ - orgId, - chatId, - messageId: KinMessageId.make(widgetId), - author: { _tag: "system" }, - kind: "system", - body: state.text, - intent: null, - mentions: [], - replyTo: null, - widgetId: null, - floorRunId: null, - status: "posted", - sentBy: null, - clientMessageId: null, - createdAt: at, - editedAt: null, - }); - const widget = { - orgId, - chatId, - messageId: message.messageId, - widgetId, - widgetClass: "native" as const, - type: "deferral", - state, - source: null, - feed: null, - awaiting: [], - pinnedAt: null, - closedAt: null, - updatedAt: at, - }; - yield* repo.insertWidget(widget); - yield* publish({ type: "message.posted", orgId, message }); - yield* publish({ type: "widget.updated", orgId, widget: { ...widget, revision: 1 } }); + let chatId: KinChatId | null = null; + if (item.chat !== undefined) { + const [chat] = yield* sql`SELECT 1 FROM kin_chats + WHERE org_id = ${orgId} AND chat_id = ${item.chat} ORDER BY chat_id`; + if (chat === undefined) + yield* Effect.logWarning("Deferral chat is not in its organization", { orgId, itemId }); + else chatId = KinChatId.make(item.chat); + } + yield* cards.write({ orgId, itemId, state, chatId, now }); }); const apply = Effect.fn("KinTriggerService.apply")(function* ( @@ -272,6 +228,7 @@ const make = Effect.gen(function* () { plan: DeferralPlan, timeZone: string, now: number, + github: ReadonlyArray | null, ) { return yield* sql.withTransaction( Effect.gen(function* () { @@ -285,7 +242,7 @@ const make = Effect.gen(function* () { yield* writeTrigger(orgId, itemId, phase, runAt, timeZone, now); } if (plan.armed && item !== undefined) { - const snapshot = yield* encodeSnapshot(deferralSnapshot(item)); + const snapshot = yield* encodeSnapshot(deferralSnapshot(item, github)); const inserted = yield* sql` INSERT INTO kin_deferral_runs (org_id, item_id, state, due_at, on_due, snapshot_json, updated_at) VALUES (${orgId}, ${itemId}, 'armed', ${plan.dueAt}, ${plan.onDue}, ${snapshot}, ${now}) @@ -318,7 +275,9 @@ const make = Effect.gen(function* () { if (item.kind !== "deferral") return "unchanged" as const; const timeZone = yield* cache.ownerTimeZone(item.orgId); const now = yield* Clock.currentTimeMillis; - return yield* apply(item.orgId, item.id, item, planDeferral(item), timeZone, now).pipe( + const plan = planDeferral(item); + const github = plan.armed ? yield* readGithubLinks(item.orgId, item.links) : null; + return yield* apply(item.orgId, item.id, item, plan, timeZone, now, github).pipe( wrap(item.orgId), ); }), @@ -340,7 +299,12 @@ const make = Effect.gen(function* () { const now = yield* Clock.currentTimeMillis; const failures: Array = []; for (const [itemId, item, plan] of pending) - yield* apply(orgId, itemId, item, plan, timeZone, now).pipe( + yield* ( + plan.armed && item !== undefined + ? readGithubLinks(orgId, item.links) + : Effect.succeed(null) + ).pipe( + Effect.flatMap((github) => apply(orgId, itemId, item, plan, timeZone, now, github)), wrap(orgId), Effect.catch((error) => { failures.push(error); diff --git a/apps/server/src/kin/triggers/deferralFlow.ts b/apps/server/src/kin/triggers/deferralFlow.ts new file mode 100644 index 0000000000000..a968130099f7d --- /dev/null +++ b/apps/server/src/kin/triggers/deferralFlow.ts @@ -0,0 +1,387 @@ +import { + KinChatId, + KinDeferralCard, + KinMemoryItemId, + KinMessageId, + KinOrgId, + KinPersonId, + type KinTriggerFired, + type MemoryItemHead, +} from "@t3tools/contracts"; +import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; +import * as Clock from "effect/Clock"; +import * as Context from "effect/Context"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; +import * as SqlClient from "effect/sql/SqlClient"; +import { isSqlError } from "effect/sql/SqlError"; + +import { PersistenceDecodeError, toPersistenceSqlError } from "../../persistence/Errors.ts"; +import * as Scheduler from "../../scheduling/Scheduler.ts"; +import * as Chat from "../chat/KinChatService.ts"; +import { WorkerLaunchGate } from "../lead/Workers.ts"; +import * as Memory from "../memory/Memory.ts"; +import * as MemoryCache from "../memory/MemoryCache.ts"; +import { readGithubLinks } from "./deferralGithub.ts"; +import { + DeferralSnapshot, + GRACE_MS, + HEADS_UP_MS, + LATE_NOTE_MS, + KinDeferralStopSwitch, + deferralTriggerId, + lateNote, + ownerDateTime, + ownerTime, + snapshotChanges, + makeDeferralCards, + makeDeferralStarter, +} from "./deferrals.ts"; +import * as KinTriggerInbox from "./KinTriggerInbox.ts"; +import * as Pings from "./pings.ts"; + +export class KinDeferralFlowError extends Schema.TaggedError()( + "KinDeferralFlowError", + { orgId: KinOrgId, cause: Schema.Defect() }, +) {} + +export class KinDeferralFlow extends Context.Service< + KinDeferralFlow, + { + /** Consumes this org's pending deferral fires: recheck, heads-up, grace, start or ask. */ + readonly drain: (orgId: KinOrgId) => Effect.Effect; + } +>()("t3/kin/triggers/deferralFlow/KinDeferralFlow") {} + +interface Run { + readonly state: string; + readonly dueAt: number | null; + readonly onDue: string; + readonly snapshot: string; +} +type Step = + | { readonly kind: "drop" } + | { readonly kind: "heads_up"; readonly dueAt: number } + | { readonly kind: "due"; readonly dueAt: number; readonly grace: boolean }; + +/** Which step a fire takes, from its run row. Disarmed, moved and handled runs drop it. */ +function classifyFire( + run: Run | undefined, + phase: "heads_up" | "due", + fireAt: number, + now: number, +): Step { + if (run === undefined || run.state === "disarmed" || run.dueAt === null) return { kind: "drop" }; + const dueAt = run.dueAt; + if (phase === "heads_up") + return run.onDue === "start" && + run.state === "armed" && + fireAt === dueAt - HEADS_UP_MS && + now < dueAt + ? { kind: "heads_up", dueAt } + : { kind: "drop" }; + if (fireAt === dueAt) + return run.state === "armed" || (run.onDue === "start" && run.state === "heads_up") + ? { kind: "due", dueAt, grace: false } + : { kind: "drop" }; + // A late due fire re-arms the due trigger at the end of its grace window. + return run.onDue === "start" && run.state === "grace" && fireAt > dueAt + ? { kind: "due", dueAt, grace: true } + : { kind: "drop" }; +} + +const decodeOnDue = Schema.decodeUnknownEffect(KinDeferralCard.fields.onDue); +const decodeSnapshot = Schema.decodeUnknownEffect(Schema.fromJsonString(DeferralSnapshot)); +const isDecodeError = Schema.is(PersistenceDecodeError); +const isPermanent = (error: unknown) => { + let current = error; + for (let depth = 0; depth < 8 && typeof current === "object" && current !== null; depth++) { + if (Schema.isSchemaError(current) || isDecodeError(current)) return true; + current = "cause" in current ? current.cause : undefined; + } + return false; +}; +const iso = (ms: number) => DateTime.formatIso(DateTime.makeUnsafe(ms)); +const levels = ["L0", "L1", "L2", "L3"]; + +const make = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const inbox = yield* KinTriggerInbox.KinTriggerInbox; + const memory = yield* Memory.Memory; + const cache = yield* MemoryCache.MemoryCache; + const chat = yield* Chat.KinChatService; + const pings = yield* Pings.KinPings; + const ticks = yield* Scheduler.Scheduler; + const cards = yield* makeDeferralCards; + const start = yield* makeDeferralStarter; + + const readRun = (orgId: KinOrgId, itemId: KinMemoryItemId) => + sql`SELECT state, due_at AS "dueAt", on_due AS "onDue", snapshot_json AS snapshot + FROM kin_deferral_runs WHERE org_id = ${orgId} AND item_id = ${itemId} ORDER BY item_id`.pipe( + Effect.map((rows) => rows[0]), + ); + const readItem = (orgId: KinOrgId, itemId: KinMemoryItemId) => + memory.readItemQuietly(orgId, itemId).pipe( + Effect.map((item): MemoryItemHead | undefined => item.head), + Effect.catchTags({ + MemoryNotFound: () => Effect.undefined, + MemoryDuplicateId: () => Effect.undefined, + }), + ); + // The item's chat when it belongs to the org, else the owner's direct chat. + const cardChat = Effect.fn("KinDeferralFlow.cardChat")(function* ( + orgId: KinOrgId, + item: MemoryItemHead | undefined, + ) { + if (item?.chat !== undefined) { + const [own] = yield* sql`SELECT 1 FROM kin_chats + WHERE org_id = ${orgId} AND chat_id = ${item.chat} ORDER BY chat_id`; + if (own !== undefined) return KinChatId.make(item.chat); + } + const [lead] = yield* sql<{ + chatId: string; + }>`SELECT l.owner_chat_id AS "chatId" FROM kin_leads l + JOIN kin_agents a ON a.org_id = l.org_id AND a.agent_id = l.agent_id + WHERE l.org_id = ${orgId} AND a.retired_at IS NULL ORDER BY l.agent_id LIMIT 1`; + return lead === undefined ? null : KinChatId.make(lead.chatId); + }); + + const prepareChase = Effect.fn("KinDeferralFlow.prepareChase")(function* ( + orgId: KinOrgId, + item: MemoryItemHead, + dueAt: number, + timeZone: string, + ) { + const handle = item.owner ?? item.said_by; + const [person] = yield* sql<{ personId: string }>`SELECT person_id AS "personId" + FROM kin_people WHERE org_id = ${orgId} AND handle = ${handle} ORDER BY person_id`; + const [owner] = yield* sql<{ owner: string; chatId: string }>`SELECT a.owner_person_id AS owner, + l.owner_chat_id AS "chatId" FROM kin_leads l + JOIN kin_agents a ON a.org_id = l.org_id AND a.agent_id = l.agent_id + WHERE l.org_id = ${orgId} AND a.retired_at IS NULL ORDER BY l.agent_id LIMIT 1`; + if (person === undefined || owner === undefined || item.chat === undefined) + return `I couldn't chase ${handle}`; + const put = yield* pings + .put({ + orgId, + person: KinPersonId.make(person.personId), + itemId: item.id, + chatId: item.chat, + owner: KinPersonId.make(owner.owner), + ownerChatId: KinChatId.make(owner.chatId), + nextStep: `${item.title} was due ${ownerDateTime(dueAt, timeZone)}. What's the status?`, + }) + .pipe(Effect.result); + if (put._tag === "Failure") { + yield* Effect.logWarning("Kin deferral chase failed", { orgId, itemId: item.id }); + return `I couldn't chase ${handle}`; + } + return null; + }); + + const process = Effect.fn("KinDeferralFlow.process")(function* ( + orgId: KinOrgId, + fire: KinTriggerFired, + itemId: KinMemoryItemId, + phase: "heads_up" | "due", + now: number, + timeZone: string, + ) { + const fireAt = Date.parse(fire.dueAt); + const run = yield* readRun(orgId, itemId).pipe(NodeSqliteClient.readOnly); + const step = classifyFire(run, phase, fireAt, now); + if (step.kind === "drop" || run === undefined) + return yield* inbox.markConsumed(orgId, [fire.fireId]); + + // Read-only or idempotent I/O; a replay repeats it. + const item = yield* readItem(orgId, itemId); + const onDue = run.onDue; + const decodedOnDue = yield* decodeOnDue(onDue); + const gates: Array = []; + if (onDue === "start") { + const snapshot = yield* decodeSnapshot(run.snapshot); + const github = yield* readGithubLinks(orgId, item?.links ?? snapshot.links); + gates.push(...snapshotChanges(snapshot, item, github)); + if (yield* (yield* KinDeferralStopSwitch).isStopped(orgId)) + gates.push("the stop switch is on"); + if (!(yield* WorkerLaunchGate)) gates.push("workers can't launch yet"); + } + const chaseFailure = + onDue === "chase" && step.kind === "due" + ? item === undefined + ? "the item is gone" + : yield* prepareChase(orgId, item, step.dueAt, timeZone) + : null; + + let chased = false; + yield* sql.withTransaction( + Effect.gen(function* () { + const pending = yield* sql`SELECT fire_id FROM kin_trigger_inbox WHERE org_id = ${orgId} + AND fire_id = ${fire.fireId} AND consumed_at IS NULL ORDER BY fire_id`; + if (pending.length === 0) return; + const reasons = [...gates]; + const current = yield* readRun(orgId, itemId); + // Moved or handled since the reads above: classify again on the next tick. + if (current?.state !== run.state || current.dueAt !== run.dueAt) return; + const dueAt = step.dueAt; + const title = item?.title ?? itemId; + const card = (phase: KinDeferralCard["phase"], text: string, graceEndsAt?: number) => ({ + itemId, + title, + dueAt: iso(dueAt), + onDue: decodedOnDue, + text, + phase, + ...(graceEndsAt === undefined ? {} : { graceEndsAt: iso(graceEndsAt) }), + }); + const setState = (state: string) => + sql`UPDATE kin_deferral_runs SET state = ${state}, updated_at = ${now} + WHERE org_id = ${orgId} AND item_id = ${itemId} AND due_at = ${dueAt}`; + const post = Effect.fn("KinDeferralFlow.post")(function* (state: KinDeferralCard) { + const chatId = yield* cardChat(orgId, item); + yield* cards.write({ orgId, itemId, state, chatId, now }); + }); + const ask = Effect.fn("KinDeferralFlow.ask")(function* (why: ReadonlyArray) { + const reason = why.length === 0 ? "" : ` I won't start it on my own: ${why.join("; ")}.`; + yield* post( + card( + "asked", + `${title} is due at ${ownerTime(dueAt, now, timeZone)}. Start it, move it to Monday, or cancel it?${reason}${lateNote(dueAt, now, timeZone)}`, + ), + ); + yield* setState("asked"); + }); + + if (onDue === "start") { + const [lead] = yield* sql<{ level: string }>`SELECT l.level FROM kin_leads l + JOIN kin_agents a ON a.org_id = l.org_id AND a.agent_id = l.agent_id + WHERE l.org_id = ${orgId} AND a.retired_at IS NULL ORDER BY l.agent_id LIMIT 1`; + if (lead === undefined) reasons.push("there is no active lead"); + else if (levels.indexOf(lead.level) < levels.indexOf("L2")) + reasons.push(`Kin is at ${lead.level}, below L2`); + } + + if (step.kind === "heads_up") { + if (reasons.length > 0) yield* ask(reasons); + else { + yield* post( + card("heads_up", `Starting ${title} at ${ownerTime(dueAt, now, timeZone)}.`), + ); + yield* setState("heads_up"); + } + } else if (onDue === "start") { + const anchor = step.grace ? fireAt : dueAt; + const late = now - anchor; + if (late > GRACE_MS) yield* ask([...reasons, "it's more than 30 minutes late"]); + else if (reasons.length > 0) yield* ask(reasons); + else if (!step.grace && (run.state === "armed" || late > LATE_NOTE_MS)) { + // No heads-up was shown, or it was shown too long ago: give the grace from now. + const graceEnd = now + GRACE_MS; + yield* post( + card( + "heads_up", + `Starting ${title} at ${ownerTime(graceEnd, now, timeZone)}.${lateNote(dueAt, now, timeZone)}`, + graceEnd, + ), + ); + yield* sql`UPDATE kin_triggers SET next_run_at = ${graceEnd}, + updated_at = max(updated_at, ${now}) + WHERE org_id = ${orgId} AND trigger_id = ${deferralTriggerId(itemId, "due")} + AND enabled = 1`; + yield* setState("grace"); + } else { + const started = yield* start({ + orgId, + itemId, + dueAt, + card: card("started", ""), + provenance: { _tag: "system" }, + now, + }); + if (!started) yield* ask(["there is no active lead"]); + } + } else if (onDue === "ask") yield* ask([]); + else if (onDue === "chase") { + if (chaseFailure !== null) yield* ask([chaseFailure]); + else { + yield* setState("chased"); + chased = true; + } + } else { + const chatId = yield* cardChat(orgId, item); + if (chatId !== null) + yield* chat.notice({ + orgId, + chatId, + messageId: KinMessageId.make(`deferral-remind:${orgId}:${fire.fireId}`), + body: `Reminder: ${title}.${lateNote(dueAt, now, timeZone)}`, + }); + yield* setState("reminded"); + } + yield* inbox.markConsumed(orgId, [fire.fireId]); + }), + ); + if (chased) + yield* pings + .sweep(orgId) + .pipe( + Effect.catch((cause) => + Effect.logWarning("Kin deferral chase sweep failed", { orgId, cause }), + ), + ); + }); + + const drain = Effect.fn("KinDeferralFlow.drain")((orgId: KinOrgId) => + Effect.gen(function* () { + const now = yield* Clock.currentTimeMillis; + const timeZone = yield* cache.ownerTimeZone(orgId); + const fires = yield* inbox.listPending(orgId, 100, "deferral"); + // Catch-up fires share received_at; handle them by due time. + for (const fire of fires.toSorted( + (a, b) => + Date.parse(a.dueAt) - Date.parse(b.dueAt) || + (a.fireId < b.fireId ? -1 : a.fireId > b.fireId ? 1 : 0), + )) { + if (fire.payload.kind !== "time" || fire.payload.subject.type !== "deferral") continue; + const { itemId, phase } = fire.payload.subject; + yield* process(orgId, fire, itemId, phase, now, timeZone).pipe( + Effect.mapError((cause) => + isSqlError(cause) ? toPersistenceSqlError("KinDeferralFlow.drain")(cause) : cause, + ), + Effect.catch((cause) => + Effect.gen(function* () { + const consume = + isPermanent(cause) || now - Date.parse(fire.dueAt) > 24 * 60 * 60 * 1000; + yield* (consume ? Effect.logWarning : Effect.logDebug)( + "Kin deferral fire processing failed", + { orgId, fireId: fire.fireId, cause, disposition: consume ? "consumed" : "retry" }, + ); + if (consume) yield* inbox.markConsumed(orgId, [fire.fireId]); + }), + ), + ); + } + }).pipe(Effect.mapError((cause) => new KinDeferralFlowError({ orgId, cause }))), + ); + + yield* ticks.register( + "kin-deferrals", + Effect.gen(function* () { + // Scheduler maintenance scans organizations; each drain is org-scoped. + const orgs = yield* sql<{ orgId: string }>`SELECT org_id AS "orgId" FROM kin_orgs + ORDER BY org_id`; + for (const { orgId } of orgs) + yield* drain(KinOrgId.make(orgId)).pipe( + Effect.catch((cause) => Effect.logWarning("Kin deferral tick failed", { orgId, cause })), + ); + }).pipe(Effect.catch((cause) => Effect.logWarning("Kin deferral scan failed", { cause }))), + ); + return KinDeferralFlow.of({ drain }); +}); + +export const layer = Layer.effect(KinDeferralFlow, make).pipe( + Layer.provide(Pings.layer), + Layer.provide(MemoryCache.layer), +); diff --git a/apps/server/src/kin/triggers/deferralGithub.ts b/apps/server/src/kin/triggers/deferralGithub.ts new file mode 100644 index 0000000000000..cb0965274ee55 --- /dev/null +++ b/apps/server/src/kin/triggers/deferralGithub.ts @@ -0,0 +1,73 @@ +import { KinOrgId } from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; + +export const GithubLinkState = Schema.Struct({ + link: Schema.String, + state: Schema.Literals(["open", "closed", "merged"]), + assignees: Schema.Array(Schema.String), + linkedPrs: Schema.Array(Schema.Int), +}); +export type GithubLinkState = typeof GithubLinkState.Type; + +export interface GithubRef { + readonly repo: string; + readonly number: number; +} + +export class KinDeferralGithubError extends Schema.TaggedError()( + "KinDeferralGithubError", + { orgId: KinOrgId, reason: Schema.Literals(["unavailable", "not_found"]) }, +) {} + +export interface KinDeferralGithubReaderShape { + /** Read-only: one issue or PR's state, assignees and linked PR numbers. */ + readonly read: ( + orgId: KinOrgId, + ref: GithubRef, + ) => Effect.Effect, KinDeferralGithubError>; +} + +// Kin has no GitHub client yet, so production reads fail and every recheck with +// GitHub links reports "couldn't verify", which asks instead of starting. +export const KinDeferralGithubReader = Context.Reference( + "t3/kin/triggers/deferralGithub/KinDeferralGithubReader", + { + defaultValue: () => ({ + read: (orgId) => Effect.fail(new KinDeferralGithubError({ orgId, reason: "unavailable" })), + }), + }, +); + +const urlRef = /^https:\/\/github\.com\/([\w.-]+\/[\w.-]+)\/(?:issues|pull)\/(\d+)\/?$/; +const shortRef = /^([\w.-]+\/[\w.-]+)#(\d+)$/; + +export function githubRef(link: string): GithubRef | undefined { + const match = urlRef.exec(link) ?? shortRef.exec(link); + if (match === null) return undefined; + const number = Number(match[2]); + return Number.isSafeInteger(number) && number > 0 ? { repo: match[1]!, number } : undefined; +} + +/** Reads every GitHub link; `null` means at least one couldn't be read. */ +export const readGithubLinks = Effect.fn("KinDeferralGithub.readLinks")(function* ( + orgId: KinOrgId, + links: ReadonlyArray, +) { + const reader = yield* KinDeferralGithubReader; + const states: Array = []; + for (const link of links) { + const ref = githubRef(link); + if (ref === undefined) continue; + const state = yield* reader.read(orgId, ref).pipe(Effect.option); + if (state._tag === "None") return null; + states.push({ + link, + state: state.value.state, + assignees: state.value.assignees.toSorted(), + linkedPrs: state.value.linkedPrs.toSorted((a, b) => a - b), + }); + } + return states; +}); diff --git a/apps/server/src/kin/triggers/deferrals.test.ts b/apps/server/src/kin/triggers/deferrals.test.ts index 706d45c58f43c..618494f76185c 100644 --- a/apps/server/src/kin/triggers/deferrals.test.ts +++ b/apps/server/src/kin/triggers/deferrals.test.ts @@ -2,6 +2,7 @@ import { assert, it } from "@effect/vitest"; import { KinAgentId, KinChatId, + KinDeferralCard, KinEvent, KinEventId, KinMemberId, @@ -9,7 +10,9 @@ import { KinPersonId, KinWakeupId, KinWidgetId, + type MemoryItemHead, } from "@t3tools/contracts"; +import * as Clock from "effect/Clock"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; @@ -19,106 +22,198 @@ import * as TestClock from "effect/testing/TestClock"; import * as Repository from "../chat/KinChatRepository.ts"; import * as Queue from "../lead/KinEventQueue.ts"; +import { WorkerLaunchGate } from "../lead/Workers.ts"; import * as MemoryWriter from "../memory/MemoryWriter.ts"; +import * as DeferralFlow from "./deferralFlow.ts"; +import { + KinDeferralGithubError, + KinDeferralGithubReader, + type GithubLinkState, +} from "./deferralGithub.ts"; import * as Deferrals from "./deferrals.ts"; +import * as TriggerScheduler from "./KinTriggerScheduler.ts"; import * as TriggerService from "./KinTriggerService.ts"; import { fixture, item, orgId, otherOrg, seed } from "./routines.testkit.ts"; const encodeJson = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); const decodeEvent = Schema.decodeSync(KinEvent); -const deferral = item("def-1", { - kind: "deferral", - path: "deferrals/def-1.md", - due: "2026-11-13T09:00+05:30", - on_due: "start", - chat: KinChatId.make("chat-a"), +const decodeCard = Schema.decodeSync(Schema.fromJsonString(KinDeferralCard)); +const issue = "https://github.com/acme/api/issues/12"; +const makeDeferral = (id: string, overrides: Partial = {}) => + item(id, { + kind: "deferral", + path: `deferrals/${id}.md`, + due: "2026-11-13T09:00+05:30", + on_due: "start", + chat: KinChatId.make("chat-a"), + links: [], + ...overrides, + }); +const deferral = makeDeferral("def-1"); +const widgetFor = (itemId: string) => KinWidgetId.make(`deferral:${orgId}:${itemId}`); +const widgetId = widgetFor("def-1"); +const dueAt = Date.parse("2026-11-13T03:30:00.000Z"); +const headsUpAt = dueAt - 30 * 60_000; + +const layerFor = (items: ReadonlyArray) => + DeferralFlow.layer.pipe( + Layer.provideMerge(Deferrals.layer), + Layer.provideMerge(TriggerService.layer), + Layer.provideMerge(fixture(items)), + ); + +const fakeGithub = (states: Map>) => ({ + read: (org: typeof orgId, ref: { readonly repo: string; readonly number: number }) => { + const state = states.get(`${ref.repo}#${ref.number}`); + return state === undefined + ? Effect.fail(new KinDeferralGithubError({ orgId: org, reason: "not_found" })) + : Effect.succeed(state); + }, +}); +// Stands in for LEAD-11, the worker release gate and #44's GitHub client. +const openGates = + (states: Map>, stopped = false) => + (effect: Effect.Effect) => + effect.pipe( + Effect.provideService(WorkerLaunchGate, true), + Effect.provideService(Deferrals.KinDeferralStopSwitch, { + isStopped: () => Effect.succeed(stopped), + }), + Effect.provideService(KinDeferralGithubReader, fakeGithub(states)), + ); + +const setup = Effect.fn("test.deferralFlowSetup")(function* (level = "L2") { + yield* seed(); + const sql = yield* SqlClient.SqlClient; + yield* sql`INSERT INTO kin_trigger_settings VALUES (${orgId}, 'Asia/Kolkata', '08:30', '18:30', '03:00', 0, 1, '09:00', '{"from":"21:00","to":"08:00"}')`; + yield* sql`UPDATE kin_leads SET level = ${level} WHERE org_id = ${orgId}`; + yield* TestClock.setTime(Date.parse("2026-11-09T09:00:00.000Z")); +}); + +const tick = Effect.fn("test.deferralTick")(function* (at: number) { + yield* TestClock.setTime(at); + yield* (yield* TriggerScheduler.KinTriggerScheduler).runDue(); + yield* (yield* DeferralFlow.KinDeferralFlow).drain(orgId); +}); + +const runOf = Effect.fn("test.deferralRun")(function* (itemId: string) { + const sql = yield* SqlClient.SqlClient; + const [row] = yield* sql<{ state: string; dueAt: number }>`SELECT state, due_at AS "dueAt" + FROM kin_deferral_runs WHERE org_id = ${orgId} AND item_id = ${itemId} ORDER BY item_id`; + return row; +}); + +const cardOf = Effect.fn("test.deferralCard")(function* (itemId: string) { + const repo = yield* Repository.KinChatRepository; + const widget = Option.getOrThrow(yield* repo.getWidget({ orgId, widgetId: widgetFor(itemId) })); + return { ...decodeCard(encodeJson(widget.state)), revision: widget.revision }; +}); + +const starts = Effect.fn("test.deferralStarts")(function* () { + const sql = yield* SqlClient.SqlClient; + return yield* sql<{ dedupeKey: string; json: string }>`SELECT dedupe_key AS "dedupeKey", json + FROM kin_events WHERE org_id = ${orgId} AND tag = 'deferral.due' ORDER BY event_id`; +}); + +const pendingFires = Effect.fn("test.deferralPending")(function* () { + const sql = yield* SqlClient.SqlClient; + return yield* sql<{ fireId: string }>`SELECT fire_id AS "fireId" FROM kin_trigger_inbox + WHERE org_id = ${orgId} AND consumed_at IS NULL ORDER BY fire_id`; }); -const widgetId = KinWidgetId.make(`deferral:${orgId}:def-1`); + +const tapOnce = Effect.fn("test.deferralTap")(function* ( + actionId: string, + label: string, + staged: Array, + target: KinWidgetId = widgetId, + value = "def-1", +) { + const sql = yield* SqlClient.SqlClient; + const repo = yield* Repository.KinChatRepository; + const queue = yield* Queue.KinEventQueue; + const deferrals = yield* Deferrals.Deferrals; + const writer = Layer.mock(MemoryWriter.MemoryWriter)({ + stage: (_org, _wakeup, change) => Effect.sync(() => void staged.push(change)), + }); + const memberId = "owner-member-a"; + const revision = Option.match(yield* repo.getWidget({ orgId, widgetId: target }), { + onNone: () => 1, + onSome: (widget) => widget.revision, + }); + const at = new Date(yield* Clock.currentTimeMillis).toISOString(); + // Stands in for the gated dispatcher: chat act refuses deferral taps today. + const message = yield* repo.insertMessage({ + orgId, + chatId: KinChatId.make("chat-a"), + messageId: KinMessageId.make(`tap-${actionId}-${label}`), + author: { + _tag: "human", + personId: KinPersonId.make("owner-a"), + memberId: KinMemberId.make(memberId), + }, + kind: "widget_action", + body: encodeJson({ widgetId: target, revision, actionId, value }), + intent: null, + mentions: [], + replyTo: KinMessageId.make(target), + widgetId: target, + floorRunId: null, + status: "posted", + sentBy: KinPersonId.make("owner-a"), + clientMessageId: { kind: "act", id: `${actionId}-${label}` }, + createdAt: at, + editedAt: null, + }); + const eventId = `${actionId}-${label}`; + yield* queue.enqueue( + decodeEvent({ + _tag: "widget.action", + orgId, + eventId, + agentId: "lead-a", + receivedAt: at, + dedupeKey: eventId, + provenance: { _tag: "human", personId: "owner-a", memberId }, + chatId: "chat-a", + messageId: message.messageId, + actionId, + }), + ); + yield* TestClock.adjust("20 seconds"); + const wakeupId = KinWakeupId.make(`wake-${eventId}`); + yield* queue.claim({ orgId, agentId: KinAgentId.make("lead-a"), wakeupId, model: "fixture" }); + if (label === "demoted") + yield* sql`UPDATE kin_chat_members SET role = 'member' WHERE org_id = ${orgId} AND member_id = ${memberId}`; + const result = yield* deferrals + .act(orgId, wakeupId, KinEventId.make(eventId)) + .pipe(Effect.provide(writer), Effect.result); + const forged = yield* deferrals + .act(otherOrg, wakeupId, KinEventId.make(eventId)) + .pipe(Effect.provide(writer), Effect.flip); + assert.equal(forged._tag, "DeferralError"); + yield* queue.finish({ orgId, wakeupId, status: "completed" }); + return result; +}); +const tap = (...args: Parameters) => Effect.scoped(tapOnce(...args)); it.effect("applies verified owner taps on the confirm card and ignores forged ones", () => Effect.gen(function* () { - yield* seed(); - yield* TestClock.setTime(Date.parse("2026-11-09T09:00:00.000Z")); - const sql = yield* SqlClient.SqlClient; + yield* setup(); const repo = yield* Repository.KinChatRepository; - const queue = yield* Queue.KinEventQueue; - const deferrals = yield* Deferrals.Deferrals; yield* (yield* TriggerService.KinTriggerService).syncItem(deferral); - const staged: MemoryWriter.MemoryChange[] = []; - const writer = Layer.mock(MemoryWriter.MemoryWriter)({ - stage: (_org, _wakeup, change) => Effect.sync(() => void staged.push(change)), - }); - const tapOnce = Effect.fn("test.deferralTap")(function* ( - actionId: string, - label: string, - target: KinWidgetId = widgetId, - ) { - const memberId = "owner-member-a"; - // Stands in for the gated dispatcher: chat act refuses deferral taps today. - const message = yield* repo.insertMessage({ - orgId, - chatId: KinChatId.make("chat-a"), - messageId: KinMessageId.make(`tap-${actionId}-${label}`), - author: { - _tag: "human", - personId: KinPersonId.make("owner-a"), - memberId: KinMemberId.make(memberId), - }, - kind: "widget_action", - body: encodeJson({ widgetId: target, revision: 1, actionId, value: "def-1" }), - intent: null, - mentions: [], - replyTo: KinMessageId.make(target), - widgetId: target, - floorRunId: null, - status: "posted", - sentBy: KinPersonId.make("owner-a"), - clientMessageId: { kind: "act", id: `${actionId}-${label}` }, - createdAt: "2026-11-09T09:00:00.000Z", - editedAt: null, - }); - const eventId = `${actionId}-${label}`; - yield* queue.enqueue( - decodeEvent({ - _tag: "widget.action", - orgId, - eventId, - agentId: "lead-a", - receivedAt: "2026-11-09T09:00:00.000Z", - dedupeKey: eventId, - provenance: { _tag: "human", personId: "owner-a", memberId }, - chatId: "chat-a", - messageId: message.messageId, - actionId, - }), - ); - yield* TestClock.adjust("20 seconds"); - const wakeupId = KinWakeupId.make(`wake-${eventId}`); - yield* queue.claim({ orgId, agentId: KinAgentId.make("lead-a"), wakeupId, model: "fixture" }); - if (label === "demoted") - yield* sql`UPDATE kin_chat_members SET role = 'member' WHERE org_id = ${orgId} AND member_id = ${memberId}`; - const result = yield* deferrals - .act(orgId, wakeupId, KinEventId.make(eventId)) - .pipe(Effect.provide(writer), Effect.result); - const forged = yield* deferrals - .act(otherOrg, wakeupId, KinEventId.make(eventId)) - .pipe(Effect.provide(writer), Effect.flip); - assert.equal(forged._tag, "DeferralError"); - yield* queue.finish({ orgId, wakeupId, status: "completed" }); - return result; - }); - const tap = (actionId: string, label: string, target?: KinWidgetId) => - Effect.scoped(tapOnce(actionId, label, target)); + const staged: Array = []; - const remind = yield* tap("remind", "owner"); + const remind = yield* tap("remind", "owner", staged); assert.equal(remind._tag === "Success" && remind.success.action, "remind"); assert.lengthOf(staged, 1); assert.equal(staged[0]?.sourceEventId, "remind-owner"); assert.include(staged[0]!.content, "on_due: remind"); - const change = yield* tap("change", "owner"); + const change = yield* tap("change", "owner", staged); assert.equal(change._tag === "Success" && change.success.action, "change"); assert.lengthOf(staged, 1); + // Heads-up actions don't apply to the confirm card. + assert.equal((yield* tap("start_now", "confirm", staged))._tag, "Failure"); const real = Option.getOrThrow(yield* repo.getWidget({ orgId, widgetId })); const forgedId = KinWidgetId.make("model-card"); @@ -155,18 +250,262 @@ it.effect("applies verified owner taps on the confirm card and ignores forged on closedAt: null, updatedAt: "2026-11-09T09:00:00.000Z", }); - assert.equal((yield* tap("remind", "forged", forgedId))._tag, "Failure"); + assert.equal((yield* tap("remind", "forged", staged, forgedId))._tag, "Failure"); assert.lengthOf(staged, 1); - const member = yield* tap("remind", "demoted"); + const member = yield* tap("remind", "demoted", staged); assert.equal(member._tag, "Failure"); assert.lengthOf(staged, 1); + }).pipe(Effect.provide(layerFor([deferral]))), +); + +it.effect("unchanged recheck, heads-up, silence, then starts once at due", () => { + const github = new Map([ + ["acme/api#12", { state: "open" as const, assignees: ["bob"], linkedPrs: [] }], + ]); + const linked = makeDeferral("def-1", { links: [issue] }); + return Effect.gen(function* () { + yield* setup(); + const sql = yield* SqlClient.SqlClient; + yield* (yield* TriggerService.KinTriggerService).syncItem(linked); + + yield* tick(headsUpAt); + assert.equal((yield* runOf("def-1"))?.state, "heads_up"); + const headsUp = yield* cardOf("def-1"); + assert.equal(headsUp.phase, "heads_up"); + assert.equal(headsUp.text, "Starting def-1 at 9:00."); + + yield* tick(headsUpAt + 15 * 60_000); + assert.equal((yield* runOf("def-1"))?.state, "heads_up"); + assert.lengthOf(yield* starts(), 0); + + yield* tick(dueAt); + assert.equal((yield* runOf("def-1"))?.state, "started"); + assert.equal((yield* cardOf("def-1")).phase, "started"); + const [started] = yield* starts(); + assert.equal(started?.dedupeKey, `deferral:def-1:due:${dueAt}`); + assert.lengthOf(yield* pendingFires(), 0); + + // A replay of the due fire after a crash can't start a second worker. + yield* sql`UPDATE kin_trigger_inbox SET consumed_at = NULL WHERE org_id = ${orgId}`; + yield* sql`UPDATE kin_deferral_runs SET state = 'heads_up' WHERE org_id = ${orgId}`; + yield* tick(dueAt); + assert.lengthOf(yield* starts(), 1); + }).pipe(openGates(github), Effect.provide(layerFor([linked]))); +}); + +it.effect("asks instead of starting when a linked issue closes before due", () => { + const github = new Map([ + ["acme/api#12", { state: "open" as const, assignees: [], linkedPrs: [] }], + ]); + const linked = makeDeferral("def-1", { links: [issue] }); + return Effect.gen(function* () { + yield* setup(); + yield* (yield* TriggerService.KinTriggerService).syncItem(linked); + yield* tick(headsUpAt); + assert.equal((yield* runOf("def-1"))?.state, "heads_up"); + + github.set("acme/api#12", { state: "closed", assignees: [], linkedPrs: [41] }); + yield* tick(dueAt); + assert.equal((yield* runOf("def-1"))?.state, "asked"); + const asked = yield* cardOf("def-1"); + assert.equal(asked.phase, "asked"); + assert.include(asked.text, "acme/api#12 is closed"); + assert.include(asked.text, "PR #41 was linked to acme/api#12"); + assert.lengthOf(yield* starts(), 0); + }).pipe(openGates(github), Effect.provide(layerFor([linked]))); +}); + +it.effect("asks when the stop switch is on, and production defaults never start", () => { + const stopped = makeDeferral("def-1"); + const linked = makeDeferral("def-2", { + links: [issue], + due: "2026-11-13T10:00+05:30", + }); + return Effect.gen(function* () { + yield* setup("L0"); + const service = yield* TriggerService.KinTriggerService; + yield* service.syncItem(stopped); + yield* tick(headsUpAt); + yield* tick(dueAt); + assert.equal((yield* runOf("def-1"))?.state, "asked"); + assert.include((yield* cardOf("def-1")).text, "the stop switch is on"); + + // No stop switch, GitHub reader, worker release or L2 yet: every gate asks. + yield* service + .syncItem(linked) + .pipe(Effect.provideService(KinDeferralGithubReader, KinDeferralGithubReader.defaultValue())); + yield* tick(dueAt + 30 * 60_000); + yield* tick(dueAt + 60 * 60_000); + assert.equal((yield* runOf("def-2"))?.state, "asked"); + const text = (yield* cardOf("def-2")).text; + for (const reason of [ + "couldn't verify GitHub", + "the stop switch is on", + "workers can't launch yet", + "Kin is at L0, below L2", + ]) + assert.include(text, reason); + assert.lengthOf(yield* starts(), 0); }).pipe( - Effect.provide( - Deferrals.layer.pipe( - Layer.provideMerge(TriggerService.layer), - Layer.provideMerge(fixture([deferral])), - ), - ), - ), + Effect.provideService(Deferrals.KinDeferralStopSwitch, { + isStopped: () => Effect.succeed(true), + }), + Effect.provide(layerFor([stopped, linked])), + ); +}); + +it.effect( + "heads-up taps: [Monday] re-arms next Monday 09:00, [Cancel] cancels, [Start now] starts", + () => { + const items = [deferral]; + return Effect.gen(function* () { + yield* setup(); + const service = yield* TriggerService.KinTriggerService; + yield* service.syncItem(deferral); + yield* tick(headsUpAt); + assert.equal((yield* cardOf("def-1")).phase, "heads_up"); + const staged: Array = []; + + const monday = yield* tap("monday", "owner", staged); + assert.equal( + monday._tag === "Success" && monday.success.due, + "2026-11-16T09:00:00.000+05:30", + ); + assert.include(staged[0]!.content, "due: 2026-11-16T09:00:00.000+05:30"); + const cancel = yield* tap("cancel", "owner", staged); + assert.equal(cancel._tag, "Success"); + assert.include(staged[1]!.content, "status: cancelled"); + + const start = yield* tap("start_now", "owner", staged); + assert.equal(start._tag === "Success" && start.success.action, "start_now"); + assert.equal((yield* runOf("def-1"))?.state, "started"); + const [started] = yield* starts(); + assert.equal(started?.dedupeKey, `deferral:def-1:due:${dueAt}`); + assert.include(started!.json, '"_tag":"human"'); + assert.equal((yield* tap("start_now", "again", staged))._tag, "Failure"); + // The due fire finds the run started and starts nothing more. + yield* tick(dueAt); + assert.lengthOf(yield* starts(), 1); + + // The [Monday] commit reaches the cache hook and re-arms the triggers. + items[0] = makeDeferral("def-1", { due: "2026-11-16T09:00:00.000+05:30" }); + assert.equal(yield* service.syncItem(items[0]), "armed"); + const nextDue = Date.parse("2026-11-16T03:30:00.000Z"); + const sql = yield* SqlClient.SqlClient; + const triggers = yield* sql<{ id: string; nextRunAt: number | null }>`SELECT trigger_id AS id, + next_run_at AS "nextRunAt" FROM kin_triggers WHERE org_id = ${orgId} ORDER BY trigger_id`; + assert.deepStrictEqual(triggers, [ + { id: "deferral:def-1:due", nextRunAt: nextDue }, + { id: "deferral:def-1:heads_up", nextRunAt: nextDue - 30 * 60_000 }, + ]); + assert.deepStrictEqual(yield* runOf("def-1"), { state: "armed", dueAt: nextDue }); + const card = yield* cardOf("def-1"); + assert.equal(card.phase, undefined); + assert.equal(card.text, "Deferred to Mon 16 Nov, 9:00. I'll start then."); + }).pipe(openGates(new Map()), Effect.provide(layerFor(items))); + }, +); + +it.effect( + "late due fires: two hours late asks; within the grace, starts after a grace from now", + () => { + const late = makeDeferral("def-1"); + const graced = makeDeferral("def-2", { due: "2026-11-13T10:00+05:30" }); + const gracedDue = Date.parse("2026-11-13T04:30:00.000Z"); + return Effect.gen(function* () { + yield* setup(); + const service = yield* TriggerService.KinTriggerService; + yield* service.syncItem(late); + yield* service.syncItem(graced); + + const wake = gracedDue + 15 * 60_000; + yield* tick(wake); + assert.equal((yield* runOf("def-1"))?.state, "asked"); + const asked = (yield* cardOf("def-1")).text; + assert.include(asked, "it's more than 30 minutes late"); + assert.include(asked, "This was due at 9:00."); + + assert.equal((yield* runOf("def-2"))?.state, "grace"); + const grace = yield* cardOf("def-2"); + assert.equal(grace.phase, "heads_up"); + assert.equal(grace.text, "Starting def-2 at 10:45. This was due at 10:00."); + assert.equal(grace.graceEndsAt, new Date(wake + 30 * 60_000).toISOString()); + assert.lengthOf(yield* starts(), 0); + assert.lengthOf(yield* pendingFires(), 0); + + yield* tick(wake + 30 * 60_000); + assert.equal((yield* runOf("def-2"))?.state, "started"); + const [started] = yield* starts(); + assert.equal(started?.dedupeKey, `deferral:def-2:due:${gracedDue}`); + assert.equal((yield* runOf("def-1"))?.state, "asked"); + }).pipe(openGates(new Map()), Effect.provide(layerFor([late, graced]))); + }, ); + +it.effect("drops queued fires for disarmed or moved deferrals", () => { + const items = [ + makeDeferral("def-1", { on_due: "remind" }), + makeDeferral("def-2", { on_due: "remind" }), + ]; + return Effect.gen(function* () { + yield* setup(); + const sql = yield* SqlClient.SqlClient; + const service = yield* TriggerService.KinTriggerService; + for (const entry of items) yield* service.syncItem(entry); + yield* TestClock.setTime(dueAt); + yield* (yield* TriggerScheduler.KinTriggerScheduler).runDue(); + assert.lengthOf(yield* pendingFires(), 2); + + assert.equal( + yield* service.syncItem(makeDeferral("def-1", { on_due: "remind", status: "done" })), + "disarmed", + ); + const moved = makeDeferral("def-2", { on_due: "remind", due: "2026-11-14T09:00+05:30" }); + items[1] = moved; + assert.equal(yield* service.syncItem(moved), "armed"); + yield* (yield* DeferralFlow.KinDeferralFlow).drain(orgId); + assert.lengthOf(yield* pendingFires(), 0); + assert.equal((yield* runOf("def-1"))?.state, "disarmed"); + assert.equal((yield* runOf("def-2"))?.state, "armed"); + const reminders = () => + sql<{ body: string }>`SELECT body FROM kin_messages WHERE org_id = ${orgId} + AND body LIKE 'Reminder:%' ORDER BY seq`; + assert.lengthOf(yield* reminders(), 0); + + yield* tick(Date.parse("2026-11-14T03:30:00.000Z")); + assert.deepStrictEqual(yield* reminders(), [{ body: "Reminder: def-2." }]); + }).pipe(Effect.provide(layerFor(items))); +}); + +it.effect("remind, ask and chase act at due, noting lateness", () => { + const remind = makeDeferral("def-r", { on_due: "remind" }); + const ask = makeDeferral("def-a", { on_due: "ask" }); + const chase = makeDeferral("def-c", { on_due: "chase", owner: "owner" }); + return Effect.gen(function* () { + yield* setup(); + const sql = yield* SqlClient.SqlClient; + const service = yield* TriggerService.KinTriggerService; + for (const entry of [remind, ask, chase]) yield* service.syncItem(entry); + yield* tick(dueAt + 20 * 60_000); + + const messages = yield* sql<{ body: string }>`SELECT body FROM kin_messages + WHERE org_id = ${orgId} AND kind = 'system' ORDER BY seq`; + assert.isTrue(messages.some(({ body }) => body === "Reminder: def-r. This was due at 9:00.")); + assert.isTrue( + messages.some(({ body }) => + body.includes("def-c was due Fri 13 Nov, 9:00. What's the status?"), + ), + ); + assert.equal((yield* runOf("def-r"))?.state, "reminded"); + assert.equal((yield* runOf("def-a"))?.state, "asked"); + const asked = yield* cardOf("def-a"); + assert.equal(asked.phase, "asked"); + assert.include(asked.text, "This was due at 9:00."); + assert.equal((yield* runOf("def-c"))?.state, "chased"); + const ledger = + yield* sql`SELECT item_id FROM kin_ping_ledger WHERE org_id = ${orgId} ORDER BY item_id`; + assert.lengthOf(ledger, 1); + assert.lengthOf(yield* starts(), 0); + }).pipe(Effect.provide(layerFor([remind, ask, chase]))); +}); diff --git a/apps/server/src/kin/triggers/deferrals.ts b/apps/server/src/kin/triggers/deferrals.ts index a90798dab3ef9..c626ebf35b60c 100644 --- a/apps/server/src/kin/triggers/deferrals.ts +++ b/apps/server/src/kin/triggers/deferrals.ts @@ -1,27 +1,41 @@ import { + KinAgentId, + type KinAuthor, KinChatId, KinDeferralCard, + KinEvent, KinEventId, KinMemoryItemId, + KinMessageId, KinOrgId, KinTriggerId, + KinWidgetId, type MemoryItemHead, } from "@t3tools/contracts"; +import * as Clock from "effect/Clock"; import * as Context from "effect/Context"; +import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; import * as SqlClient from "effect/sql/SqlClient"; import { stringify } from "yaml"; +import { KinChatHub } from "../chat/KinChatHub.ts"; import * as Repository from "../chat/KinChatRepository.ts"; +import * as KinEventQueue from "../lead/KinEventQueue.ts"; import * as Memory from "../memory/Memory.ts"; +import * as MemoryCache from "../memory/MemoryCache.ts"; import * as MemoryWriter from "../memory/MemoryWriter.ts"; import { memoryDateMillis } from "../memory/frontMatter.ts"; import { deferralCardActions } from "../widgets/deferralCard.ts"; +import { GithubLinkState, githubRef } from "./deferralGithub.ts"; import { verifyOwnerTap } from "./ownerCheck.ts"; -const HEADS_UP_MS = 30 * 60_000; +export const HEADS_UP_MS = 30 * 60_000; +export const GRACE_MS = 30 * 60_000; +export const LATE_NOTE_MS = 10 * 60_000; export type OnDue = KinDeferralCard["onDue"]; export type DeferralPhase = "heads_up" | "due"; @@ -35,6 +49,9 @@ export interface DeferralPlan { export const deferralTriggerId = (itemId: string, phase: DeferralPhase) => KinTriggerId.make(`deferral:${itemId}:${phase}`); export const deferralWidgetId = (orgId: string, itemId: string) => `deferral:${orgId}:${itemId}`; +/** The due fire's id: starting is keyed by it, so taps and replays share one start. */ +const deferralStartKey = (itemId: string, dueAt: number) => + `${deferralTriggerId(itemId, "due")}:${dueAt}`; // A deferral without on_due only reminds: silence never starts work. Pre-epoch run // times would fail every scheduler tick, so they leave the item unarmed. @@ -59,18 +76,66 @@ export function planDeferral( return { armed, dueAt, onDue, phases }; } -/** The item half of the recheck baseline; TRIG-8 adds the GitHub half. */ +/** The recheck baseline. `github` is null when GitHub couldn't be read at arm time. */ export const DeferralSnapshot = Schema.Struct({ recordedAt: Schema.String, supersededBy: Schema.NullOr(KinMemoryItemId), links: Schema.Array(Schema.String), + github: Schema.optional(Schema.NullOr(Schema.Array(GithubLinkState))), }); -export const deferralSnapshot = (item: MemoryItemHead): typeof DeferralSnapshot.Type => ({ +export type DeferralSnapshot = typeof DeferralSnapshot.Type; +export const deferralSnapshot = ( + item: MemoryItemHead, + github: ReadonlyArray | null, +): DeferralSnapshot => ({ recordedAt: item.recorded_at, supersededBy: item.superseded_by, links: item.links, + github, }); +const label = (link: string) => { + const ref = githubRef(link); + return ref === undefined ? link : `${ref.repo}#${ref.number}`; +}; + +/** What changed since arm time; empty means unchanged. */ +export function snapshotChanges( + before: DeferralSnapshot, + item: MemoryItemHead | undefined, + github: ReadonlyArray | null, +): ReadonlyArray { + if (item === undefined) return ["the item is gone"]; + const changes: Array = []; + if (item.status !== "open") changes.push(`the item is ${item.status}`); + if (item.invalid_at !== null) changes.push("the item was invalidated"); + if (item.superseded_by !== before.supersededBy) + changes.push(`the item was superseded by ${item.superseded_by}`); + if (item.recorded_at !== before.recordedAt) changes.push("the item was edited"); + const sameLinks = + item.links.length === before.links.length && + item.links.every((link) => before.links.includes(link)); + if (!sameLinks) changes.push("its links changed"); + if (before.github === undefined || before.github === null) + changes.push("changed: couldn't verify GitHub when it was deferred"); + else if (github === null) changes.push("changed: couldn't verify GitHub"); + else + for (const previous of before.github) { + const current = github.find((entry) => entry.link === previous.link); + const name = label(previous.link); + if (current === undefined) continue; + if (current.state !== previous.state) changes.push(`${name} is ${current.state}`); + for (const person of current.assignees) + if (!previous.assignees.includes(person)) changes.push(`${person} was assigned to ${name}`); + for (const person of previous.assignees) + if (!current.assignees.includes(person)) + changes.push(`${person} was unassigned from ${name}`); + for (const pr of current.linkedPrs) + if (!previous.linkedPrs.includes(pr)) changes.push(`PR #${pr} was linked to ${name}`); + } + return changes; +} + const then: Record = { start: "I'll start then.", remind: "I'll remind you then.", @@ -78,23 +143,196 @@ const then: Record = { chase: "I'll chase it then.", }; -export function confirmText(dueAt: number, onDue: OnDue, timeZone: string): string { - const parts = Object.fromEntries( +const parts = (at: number, timeZone: string) => + Object.fromEntries( new Intl.DateTimeFormat("en-GB", { timeZone, weekday: "short", day: "numeric", month: "short", + year: "numeric", hour: "numeric", minute: "2-digit", hourCycle: "h23", }) - .formatToParts(dueAt) + .formatToParts(at) .map((part) => [part.type, part.value]), ); - return `Deferred to ${parts.weekday} ${parts.day} ${parts.month}, ${Number(parts.hour)}:${parts.minute}. ${then[onDue]}`; + +export function confirmText(dueAt: number, onDue: OnDue, timeZone: string): string { + return `Deferred to ${ownerDateTime(dueAt, timeZone)}. ${then[onDue]}`; +} + +/** Owner-local "Fri 13 Nov, 9:00". */ +export function ownerDateTime(at: number, timeZone: string): string { + const p = parts(at, timeZone); + return `${p.weekday} ${p.day} ${p.month}, ${Number(p.hour)}:${p.minute}`; +} + +/** Owner-local "9:00", with the day when it isn't today. */ +export function ownerTime(at: number, now: number, timeZone: string): string { + const p = parts(at, timeZone); + const today = parts(now, timeZone); + return p.day === today.day && p.month === today.month && p.year === today.year + ? `${Number(p.hour)}:${p.minute}` + : ownerDateTime(at, timeZone); +} + +export const lateNote = (dueAt: number, now: number, timeZone: string) => + now - dueAt > LATE_NOTE_MS ? ` This was due at ${ownerTime(dueAt, now, timeZone)}.` : ""; + +/** Next Monday 09:00 owner time, strictly after today, as a front-matter date-time. */ +function nextMondayDue(now: number, timeZone: string): string { + const today = DateTime.makeZonedUnsafe(now, { timeZone }); + const weekDay = DateTime.toParts(today).weekDay; + const ahead = (8 - weekDay) % 7 || 7; + const monday = DateTime.setParts(DateTime.add(today, { days: ahead }), { + hour: 9, + minute: 0, + second: 0, + millisecond: 0, + }); + return DateTime.formatIsoOffset(monday); } +// LEAD-11's stop switch isn't built; until it is every org reads as stopped, so +// silence never starts work. +export const KinDeferralStopSwitch = Context.Reference<{ + readonly isStopped: (orgId: KinOrgId) => Effect.Effect; +}>("t3/kin/triggers/deferrals/KinDeferralStopSwitch", { + defaultValue: () => ({ isStopped: () => Effect.succeed(true) }), +}); + +const iso = (ms: number) => DateTime.formatIso(DateTime.makeUnsafe(ms)); +const decodeEvent = Schema.decodeUnknownEffect(KinEvent); + +/** Code posts deferral cards, never the model. */ +export const makeDeferralCards = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const repo = yield* Repository.KinChatRepository; + const hub = yield* Effect.serviceOption(KinChatHub); + const publish = (event: Parameters[0]) => + Option.isSome(hub) ? sql.afterCommit(hub.value.publish(event)) : Effect.void; + + /** Updates the item's card in place, or posts it to `chatId`. */ + const write = Effect.fn("Deferrals.writeCard")(function* (input: { + readonly orgId: KinOrgId; + readonly itemId: KinMemoryItemId; + readonly state: KinDeferralCard; + readonly chatId: KinChatId | null; + readonly now: number; + }) { + const { orgId, state } = input; + const widgetId = KinWidgetId.make(deferralWidgetId(orgId, input.itemId)); + const existing = yield* repo.getWidget({ orgId, widgetId }); + const at = iso(input.now); + if (Option.isSome(existing)) { + const widget = yield* repo.updateWidget({ + ...existing.value, + expectedRevision: existing.value.revision, + state, + closedAt: null, + updatedAt: at, + }); + yield* publish({ type: "widget.updated", orgId, widget }); + return true; + } + if (input.chatId === null) return false; + const message = yield* repo.insertMessage({ + orgId, + chatId: input.chatId, + messageId: KinMessageId.make(widgetId), + author: { _tag: "system" }, + kind: "system", + body: state.text, + intent: null, + mentions: [], + replyTo: null, + widgetId: null, + floorRunId: null, + status: "posted", + sentBy: null, + clientMessageId: null, + createdAt: at, + editedAt: null, + }); + const widget = { + orgId, + chatId: input.chatId, + messageId: message.messageId, + widgetId, + widgetClass: "native" as const, + type: "deferral", + state, + source: null, + feed: null, + awaiting: [], + pinnedAt: null, + closedAt: null, + updatedAt: at, + }; + yield* repo.insertWidget(widget); + yield* publish({ type: "message.posted", orgId, message }); + yield* publish({ type: "widget.updated", orgId, widget: { ...widget, revision: 1 } }); + return true; + }); + return { write, publish }; +}); + +const hex = (value: string) => + Array.from(new TextEncoder().encode(value), (byte) => byte.toString(16).padStart(2, "0")).join( + "", + ); + +export const makeDeferralStarter = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const queue = yield* KinEventQueue.KinEventQueue; + const cards = yield* makeDeferralCards; + /** + * Marks the run started and hands the work to the lead as `deferral.due`, inside the + * caller's transaction. The event dedupes on the due fire id, so a replay, a late fire + * and a [Start now] tap share one start. + */ + return Effect.fn("Deferrals.start")(function* (input: { + readonly orgId: KinOrgId; + readonly itemId: KinMemoryItemId; + readonly dueAt: number; + readonly card: KinDeferralCard; + readonly provenance: KinAuthor; + readonly now: number; + }) { + const { orgId, itemId, dueAt, now } = input; + const [lead] = yield* sql<{ agentId: string }>`SELECT l.agent_id AS "agentId" FROM kin_leads l + JOIN kin_agents a ON a.org_id = l.org_id AND a.agent_id = l.agent_id + WHERE l.org_id = ${orgId} AND a.retired_at IS NULL ORDER BY l.agent_id LIMIT 1`; + if (lead === undefined) return false; + const dedupeKey = deferralStartKey(itemId, dueAt); + yield* queue.enqueue( + yield* decodeEvent({ + _tag: "deferral.due", + eventId: KinEventId.make(`deferral_${hex(`${orgId.length}:${orgId}${dedupeKey}`)}`), + orgId, + agentId: KinAgentId.make(lead.agentId), + receivedAt: iso(now), + provenance: input.provenance, + dedupeKey, + subject: { type: "deferral", itemId, phase: "due" }, + lateByMs: Math.max(0, now - dueAt), + }), + ); + yield* sql`UPDATE kin_deferral_runs SET state = 'started', updated_at = ${now} + WHERE org_id = ${orgId} AND item_id = ${itemId} AND due_at = ${dueAt}`; + yield* cards.write({ + orgId, + itemId, + state: { ...input.card, text: `Started ${input.card.title}.`, phase: "started" }, + chatId: null, + now, + }); + return true; + }); +}); + export class DeferralError extends Schema.TaggedError()("DeferralError", { orgId: KinOrgId, cause: Schema.Defect(), @@ -104,11 +342,21 @@ const decodeCard = Schema.decodeUnknownEffect(KinDeferralCard); const decodeAction = Schema.decodeUnknownEffect( Schema.Struct({ actionId: Schema.Literals(deferralCardActions), value: KinMemoryItemId }), ); +type DeferralAction = (typeof deferralCardActions)[number]; +const actionsFor = (phase: KinDeferralCard["phase"]): ReadonlyArray => + phase === undefined + ? ["change", "remind"] + : phase === "started" + ? [] + : ["start_now", "monday", "cancel"]; export class Deferrals extends Context.Service< Deferrals, { - /** Applies a verified owner tap on a deferral card. [Change] hands the item to the lead. */ + /** + * Applies a verified owner tap on a deferral card. [Change] hands the item to the lead; + * [Just remind me], [Monday] and [Cancel] stage memory writes; [Start now] starts. + */ readonly act: ( orgId: KinOrgId, wakeupId: string, @@ -117,7 +365,8 @@ export class Deferrals extends Context.Service< { readonly chatId: KinChatId; readonly itemId: KinMemoryItemId; - readonly action: (typeof deferralCardActions)[number]; + readonly action: DeferralAction; + readonly due?: string; }, DeferralError, MemoryWriter.MemoryWriter @@ -127,8 +376,10 @@ export class Deferrals extends Context.Service< const make = Effect.gen(function* () { const memory = yield* Memory.Memory; + const cache = yield* MemoryCache.MemoryCache; const sql = yield* SqlClient.SqlClient; const repo = yield* Repository.KinChatRepository; + const start = yield* makeDeferralStarter; return Deferrals.of({ act: Effect.fn("Deferrals.act")((orgId, wakeupId, eventId) => Effect.gen(function* () { @@ -144,23 +395,64 @@ const make = Effect.gen(function* () { tap.widget.widgetId !== deferralWidgetId(orgId, card.itemId) ) return yield* Effect.fail("unknown_item" as const); - if (action.actionId === "remind") { - const item = yield* memory.readItemQuietly(orgId, card.itemId); - if (item.head.kind !== "deferral" || item.head.status !== "open") - return yield* Effect.fail("not_open" as const); - if (item.head.on_due !== "remind") { - const { orgId: _org, path, ...frontMatter } = item.head; - yield* writer.stage(orgId, wakeupId, { - path, - content: `---\n${stringify({ ...frontMatter, on_due: "remind" })}---\n${item.body}`, - sourceEventId: eventId, - }); + if (!actionsFor(card.phase).includes(action.actionId)) + return yield* Effect.fail("stale_card" as const); + const result = { chatId: tap.chatId, itemId: card.itemId, action: action.actionId }; + if (action.actionId === "change") return result; + const item = yield* memory.readItemQuietly(orgId, card.itemId); + if (item.head.kind !== "deferral" || item.head.status !== "open") + return yield* Effect.fail("not_open" as const); + const { orgId: _org, path, ...frontMatter } = item.head; + const stage = (fields: Partial) => + writer.stage(orgId, wakeupId, { + path, + content: `---\n${stringify({ ...frontMatter, ...fields })}---\n${item.body}`, + sourceEventId: eventId, + }); + switch (action.actionId) { + case "remind": + if (item.head.on_due !== "remind") yield* stage({ on_due: "remind" }); + return result; + case "cancel": + yield* stage({ status: "cancelled" }); + return result; + case "monday": { + const due = nextMondayDue( + yield* Clock.currentTimeMillis, + yield* cache.ownerTimeZone(orgId), + ); + yield* stage({ due }); + return { ...result, due }; + } + case "start_now": { + const dueAt = Date.parse(card.dueAt); + if (yield* (yield* KinDeferralStopSwitch).isStopped(orgId)) + return yield* Effect.fail("stopped" as const); + const now = yield* Clock.currentTimeMillis; + const started = yield* sql.withTransaction( + Effect.gen(function* () { + const [run] = yield* sql<{ state: string }>`SELECT state FROM kin_deferral_runs + WHERE org_id = ${orgId} AND item_id = ${card.itemId} AND due_at = ${dueAt} + ORDER BY item_id`; + if (run === undefined || !["heads_up", "asked", "grace"].includes(run.state)) + return false; + return yield* start({ + orgId, + itemId: card.itemId, + dueAt, + card, + provenance: tap.provenance, + now, + }); + }), + ); + if (!started) return yield* Effect.fail("not_pending" as const); + return result; } } - return { chatId: tap.chatId, itemId: card.itemId, action: action.actionId }; }).pipe(Effect.mapError((cause) => new DeferralError({ orgId, cause }))), ), }); }); -export const layer = Layer.effect(Deferrals, make); +export const layer = Layer.effect(Deferrals, make).pipe(Layer.provide(MemoryCache.layer)); diff --git a/apps/server/src/kin/triggers/ownerCheck.ts b/apps/server/src/kin/triggers/ownerCheck.ts index d554c4d7d843b..0974e9c19bad9 100644 --- a/apps/server/src/kin/triggers/ownerCheck.ts +++ b/apps/server/src/kin/triggers/ownerCheck.ts @@ -153,7 +153,7 @@ export const verifyOwnerTap = Effect.fn("OwnerCheck.verifyOwnerTap")(function* ( event.actionId !== action.actionId ) return yield* Effect.fail("invalid_card" as const); - return { action, chatId: event.chatId, widget: widget.value }; + return { action, chatId: event.chatId, widget: widget.value, provenance: event.provenance }; }), ) .pipe( diff --git a/apps/server/src/kin/widgets/deferralCard.ts b/apps/server/src/kin/widgets/deferralCard.ts index c0e644b6879a4..efc3a7aea3472 100644 --- a/apps/server/src/kin/widgets/deferralCard.ts +++ b/apps/server/src/kin/widgets/deferralCard.ts @@ -1,4 +1,4 @@ // Taps need the owner-tap dispatcher that also gates the owner check; see the // deferral card release gate in docs/kin/specs/triggers.md. export const deferralCardActionsReady = false; -export const deferralCardActions = ["change", "remind"] as const; +export const deferralCardActions = ["change", "remind", "start_now", "monday", "cancel"] as const; diff --git a/docs/kin/specs/triggers.md b/docs/kin/specs/triggers.md index 29f8df10d8b52..dc26caafd6bc2 100644 --- a/docs/kin/specs/triggers.md +++ b/docs/kin/specs/triggers.md @@ -205,7 +205,7 @@ Arming comes from memory. When the front-matter cache (LEAD) sees a `deferrals/* Item edits move or disarm the triggers: a new `due` re-arms; `status` other than `open` or a set `invalid_at` disables them. A due time already past at save fires at once, marked late; no "pick a future time" error. A missing `on_due` means `remind`, so silence never starts work. -The cache hook is `MemorySyncHooks` in [`Memory.ts`](../../../apps/server/src/kin/memory/Memory.ts): each new head goes once to `KinTriggerService.syncOrg`, which also disarms armed deferrals missing from the snapshot or sharing a duplicate id. A failing item is logged without blocking the others, and the head retries on the next two loads, then waits for the next head. The deferral `trigger_reconcile` described below is not built yet, so until it is, a new head is the only recovery. A due time whose fires would land before 1970 never arms; `kin_defer` accepts only years 2000–2099. A commit to the default branch moves the cache to the new head and reloads it, so a `kin_defer` item arms when its wake-up commits, not at the next wake-up; shadow (L0) commits never arm. Trigger ids are `deferral::heads_up|due`. Re-syncing an unchanged `due` writes nothing, so a fired `once` is never re-armed. The run row in `kin_deferral_runs` (migration K016) holds the snapshot and an `armed`/`disarmed` state; TRIG-8 owns the due-flow states. The snapshot holds the item half only (`recorded_at`, `superseded_by`, links). There is no GitHub reader in Kin yet, so TRIG-8 must add the GitHub half (issue/PR state, assignees, linked PRs) at arm time before the due flow ships. +The cache hook is `MemorySyncHooks` in [`Memory.ts`](../../../apps/server/src/kin/memory/Memory.ts): each new head goes once to `KinTriggerService.syncOrg`, which also disarms armed deferrals missing from the snapshot or sharing a duplicate id. A failing item is logged without blocking the others, and the head retries on the next two loads, then waits for the next head. The deferral `trigger_reconcile` described below is not built yet, so until it is, a new head is the only recovery. A due time whose fires would land before 1970 never arms; `kin_defer` accepts only years 2000–2099. A commit to the default branch moves the cache to the new head and reloads it, so a `kin_defer` item arms when its wake-up commits, not at the next wake-up; shadow (L0) commits never arm. Trigger ids are `deferral::heads_up|due`. Re-syncing an unchanged `due` writes nothing, so a fired `once` is never re-armed. The run row in `kin_deferral_runs` (migration K016) holds the snapshot and the flow state (below). The snapshot holds the item half (`recorded_at`, `superseded_by`, links) and the GitHub half (state, assignees and linked PRs of each `github.com///issues|pull/` or `/#` link), read at arm time through `KinDeferralGithubReader` in [`deferralGithub.ts`](../../../apps/server/src/kin/triggers/deferralGithub.ts). The GitHub half is `null` when any link couldn't be read. That reader is a narrow read-only seam (one issue or PR per call) for #44's REST client to back; until then its default fails with `unavailable`. **Confirm.** When the lead calls `kin_defer` (LEAD), Kin writes the item, arms the triggers, and the lead posts a `KinDeferralCard` widget (CHAT envelope): "Deferred to Fri 13 Nov, 9:00. I'll start then." [Change] [Just remind me]. [Just remind me] sets `on_due: remind`, and the triggers re-arm. [Change] writes nothing: it hands the item to the lead, which asks the owner what to change, marks the old item `cancelled` and calls `kin_defer` for the new one. @@ -214,8 +214,8 @@ Code posts the card, never the model. It goes to the item's `chat` when that cha **Deferral card release gate (TRIG-7):** cards post live, but `KinChatService.act` refuses deferral-card taps (`deferralCardActionsReady = false`) until the owner-tap dispatcher from the owner-check gate exists. `Deferrals.act` already verifies the tap with the same `verifyOwnerTap` as the owner check. Before enabling taps: - the owner-check gate's dispatcher steps must land (chat act enqueues `widget.action`; `WakeupRunner` intercepts card actions without showing them to the model); -- `WakeupRunner` must run `Deferrals.act` for `deferral` actions, then `MemoryWriter.commitWakeup` (the commit re-arms through the cache hook), then queue `finish`. [Change] goes to the lead as an owner message about the item; -- production `MemoryWriterAuthority.resolve` must grant `kind: "deferral_card"` (item id, action `remind`) only after verifying the claimed owner tap. With that grant, the writer lets `on_due` change, and only to `remind`; +- `WakeupRunner` must run `Deferrals.act` for `deferral` actions, then `MemoryWriter.commitWakeup` (the commit re-arms through the cache hook), then queue `finish`. [Change] goes to the lead as an owner message about the item. `act` accepts only the current card's actions: [Change] and [Just remind me] on the confirm card, [Start now], [Monday] and [Cancel] on heads-up and asked cards; +- production `MemoryWriterAuthority.resolve` must grant `kind: "deferral_card"` only after verifying the claimed owner tap: action `remind` lets `on_due` change, and only to `remind`; action `monday` carries the `due` that `act` returned and lets `due` change only to that value. [Cancel] sets `status: cancelled`, which any owner write may do; - web and mobile must render the native card and its closed state. **Flow for `start`** (`kin_deferral_runs.state`): @@ -233,11 +233,13 @@ armed ──heads_up fire──▶ recheck ──changed──▶ asked (decis started ─▶ worker via delegate_task (LEAD) ─▶ closed with PR and proof ``` -- **The recheck is code.** It runs `gh api` reads for every GitHub link plus a read of the memory item. Any difference from the snapshot means "changed": an issue closed, an assignee added, another linked PR, or the item superseded. The ask then names what changed. -- **Late.** If the machine woke after `due`, the `heads_up` fire is skipped. A `due` fire within the 30-minute grace window posts the same card with "This was due at 9:00" and a grace from now; silence still means go. An older fire asks instead of starting. -- **Stale fires.** Fires wait in the inbox until TRIG-8 consumes them. TRIG-8 drops a fire whose run row is `disarmed` or whose `due_at` no longer equals the fire's due instant. -- **Level and stop switch.** Below L2, or with the org's stop switch on, `start` becomes `ask`. Silence never starts work then. -- **Idempotent start.** Starting is a Work-class action keyed by `fireId`, so a replay can't start two workers. +- **Consumer.** `KinDeferralFlow` in [`deferralFlow.ts`](../../../apps/server/src/kin/triggers/deferralFlow.ts) drains each org's pending deferral fires on its own `Scheduler` tick, in due order. Like routines, it does read-only or idempotent I/O first (memory read, GitHub reads, gate checks, the chase's `KinPings.put`), then one transaction that rechecks the run row, writes the state and card, enqueues, and marks the fire consumed. Decode failures, and fires more than 24 hours past due, are consumed; other failures retry on the next tick. Code posts every card: the item's card is updated in place, or posted to the item's chat, else the owner's direct chat. +- **The recheck is code.** It reads the memory item and every GitHub link through `KinDeferralGithubReader`. Any difference from the snapshot means "changed": an issue closed, an assignee added or removed, another linked PR, or the item edited or superseded. A GitHub half that couldn't be read, at arm time or now, counts as "changed: couldn't verify". The ask names what changed. +- **Late.** If `due` has passed when the `heads_up` fire is consumed, it is skipped. A `due` fire more than 10 minutes late notes "This was due at 9:00". A `start` due fire within the 30-minute grace window, or one with no heads-up shown, posts the heads-up card again with a grace from now: the `due` trigger re-arms at the grace end (state `grace`, `graceEndsAt` on the card), and silence still means go. An older fire asks instead of starting. `remind`, `ask` and `chase` act at due only and carry the same note. +- **Stale fires.** Fires wait in the inbox until the consumer handles them. It consumes, and otherwise ignores, a fire whose run row is missing or `disarmed`, whose `due_at` no longer equals the fire's due instant (a grace fire is the one exception), or whose run already left the state that fire expects. +- **Gates.** `start` becomes `ask` below L2, with the org's stop switch on, while `workerLaunchReady` is false, or with no active lead. LEAD-11 isn't built, so `KinDeferralStopSwitch` defaults to stopped. With today's defaults, silence never starts work. [Start now] is an owner decision, so it ignores the level but still refuses while stopped. +- **Idempotent start.** Starting sets `started` and enqueues `deferral.due` (phase `due`) for the lead to delegate, deduplicated by the due fire's id `deferral::due:`. A replay, a grace fire and a [Start now] tap share that key, so they can't start two workers. +- **States.** `armed`, `heads_up`, `grace`, `asked`, `started` (`start`); `reminded`, `asked`, `chased` (`remind`, `ask`, `chase`); `disarmed`. A re-arm from memory returns the run to `armed`. - **Bring it up early.** When a message touches the topic before the due time, the lead mentions it (LEAD, using the cache). The Monday morning brief lists deferrals due this week. **Follow-ups (deferred to TRIG-8/10; not implemented by TRIG-11).** A ping with no answer creates a follow-up trigger (`followup`, `once`, next working morning at the owner's brief time). That is the "follow-up = a deferral created by a ping" row of section 6. @@ -477,7 +479,7 @@ Kin runs on one always-on machine, the **Kin host**. The laptop and phone are cl - `packages/contracts/src/kinTriggers.ts` - `apps/server/src/kin/triggers/KinTriggerStore.ts`, `KinTriggerScheduler.ts`, `KinTriggerService.ts`, `KinTriggerSink.ts`, `KinTriggerInbox.ts` -- `apps/server/src/kin/triggers/githubWebhooks.ts`, `githubEvents.ts`, `deferrals.ts`, `routines.ts`, `watchdogs.ts`, `pings.ts`, `ownerCheck.ts` +- `apps/server/src/kin/triggers/githubWebhooks.ts`, `githubEvents.ts`, `deferrals.ts`, `deferralFlow.ts`, `deferralGithub.ts`, `routines.ts`, `watchdogs.ts`, `pings.ts`, `ownerCheck.ts` - `apps/server/src/kin/memory/freshness.ts`, `buildIndex.ts`, `secretScan.ts`, `purge.ts` - `apps/server/src/kin/dreaming/DreamRunner.ts`, `DreamSession.ts`, `material.ts`, `checkDream.ts`, `classifyDream.ts` - `apps/server/src/kin/contract/*.kin-contract.test.ts`, `apps/mobile/src/features/chats/contract/mcpAppWebView.kin-contract.test.ts` diff --git a/packages/contracts/src/kinLead.ts b/packages/contracts/src/kinLead.ts index c3594ec11f9d0..a7afdf0a92baf 100644 --- a/packages/contracts/src/kinLead.ts +++ b/packages/contracts/src/kinLead.ts @@ -148,6 +148,10 @@ export const KinDeferralCard = Schema.Struct({ dueAt: Schema.toEncoded(CanonicalUtcTimestamp), onDue: Schema.Literals(["remind", "start", "ask", "chase"]), text: Schema.String, + // Absent on the confirm card ([Change] [Just remind me]); heads_up and asked offer + // [Start now] [Monday] [Cancel]; started offers nothing. + phase: Schema.optional(Schema.Literals(["heads_up", "asked", "started"])), + graceEndsAt: Schema.optional(Schema.toEncoded(CanonicalUtcTimestamp)), }); export type KinDeferralCard = typeof KinDeferralCard.Type; From 35c6081579fd7c721f823311e0f8263ea4bd99ba Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Sun, 8 Nov 2026 21:48:45 +0530 Subject: [PATCH 2/7] test(kin): use DateTime in deferral flow tests Co-Authored-By: Claude Opus 5.5 --- apps/server/src/kin/triggers/deferrals.test.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/apps/server/src/kin/triggers/deferrals.test.ts b/apps/server/src/kin/triggers/deferrals.test.ts index 618494f76185c..a7cc31243e963 100644 --- a/apps/server/src/kin/triggers/deferrals.test.ts +++ b/apps/server/src/kin/triggers/deferrals.test.ts @@ -13,6 +13,7 @@ import { type MemoryItemHead, } from "@t3tools/contracts"; import * as Clock from "effect/Clock"; +import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; @@ -140,7 +141,7 @@ const tapOnce = Effect.fn("test.deferralTap")(function* ( onNone: () => 1, onSome: (widget) => widget.revision, }); - const at = new Date(yield* Clock.currentTimeMillis).toISOString(); + const at = DateTime.formatIso(DateTime.makeUnsafe(yield* Clock.currentTimeMillis)); // Stands in for the gated dispatcher: chat act refuses deferral taps today. const message = yield* repo.insertMessage({ orgId, @@ -295,8 +296,8 @@ it.effect("unchanged recheck, heads-up, silence, then starts once at due", () => }); it.effect("asks instead of starting when a linked issue closes before due", () => { - const github = new Map([ - ["acme/api#12", { state: "open" as const, assignees: [], linkedPrs: [] }], + const github = new Map>([ + ["acme/api#12", { state: "open", assignees: [], linkedPrs: [] }], ]); const linked = makeDeferral("def-1", { links: [issue] }); return Effect.gen(function* () { @@ -430,7 +431,7 @@ it.effect( const grace = yield* cardOf("def-2"); assert.equal(grace.phase, "heads_up"); assert.equal(grace.text, "Starting def-2 at 10:45. This was due at 10:00."); - assert.equal(grace.graceEndsAt, new Date(wake + 30 * 60_000).toISOString()); + assert.equal(grace.graceEndsAt, DateTime.formatIso(DateTime.makeUnsafe(wake + 30 * 60_000))); assert.lengthOf(yield* starts(), 0); assert.lengthOf(yield* pendingFires(), 0); From b5c6187cb254dac6b0acaf7acf7b5ace3c5a8d8a Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Sun, 8 Nov 2026 23:08:05 +0530 Subject: [PATCH 3/7] fix(kin): deferral chases always ping, grace and start stay single - Put the chase's ping inside the flow transaction and sweep each org's pings on every kin-deferrals tick, so quiet hours or a crash can't lose a chase. - Accept a grace fire only at the grace end recorded in the run snapshot. - [Start now] refuses while stopped, while workers can't launch, or below L2. - deferral.due takes its time fields from the fire; a same-key start from another path counts as already started. - syncItem reads GitHub only when the run will be (re-)armed. - Tell the owner before consuming a fire that keeps failing. Co-Authored-By: Claude Opus 5.5 --- .../kin/triggers/KinTriggerService.test.ts | 26 ++++- .../src/kin/triggers/KinTriggerService.ts | 3 + apps/server/src/kin/triggers/deferralFlow.ts | 106 +++++++++++------ .../server/src/kin/triggers/deferrals.test.ts | 107 ++++++++++++++++-- apps/server/src/kin/triggers/deferrals.ts | 56 ++++++--- apps/server/src/kin/triggers/pings.ts | 5 +- docs/kin/specs/triggers.md | 12 +- 7 files changed, 242 insertions(+), 73 deletions(-) diff --git a/apps/server/src/kin/triggers/KinTriggerService.test.ts b/apps/server/src/kin/triggers/KinTriggerService.test.ts index 07fe4e79dee00..f75ecc9f8da2e 100644 --- a/apps/server/src/kin/triggers/KinTriggerService.test.ts +++ b/apps/server/src/kin/triggers/KinTriggerService.test.ts @@ -21,6 +21,7 @@ import * as Repository from "../chat/KinChatRepository.ts"; import * as JudgeShadow from "../floor/JudgeShadow.ts"; import * as Memory from "../memory/Memory.ts"; import * as Persistence from "../persistence/KinPersistence.ts"; +import { KinDeferralGithubReader } from "./deferralGithub.ts"; import * as TriggerService from "./KinTriggerService.ts"; import { item, orgId, otherOrg, seed } from "./routines.testkit.ts"; @@ -78,10 +79,25 @@ const card = Effect.fn("test.deferralCard")(function* (org: string) { return rows.map((row) => ({ ...row, state: decodeCard(row.state) })); }); -it.effect("arms from front matter, re-arms on edits, and re-syncs without writes", () => - Effect.gen(function* () { +it.effect("arms from front matter, re-arms on edits, and re-syncs without writes", () => { + let reads = 0; + const reader = { + read: () => + Effect.sync(() => { + reads++; + return { state: "open" as const, assignees: [], linkedPrs: [] }; + }), + }; + const linked = (overrides: Partial = {}) => + deferral({ links: ["https://github.com/acme/api/issues/12"], ...overrides }); + return Effect.gen(function* () { const { sql, service } = yield* setup(); - assert.equal(yield* service.syncItem(deferral()), "armed"); + assert.equal(yield* service.syncItem(linked()), "armed"); + assert.equal(reads, 1); + // Unchanged re-syncs, per item or per head, don't read GitHub. + assert.equal(yield* service.syncItem(linked()), "unchanged"); + yield* service.syncOrg(orgId, [linked()]); + assert.equal(reads, 1); const armed = yield* triggers(orgId); assert.deepEqual( armed.map(({ id, enabled, nextRunAt }) => [id, enabled, nextRunAt]), @@ -148,8 +164,8 @@ it.effect("arms from front matter, re-arms on edits, and re-syncs without writes [0, null], ], ); - }).pipe(Effect.provide(layer)), -); + }).pipe(Effect.provideService(KinDeferralGithubReader, reader), Effect.provide(layer)); +}); it.effect.each([ ["done", { status: "done" }], diff --git a/apps/server/src/kin/triggers/KinTriggerService.ts b/apps/server/src/kin/triggers/KinTriggerService.ts index db11142450c4b..94012874f50d6 100644 --- a/apps/server/src/kin/triggers/KinTriggerService.ts +++ b/apps/server/src/kin/triggers/KinTriggerService.ts @@ -276,6 +276,9 @@ const make = Effect.gen(function* () { const timeZone = yield* cache.ownerTimeZone(item.orgId); const now = yield* Clock.currentTimeMillis; const plan = planDeferral(item); + // GitHub is read only when the run will be (re-)armed, not on every head. + const stored = yield* readStored(item.orgId, item.id).pipe(wrap(item.orgId)); + if (isCurrent(plan, stored.get(item.id))) return "unchanged" as const; const github = plan.armed ? yield* readGithubLinks(item.orgId, item.links) : null; return yield* apply(item.orgId, item.id, item, plan, timeZone, now, github).pipe( wrap(item.orgId), diff --git a/apps/server/src/kin/triggers/deferralFlow.ts b/apps/server/src/kin/triggers/deferralFlow.ts index a968130099f7d..005d3ec04186c 100644 --- a/apps/server/src/kin/triggers/deferralFlow.ts +++ b/apps/server/src/kin/triggers/deferralFlow.ts @@ -36,6 +36,7 @@ import { ownerDateTime, ownerTime, snapshotChanges, + startsAtLevel, makeDeferralCards, makeDeferralStarter, } from "./deferrals.ts"; @@ -50,7 +51,10 @@ export class KinDeferralFlowError extends Schema.TaggedError Effect.Effect; } >()("t3/kin/triggers/deferralFlow/KinDeferralFlow") {} @@ -69,6 +73,7 @@ type Step = /** Which step a fire takes, from its run row. Disarmed, moved and handled runs drop it. */ function classifyFire( run: Run | undefined, + graceEndsAt: number | undefined, phase: "heads_up" | "due", fireAt: number, now: number, @@ -87,13 +92,14 @@ function classifyFire( ? { kind: "due", dueAt, grace: false } : { kind: "drop" }; // A late due fire re-arms the due trigger at the end of its grace window. - return run.onDue === "start" && run.state === "grace" && fireAt > dueAt + return run.onDue === "start" && run.state === "grace" && fireAt === graceEndsAt ? { kind: "due", dueAt, grace: true } : { kind: "drop" }; } const decodeOnDue = Schema.decodeUnknownEffect(KinDeferralCard.fields.onDue); const decodeSnapshot = Schema.decodeUnknownEffect(Schema.fromJsonString(DeferralSnapshot)); +const encodeSnapshot = Schema.encodeEffect(Schema.fromJsonString(DeferralSnapshot)); const isDecodeError = Schema.is(PersistenceDecodeError); const isPermanent = (error: unknown) => { let current = error; @@ -104,7 +110,6 @@ const isPermanent = (error: unknown) => { return false; }; const iso = (ms: number) => DateTime.formatIso(DateTime.makeUnsafe(ms)); -const levels = ["L0", "L1", "L2", "L3"]; const make = Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; @@ -191,8 +196,9 @@ const make = Effect.gen(function* () { ) { const fireAt = Date.parse(fire.dueAt); const run = yield* readRun(orgId, itemId).pipe(NodeSqliteClient.readOnly); - const step = classifyFire(run, phase, fireAt, now); - if (step.kind === "drop" || run === undefined) + const snapshot = run === undefined ? undefined : yield* decodeSnapshot(run.snapshot); + const step = classifyFire(run, snapshot?.graceEndsAt, phase, fireAt, now); + if (step.kind === "drop" || run === undefined || snapshot === undefined) return yield* inbox.markConsumed(orgId, [fire.fireId]); // Read-only or idempotent I/O; a replay repeats it. @@ -201,21 +207,12 @@ const make = Effect.gen(function* () { const decodedOnDue = yield* decodeOnDue(onDue); const gates: Array = []; if (onDue === "start") { - const snapshot = yield* decodeSnapshot(run.snapshot); const github = yield* readGithubLinks(orgId, item?.links ?? snapshot.links); gates.push(...snapshotChanges(snapshot, item, github)); if (yield* (yield* KinDeferralStopSwitch).isStopped(orgId)) gates.push("the stop switch is on"); if (!(yield* WorkerLaunchGate)) gates.push("workers can't launch yet"); } - const chaseFailure = - onDue === "chase" && step.kind === "due" - ? item === undefined - ? "the item is gone" - : yield* prepareChase(orgId, item, step.dueAt, timeZone) - : null; - - let chased = false; yield* sql.withTransaction( Effect.gen(function* () { const pending = yield* sql`SELECT fire_id FROM kin_trigger_inbox WHERE org_id = ${orgId} @@ -224,7 +221,12 @@ const make = Effect.gen(function* () { const reasons = [...gates]; const current = yield* readRun(orgId, itemId); // Moved or handled since the reads above: classify again on the next tick. - if (current?.state !== run.state || current.dueAt !== run.dueAt) return; + if ( + current?.state !== run.state || + current.dueAt !== run.dueAt || + current.snapshot !== run.snapshot + ) + return; const dueAt = step.dueAt; const title = item?.title ?? itemId; const card = (phase: KinDeferralCard["phase"], text: string, graceEndsAt?: number) => ({ @@ -259,8 +261,7 @@ const make = Effect.gen(function* () { JOIN kin_agents a ON a.org_id = l.org_id AND a.agent_id = l.agent_id WHERE l.org_id = ${orgId} AND a.retired_at IS NULL ORDER BY l.agent_id LIMIT 1`; if (lead === undefined) reasons.push("there is no active lead"); - else if (levels.indexOf(lead.level) < levels.indexOf("L2")) - reasons.push(`Kin is at ${lead.level}, below L2`); + else if (!startsAtLevel(lead.level)) reasons.push(`Kin is at ${lead.level}, below L2`); } if (step.kind === "heads_up") { @@ -290,6 +291,9 @@ const make = Effect.gen(function* () { updated_at = max(updated_at, ${now}) WHERE org_id = ${orgId} AND trigger_id = ${deferralTriggerId(itemId, "due")} AND enabled = 1`; + const graced = yield* encodeSnapshot({ ...snapshot, graceEndsAt: graceEnd }); + yield* sql`UPDATE kin_deferral_runs SET snapshot_json = ${graced} + WHERE org_id = ${orgId} AND item_id = ${itemId} AND due_at = ${dueAt}`; yield* setState("grace"); } else { const started = yield* start({ @@ -298,17 +302,20 @@ const make = Effect.gen(function* () { dueAt, card: card("started", ""), provenance: { _tag: "system" }, + at: Date.parse(fire.firedAt), now, }); if (!started) yield* ask(["there is no active lead"]); } } else if (onDue === "ask") yield* ask([]); else if (onDue === "chase") { - if (chaseFailure !== null) yield* ask([chaseFailure]); - else { - yield* setState("chased"); - chased = true; - } + // Inside the transaction, so a chased run always has its ping row. + const failure = + item === undefined + ? "the item is gone" + : yield* prepareChase(orgId, item, dueAt, timeZone); + if (failure !== null) yield* ask([failure]); + else yield* setState("chased"); } else { const chatId = yield* cardChat(orgId, item); if (chatId !== null) @@ -323,18 +330,31 @@ const make = Effect.gen(function* () { yield* inbox.markConsumed(orgId, [fire.fireId]); }), ); - if (chased) - yield* pings - .sweep(orgId) - .pipe( - Effect.catch((cause) => - Effect.logWarning("Kin deferral chase sweep failed", { orgId, cause }), - ), - ); }); - const drain = Effect.fn("KinDeferralFlow.drain")((orgId: KinOrgId) => - Effect.gen(function* () { + // Best effort: the owner hears about a deferral Kin gave up on. + const noticeFailure = Effect.fn("KinDeferralFlow.noticeFailure")(function* ( + orgId: KinOrgId, + fire: KinTriggerFired, + itemId: KinMemoryItemId, + timeZone: string, + ) { + const run = yield* readRun(orgId, itemId); + if (run === undefined || !["start", "ask", "chase"].includes(run.onDue)) return; + const chatId = yield* cardChat(orgId, undefined); + if (chatId === null) return; + const title = (yield* readItem(orgId, itemId))?.title ?? itemId; + const dueAt = run.dueAt ?? Date.parse(fire.dueAt); + yield* chat.notice({ + orgId, + chatId, + messageId: KinMessageId.make(`deferral-failed:${orgId}:${fire.fireId}`), + body: `I couldn't handle the deferral ${title} due ${ownerDateTime(dueAt, timeZone)}.`, + }); + }); + + const drain = Effect.fn("KinDeferralFlow.drain")( + function* (orgId: KinOrgId) { const now = yield* Clock.currentTimeMillis; const timeZone = yield* cache.ownerTimeZone(orgId); const fires = yield* inbox.listPending(orgId, 100, "deferral"); @@ -358,12 +378,30 @@ const make = Effect.gen(function* () { "Kin deferral fire processing failed", { orgId, fireId: fire.fireId, cause, disposition: consume ? "consumed" : "retry" }, ); - if (consume) yield* inbox.markConsumed(orgId, [fire.fireId]); + if (!consume) return; + yield* noticeFailure(orgId, fire, itemId, timeZone).pipe( + Effect.catch((cause) => + Effect.logWarning("Kin deferral failure notice failed", { + orgId, + fireId: fire.fireId, + cause, + }), + ), + ); + yield* inbox.markConsumed(orgId, [fire.fireId]); }), ), ); } - }).pipe(Effect.mapError((cause) => new KinDeferralFlowError({ orgId, cause }))), + // The only pings sweeper: chases and their day-two escalations deliver from here. + yield* pings + .sweep(orgId) + .pipe( + Effect.catch((cause) => Effect.logWarning("Kin pings sweep failed", { orgId, cause })), + ); + }, + (effect, orgId) => + effect.pipe(Effect.mapError((cause) => new KinDeferralFlowError({ orgId, cause }))), ); yield* ticks.register( diff --git a/apps/server/src/kin/triggers/deferrals.test.ts b/apps/server/src/kin/triggers/deferrals.test.ts index a7cc31243e963..ee91d641900e0 100644 --- a/apps/server/src/kin/triggers/deferrals.test.ts +++ b/apps/server/src/kin/triggers/deferrals.test.ts @@ -265,10 +265,21 @@ it.effect("unchanged recheck, heads-up, silence, then starts once at due", () => ["acme/api#12", { state: "open" as const, assignees: ["bob"], linkedPrs: [] }], ]); const linked = makeDeferral("def-1", { links: [issue] }); + const foreign = makeDeferral("def-b", { orgId: otherOrg, chat: KinChatId.make("chat-b") }); return Effect.gen(function* () { yield* setup(); const sql = yield* SqlClient.SqlClient; - yield* (yield* TriggerService.KinTriggerService).syncItem(linked); + const service = yield* TriggerService.KinTriggerService; + yield* service.syncItem(linked); + yield* service.syncItem(foreign); + const foreignState = () => + Effect.all({ + runs: sql`SELECT state, due_at FROM kin_deferral_runs WHERE org_id = ${otherOrg} + ORDER BY item_id`, + cards: sql`SELECT revision, state_json FROM kin_widgets WHERE org_id = ${otherOrg} + ORDER BY widget_id`, + }); + const foreignBefore = yield* foreignState(); yield* tick(headsUpAt); assert.equal((yield* runOf("def-1"))?.state, "heads_up"); @@ -287,12 +298,20 @@ it.effect("unchanged recheck, heads-up, silence, then starts once at due", () => assert.equal(started?.dedupeKey, `deferral:def-1:due:${dueAt}`); assert.lengthOf(yield* pendingFires(), 0); - // A replay of the due fire after a crash can't start a second worker. + // A later replay of the due fire after a crash enqueues the same event, not a second worker. yield* sql`UPDATE kin_trigger_inbox SET consumed_at = NULL WHERE org_id = ${orgId}`; yield* sql`UPDATE kin_deferral_runs SET state = 'heads_up' WHERE org_id = ${orgId}`; - yield* tick(dueAt); + yield* tick(dueAt + 5 * 60_000); assert.lengthOf(yield* starts(), 1); - }).pipe(openGates(github), Effect.provide(layerFor([linked]))); + assert.equal((yield* runOf("def-1"))?.state, "started"); + assert.lengthOf(yield* pendingFires(), 0); + + // Org a's ticks never touch org b's run, card or fires. + assert.deepStrictEqual(yield* foreignState(), foreignBefore); + const foreignFires = yield* sql`SELECT fire_id FROM kin_trigger_inbox + WHERE org_id = ${otherOrg} AND consumed_at IS NULL ORDER BY fire_id`; + assert.lengthOf(foreignFires, 2); + }).pipe(openGates(github), Effect.provide(layerFor([linked, foreign]))); }); it.effect("asks instead of starting when a linked issue closes before due", () => { @@ -378,6 +397,29 @@ it.effect( assert.equal(cancel._tag, "Success"); assert.include(staged[1]!.content, "status: cancelled"); + // An owner tap isn't a level grant: [Start now] passes the same gates as silence. + const sql = yield* SqlClient.SqlClient; + const before = { run: yield* runOf("def-1"), card: yield* cardOf("def-1") }; + const refusals = [ + yield* tap("start_now", "stopped", staged).pipe( + Effect.provideService(Deferrals.KinDeferralStopSwitch, { + isStopped: () => Effect.succeed(true), + }), + ), + yield* tap("start_now", "gated", staged).pipe( + Effect.provideService(WorkerLaunchGate, false), + ), + ]; + yield* sql`UPDATE kin_leads SET level = 'L1' WHERE org_id = ${orgId}`; + refusals.push(yield* tap("start_now", "level", staged)); + yield* sql`UPDATE kin_leads SET level = 'L2' WHERE org_id = ${orgId}`; + assert.deepStrictEqual( + refusals.map((result) => result._tag === "Failure" && result.failure.cause), + ["stopped", "workers_blocked", "below_l2"], + ); + assert.deepStrictEqual({ run: yield* runOf("def-1"), card: yield* cardOf("def-1") }, before); + assert.lengthOf(yield* starts(), 0); + const start = yield* tap("start_now", "owner", staged); assert.equal(start._tag === "Success" && start.success.action, "start_now"); assert.equal((yield* runOf("def-1"))?.state, "started"); @@ -393,7 +435,6 @@ it.effect( items[0] = makeDeferral("def-1", { due: "2026-11-16T09:00:00.000+05:30" }); assert.equal(yield* service.syncItem(items[0]), "armed"); const nextDue = Date.parse("2026-11-16T03:30:00.000Z"); - const sql = yield* SqlClient.SqlClient; const triggers = yield* sql<{ id: string; nextRunAt: number | null }>`SELECT trigger_id AS id, next_run_at AS "nextRunAt" FROM kin_triggers WHERE org_id = ${orgId} ORDER BY trigger_id`; assert.deepStrictEqual(triggers, [ @@ -448,15 +489,22 @@ it.effect("drops queued fires for disarmed or moved deferrals", () => { const items = [ makeDeferral("def-1", { on_due: "remind" }), makeDeferral("def-2", { on_due: "remind" }), + makeDeferral("def-3"), ]; return Effect.gen(function* () { yield* setup(); const sql = yield* SqlClient.SqlClient; const service = yield* TriggerService.KinTriggerService; + const scheduler = yield* TriggerScheduler.KinTriggerScheduler; for (const entry of items) yield* service.syncItem(entry); yield* TestClock.setTime(dueAt); - yield* (yield* TriggerScheduler.KinTriggerScheduler).runDue(); - assert.lengthOf(yield* pendingFires(), 2); + yield* scheduler.runDue(); + assert.lengthOf(yield* pendingFires(), 4); + + // def-3's 9:00 due fire is still queued when due moves to 8:50, already past. + items[2] = makeDeferral("def-3", { due: "2026-11-13T08:50+05:30" }); + assert.equal(yield* service.syncItem(items[2]), "armed"); + yield* scheduler.runDue(); assert.equal( yield* service.syncItem(makeDeferral("def-1", { on_due: "remind", status: "done" })), @@ -469,6 +517,12 @@ it.effect("drops queued fires for disarmed or moved deferrals", () => { assert.lengthOf(yield* pendingFires(), 0); assert.equal((yield* runOf("def-1"))?.state, "disarmed"); assert.equal((yield* runOf("def-2"))?.state, "armed"); + // 8:50 is handled and goes to grace; the old 9:00 fire isn't taken for the grace fire. + assert.equal((yield* runOf("def-3"))?.state, "grace"); + assert.lengthOf(yield* starts(), 0); + yield* tick(dueAt + 30 * 60_000); + assert.equal((yield* runOf("def-3"))?.state, "started"); + assert.lengthOf(yield* starts(), 1); const reminders = () => sql<{ body: string }>`SELECT body FROM kin_messages WHERE org_id = ${orgId} AND body LIKE 'Reminder:%' ORDER BY seq`; @@ -476,19 +530,41 @@ it.effect("drops queued fires for disarmed or moved deferrals", () => { yield* tick(Date.parse("2026-11-14T03:30:00.000Z")); assert.deepStrictEqual(yield* reminders(), [{ body: "Reminder: def-2." }]); - }).pipe(Effect.provide(layerFor(items))); + }).pipe(openGates(new Map()), Effect.provide(layerFor(items))); }); it.effect("remind, ask and chase act at due, noting lateness", () => { const remind = makeDeferral("def-r", { on_due: "remind" }); const ask = makeDeferral("def-a", { on_due: "ask" }); const chase = makeDeferral("def-c", { on_due: "chase", owner: "owner" }); + const quiet = makeDeferral("def-q", { + on_due: "chase", + owner: "owner", + due: "2026-11-13T07:00+05:30", + }); + const broken = makeDeferral("def-x", { on_due: "ask" }); + const items = [remind, ask, chase, quiet, broken]; return Effect.gen(function* () { yield* setup(); const sql = yield* SqlClient.SqlClient; const service = yield* TriggerService.KinTriggerService; - for (const entry of [remind, ask, chase]) yield* service.syncItem(entry); + for (const entry of items) yield* service.syncItem(entry); + const quietPings = () => + sql`SELECT body FROM kin_messages WHERE org_id = ${orgId} + AND body LIKE '%def-q was due%' ORDER BY seq`; + + // A chase due in quiet hours waits for a later tick, then pings once. + yield* tick(Date.parse("2026-11-13T01:30:00.000Z")); + assert.equal((yield* runOf("def-q"))?.state, "chased"); + assert.lengthOf(yield* quietPings(), 0); + yield* tick(Date.parse("2026-11-13T02:35:00.000Z")); + assert.lengthOf(yield* quietPings(), 1); + + yield* sql`UPDATE kin_deferral_runs SET snapshot_json = 'not json' + WHERE org_id = ${orgId} AND item_id = 'def-x'`; yield* tick(dueAt + 20 * 60_000); + yield* tick(dueAt + 25 * 60_000); + assert.lengthOf(yield* quietPings(), 1); const messages = yield* sql<{ body: string }>`SELECT body FROM kin_messages WHERE org_id = ${orgId} AND kind = 'system' ORDER BY seq`; @@ -506,7 +582,16 @@ it.effect("remind, ask and chase act at due, noting lateness", () => { assert.equal((yield* runOf("def-c"))?.state, "chased"); const ledger = yield* sql`SELECT item_id FROM kin_ping_ledger WHERE org_id = ${orgId} ORDER BY item_id`; - assert.lengthOf(ledger, 1); + assert.lengthOf(ledger, 2); assert.lengthOf(yield* starts(), 0); - }).pipe(Effect.provide(layerFor([remind, ask, chase]))); + + // A fire that can't be handled is consumed after telling the owner. + assert.isTrue( + messages.some( + ({ body }) => body === "I couldn't handle the deferral def-x due Fri 13 Nov, 9:00.", + ), + ); + assert.equal((yield* runOf("def-x"))?.state, "armed"); + assert.lengthOf(yield* pendingFires(), 0); + }).pipe(Effect.provide(layerFor(items))); }); diff --git a/apps/server/src/kin/triggers/deferrals.ts b/apps/server/src/kin/triggers/deferrals.ts index c626ebf35b60c..f938297d6c850 100644 --- a/apps/server/src/kin/triggers/deferrals.ts +++ b/apps/server/src/kin/triggers/deferrals.ts @@ -25,6 +25,7 @@ import { stringify } from "yaml"; import { KinChatHub } from "../chat/KinChatHub.ts"; import * as Repository from "../chat/KinChatRepository.ts"; import * as KinEventQueue from "../lead/KinEventQueue.ts"; +import { WorkerLaunchGate } from "../lead/Workers.ts"; import * as Memory from "../memory/Memory.ts"; import * as MemoryCache from "../memory/MemoryCache.ts"; import * as MemoryWriter from "../memory/MemoryWriter.ts"; @@ -82,6 +83,8 @@ export const DeferralSnapshot = Schema.Struct({ supersededBy: Schema.NullOr(KinMemoryItemId), links: Schema.Array(Schema.String), github: Schema.optional(Schema.NullOr(Schema.Array(GithubLinkState))), + /** The re-armed due fire's instant while the run is in `grace`. */ + graceEndsAt: Schema.optional(Schema.Number), }); export type DeferralSnapshot = typeof DeferralSnapshot.Type; export const deferralSnapshot = ( @@ -203,6 +206,9 @@ export const KinDeferralStopSwitch = Context.Reference<{ defaultValue: () => ({ isStopped: () => Effect.succeed(true) }), }); +/** The lead level work may start at; an owner tap is not a level grant. */ +export const startsAtLevel = (level: string) => level === "L2" || level === "L3"; + const iso = (ms: number) => DateTime.formatIso(DateTime.makeUnsafe(ms)); const decodeEvent = Schema.decodeUnknownEffect(KinEvent); @@ -291,7 +297,8 @@ export const makeDeferralStarter = Effect.gen(function* () { /** * Marks the run started and hands the work to the lead as `deferral.due`, inside the * caller's transaction. The event dedupes on the due fire id, so a replay, a late fire - * and a [Start now] tap share one start. + * and a [Start now] tap share one start. `at` is the fire's time, so a replay enqueues + * the same event. */ return Effect.fn("Deferrals.start")(function* (input: { readonly orgId: KinOrgId; @@ -299,27 +306,39 @@ export const makeDeferralStarter = Effect.gen(function* () { readonly dueAt: number; readonly card: KinDeferralCard; readonly provenance: KinAuthor; + readonly at: number; readonly now: number; }) { - const { orgId, itemId, dueAt, now } = input; + const { orgId, itemId, dueAt, at, now } = input; const [lead] = yield* sql<{ agentId: string }>`SELECT l.agent_id AS "agentId" FROM kin_leads l JOIN kin_agents a ON a.org_id = l.org_id AND a.agent_id = l.agent_id WHERE l.org_id = ${orgId} AND a.retired_at IS NULL ORDER BY l.agent_id LIMIT 1`; if (lead === undefined) return false; const dedupeKey = deferralStartKey(itemId, dueAt); - yield* queue.enqueue( - yield* decodeEvent({ - _tag: "deferral.due", - eventId: KinEventId.make(`deferral_${hex(`${orgId.length}:${orgId}${dedupeKey}`)}`), - orgId, - agentId: KinAgentId.make(lead.agentId), - receivedAt: iso(now), - provenance: input.provenance, - dedupeKey, - subject: { type: "deferral", itemId, phase: "due" }, - lateByMs: Math.max(0, now - dueAt), - }), - ); + yield* queue + .enqueue( + yield* decodeEvent({ + _tag: "deferral.due", + eventId: KinEventId.make(`deferral_${hex(`${orgId.length}:${orgId}${dedupeKey}`)}`), + orgId, + agentId: KinAgentId.make(lead.agentId), + receivedAt: iso(at), + provenance: input.provenance, + dedupeKey, + subject: { type: "deferral", itemId, phase: "due" }, + lateByMs: Math.max(0, at - dueAt), + }), + ) + .pipe( + // A start under this key from another path (tap vs fire) already handed the work over. + Effect.catchTags({ + KinEventQueueConflict: Effect.fn("Deferrals.startConflict")(function* (conflict) { + const [stored] = yield* sql`SELECT 1 FROM kin_events WHERE org_id = ${orgId} + AND dedupe_key = ${dedupeKey} AND tag = 'deferral.due' ORDER BY event_id`; + if (stored === undefined) return yield* conflict; + }), + }), + ); yield* sql`UPDATE kin_deferral_runs SET state = 'started', updated_at = ${now} WHERE org_id = ${orgId} AND item_id = ${itemId} AND due_at = ${dueAt}`; yield* cards.write({ @@ -428,6 +447,12 @@ const make = Effect.gen(function* () { const dueAt = Date.parse(card.dueAt); if (yield* (yield* KinDeferralStopSwitch).isStopped(orgId)) return yield* Effect.fail("stopped" as const); + if (!(yield* WorkerLaunchGate)) return yield* Effect.fail("workers_blocked" as const); + const [lead] = yield* sql<{ level: string }>`SELECT l.level FROM kin_leads l + JOIN kin_agents a ON a.org_id = l.org_id AND a.agent_id = l.agent_id + WHERE l.org_id = ${orgId} AND a.retired_at IS NULL ORDER BY l.agent_id LIMIT 1`; + if (lead === undefined || !startsAtLevel(lead.level)) + return yield* Effect.fail("below_l2" as const); const now = yield* Clock.currentTimeMillis; const started = yield* sql.withTransaction( Effect.gen(function* () { @@ -442,6 +467,7 @@ const make = Effect.gen(function* () { dueAt, card, provenance: tap.provenance, + at: now, now, }); }), diff --git a/apps/server/src/kin/triggers/pings.ts b/apps/server/src/kin/triggers/pings.ts index aff49b6764027..4c4ef19744807 100644 --- a/apps/server/src/kin/triggers/pings.ts +++ b/apps/server/src/kin/triggers/pings.ts @@ -207,6 +207,9 @@ const make = Effect.gen(function* () { }), sweep: Effect.fn("KinPings.sweep")( function* (orgId) { + const rows = + yield* sql`SELECT * FROM kin_ping_items WHERE org_id = ${orgId} AND stopped_at IS NULL ORDER BY person, item_id`; + if (rows.length === 0) return; const now = yield* Clock.currentTimeMillis; const zone = yield* cache.ownerTimeZone(orgId); const day = ownerDay(now, zone); @@ -217,8 +220,6 @@ const make = Effect.gen(function* () { const hours = defaultHours._tag === "Some" ? defaultHours.value : { from: "21:00", to: "08:00" }; const people = yield* memory.query(orgId, { kind: "person", excludeInvalid: true }); - const rows = - yield* sql`SELECT * FROM kin_ping_items WHERE org_id = ${orgId} AND stopped_at IS NULL ORDER BY person, item_id`; const candidates: Array<{ row: Row; request: Request; diff --git a/docs/kin/specs/triggers.md b/docs/kin/specs/triggers.md index dc26caafd6bc2..a3647846ff330 100644 --- a/docs/kin/specs/triggers.md +++ b/docs/kin/specs/triggers.md @@ -233,12 +233,12 @@ armed ──heads_up fire──▶ recheck ──changed──▶ asked (decis started ─▶ worker via delegate_task (LEAD) ─▶ closed with PR and proof ``` -- **Consumer.** `KinDeferralFlow` in [`deferralFlow.ts`](../../../apps/server/src/kin/triggers/deferralFlow.ts) drains each org's pending deferral fires on its own `Scheduler` tick, in due order. Like routines, it does read-only or idempotent I/O first (memory read, GitHub reads, gate checks, the chase's `KinPings.put`), then one transaction that rechecks the run row, writes the state and card, enqueues, and marks the fire consumed. Decode failures, and fires more than 24 hours past due, are consumed; other failures retry on the next tick. Code posts every card: the item's card is updated in place, or posted to the item's chat, else the owner's direct chat. +- **Consumer.** `KinDeferralFlow` in [`deferralFlow.ts`](../../../apps/server/src/kin/triggers/deferralFlow.ts) drains each org's pending deferral fires on its own `Scheduler` tick, in due order. Like routines, it does read-only or idempotent I/O first (memory read, GitHub reads, gate checks), then one transaction that rechecks the run row, writes the state and card, puts the chase's ping, enqueues, and marks the fire consumed. A failing fire retries on the next tick; one that fails to decode, or still fails 24 hours past due, is consumed after a best-effort server notice to the owner ("I couldn't handle the deferral …"). After the fires, the same tick runs `KinPings.sweep` for the org, so a chase that lands in quiet hours delivers when they end. Code posts every card: the item's card is updated in place, or posted to the item's chat, else the owner's direct chat. - **The recheck is code.** It reads the memory item and every GitHub link through `KinDeferralGithubReader`. Any difference from the snapshot means "changed": an issue closed, an assignee added or removed, another linked PR, or the item edited or superseded. A GitHub half that couldn't be read, at arm time or now, counts as "changed: couldn't verify". The ask names what changed. -- **Late.** If `due` has passed when the `heads_up` fire is consumed, it is skipped. A `due` fire more than 10 minutes late notes "This was due at 9:00". A `start` due fire within the 30-minute grace window, or one with no heads-up shown, posts the heads-up card again with a grace from now: the `due` trigger re-arms at the grace end (state `grace`, `graceEndsAt` on the card), and silence still means go. An older fire asks instead of starting. `remind`, `ask` and `chase` act at due only and carry the same note. -- **Stale fires.** Fires wait in the inbox until the consumer handles them. It consumes, and otherwise ignores, a fire whose run row is missing or `disarmed`, whose `due_at` no longer equals the fire's due instant (a grace fire is the one exception), or whose run already left the state that fire expects. -- **Gates.** `start` becomes `ask` below L2, with the org's stop switch on, while `workerLaunchReady` is false, or with no active lead. LEAD-11 isn't built, so `KinDeferralStopSwitch` defaults to stopped. With today's defaults, silence never starts work. [Start now] is an owner decision, so it ignores the level but still refuses while stopped. -- **Idempotent start.** Starting sets `started` and enqueues `deferral.due` (phase `due`) for the lead to delegate, deduplicated by the due fire's id `deferral::due:`. A replay, a grace fire and a [Start now] tap share that key, so they can't start two workers. +- **Late.** If `due` has passed when the `heads_up` fire is consumed, it is skipped. A `due` fire more than 10 minutes late notes "This was due at 9:00". A `start` due fire within the 30-minute grace window, or one with no heads-up shown, posts the heads-up card again with a grace from now: the `due` trigger re-arms at the grace end (state `grace`, `graceEndsAt` on the card and in the run's snapshot), and silence still means go. An older fire asks instead of starting. `remind`, `ask` and `chase` act at due only and carry the same note. +- **Stale fires.** Fires wait in the inbox until the consumer handles them. It consumes, and otherwise ignores, a fire whose run row is missing or `disarmed`, whose `due_at` no longer equals the fire's due instant (a grace fire is the one exception, and only when its instant equals the recorded grace end), or whose run already left the state that fire expects. +- **Gates.** `start` becomes `ask` below L2, with the org's stop switch on, while `workerLaunchReady` is false, or with no active lead. LEAD-11 isn't built, so `KinDeferralStopSwitch` defaults to stopped. With today's defaults, silence never starts work. [Start now] is an owner tap, not a level grant: it refuses, leaving the run and card as they were, while stopped, while `workerLaunchReady` is false, or below L2. +- **Idempotent start.** Starting sets `started` and enqueues `deferral.due` (phase `due`) for the lead to delegate, deduplicated by the due fire's id `deferral::due:`. A fire's event takes its time fields from the fire, so a replay enqueues the identical event; a start that finds another path's `deferral.due` under the same key counts it as already started. A replay, a grace fire and a [Start now] tap can't start two workers. - **States.** `armed`, `heads_up`, `grace`, `asked`, `started` (`start`); `reminded`, `asked`, `chased` (`remind`, `ask`, `chase`); `disarmed`. A re-arm from memory returns the run to `armed`. - **Bring it up early.** When a message touches the topic before the due time, the lead mentions it (LEAD, using the cache). The Monday morning brief lists deferrals due this week. @@ -250,7 +250,7 @@ armed ──heads_up fire──▶ recheck ──changed──▶ asked (decis TRIG-11 persists requests in `kin_ping_items` before delivery; quiet sweeps leave them pending, with no sent ledger row. Each later sweep rechecks the current owner day, item state and person settings. Delivery and the sent ledger commit in one SQLite transaction, so restart retries cannot duplicate a committed batch. Equal quiet-hour bounds disable quiet hours; windows include the start and exclude the end. Missing or invalid person zones use the org zone; missing or invalid windows use the org quiet hours (21:00–08:00 before settings exist). -An answer is a posted human message by that recipient replying (`reply_to`) to a ping containing the item, or any later message by that recipient in the chat when that ping contained exactly one item. Reads, system notices and agent messages are not answers. Answering stops only that recipient's item; other recipients stop through their own answers or item closure. A non-open, invalid or deleted item stops all recipients. The escalation resolves when the item closes or no recipients remain pending. A new `put` for a stopped, open item starts a fresh lifecycle for that recipient, resetting delivery membership but preserving the daily ledger: at most once per item per owner day; resolved escalation anchors can then be replaced. Day two means a later owner-calendar date, not 24 elapsed hours. Notices are server-only `KinChatService.notice` system messages, never model-authored or human-impersonated posts. Watchdog and deferral callers will compose the ping service with Memory; TRIG-11 starts no background sweeper. +An answer is a posted human message by that recipient replying (`reply_to`) to a ping containing the item, or any later message by that recipient in the chat when that ping contained exactly one item. Reads, system notices and agent messages are not answers. Answering stops only that recipient's item; other recipients stop through their own answers or item closure. A non-open, invalid or deleted item stops all recipients. The escalation resolves when the item closes or no recipients remain pending. A new `put` for a stopped, open item starts a fresh lifecycle for that recipient, resetting delivery membership but preserving the daily ledger: at most once per item per owner day; resolved escalation anchors can then be replaced. Day two means a later owner-calendar date, not 24 elapsed hours. Notices are server-only `KinChatService.notice` system messages, never model-authored or human-impersonated posts. Watchdog and deferral callers compose the ping service with Memory. TRIG-11 starts no sweeper of its own; the deferral flow's `kin-deferrals` tick (TRIG-8) runs `KinPings.sweep` for every org. ### Routines (`fixed_time` in the owner's zone) From 730b3251f81a8a22a26813d7cdcf6bf9c96aff57 Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Sun, 8 Nov 2026 23:27:48 +0530 Subject: [PATCH 4/7] fix(kin): keep memory reads out of deferral write transactions Chases pass the item head to pings.put inside the flow transaction, run state changes guard on the state read, failing fires back off, and the pings sweep is throttled and runs even when listing fires fails. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/kin/triggers/deferralFlow.ts | 161 ++++++++++++------- apps/server/src/kin/triggers/deferrals.ts | 23 ++- apps/server/src/kin/triggers/pings.ts | 24 +-- 3 files changed, 137 insertions(+), 71 deletions(-) diff --git a/apps/server/src/kin/triggers/deferralFlow.ts b/apps/server/src/kin/triggers/deferralFlow.ts index 005d3ec04186c..485db8b7e47eb 100644 --- a/apps/server/src/kin/triggers/deferralFlow.ts +++ b/apps/server/src/kin/triggers/deferralFlow.ts @@ -110,6 +110,10 @@ const isPermanent = (error: unknown) => { return false; }; const iso = (ms: number) => DateTime.formatIso(DateTime.makeUnsafe(ms)); +const SWEEP_EVERY_MS = 60_000; +const RETRY_FIRST_MS = 60_000; +const RETRY_MAX_MS = 15 * 60_000; +const GIVE_UP_MS = 24 * 60 * 60 * 1000; const make = Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; @@ -121,6 +125,9 @@ const make = Effect.gen(function* () { const ticks = yield* Scheduler.Scheduler; const cards = yield* makeDeferralCards; const start = yield* makeDeferralStarter; + // Per process: the 24 h give-up is the durable bound, so a restart only retries sooner. + const retries = new Map(); + const sweptAt = new Map(); const readRun = (orgId: KinOrgId, itemId: KinMemoryItemId) => sql`SELECT state, due_at AS "dueAt", on_due AS "onDue", snapshot_json AS snapshot @@ -169,15 +176,18 @@ const make = Effect.gen(function* () { if (person === undefined || owner === undefined || item.chat === undefined) return `I couldn't chase ${handle}`; const put = yield* pings - .put({ - orgId, - person: KinPersonId.make(person.personId), - itemId: item.id, - chatId: item.chat, - owner: KinPersonId.make(owner.owner), - ownerChatId: KinChatId.make(owner.chatId), - nextStep: `${item.title} was due ${ownerDateTime(dueAt, timeZone)}. What's the status?`, - }) + .put( + { + orgId, + person: KinPersonId.make(person.personId), + itemId: item.id, + chatId: item.chat, + owner: KinPersonId.make(owner.owner), + ownerChatId: KinChatId.make(owner.chatId), + nextStep: `${item.title} was due ${ownerDateTime(dueAt, timeZone)}. What's the status?`, + }, + item, + ) .pipe(Effect.result); if (put._tag === "Failure") { yield* Effect.logWarning("Kin deferral chase failed", { orgId, itemId: item.id }); @@ -199,7 +209,7 @@ const make = Effect.gen(function* () { const snapshot = run === undefined ? undefined : yield* decodeSnapshot(run.snapshot); const step = classifyFire(run, snapshot?.graceEndsAt, phase, fireAt, now); if (step.kind === "drop" || run === undefined || snapshot === undefined) - return yield* inbox.markConsumed(orgId, [fire.fireId]); + return yield* inbox.markConsumed(orgId, [fire.fireId]).pipe(Effect.as(false)); // Read-only or idempotent I/O; a replay repeats it. const item = yield* readItem(orgId, itemId); @@ -213,11 +223,12 @@ const make = Effect.gen(function* () { gates.push("the stop switch is on"); if (!(yield* WorkerLaunchGate)) gates.push("workers can't launch yet"); } - yield* sql.withTransaction( + // No memory or GitHub I/O below: the memory lock can wait on this writer. + return yield* sql.withTransaction( Effect.gen(function* () { const pending = yield* sql`SELECT fire_id FROM kin_trigger_inbox WHERE org_id = ${orgId} AND fire_id = ${fire.fireId} AND consumed_at IS NULL ORDER BY fire_id`; - if (pending.length === 0) return; + if (pending.length === 0) return false; const reasons = [...gates]; const current = yield* readRun(orgId, itemId); // Moved or handled since the reads above: classify again on the next tick. @@ -226,7 +237,7 @@ const make = Effect.gen(function* () { current.dueAt !== run.dueAt || current.snapshot !== run.snapshot ) - return; + return false; const dueAt = step.dueAt; const title = item?.title ?? itemId; const card = (phase: KinDeferralCard["phase"], text: string, graceEndsAt?: number) => ({ @@ -240,7 +251,8 @@ const make = Effect.gen(function* () { }); const setState = (state: string) => sql`UPDATE kin_deferral_runs SET state = ${state}, updated_at = ${now} - WHERE org_id = ${orgId} AND item_id = ${itemId} AND due_at = ${dueAt}`; + WHERE org_id = ${orgId} AND item_id = ${itemId} AND due_at = ${dueAt} + AND state = ${run.state}`; const post = Effect.fn("KinDeferralFlow.post")(function* (state: KinDeferralCard) { const chatId = yield* cardChat(orgId, item); yield* cards.write({ orgId, itemId, state, chatId, now }); @@ -293,7 +305,8 @@ const make = Effect.gen(function* () { AND enabled = 1`; const graced = yield* encodeSnapshot({ ...snapshot, graceEndsAt: graceEnd }); yield* sql`UPDATE kin_deferral_runs SET snapshot_json = ${graced} - WHERE org_id = ${orgId} AND item_id = ${itemId} AND due_at = ${dueAt}`; + WHERE org_id = ${orgId} AND item_id = ${itemId} AND due_at = ${dueAt} + AND state = ${run.state}`; yield* setState("grace"); } else { const started = yield* start({ @@ -304,8 +317,9 @@ const make = Effect.gen(function* () { provenance: { _tag: "system" }, at: Date.parse(fire.firedAt), now, + from: run.state, }); - if (!started) yield* ask(["there is no active lead"]); + if (!started) return false; } } else if (onDue === "ask") yield* ask([]); else if (onDue === "chase") { @@ -315,7 +329,11 @@ const make = Effect.gen(function* () { ? "the item is gone" : yield* prepareChase(orgId, item, dueAt, timeZone); if (failure !== null) yield* ask([failure]); - else yield* setState("chased"); + else { + yield* setState("chased"); + yield* inbox.markConsumed(orgId, [fire.fireId]); + return true; + } } else { const chatId = yield* cardChat(orgId, item); if (chatId !== null) @@ -328,6 +346,7 @@ const make = Effect.gen(function* () { yield* setState("reminded"); } yield* inbox.markConsumed(orgId, [fire.fireId]); + return false; }), ); }); @@ -356,49 +375,77 @@ const make = Effect.gen(function* () { const drain = Effect.fn("KinDeferralFlow.drain")( function* (orgId: KinOrgId) { const now = yield* Clock.currentTimeMillis; - const timeZone = yield* cache.ownerTimeZone(orgId); - const fires = yield* inbox.listPending(orgId, 100, "deferral"); - // Catch-up fires share received_at; handle them by due time. - for (const fire of fires.toSorted( - (a, b) => - Date.parse(a.dueAt) - Date.parse(b.dueAt) || - (a.fireId < b.fireId ? -1 : a.fireId > b.fireId ? 1 : 0), - )) { - if (fire.payload.kind !== "time" || fire.payload.subject.type !== "deferral") continue; - const { itemId, phase } = fire.payload.subject; - yield* process(orgId, fire, itemId, phase, now, timeZone).pipe( - Effect.mapError((cause) => - isSqlError(cause) ? toPersistenceSqlError("KinDeferralFlow.drain")(cause) : cause, - ), - Effect.catch((cause) => - Effect.gen(function* () { - const consume = - isPermanent(cause) || now - Date.parse(fire.dueAt) > 24 * 60 * 60 * 1000; - yield* (consume ? Effect.logWarning : Effect.logDebug)( - "Kin deferral fire processing failed", - { orgId, fireId: fire.fireId, cause, disposition: consume ? "consumed" : "retry" }, - ); - if (!consume) return; - yield* noticeFailure(orgId, fire, itemId, timeZone).pipe( - Effect.catch((cause) => - Effect.logWarning("Kin deferral failure notice failed", { + let chased = false; + // The only pings sweeper: chases and their day-two escalations deliver from here. + const sweep = Effect.suspend(() => { + const last = sweptAt.get(orgId); + if (!chased && last !== undefined && now - last < SWEEP_EVERY_MS) return Effect.void; + sweptAt.set(orgId, now); + return pings + .sweep(orgId) + .pipe( + Effect.catch((cause) => Effect.logWarning("Kin pings sweep failed", { orgId, cause })), + ); + }); + yield* Effect.gen(function* () { + const timeZone = yield* cache.ownerTimeZone(orgId); + const fires = yield* inbox.listPending(orgId, 100, "deferral"); + // Catch-up fires share received_at; handle them by due time. + for (const fire of fires.toSorted( + (a, b) => + Date.parse(a.dueAt) - Date.parse(b.dueAt) || + (a.fireId < b.fireId ? -1 : a.fireId > b.fireId ? 1 : 0), + )) { + if (fire.payload.kind !== "time" || fire.payload.subject.type !== "deferral") continue; + const { itemId, phase } = fire.payload.subject; + const retry = retries.get(fire.fireId); + const giveUp = now - Date.parse(fire.dueAt) > GIVE_UP_MS; + if (retry !== undefined && now < retry.notBefore && !giveUp) continue; + yield* process(orgId, fire, itemId, phase, now, timeZone).pipe( + Effect.tap((put) => + Effect.sync(() => { + retries.delete(fire.fireId); + if (put) chased = true; + }), + ), + Effect.mapError((cause) => + isSqlError(cause) ? toPersistenceSqlError("KinDeferralFlow.drain")(cause) : cause, + ), + Effect.catch((cause) => + Effect.gen(function* () { + const consume = isPermanent(cause) || giveUp; + const failures = (retry?.failures ?? 0) + 1; + yield* (consume || failures === 1 ? Effect.logWarning : Effect.logDebug)( + "Kin deferral fire processing failed", + { orgId, fireId: fire.fireId, cause, - }), - ), - ); - yield* inbox.markConsumed(orgId, [fire.fireId]); - }), - ), - ); - } - // The only pings sweeper: chases and their day-two escalations deliver from here. - yield* pings - .sweep(orgId) - .pipe( - Effect.catch((cause) => Effect.logWarning("Kin pings sweep failed", { orgId, cause })), - ); + failures, + disposition: consume ? "consumed" : "retry", + }, + ); + if (!consume) { + const delay = Math.min(RETRY_MAX_MS, RETRY_FIRST_MS * 2 ** (failures - 1)); + retries.set(fire.fireId, { notBefore: now + delay, failures }); + return; + } + retries.delete(fire.fireId); + yield* noticeFailure(orgId, fire, itemId, timeZone).pipe( + Effect.catch((cause) => + Effect.logWarning("Kin deferral failure notice failed", { + orgId, + fireId: fire.fireId, + cause, + }), + ), + ); + yield* inbox.markConsumed(orgId, [fire.fireId]); + }), + ), + ); + } + }).pipe(Effect.ensuring(sweep)); }, (effect, orgId) => effect.pipe(Effect.mapError((cause) => new KinDeferralFlowError({ orgId, cause }))), diff --git a/apps/server/src/kin/triggers/deferrals.ts b/apps/server/src/kin/triggers/deferrals.ts index f938297d6c850..2f93e4036c923 100644 --- a/apps/server/src/kin/triggers/deferrals.ts +++ b/apps/server/src/kin/triggers/deferrals.ts @@ -308,12 +308,18 @@ export const makeDeferralStarter = Effect.gen(function* () { readonly provenance: KinAuthor; readonly at: number; readonly now: number; + /** The run state the caller read in this transaction. */ + readonly from: string; }) { const { orgId, itemId, dueAt, at, now } = input; const [lead] = yield* sql<{ agentId: string }>`SELECT l.agent_id AS "agentId" FROM kin_leads l JOIN kin_agents a ON a.org_id = l.org_id AND a.agent_id = l.agent_id WHERE l.org_id = ${orgId} AND a.retired_at IS NULL ORDER BY l.agent_id LIMIT 1`; if (lead === undefined) return false; + const moved = yield* sql`UPDATE kin_deferral_runs SET state = 'started', updated_at = ${now} + WHERE org_id = ${orgId} AND item_id = ${itemId} AND due_at = ${dueAt} + AND state = ${input.from} RETURNING item_id`; + if (moved.length === 0) return false; const dedupeKey = deferralStartKey(itemId, dueAt); yield* queue .enqueue( @@ -333,14 +339,22 @@ export const makeDeferralStarter = Effect.gen(function* () { // A start under this key from another path (tap vs fire) already handed the work over. Effect.catchTags({ KinEventQueueConflict: Effect.fn("Deferrals.startConflict")(function* (conflict) { - const [stored] = yield* sql`SELECT 1 FROM kin_events WHERE org_id = ${orgId} - AND dedupe_key = ${dedupeKey} AND tag = 'deferral.due' ORDER BY event_id`; + const [stored] = yield* sql<{ agentId: string }>`SELECT agent_id AS "agentId" + FROM kin_events WHERE org_id = ${orgId} AND dedupe_key = ${dedupeKey} + AND tag = 'deferral.due' ORDER BY event_id`; if (stored === undefined) return yield* conflict; + if (stored.agentId !== lead.agentId) { + yield* Effect.logWarning("Kin deferral start went to another lead", { + orgId, + itemId, + agentId: lead.agentId, + storedAgentId: stored.agentId, + }); + return yield* conflict; + } }), }), ); - yield* sql`UPDATE kin_deferral_runs SET state = 'started', updated_at = ${now} - WHERE org_id = ${orgId} AND item_id = ${itemId} AND due_at = ${dueAt}`; yield* cards.write({ orgId, itemId, @@ -469,6 +483,7 @@ const make = Effect.gen(function* () { provenance: tap.provenance, at: now, now, + from: run.state, }); }), ); diff --git a/apps/server/src/kin/triggers/pings.ts b/apps/server/src/kin/triggers/pings.ts index 4c4ef19744807..30e220ae4b5d0 100644 --- a/apps/server/src/kin/triggers/pings.ts +++ b/apps/server/src/kin/triggers/pings.ts @@ -8,6 +8,7 @@ import { KinPersonId, KinTriggerSettings, KinWaitingId, + type MemoryItemHead, } from "@t3tools/contracts"; import * as SecureRandom from "effect/crypto/SecureRandom"; import * as Clock from "effect/Clock"; @@ -63,7 +64,8 @@ type Error = export class KinPings extends Context.Service< KinPings, { - readonly put: (input: Request) => Effect.Effect; + /** Pass `item` when calling inside a write transaction: reading memory there inverts the lock order. */ + readonly put: (input: Request, item?: MemoryItemHead) => Effect.Effect; readonly sweep: (orgId: KinOrgId) => Effect.Effect; } >()("t3/kin/triggers/pings/KinPings") {} @@ -173,11 +175,16 @@ const make = Effect.gen(function* () { }); }); return KinPings.of({ - put: Effect.fn("KinPings.put")(function* (input) { + put: Effect.fn("KinPings.put")(function* (input, known) { const r = yield* decodeRequest(input).pipe(Effect.mapError(() => invalid)); - const item = yield* memory - .readItemQuietly(r.orgId, r.itemId) - .pipe(Effect.catchTags({ MemoryNotFound: () => missing })); + if (known !== undefined && (known.orgId !== r.orgId || known.id !== r.itemId)) + return yield* invalid; + const head = + known ?? + (yield* memory.readItemQuietly(r.orgId, r.itemId).pipe( + Effect.map((item) => item.head), + Effect.catchTags({ MemoryNotFound: () => missing }), + )); const json = yield* encodeRequest(r).pipe(Effect.mapError(() => invalid)); return yield* transaction( Effect.gen(function* () { @@ -186,11 +193,7 @@ const make = Effect.gen(function* () { const rows = yield* sql`SELECT * FROM kin_ping_items WHERE org_id = ${r.orgId} AND person = ${r.person} AND item_id = ${r.itemId} ORDER BY item_id`; const previous = rows[0]; - if ( - previous?.stopped_at != null && - item.head.status === "open" && - item.head.invalid_at === null - ) { + if (previous?.stopped_at != null && head.status === "open" && head.invalid_at === null) { yield* sql`DELETE FROM kin_ping_deliveries WHERE org_id = ${r.orgId} AND person = ${r.person} AND item_id = ${r.itemId}`; yield* sql`DELETE FROM kin_ping_escalations WHERE org_id = ${r.orgId} AND item_id = ${r.itemId} AND waiting_id IN (SELECT waiting_id FROM kin_waiting WHERE org_id = ${r.orgId} AND resolved_at IS NOT NULL ORDER BY waiting_id)`; @@ -273,6 +276,7 @@ const make = Effect.gen(function* () { } candidates.push(result.success); } + if (candidates.length === 0) return; yield* transaction( Effect.gen(function* () { const batches = new Map(); From 8304b18a1a93033104366d4e4d58ce58531ad128 Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Sun, 8 Nov 2026 23:44:01 +0530 Subject: [PATCH 5/7] test(kin): chase through the real memory lock, back off and throttle the sweep Moves the fake memory remote into a shared fixture so deferral tests run the real Memory layer, and documents the transaction rule in the spec. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/kin/memory/Memory.test.ts | 134 +------------- .../src/kin/memory/__fixtures__/repository.ts | 139 ++++++++++++++ .../server/src/kin/triggers/deferrals.test.ts | 173 +++++++++++++++++- .../src/kin/triggers/routines.testkit.ts | 43 +++-- docs/kin/specs/triggers.md | 2 +- 5 files changed, 335 insertions(+), 156 deletions(-) create mode 100644 apps/server/src/kin/memory/__fixtures__/repository.ts diff --git a/apps/server/src/kin/memory/Memory.test.ts b/apps/server/src/kin/memory/Memory.test.ts index 942b89ffbba43..6ea0139003f49 100644 --- a/apps/server/src/kin/memory/Memory.test.ts +++ b/apps/server/src/kin/memory/Memory.test.ts @@ -1,6 +1,4 @@ import * as NodeZlib from "node:zlib"; -// @effect-diagnostics-next-line nodeBuiltinImport:off -- Git object identity uses the native byte-oriented hash API. -import * as NodeCrypto from "node:crypto"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { expect, it } from "@effect/vitest"; import { KinAgentId, KinWakeupId, KinMemoryItemId, KinOrgId } from "@t3tools/contracts"; @@ -12,7 +10,6 @@ import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Logger from "effect/Logger"; import * as TestClock from "effect/testing/TestClock"; -import { ChildProcessSpawner } from "effect/process"; import * as VcsProcess from "../../vcs/VcsProcess.ts"; import * as KinPersistence from "../persistence/KinPersistence.ts"; import * as Sqlite from "../../persistence/Sqlite.ts"; @@ -23,6 +20,7 @@ import * as MemoryCache from "./MemoryCache.ts"; import * as MemoryConfig from "./MemoryConfig.ts"; import * as ServerConfig from "../../config.ts"; import { fixtureItem } from "./__fixtures__/items.ts"; +import { fakeRepository } from "./__fixtures__/repository.ts"; const memoryTestLayer = Memory.layer.pipe( Layer.provide( @@ -44,136 +42,6 @@ const databaseLayer = Layer.effectDiscard( }), ).pipe(Layer.provideMerge(KinPersistence.layer.pipe(Layer.provideMerge(Sqlite.layerMemory)))); -const fakeRepository = (fs: FileSystem.FileSystem) => { - const files = new Map(); - const objects = new Map(); - const object = (type: string, bytes: Buffer) => { - const sha = NodeCrypto.createHash("sha1") - .update(`${type} ${bytes.length}\0`) - .update(bytes) - .digest("hex"); - objects.set(sha, { type, bytes }); - return sha; - }; - const head = () => { - const tree = (prefix: string): string => { - const names = [ - ...new Set( - [...files.keys()] - .filter((name) => name.startsWith(prefix)) - .map((name) => name.slice(prefix.length).split("/")[0] ?? ""), - ), - ].sort(); - return object( - "tree", - Buffer.concat( - names.map((name) => { - const text = files.get(prefix + name); - const sha = - text === undefined ? tree(prefix + name + "/") : object("blob", Buffer.from(text)); - return Buffer.concat([ - Buffer.from(`${text === undefined ? "40000" : "100644"} ${name}\0`), - Buffer.from(sha, "hex"), - ]); - }), - ), - ); - }; - return object("commit", Buffer.from(`tree ${tree("")}\n\nfixture ${revision}\n`)); - }; - let validations = 0; - let treeReads = 0; - let blobReads = 0; - let revision = 0; - let corruptNextBatch = false; - const layer = Layer.succeed( - VcsProcess.VcsProcess, - VcsProcess.VcsProcess.of({ - run: Effect.fn("Test.git")(({ args: rawArgs, stdin, cwd, onStdoutChunk }) => { - const args: string[] = []; - for (let i = 0; i < rawArgs.length; i++) { - const arg = rawArgs[i]; - if (arg === "-c") { - i++; - continue; - } - if ( - arg && - !arg.startsWith("--git-dir=") && - !arg.startsWith("--work-tree=") && - arg !== "--no-replace-objects" - ) - args.push(arg); - } - let stdout = ""; - switch (args[0]) { - case "fetch": - case "rev-parse": - if (args.includes("refs/heads/main")) validations++; - stdout = args.includes("--show-toplevel") - ? cwd - : args.includes("--git-common-dir") - ? cwd.replace(/memory-read$/, "memory/.git") - : head(); - break; - case "ls-remote": - stdout = `${head()}\trefs/heads/main\n`; - break; - case "cat-file": { - const chunks = (stdin?.trim().split("\n") ?? []).map((sha) => { - const entry = objects.get(sha); - if (!entry) return Buffer.from(`${sha} missing\n`); - if (args.includes("--batch-check")) - return Buffer.from(`${sha} ${entry.type} ${entry.bytes.length}\n`); - if (entry.type === "blob") blobReads++; - if (entry.type === "tree") treeReads++; - return Buffer.concat([ - Buffer.from(`${sha} ${entry.type} ${entry.bytes.length}\n`), - entry.bytes, - Buffer.from("\n"), - ]); - }); - if (corruptNextBatch) { - chunks.unshift(Buffer.from("invalid header\n")); - corruptNextBatch = false; - } - if (args.includes("--batch-check")) stdout = Buffer.concat(chunks).toString("utf8"); - else onStdoutChunk?.(Buffer.concat(chunks)); - break; - } - } - onStdoutChunk?.(Buffer.from(stdout)); - return Effect.gen(function* () { - if (args[0] === "clone" || args[0] === "clone-snapshot") { - yield* fs - .makeDirectory(`${args.at(-1)}/.git/objects`, { recursive: true }) - .pipe(Effect.orDie); - } - return { - exitCode: ChildProcessSpawner.ExitCode(0), - stdout, - stderr: "", - stdoutTruncated: false, - stderrTruncated: false, - }; - }); - }), - }), - ); - return { - files, - head, - layer, - commit: () => { - revision++; - return head(); - }, - counts: () => ({ treeReads, blobReads, validations }), - corruptBatch: () => { - corruptNextBatch = true; - }, - }; -}; const configLayer = (baseDirectory: string, repoUrl: string) => Layer.succeed(MemoryConfig.MemoryConfig, { baseDirectory, diff --git a/apps/server/src/kin/memory/__fixtures__/repository.ts b/apps/server/src/kin/memory/__fixtures__/repository.ts new file mode 100644 index 0000000000000..407c90b1d0523 --- /dev/null +++ b/apps/server/src/kin/memory/__fixtures__/repository.ts @@ -0,0 +1,139 @@ +// @effect-diagnostics-next-line nodeBuiltinImport:off -- Git object identity uses the native byte-oriented hash API. +import * as NodeCrypto from "node:crypto"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import { ChildProcessSpawner } from "effect/process"; +import * as VcsProcess from "../../../vcs/VcsProcess.ts"; + +/** An in-memory git remote: `files` is the tree, `commit()` makes a new head. */ +export const fakeRepository = (fs: FileSystem.FileSystem) => { + const files = new Map(); + const objects = new Map(); + const object = (type: string, bytes: Buffer) => { + const sha = NodeCrypto.createHash("sha1") + .update(`${type} ${bytes.length}\0`) + .update(bytes) + .digest("hex"); + objects.set(sha, { type, bytes }); + return sha; + }; + const head = () => { + const tree = (prefix: string): string => { + const names = [ + ...new Set( + [...files.keys()] + .filter((name) => name.startsWith(prefix)) + .map((name) => name.slice(prefix.length).split("/")[0] ?? ""), + ), + ].sort(); + return object( + "tree", + Buffer.concat( + names.map((name) => { + const text = files.get(prefix + name); + const sha = + text === undefined ? tree(prefix + name + "/") : object("blob", Buffer.from(text)); + return Buffer.concat([ + Buffer.from(`${text === undefined ? "40000" : "100644"} ${name}\0`), + Buffer.from(sha, "hex"), + ]); + }), + ), + ); + }; + return object("commit", Buffer.from(`tree ${tree("")}\n\nfixture ${revision}\n`)); + }; + let validations = 0; + let treeReads = 0; + let blobReads = 0; + let revision = 0; + let corruptNextBatch = false; + const layer = Layer.succeed( + VcsProcess.VcsProcess, + VcsProcess.VcsProcess.of({ + run: Effect.fn("Test.git")(({ args: rawArgs, stdin, cwd, onStdoutChunk }) => { + const args: string[] = []; + for (let i = 0; i < rawArgs.length; i++) { + const arg = rawArgs[i]; + if (arg === "-c") { + i++; + continue; + } + if ( + arg && + !arg.startsWith("--git-dir=") && + !arg.startsWith("--work-tree=") && + arg !== "--no-replace-objects" + ) + args.push(arg); + } + let stdout = ""; + switch (args[0]) { + case "fetch": + case "rev-parse": + if (args.includes("refs/heads/main")) validations++; + stdout = args.includes("--show-toplevel") + ? cwd + : args.includes("--git-common-dir") + ? cwd.replace(/memory-read$/, "memory/.git") + : head(); + break; + case "ls-remote": + stdout = `${head()}\trefs/heads/main\n`; + break; + case "cat-file": { + const chunks = (stdin?.trim().split("\n") ?? []).map((sha) => { + const entry = objects.get(sha); + if (!entry) return Buffer.from(`${sha} missing\n`); + if (args.includes("--batch-check")) + return Buffer.from(`${sha} ${entry.type} ${entry.bytes.length}\n`); + if (entry.type === "blob") blobReads++; + if (entry.type === "tree") treeReads++; + return Buffer.concat([ + Buffer.from(`${sha} ${entry.type} ${entry.bytes.length}\n`), + entry.bytes, + Buffer.from("\n"), + ]); + }); + if (corruptNextBatch) { + chunks.unshift(Buffer.from("invalid header\n")); + corruptNextBatch = false; + } + if (args.includes("--batch-check")) stdout = Buffer.concat(chunks).toString("utf8"); + else onStdoutChunk?.(Buffer.concat(chunks)); + break; + } + } + onStdoutChunk?.(Buffer.from(stdout)); + return Effect.gen(function* () { + if (args[0] === "clone" || args[0] === "clone-snapshot") { + yield* fs + .makeDirectory(`${args.at(-1)}/.git/objects`, { recursive: true }) + .pipe(Effect.orDie); + } + return { + exitCode: ChildProcessSpawner.ExitCode(0), + stdout, + stderr: "", + stdoutTruncated: false, + stderrTruncated: false, + }; + }); + }), + }), + ); + return { + files, + head, + layer, + commit: () => { + revision++; + return head(); + }, + counts: () => ({ treeReads, blobReads, validations }), + corruptBatch: () => { + corruptNextBatch = true; + }, + }; +}; diff --git a/apps/server/src/kin/triggers/deferrals.test.ts b/apps/server/src/kin/triggers/deferrals.test.ts index ee91d641900e0..e5812d9531013 100644 --- a/apps/server/src/kin/triggers/deferrals.test.ts +++ b/apps/server/src/kin/triggers/deferrals.test.ts @@ -1,3 +1,4 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, it } from "@effect/vitest"; import { KinAgentId, @@ -15,8 +16,11 @@ import { import * as Clock from "effect/Clock"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; +import * as Logger from "effect/Logger"; import * as Option from "effect/Option"; +import * as References from "effect/References"; import * as Schema from "effect/Schema"; import * as SqlClient from "effect/sql/SqlClient"; import * as TestClock from "effect/testing/TestClock"; @@ -24,6 +28,10 @@ import * as TestClock from "effect/testing/TestClock"; import * as Repository from "../chat/KinChatRepository.ts"; import * as Queue from "../lead/KinEventQueue.ts"; import { WorkerLaunchGate } from "../lead/Workers.ts"; +import { fixtureItem } from "../memory/__fixtures__/items.ts"; +import { fakeRepository } from "../memory/__fixtures__/repository.ts"; +import * as Memory from "../memory/Memory.ts"; +import * as MemoryConfig from "../memory/MemoryConfig.ts"; import * as MemoryWriter from "../memory/MemoryWriter.ts"; import * as DeferralFlow from "./deferralFlow.ts"; import { @@ -56,11 +64,14 @@ const widgetId = widgetFor("def-1"); const dueAt = Date.parse("2026-11-13T03:30:00.000Z"); const headsUpAt = dueAt - 30 * 60_000; -const layerFor = (items: ReadonlyArray) => +const layerFor = ( + items: ReadonlyArray, + options: Parameters>[1] = {}, +) => DeferralFlow.layer.pipe( Layer.provideMerge(Deferrals.layer), Layer.provideMerge(TriggerService.layer), - Layer.provideMerge(fixture(items)), + Layer.provideMerge(fixture(items, options)), ); const fakeGithub = (states: Map>) => ({ @@ -595,3 +606,161 @@ it.effect("remind, ask and chase act at due, noting lateness", () => { assert.lengthOf(yield* pendingFires(), 0); }).pipe(Effect.provide(layerFor(items))); }); + +it.effect("chases through the real memory lock without reading memory inside the write", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.realPath(yield* fs.makeTempDirectoryScoped()); + const repository = fakeRepository(fs); + for (const dir of ["memory/.git/objects", "memory/.git/refs", "memory-read"]) + yield* fs.makeDirectory(`${root}/a/${dir}`, { recursive: true }); + const memoryItem = (id: string, due: string, onDue: string) => + fixtureItem({ + id, + due, + on_due: onDue, + owner: "owner", + said_by: "owner", + chat: "chat-a", + links: [], + }); + repository.files.set( + "deferrals/def-c.md", + memoryItem("def-c", "2026-11-13T09:00+05:30", "chase"), + ); + const writing: Array = []; + const spied = Layer.effect( + Memory.Memory, + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const memory = yield* Memory.Memory; + // Any memory call while this fiber holds the SQLite writer can deadlock against load. + const spy = (name: string, effect: Effect.Effect) => + Effect.serviceOption(sql.transactionService).pipe( + Effect.tap((open) => Effect.sync(() => Option.isSome(open) && writing.push(name))), + Effect.andThen(effect), + ); + return Memory.Memory.of({ + ...memory, + query: (org, q) => spy("query", memory.query(org, q)), + readItem: (org, id) => spy("readItem", memory.readItem(org, id)), + readItemQuietly: (org, id) => spy("readItemQuietly", memory.readItemQuietly(org, id)), + }); + }), + ).pipe( + Layer.provide(Memory.layer), + Layer.provide(Layer.succeed(Memory.MemoryRecoveryHooks, { notify: () => Effect.void })), + Layer.provide(TriggerService.syncHooksLayer.pipe(Layer.provide(TriggerService.layer))), + Layer.provide( + Layer.succeed(MemoryConfig.MemoryConfig, { + baseDirectory: root, + repositories: [{ orgId, repoUrl: "unused", path: "a" }], + }), + ), + Layer.provide(repository.layer), + Layer.provide(NodeServices.layer), + ); + yield* Effect.gen(function* () { + yield* setup(); + const sql = yield* SqlClient.SqlClient; + const memory = yield* Memory.Memory; + yield* memory.anchor(orgId, "main", repository.head()); + yield* memory.query(orgId, {}); + assert.equal((yield* runOf("def-c"))?.state, "armed"); + + // A new head lands before the chase, so its load re-syncs triggers under the memory lock. + repository.files.set( + "deferrals/def-n.md", + memoryItem("def-n", "2026-11-20T09:00+05:30", "remind"), + ); + yield* memory.record(orgId, "main", yield* memory.anchor(orgId, "main"), repository.commit()); + yield* tick(dueAt); + + assert.equal((yield* runOf("def-c"))?.state, "chased"); + assert.equal((yield* runOf("def-n"))?.state, "armed"); + const pings = yield* sql`SELECT person FROM kin_ping_items + WHERE org_id = ${orgId} AND item_id = 'def-c' ORDER BY person`; + assert.deepStrictEqual(pings, [{ person: "owner-a" }]); + assert.deepStrictEqual(writing, []); + }).pipe(Effect.provide(layerFor([], { memory: spied }))); + }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), +); + +it.effect("backs off a failing fire and warns once", () => { + const remind = makeDeferral("def-r", { on_due: "remind" }); + const failures: Array = []; + const logger = Logger.layer( + [ + Logger.make(({ message, logLevel }) => { + if (String(message).includes("Kin deferral fire processing failed")) + failures.push(logLevel); + }), + ], + { mergeWithExisting: false }, + ); + return Effect.gen(function* () { + yield* setup(); + const sql = yield* SqlClient.SqlClient; + yield* (yield* TriggerService.KinTriggerService).syncItem(remind); + yield* sql`CREATE TEMP TRIGGER fail_remind BEFORE INSERT ON kin_messages + BEGIN SELECT RAISE(ABORT, 'fixture failure'); END`; + for (const seconds of [0, 5, 60, 90, 180]) yield* tick(dueAt + seconds * 1000); + // Retries at +1 and +3 minutes, each after a longer wait. + assert.deepStrictEqual(failures, ["Warn", "Debug", "Debug"]); + yield* sql`DROP TRIGGER fail_remind`; + yield* tick(dueAt + 400_000); + assert.lengthOf(failures, 3); + yield* tick(dueAt + 420_000); + assert.equal((yield* runOf("def-r"))?.state, "reminded"); + assert.lengthOf(yield* pendingFires(), 0); + }).pipe( + Effect.provide(layerFor([remind])), + Effect.provide(logger), + Effect.provideService(References.MinimumLogLevel, "Debug"), + ); +}); + +it.effect("sweeps pings once a minute unless a chase just put one", () => { + const chase = makeDeferral("def-c", { on_due: "chase", owner: "owner" }); + const items = [chase]; + let sweeps = 0; + const query = () => (org: typeof orgId) => + Effect.sync(() => { + sweeps++; + return items + .filter((entry) => entry.orgId === org) + .map((entry) => ({ ...entry, duplicate: false })); + }); + return Effect.gen(function* () { + yield* setup(); + const sql = yield* SqlClient.SqlClient; + const service = yield* TriggerService.KinTriggerService; + yield* service.syncItem(chase); + yield* tick(dueAt); + assert.equal((yield* runOf("def-c"))?.state, "chased"); + assert.equal(sweeps, 1); + yield* tick(dueAt + 5_000); + assert.equal(sweeps, 1); + yield* tick(dueAt + 65_000); + assert.equal(sweeps, 2); + + // Re-chasing with a new due while the first ping is pending asks instead. + items[0] = makeDeferral("def-c", { + on_due: "chase", + owner: "owner", + due: "2026-11-14T09:00+05:30", + }); + assert.equal(yield* service.syncItem(items[0]), "armed"); + yield* tick(dueAt + 24 * 60 * 60_000); + assert.equal((yield* runOf("def-c"))?.state, "asked"); + assert.include((yield* cardOf("def-c")).text, "I couldn't chase owner"); + assert.equal(sweeps, 3); + + // An undecodable inbox row fails the drain, but the sweep still runs. + yield* sql`INSERT INTO kin_trigger_inbox (fire_id, org_id, payload_json, received_at) + VALUES ('bad', ${orgId}, '{"payload":{"subject":{"type":"deferral"}}}', 0)`; + const failed = yield* tick(dueAt + 24 * 60 * 60_000 + 65_000).pipe(Effect.flip); + assert.equal(failed._tag, "KinDeferralFlowError"); + assert.equal(sweeps, 4); + }).pipe(Effect.provide(layerFor(items, { query }))); +}); diff --git a/apps/server/src/kin/triggers/routines.testkit.ts b/apps/server/src/kin/triggers/routines.testkit.ts index a1c47e4ae09af..a54c19bf9b9a9 100644 --- a/apps/server/src/kin/triggers/routines.testkit.ts +++ b/apps/server/src/kin/triggers/routines.testkit.ts @@ -43,11 +43,13 @@ export const item = (id: string, overrides: Partial = {}) => links: ["https://example.com/evidence"], ...overrides, }); -export function fixture( +export function fixture( items: ReadonlyArray = [], options: { query?: (sql: SqlClient.SqlClient) => Memory.Memory["Service"]["query"]; register?: Scheduler.Scheduler["Service"]["register"]; + /** Replaces the mock, e.g. with the real layer over a fake repository. */ + memory?: Layer.Layer; } = {}, ) { const database = Persistence.layer.pipe(Layer.provideMerge(Sqlite.layerMemory)); @@ -61,26 +63,27 @@ export function fixture( Layer.provide(Layer.mock(ThreadManagement.ThreadManagementService)({})), ); const dependencies = Layer.mergeAll( - Layer.unwrap( - Effect.map(SqlClient.SqlClient, (sql) => - Layer.mock(Memory.Memory)({ - query: - options.query?.(sql) ?? - ((orgId) => - Effect.succeed( - items - .filter((item) => item.orgId === orgId) - .map((item) => ({ ...item, duplicate: false })), - )), - readItemQuietly: (orgId, id) => { - const head = items.find((item) => item.orgId === orgId && item.id === id); - return head === undefined - ? Effect.fail(new Memory.MemoryNotFound({ orgId, id })) - : Effect.succeed({ head, body: "Body\n" }); - }, - }), + options.memory ?? + Layer.unwrap( + Effect.map(SqlClient.SqlClient, (sql) => + Layer.mock(Memory.Memory)({ + query: + options.query?.(sql) ?? + ((orgId) => + Effect.succeed( + items + .filter((item) => item.orgId === orgId) + .map((item) => ({ ...item, duplicate: false })), + )), + readItemQuietly: (orgId, id) => { + const head = items.find((item) => item.orgId === orgId && item.id === id); + return head === undefined + ? Effect.fail(new Memory.MemoryNotFound({ orgId, id })) + : Effect.succeed({ head, body: "Body\n" }); + }, + }), + ), ), - ), Chat.layer.pipe(Layer.provide(JudgeShadow.layerOff), Layer.provideMerge(Hub.layer)), TriggerScheduler.layer, ).pipe( diff --git a/docs/kin/specs/triggers.md b/docs/kin/specs/triggers.md index a3647846ff330..cdb084d6f08e8 100644 --- a/docs/kin/specs/triggers.md +++ b/docs/kin/specs/triggers.md @@ -233,7 +233,7 @@ armed ──heads_up fire──▶ recheck ──changed──▶ asked (decis started ─▶ worker via delegate_task (LEAD) ─▶ closed with PR and proof ``` -- **Consumer.** `KinDeferralFlow` in [`deferralFlow.ts`](../../../apps/server/src/kin/triggers/deferralFlow.ts) drains each org's pending deferral fires on its own `Scheduler` tick, in due order. Like routines, it does read-only or idempotent I/O first (memory read, GitHub reads, gate checks), then one transaction that rechecks the run row, writes the state and card, puts the chase's ping, enqueues, and marks the fire consumed. A failing fire retries on the next tick; one that fails to decode, or still fails 24 hours past due, is consumed after a best-effort server notice to the owner ("I couldn't handle the deferral …"). After the fires, the same tick runs `KinPings.sweep` for the org, so a chase that lands in quiet hours delivers when they end. Code posts every card: the item's card is updated in place, or posted to the item's chat, else the owner's direct chat. +- **Consumer.** `KinDeferralFlow` in [`deferralFlow.ts`](../../../apps/server/src/kin/triggers/deferralFlow.ts) drains each org's pending deferral fires on its own `Scheduler` tick, in due order. Like routines, it does read-only or idempotent I/O first (memory read, GitHub reads, gate checks), then one transaction that rechecks the run row, writes the state and card, puts the chase's ping, enqueues, and marks the fire consumed. That transaction does no memory or GitHub I/O: the memory org lock is held while a new head syncs triggers, so a memory read while holding the SQLite writer can deadlock. The chase hands `KinPings.put` the item head it already read. Every run-state change is guarded on the state that was read. A failing fire retries after 1 minute, doubling to at most 15 minutes, held per process; its first failure logs a warning. One that fails to decode, or still fails 24 hours past due, is consumed after a best-effort server notice to the owner ("I couldn't handle the deferral …"). After the fires, even when listing them fails, the tick runs `KinPings.sweep` for the org: at once when a chase just put a ping, otherwise at most once a minute. A chase that lands in quiet hours delivers when they end. Code posts every card: the item's card is updated in place, or posted to the item's chat, else the owner's direct chat. - **The recheck is code.** It reads the memory item and every GitHub link through `KinDeferralGithubReader`. Any difference from the snapshot means "changed": an issue closed, an assignee added or removed, another linked PR, or the item edited or superseded. A GitHub half that couldn't be read, at arm time or now, counts as "changed: couldn't verify". The ask names what changed. - **Late.** If `due` has passed when the `heads_up` fire is consumed, it is skipped. A `due` fire more than 10 minutes late notes "This was due at 9:00". A `start` due fire within the 30-minute grace window, or one with no heads-up shown, posts the heads-up card again with a grace from now: the `due` trigger re-arms at the grace end (state `grace`, `graceEndsAt` on the card and in the run's snapshot), and silence still means go. An older fire asks instead of starting. `remind`, `ask` and `chase` act at due only and carry the same note. - **Stale fires.** Fires wait in the inbox until the consumer handles them. It consumes, and otherwise ignores, a fire whose run row is missing or `disarmed`, whose `due_at` no longer equals the fire's due instant (a grace fire is the one exception, and only when its instant equals the recorded grace end), or whose run already left the state that fire expects. From bdaa2dbc7fe58a49b3cc46c9357b5c8b8392c306 Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Sun, 8 Nov 2026 23:47:00 +0530 Subject: [PATCH 6/7] test(kin): provide the backoff test's layers in one call Co-Authored-By: Claude Opus 5.5 --- apps/server/src/kin/triggers/deferrals.test.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/server/src/kin/triggers/deferrals.test.ts b/apps/server/src/kin/triggers/deferrals.test.ts index e5812d9531013..4e97fa0d66697 100644 --- a/apps/server/src/kin/triggers/deferrals.test.ts +++ b/apps/server/src/kin/triggers/deferrals.test.ts @@ -714,8 +714,7 @@ it.effect("backs off a failing fire and warns once", () => { assert.equal((yield* runOf("def-r"))?.state, "reminded"); assert.lengthOf(yield* pendingFires(), 0); }).pipe( - Effect.provide(layerFor([remind])), - Effect.provide(logger), + Effect.provide(Layer.merge(layerFor([remind]), logger)), Effect.provideService(References.MinimumLogLevel, "Debug"), ); }); From 95ad4e41c580b16dcde4c39611e461ef65436071 Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Mon, 9 Nov 2026 00:01:50 +0530 Subject: [PATCH 7/7] test(kin): fail any trigger test that reads memory inside a SQLite transaction Retry backoff is now keyed per org and drops entries for fires no longer pending, and the GitHub reader seam states it runs under the memory lock. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/kin/triggers/deferralFlow.ts | 21 ++++++--- .../server/src/kin/triggers/deferralGithub.ts | 5 ++- .../src/kin/triggers/routines.testkit.ts | 44 ++++++++++++------- 3 files changed, 49 insertions(+), 21 deletions(-) diff --git a/apps/server/src/kin/triggers/deferralFlow.ts b/apps/server/src/kin/triggers/deferralFlow.ts index 485db8b7e47eb..8831196d01566 100644 --- a/apps/server/src/kin/triggers/deferralFlow.ts +++ b/apps/server/src/kin/triggers/deferralFlow.ts @@ -126,7 +126,10 @@ const make = Effect.gen(function* () { const cards = yield* makeDeferralCards; const start = yield* makeDeferralStarter; // Per process: the 24 h give-up is the durable bound, so a restart only retries sooner. - const retries = new Map(); + const retries = new Map< + KinOrgId, + Map + >(); const sweptAt = new Map(); const readRun = (orgId: KinOrgId, itemId: KinMemoryItemId) => @@ -390,6 +393,14 @@ const make = Effect.gen(function* () { yield* Effect.gen(function* () { const timeZone = yield* cache.ownerTimeZone(orgId); const fires = yield* inbox.listPending(orgId, 100, "deferral"); + const orgRetries = retries.get(orgId) ?? new Map(); + retries.set(orgId, orgRetries); + // A full page may hide pending fires, so only a short one can prove an entry stale. + if (fires.length < 100) { + const pending = new Set(fires.map((fire) => fire.fireId)); + for (const fireId of orgRetries.keys()) + if (!pending.has(fireId)) orgRetries.delete(fireId); + } // Catch-up fires share received_at; handle them by due time. for (const fire of fires.toSorted( (a, b) => @@ -398,13 +409,13 @@ const make = Effect.gen(function* () { )) { if (fire.payload.kind !== "time" || fire.payload.subject.type !== "deferral") continue; const { itemId, phase } = fire.payload.subject; - const retry = retries.get(fire.fireId); + const retry = orgRetries.get(fire.fireId); const giveUp = now - Date.parse(fire.dueAt) > GIVE_UP_MS; if (retry !== undefined && now < retry.notBefore && !giveUp) continue; yield* process(orgId, fire, itemId, phase, now, timeZone).pipe( Effect.tap((put) => Effect.sync(() => { - retries.delete(fire.fireId); + orgRetries.delete(fire.fireId); if (put) chased = true; }), ), @@ -427,10 +438,10 @@ const make = Effect.gen(function* () { ); if (!consume) { const delay = Math.min(RETRY_MAX_MS, RETRY_FIRST_MS * 2 ** (failures - 1)); - retries.set(fire.fireId, { notBefore: now + delay, failures }); + orgRetries.set(fire.fireId, { notBefore: now + delay, failures }); return; } - retries.delete(fire.fireId); + orgRetries.delete(fire.fireId); yield* noticeFailure(orgId, fire, itemId, timeZone).pipe( Effect.catch((cause) => Effect.logWarning("Kin deferral failure notice failed", { diff --git a/apps/server/src/kin/triggers/deferralGithub.ts b/apps/server/src/kin/triggers/deferralGithub.ts index cb0965274ee55..5e9e388662a54 100644 --- a/apps/server/src/kin/triggers/deferralGithub.ts +++ b/apps/server/src/kin/triggers/deferralGithub.ts @@ -22,7 +22,10 @@ export class KinDeferralGithubError extends Schema.TaggedError( const dependencies = Layer.mergeAll( options.memory ?? Layer.unwrap( - Effect.map(SqlClient.SqlClient, (sql) => - Layer.mock(Memory.Memory)({ - query: - options.query?.(sql) ?? - ((orgId) => - Effect.succeed( - items - .filter((item) => item.orgId === orgId) - .map((item) => ({ ...item, duplicate: false })), - )), + Effect.map(SqlClient.SqlClient, (sql) => { + // Memory takes its org lock, which load holds while it waits for the SQLite writer. + const outsideTransaction = (name: string, effect: Effect.Effect) => + Effect.serviceOption(sql.transactionService).pipe( + Effect.flatMap((open) => + Option.isSome(open) + ? Effect.die(`Memory.${name} ran inside a SQLite transaction`) + : effect, + ), + ); + const query = + options.query?.(sql) ?? + ((orgId: KinOrgId) => + Effect.succeed( + items + .filter((item) => item.orgId === orgId) + .map((item) => ({ ...item, duplicate: false })), + )); + return Layer.mock(Memory.Memory)({ + query: (orgId, input) => outsideTransaction("query", query(orgId, input)), readItemQuietly: (orgId, id) => { const head = items.find((item) => item.orgId === orgId && item.id === id); - return head === undefined - ? Effect.fail(new Memory.MemoryNotFound({ orgId, id })) - : Effect.succeed({ head, body: "Body\n" }); + return outsideTransaction( + "readItemQuietly", + head === undefined + ? Effect.fail(new Memory.MemoryNotFound({ orgId, id })) + : Effect.succeed({ head, body: "Body\n" }), + ); }, - }), - ), + }); + }), ), Chat.layer.pipe(Layer.provide(JudgeShadow.layerOff), Layer.provideMerge(Hub.layer)), TriggerScheduler.layer,