From 2f804b48b3a2fb0de706425aed1efc7e94862ffd Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Mon, 9 Nov 2026 05:48:50 +0530 Subject: [PATCH 1/2] feat(kin): read GitHub for deferrals and watchdogs through the Kin App Back KinDeferralGithubReader (TRIG-8) and KinWatchdogGithubReader (TRIG-10) with GraphQL reads over KinGitHubApp when KIN_GITHUB_APP_ID is set and the App key is in the keychain. One read-only server-holder token per repo, cached until five minutes before expiry. Out-of-scope repos, rate limits and timeouts fail closed. Without the App, startup warns once and the unavailable/null defaults stay. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/kin/lead/layer.ts | 2 + .../server/src/kin/triggers/deferralGithub.ts | 4 +- .../src/kin/triggers/githubReaders.test.ts | 317 +++++++++++++++++ apps/server/src/kin/triggers/githubReaders.ts | 333 ++++++++++++++++++ apps/server/src/kin/triggers/watchdogs.ts | 4 +- docs/kin/specs/approvals.md | 2 +- docs/kin/specs/triggers.md | 7 +- 7 files changed, 661 insertions(+), 8 deletions(-) create mode 100644 apps/server/src/kin/triggers/githubReaders.test.ts create mode 100644 apps/server/src/kin/triggers/githubReaders.ts diff --git a/apps/server/src/kin/lead/layer.ts b/apps/server/src/kin/lead/layer.ts index 6e490580f89fa..9f5eddf119369 100644 --- a/apps/server/src/kin/lead/layer.ts +++ b/apps/server/src/kin/lead/layer.ts @@ -3,6 +3,7 @@ import * as Routines from "../triggers/routines.ts"; import * as DeferralFlow from "../triggers/deferralFlow.ts"; import * as GithubEvents from "../triggers/githubEvents.ts"; import * as Watchdogs from "../triggers/watchdogs.ts"; +import * as GithubReaders from "../triggers/githubReaders.ts"; import * as OwnerCheck from "../triggers/ownerCheck.ts"; import * as TriggerScheduler from "../triggers/KinTriggerScheduler.ts"; import * as TriggerStore from "../triggers/KinTriggerStore.ts"; @@ -160,6 +161,7 @@ const withServices = (entry: Layer.Layer) => ), ), Layer.provide(KinPersistence.layer), + Layer.provide(GithubReaders.layer), ); const registration = McpHttpServer.toolkitRegistration(KinLeadToolkit, handlers); diff --git a/apps/server/src/kin/triggers/deferralGithub.ts b/apps/server/src/kin/triggers/deferralGithub.ts index 5e9e388662a54..21fdc58b16ec3 100644 --- a/apps/server/src/kin/triggers/deferralGithub.ts +++ b/apps/server/src/kin/triggers/deferralGithub.ts @@ -32,8 +32,8 @@ export interface KinDeferralGithubReaderShape { ) => Effect.Effect, KinDeferralGithubError>; } -// Kin has no GitHub client yet, so production reads fail and every recheck with -// GitHub links reports "couldn't verify", which asks instead of starting. +// githubReaders.ts replaces this when the Kin App is configured. Without it every read fails, +// so a recheck with GitHub links reports "couldn't verify", which asks instead of starting. export const KinDeferralGithubReader = Context.Reference( "t3/kin/triggers/deferralGithub/KinDeferralGithubReader", { diff --git a/apps/server/src/kin/triggers/githubReaders.test.ts b/apps/server/src/kin/triggers/githubReaders.test.ts new file mode 100644 index 0000000000000..c6dbeb95b9eb3 --- /dev/null +++ b/apps/server/src/kin/triggers/githubReaders.test.ts @@ -0,0 +1,317 @@ +// @effect-diagnostics nodeBuiltinImport:off -- reads GitHub's vendored GraphQL schema. +import * as NodeFS from "node:fs"; + +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, describe, it } from "@effect/vitest"; +import { KinOrgId } from "@t3tools/contracts"; +import * as ConfigProvider from "effect/ConfigProvider"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as Layer from "effect/Layer"; +import * as Logger from "effect/Logger"; +import * as Redacted from "effect/Redacted"; +import * as References from "effect/References"; +import * as TestClock from "effect/testing/TestClock"; +import { HttpClient } from "effect/http"; +import { buildSchema, parse, validate } from "graphql"; + +import * as KinGitHubApp from "../github/KinGitHubApp.ts"; +import { fakeGitHub, type FakeAnswer, type RecordedRequest } from "../github/testFakes.ts"; +import { KinDeferralGithubReader, readGithubLinks } from "./deferralGithub.ts"; +import { KinGithubRepoScope } from "./githubEvents.ts"; +import * as GithubReaders from "./githubReaders.ts"; +import { KinWatchdogGithubReader } from "./watchdogs.ts"; + +const orgId = KinOrgId.make("org-1"); +const REPO = "GLDRoger/t3code"; +const TOKEN = "ghs_reader_token_never_shown"; +const NOW = Date.UTC(2026, 10, 8, 12); + +const closedIssue = { + __typename: "Issue", + number: 7, + state: "CLOSED", + assignees: { nodes: [{ login: "zoe" }, { login: "amir" }] }, + closedByPullRequestsReferences: { nodes: [{ number: 12 }, { number: 9 }] }, +}; +const mergedPr = { + __typename: "PullRequest", + number: 9, + pullState: "MERGED", + assignees: { nodes: [] }, + reviewRequests: { nodes: [] }, + timelineItems: { nodes: [] }, +}; +const openPr = { + __typename: "PullRequest", + number: 11, + pullState: "OPEN", + assignees: { nodes: [{ login: "amir" }] }, + reviewRequests: { + nodes: [{ requestedReviewer: { login: "Zoe" } }, { requestedReviewer: {} }], + }, + timelineItems: { + nodes: [ + { createdAt: "2026-11-01T00:00:00Z", requestedReviewer: { login: "zoe" } }, + { createdAt: "2026-11-05T00:00:00Z", requestedReviewer: { login: "zoe" } }, + ], + }, +}; +const fixtures = new Map([ + [7, closedIssue], + [9, mergedPr], + [11, openPr], +]); + +/** GraphQL answers from `fixtures`; numbers it lacks come back null with NOT_FOUND, as GitHub does. */ +const graphql = (request: RecordedRequest): FakeAnswer => { + const body = request.body; + const query = + typeof body === "object" && body !== null && "query" in body && typeof body.query === "string" + ? body.query + : ""; + const numbers = [...query.matchAll(/\bi(\d+): issueOrPullRequest/g)].map((m) => Number(m[1])); + return { + status: 200, + body: { + data: { + repository: Object.fromEntries(numbers.map((n) => [`i${n}`, fixtures.get(n) ?? null])), + }, + errors: numbers + .filter((n) => !fixtures.has(n)) + .map((n) => ({ type: "NOT_FOUND", path: ["repository", `i${n}`] })), + }, + }; +}; + +const setup = ( + route: (request: RecordedRequest) => FakeAnswer = graphql, + client?: Layer.Layer, +) => { + const mints: Array[0]> = []; + const github = fakeGitHub(route); + const app = Layer.succeed( + KinGitHubApp.KinGitHubApp, + KinGitHubApp.KinGitHubApp.of({ + app: Effect.die("unused"), + installations: Effect.die("unused"), + installationToken: (input) => { + mints.push(input); + return Effect.succeed({ + token: Redacted.make(TOKEN), + repo: input.repo, + expiresAt: "2026-11-08T13:00:00Z", + permissions: { ...input.permissions, metadata: "read" }, + }); + }, + }), + ); + const layer = GithubReaders.readersLayer.pipe( + Layer.provide(app), + Layer.provide(client ?? github.layer), + Layer.provide(Layer.succeed(KinGithubRepoScope, { repos: [REPO] })), + ); + return { mints, requests: github.requests, layer }; +}; + +const readers = Effect.gen(function* () { + const watchdog = yield* KinWatchdogGithubReader; + assert.isNotNull(watchdog); + return { deferral: yield* KinDeferralGithubReader, watchdog: watchdog! }; +}); + +describe("Kin GitHub readers", () => { + it("asks GitHub a query its schema accepts", () => { + const schema = buildSchema( + NodeFS.readFileSync( + new URL( + "../../../../../packages/source-control-github/schema/github.graphql", + import.meta.url, + ), + "utf8", + ), + ); + assert.deepStrictEqual( + validate(schema, parse(GithubReaders.itemsQuery([7, 9]))).map((error) => error.message), + [], + ); + }); + + it.effect("reads a merged PR and a closed issue with one batched query and one token", () => { + const { mints, requests, layer } = setup(); + return Effect.gen(function* () { + yield* TestClock.setTime(NOW); + const { watchdog } = yield* readers; + const states = yield* watchdog.readRepo(orgId, REPO, [7, 9, 11, 404]); + assert.deepStrictEqual(states, [ + { number: 7, kind: "issue", state: "closed", reviewRequests: [] }, + { number: 9, kind: "pull", state: "merged", reviewRequests: [] }, + { + number: 11, + kind: "pull", + state: "open", + reviewRequests: [{ login: "Zoe", requestedAt: Date.parse("2026-11-05T00:00:00Z") }], + }, + ]); + assert.deepStrictEqual( + yield* readGithubLinks(orgId, ["https://github.com/GLDRoger/t3code/issues/7"]), + [ + { + link: "https://github.com/GLDRoger/t3code/issues/7", + state: "closed", + assignees: ["amir", "zoe"], + linkedPrs: [9, 12], + }, + ], + ); + assert.isNull(yield* readGithubLinks(orgId, ["GLDRoger/t3code#404"])); + + assert.deepStrictEqual( + requests.map(({ method, path }) => `${method} ${path}`), + ["POST /graphql", "POST /graphql", "POST /graphql"], + ); + assert.equal(requests[0]?.authorization, `Bearer ${TOKEN}`); + assert.deepInclude(requests[0]?.body, { variables: { owner: "GLDRoger", name: "t3code" } }); + assert.deepStrictEqual(mints, [ + { repo: REPO, permissions: { issues: "read", pull_requests: "read" }, holder: "server" }, + ]); + // Reused until five minutes before it expires, then minted again. + yield* TestClock.adjust("56 minutes"); + yield* watchdog.readRepo(orgId, REPO, [9]); + assert.lengthOf(mints, 2); + }).pipe(Effect.provide(layer)); + }); + + it.effect("fails closed for a repo outside KIN_GITHUB_REPOS without calling GitHub", () => { + const { mints, requests, layer } = setup(); + return Effect.gen(function* () { + const { deferral, watchdog } = yield* readers; + const read = yield* watchdog.readRepo(orgId, "other/repo", [7]).pipe(Effect.flip); + assert.equal(read.reason, "unavailable"); + const one = yield* deferral.read(orgId, { repo: "other/repo", number: 7 }).pipe(Effect.flip); + assert.equal(one.reason, "unavailable"); + assert.lengthOf(requests, 0); + assert.lengthOf(mints, 0); + }).pipe(Effect.provide(layer)); + }); + + it.effect.each([ + { + name: "HTTP 403 with no remaining quota", + answer: { + status: 403, + headers: { "x-ratelimit-remaining": "0" }, + body: { message: "API rate limit exceeded" }, + }, + }, + { + name: "GraphQL RATE_LIMITED", + answer: { status: 200, body: { data: null, errors: [{ type: "RATE_LIMITED" }] } }, + }, + ] satisfies ReadonlyArray<{ name: string; answer: FakeAnswer }>)( + "counts a rate limit ($name) as a failed repo, once", + ({ answer }) => { + const { requests, layer } = setup(() => answer); + return Effect.gen(function* () { + const { deferral, watchdog } = yield* readers; + assert.equal( + (yield* watchdog.readRepo(orgId, REPO, [7, 9]).pipe(Effect.flip)).reason, + "unavailable", + ); + assert.equal( + (yield* deferral.read(orgId, { repo: REPO, number: 7 }).pipe(Effect.flip)).reason, + "unavailable", + ); + assert.lengthOf(requests, 2); + }).pipe(Effect.provide(layer)); + }, + ); + + it.effect("bounds each read in time", () => { + const hanging = Layer.succeed( + HttpClient.HttpClient, + HttpClient.make(() => Effect.never), + ); + const { layer } = setup(graphql, hanging); + return Effect.gen(function* () { + const { deferral, watchdog } = yield* readers; + const one = yield* deferral + .read(orgId, { repo: REPO, number: 7 }) + .pipe(Effect.flip, Effect.forkChild); + const batch = yield* watchdog.readRepo(orgId, REPO, [7]).pipe(Effect.flip, Effect.forkChild); + yield* TestClock.adjust("10 seconds"); + assert.equal((yield* Fiber.join(one)).reason, "unavailable"); + yield* TestClock.adjust("20 seconds"); + assert.equal((yield* Fiber.join(batch)).reason, "timeout"); + }).pipe(Effect.provide(layer)); + }); + + it.effect("keeps the token out of logs and errors", () => { + const lines: Array = []; + const logger = Logger.layer( + [Logger.make(({ message }) => lines.push(JSON.stringify(message)))], + { mergeWithExisting: false }, + ); + let call = 0; + const answers: ReadonlyArray = [ + { status: 401, body: { message: "Bad credentials" } }, + { status: 429, headers: { "retry-after": "60" }, body: { message: "slow down" } }, + { status: 500, body: { message: "boom" } }, + ]; + const { requests, layer } = setup(() => answers[call++ % answers.length]!); + return Effect.gen(function* () { + yield* TestClock.setTime(NOW); + const { deferral, watchdog } = yield* readers; + const errors = [ + yield* watchdog.readRepo(orgId, REPO, [7]).pipe(Effect.flip), + yield* watchdog.readRepo(orgId, REPO, [7]).pipe(Effect.flip), + yield* deferral.read(orgId, { repo: REPO, number: 7 }).pipe(Effect.flip), + ]; + assert.isTrue(requests.every((request) => request.authorization === `Bearer ${TOKEN}`)); + assert.isNotEmpty(lines); + for (const text of [ + ...lines, + ...errors.flatMap((error) => [JSON.stringify(error), String(error), error.message]), + ]) + assert.notInclude(text, TOKEN); + }).pipe( + Effect.provide(Layer.merge(layer, logger)), + Effect.provideService(References.MinimumLogLevel, "Debug"), + ); + }); + + it.effect.each([ + { name: "no App id", config: {} }, + // A missing KIN_GITHUB_OWNER_LOGIN fails the App's scope, so the keychain is never read. + { name: "an App that can't be built", config: { KIN_GITHUB_APP_ID: "123456" } }, + ])("keeps the fail-closed defaults with one warning for $name", ({ config }) => { + const warnings: Array = []; + const logger = Logger.layer( + [ + Logger.make(({ message, logLevel }) => { + if (logLevel === "Warn") warnings.push(String(message)); + }), + ], + { mergeWithExisting: false }, + ); + return Effect.gen(function* () { + assert.isNull(yield* KinWatchdogGithubReader); + const read = yield* (yield* KinDeferralGithubReader) + .read(orgId, { repo: REPO, number: 7 }) + .pipe(Effect.flip); + assert.equal(read.reason, "unavailable"); + assert.lengthOf( + warnings.filter((warning) => warning.includes("Kin GitHub reads are off")), + 1, + ); + }).pipe( + Effect.provide( + GithubReaders.layer.pipe( + Layer.provide(ConfigProvider.layer(ConfigProvider.fromUnknown(config))), + Layer.provide(NodeServices.layer), + Layer.provideMerge(logger), + ), + ), + ); + }); +}); diff --git a/apps/server/src/kin/triggers/githubReaders.ts b/apps/server/src/kin/triggers/githubReaders.ts new file mode 100644 index 0000000000000..b47cab0415389 --- /dev/null +++ b/apps/server/src/kin/triggers/githubReaders.ts @@ -0,0 +1,333 @@ +import { type KinOrgId, RepoSlug } from "@t3tools/contracts"; +import * as Cause from "effect/Cause"; +import * as Clock from "effect/Clock"; +import * as Config from "effect/Config"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import type * as Redacted from "effect/Redacted"; +import * as Schema from "effect/Schema"; +import { FetchHttpClient, HttpClient } from "effect/http"; + +import { gitHubAnswer, isGitHubRateLimit } from "../github/githubRest.ts"; +import * as Keychain from "../github/Keychain.ts"; +import * as KinGitHubApp from "../github/KinGitHubApp.ts"; +import type { KinGitHubPermissionSet } from "../github/KinGitHubPermissions.ts"; +import { KinGitHubScope } from "../github/KinGitHubScope.ts"; +import { + type GithubRef, + KinDeferralGithubError, + KinDeferralGithubReader, + type KinDeferralGithubReaderShape, +} from "./deferralGithub.ts"; +import { inRepoScope, KinGithubRepoScope, repoScopeLayer } from "./githubEvents.ts"; +import { + type GithubItemState, + KinWatchdogGithubError, + KinWatchdogGithubReader, + type KinWatchdogGithubReaderShape, +} from "./watchdogs.ts"; + +// The deferral read runs under the org memory lock, so it gets far less than a watchdog sweep. +const DEFERRAL_READ_TIMEOUT = "10 seconds"; +const WATCHDOG_READ_TIMEOUT = "30 seconds"; +const ITEMS_PER_QUERY = 50; +const TOKEN_REUSE_MARGIN_MS = 5 * 60_000; +const READ_PERMISSIONS = { + issues: "read", + pull_requests: "read", +} as const satisfies KinGitHubPermissionSet; + +// `state` is aliased on pull requests: GraphQL refuses one name with two enum types. +const ITEM_FRAGMENT = `fragment KinItem on IssueOrPullRequest { + __typename + ... on Issue { + number state + assignees(first: 50) { nodes { login } } + closedByPullRequestsReferences(first: 50, includeClosedPrs: true) { nodes { number } } + } + ... on PullRequest { + number pullState: state + assignees(first: 50) { nodes { login } } + reviewRequests(first: 20) { nodes { requestedReviewer { ... on User { login } } } } + timelineItems(last: 50, itemTypes: [REVIEW_REQUESTED_EVENT]) { + nodes { ... on ReviewRequestedEvent { createdAt requestedReviewer { ... on User { login } } } } + } + } +}`; +// Numbers are positive safe integers, so the aliases can't carry anything else into the query. +export const itemsQuery = (numbers: ReadonlyArray) => + `query($owner: String!, $name: String!) { repository(owner: $owner, name: $name) { ${numbers + .map((number) => `i${number}: issueOrPullRequest(number: ${number}) { ...KinItem }`) + .join(" ")} } }\n${ITEM_FRAGMENT}`; + +const Logins = Schema.Struct({ + nodes: Schema.Array(Schema.NullOr(Schema.Struct({ login: Schema.String }))), +}); +const Reviewer = Schema.NullOr(Schema.Struct({ login: Schema.optional(Schema.String) })); +const Issue = Schema.Struct({ + __typename: Schema.Literal("Issue"), + number: Schema.Int, + state: Schema.Literals(["OPEN", "CLOSED"]), + assignees: Logins, + closedByPullRequestsReferences: Schema.Struct({ + nodes: Schema.Array(Schema.NullOr(Schema.Struct({ number: Schema.Int }))), + }), +}); +const Pull = Schema.Struct({ + __typename: Schema.Literal("PullRequest"), + number: Schema.Int, + pullState: Schema.Literals(["OPEN", "CLOSED", "MERGED"]), + assignees: Logins, + reviewRequests: Schema.Struct({ + nodes: Schema.Array(Schema.NullOr(Schema.Struct({ requestedReviewer: Reviewer }))), + }), + timelineItems: Schema.Struct({ + nodes: Schema.Array( + Schema.NullOr( + Schema.Struct({ + createdAt: Schema.optional(Schema.String), + requestedReviewer: Schema.optional(Reviewer), + }), + ), + ), + }), +}); +const Item = Schema.Union([Issue, Pull]); +type Item = typeof Item.Type; +const GraphQlAnswer = Schema.Struct({ + data: Schema.optional( + Schema.NullOr( + Schema.Struct({ + repository: Schema.NullOr(Schema.Record(Schema.String, Schema.NullOr(Item))), + }), + ), + ), + errors: Schema.optional(Schema.Array(Schema.Struct({ type: Schema.optional(Schema.String) }))), +}); +const decodeAnswer = Schema.decodeUnknownOption(GraphQlAnswer); +const isRepoSlug = Schema.is(RepoSlug); + +class KinGithubReadFailed extends Schema.TaggedError()("KinGithubReadFailed", { + repo: Schema.String, + reason: Schema.Literals(["out_of_scope", "token", "rate_limited", "unavailable", "timeout"]), +}) {} + +const lowerState = (item: Item): GithubItemState["state"] => { + const state = item.__typename === "Issue" ? item.state : item.pullState; + return state === "OPEN" ? "open" : state === "MERGED" ? "merged" : "closed"; +}; + +/** Pending user review requests, each dated by its latest request event; undated ones are left out. */ +function reviewRequestsOf(pull: typeof Pull.Type): GithubItemState["reviewRequests"] { + const requestedAt = new Map(); + for (const event of pull.timelineItems.nodes) { + const login = event?.requestedReviewer?.login?.toLowerCase(); + const at = Date.parse(event?.createdAt ?? ""); + if (login !== undefined && Number.isFinite(at)) requestedAt.set(login, at); + } + return pull.reviewRequests.nodes.flatMap((request) => { + const login = request?.requestedReviewer?.login; + const at = login === undefined ? undefined : requestedAt.get(login.toLowerCase()); + return login === undefined || at === undefined ? [] : [{ login, requestedAt: at }]; + }); +} + +const make = Effect.gen(function* () { + const app = yield* KinGitHubApp.KinGitHubApp; + const client = yield* HttpClient.HttpClient; + const scope = yield* KinGithubRepoScope; + const tokens = new Map< + string, + { readonly token: Redacted.Redacted; readonly expiresAt: number } + >(); + + /** One read-only `server` token per repo, reused until shortly before GitHub expires it. */ + const tokenFor = Effect.fn("KinGithubReaders.token")(function* (repo: RepoSlug) { + const key = repo.toLowerCase(); + const now = yield* Clock.currentTimeMillis; + const cached = tokens.get(key); + if (cached !== undefined && now < cached.expiresAt - TOKEN_REUSE_MARGIN_MS) return cached.token; + tokens.delete(key); + const minted = yield* app + .installationToken({ repo, permissions: READ_PERMISSIONS, holder: "server" }) + .pipe( + Effect.tapError((error) => + Effect.logWarning("Kin GitHub read token refused", { repo, errorTag: error._tag }), + ), + Effect.mapError(() => new KinGithubReadFailed({ repo, reason: "token" })), + ); + const expiresAt = Date.parse(minted.expiresAt); + if (Number.isFinite(expiresAt)) tokens.set(key, { token: minted.token, expiresAt }); + return minted.token; + }); + + /** Read-only: the items GitHub knows among `numbers`, keyed by number. Never retried here. */ + const fetchItems = Effect.fn("KinGithubReaders.fetch")(function* ( + repo: string, + numbers: ReadonlyArray, + ) { + if (!isRepoSlug(repo) || !inRepoScope(scope.repos, repo)) + return yield* new KinGithubReadFailed({ repo, reason: "out_of_scope" }); + const token = yield* tokenFor(repo); + const [owner, name] = repo.split("/"); + const items = new Map(); + for (let start = 0; start < numbers.length; start += ITEMS_PER_QUERY) { + const answer = yield* gitHubAnswer({ + client, + operation: "graphql.items", + method: "POST", + path: "/graphql", + token, + body: { + query: itemsQuery(numbers.slice(start, start + ITEMS_PER_QUERY)), + variables: { owner, name }, + }, + }).pipe(Effect.mapError(() => new KinGithubReadFailed({ repo, reason: "unavailable" }))); + if (answer.status === 401) tokens.delete(repo.toLowerCase()); + const body = decodeAnswer(answer.body); + const errors = Option.isSome(body) ? (body.value.errors ?? []) : []; + if (isGitHubRateLimit(answer) || errors.some((error) => error.type === "RATE_LIMITED")) { + yield* Effect.logWarning("Kin GitHub read rate-limited; skipping the repo for now", { + repo, + }); + return yield* new KinGithubReadFailed({ repo, reason: "rate_limited" }); + } + const repository = Option.isSome(body) ? body.value.data?.repository : undefined; + // NOT_FOUND only for numbers GitHub doesn't have; those come back null and are left out. + if ( + answer.status !== 200 || + repository === undefined || + repository === null || + errors.some((error) => error.type !== "NOT_FOUND") + ) + return yield* new KinGithubReadFailed({ repo, reason: "unavailable" }); + for (const item of Object.values(repository)) if (item !== null) items.set(item.number, item); + } + return items; + }); + + const deferral: KinDeferralGithubReaderShape = { + read: Effect.fn("KinGithubReaders.deferral")( + function* (_orgId: KinOrgId, ref: GithubRef) { + const item = (yield* fetchItems(ref.repo, [ref.number])).get(ref.number); + if (item === undefined) return Option.none(); + return Option.some({ + state: lowerState(item), + assignees: item.assignees.nodes.flatMap((node) => (node === null ? [] : [node.login])), + linkedPrs: + item.__typename === "Issue" + ? item.closedByPullRequestsReferences.nodes.flatMap((node) => + node === null ? [] : [node.number], + ) + : [], + }); + }, + (effect, orgId, ref) => + effect.pipe( + Effect.timeoutOrElse({ + duration: DEFERRAL_READ_TIMEOUT, + orElse: () => + Effect.fail(new KinGithubReadFailed({ repo: ref.repo, reason: "timeout" })), + }), + Effect.mapError(() => new KinDeferralGithubError({ orgId, reason: "unavailable" })), + Effect.flatMap( + Option.match({ + onNone: () => Effect.fail(new KinDeferralGithubError({ orgId, reason: "not_found" })), + onSome: Effect.succeed, + }), + ), + ), + ), + }; + + const watchdog: KinWatchdogGithubReaderShape = { + readRepo: Effect.fn("KinGithubReaders.watchdog")( + function* (_orgId: KinOrgId, repo: string, numbers: ReadonlyArray) { + const items = yield* fetchItems(repo, numbers); + return numbers.flatMap((number): ReadonlyArray => { + const item = items.get(number); + if (item === undefined) return []; + const kind = item.__typename === "Issue" ? "issue" : "pull"; + return [ + { + number, + kind, + state: lowerState(item), + reviewRequests: item.__typename === "PullRequest" ? reviewRequestsOf(item) : [], + }, + ]; + }); + }, + (effect, orgId, repo) => + effect.pipe( + Effect.timeoutOrElse({ + duration: WATCHDOG_READ_TIMEOUT, + orElse: () => Effect.fail(new KinGithubReadFailed({ repo, reason: "timeout" })), + }), + Effect.mapError( + (failure) => + new KinWatchdogGithubError({ + orgId, + repo, + reason: failure.reason === "timeout" ? "timeout" : "unavailable", + }), + ), + ), + ), + }; + + return Context.make(KinDeferralGithubReader, deferral).pipe( + Context.add(KinWatchdogGithubReader, watchdog), + ); +}); + +/** Both readers over `KinGitHubApp`; needs the App id and its key in the keychain. */ +export const readersLayer = Layer.effectContext(make); + +const appLayer = (appId: string) => + Layer.effectDiscard( + Effect.gen(function* () { + const keychain = yield* Keychain.KinKeychain; + if (Option.isNone(yield* keychain.get("kin-github-app", appId))) + return yield* new KinGitHubApp.KinGitHubAppNotConfiguredError({ appId }); + }), + ).pipe( + Layer.provideMerge(KinGitHubApp.layer), + Layer.provideMerge( + Layer.mergeAll( + Layer.succeed(KinGitHubApp.KinGitHubAppConfig, { appId }), + KinGitHubScope.layer, + FetchHttpClient.layer, + Keychain.layer, + ), + ), + ); + +/** + * The real readers when the Kin App is configured. Otherwise, or if the App can't be built, one + * warning and the fail-closed defaults (deferrals ask, GitHub watchdogs wait). + */ +export const layer = Layer.unwrap( + Effect.gen(function* () { + const appId = yield* Config.option(Config.String("KIN_GITHUB_APP_ID")); + if (Option.isNone(appId)) { + yield* Effect.logWarning( + "Kin GitHub reads are off: KIN_GITHUB_APP_ID is unset, so deferrals ask and GitHub watchdogs wait", + ); + return Layer.empty; + } + return readersLayer.pipe( + Layer.provide(Layer.mergeAll(appLayer(appId.value), repoScopeLayer)), + Layer.catchCause((cause) => + Layer.effectDiscard( + Effect.logWarning( + "Kin GitHub reads are off: the Kin GitHub App is unavailable, so deferrals ask and GitHub watchdogs wait", + Cause.pretty(cause), + ), + ), + ), + ); + }), +); diff --git a/apps/server/src/kin/triggers/watchdogs.ts b/apps/server/src/kin/triggers/watchdogs.ts index 11e641e85edc1..3330f230220f3 100644 --- a/apps/server/src/kin/triggers/watchdogs.ts +++ b/apps/server/src/kin/triggers/watchdogs.ts @@ -86,8 +86,8 @@ export interface KinWatchdogGithubReaderShape { ) => Effect.Effect, KinWatchdogGithubError>; } -// No credentialed GitHub client exists yet (APPR-8), so the GitHub watchdogs stay off and -// `github_recheck` requests wait in the inbox until a reader backs this. +// githubReaders.ts replaces this when the Kin App is configured. Without it the GitHub watchdogs +// stay off and `github_recheck` requests wait in the inbox. export const KinWatchdogGithubReader = Context.Reference( "t3/kin/triggers/watchdogs/KinWatchdogGithubReader", { defaultValue: () => null }, diff --git a/docs/kin/specs/approvals.md b/docs/kin/specs/approvals.md index 3e7999a07e4f4..6dfcbe45bdcf4 100644 --- a/docs/kin/specs/approvals.md +++ b/docs/kin/specs/approvals.md @@ -374,7 +374,7 @@ export const KinProvenance = Schema.Struct({ | Workflows | **None** | GitHub then rejects pushes that change `.github/workflows`, so a worker can't add a workflow that reads repo secrets | | Administration, Secrets, Environments, Deployments, Members, Webhooks | **None** | | -Webhook events (consumed by the triggers spec's App webhook source, which filters to exactly these): `pull_request`, `pull_request_review`, `pull_request_review_comment`, `issue_comment`, `issues`, `check_suite`, `workflow_run`, `push`. Approvals reads check runs and statuses over REST at verify time, so it needs no `check_run` or `status` events. The App's private key is in the OS keychain (`AsyncEntry("kin-github-app", "")`, a JSON item with the PEM, webhook secret and client secret); the server finds it through `KIN_GITHUB_APP_ID`. Installation tokens last one hour; each worker gets one minted with `POST /app/installations/{id}/access_tokens` narrowed by `repositories` and `permissions` (no agent token carries `contents: write`). `KinGitHubApp.installationToken` takes exactly one repository from `KIN_GITHUB_REPOS`, a non-empty subset of the table at or below its levels, and the `holder` (`lead`, `worker` or `server`). It refuses anything else before calling GitHub, including `contents: write` for any holder but `server` (open question 7). A `server` token stays in Kin's own process for its own pushes and is never handed to an agent. That refusal only holds if `holder` is true: callers hard-code it and never take it, or anything that picks it, from model input. It also refuses when `GET /repos/{o}/{r}/installation` reports `repository_selection: "all"`. If GitHub's answer is wider than asked (more repositories, or a permission not requested or at a higher rank, beyond the `metadata: read` it always adds), or doesn't decode, it revokes the token with `DELETE /installation/token` and fails. A failed revoke is logged without the token; the token still expires within the hour. +Webhook events (consumed by the triggers spec's App webhook source, which filters to exactly these): `pull_request`, `pull_request_review`, `pull_request_review_comment`, `issue_comment`, `issues`, `check_suite`, `workflow_run`, `push`. Approvals reads check runs and statuses over REST at verify time, so it needs no `check_run` or `status` events. The App's private key is in the OS keychain (`AsyncEntry("kin-github-app", "")`, a JSON item with the PEM, webhook secret and client secret); the server finds it through `KIN_GITHUB_APP_ID`. Installation tokens last one hour; each worker gets one minted with `POST /app/installations/{id}/access_tokens` narrowed by `repositories` and `permissions` (no agent token carries `contents: write`). `KinGitHubApp.installationToken` takes exactly one repository from `KIN_GITHUB_REPOS`, a non-empty subset of the table at or below its levels, and the `holder` (`lead`, `worker` or `server`). It refuses anything else before calling GitHub, including `contents: write` for any holder but `server` (open question 7). A `server` token stays in Kin's own process, for its own pushes and the triggers spec's read-only deferral and watchdog reads, and is never handed to an agent. That refusal only holds if `holder` is true: callers hard-code it and never take it, or anything that picks it, from model input. It also refuses when `GET /repos/{o}/{r}/installation` reports `repository_selection: "all"`. If GitHub's answer is wider than asked (more repositories, or a permission not requested or at a higher rank, beyond the `metadata: read` it always adds), or doesn't decode, it revokes the token with `DELETE /installation/token` and fails. A failed revoke is logged without the token; the token still expires within the hour. `KinGitHubScope` (`apps/server/src/kin/github/KinGitHubScope.ts`) holds the scope the checks compare against: `KIN_GITHUB_OWNER_LOGIN` (the owner's personal login), `KIN_GITHUB_BOT_LOGIN` (the only login the Kin host may use, section 12) and `KIN_GITHUB_REPOS` (comma-separated `owner/name`). diff --git a/docs/kin/specs/triggers.md b/docs/kin/specs/triggers.md index 862a81305949e..f01ddfb525109 100644 --- a/docs/kin/specs/triggers.md +++ b/docs/kin/specs/triggers.md @@ -218,7 +218,7 @@ The sink verifies the delivery-log row belongs to the scheduled task. `dueAt` is **Missed while off.** The Kin host should be on, so outages are short: - **Hold on (recommended):** GitHub's request reaches the relay, the relay holds it up to 24 h, and `HeldHooksWaker` delivers it on reconnect. Replay keys stop doubles. -- **Hold off, or longer than 24 h:** GitHub gets a 530 or 502 and does not retry by itself. `github_recheck` runs at startup and every 10 minutes, compares memory with GitHub state, and finds what changed. It needs a GitHub reader, which arrives with APPR-8; until then its requests wait in the inbox. +- **Hold off, or longer than 24 h:** GitHub gets a 530 or 502 and does not retry by itself. `github_recheck` runs at startup and every 10 minutes, compares memory with GitHub state, and finds what changed. It needs the GitHub reader below; without the Kin App its requests wait in the inbox. Webhooks are a fast path; the sweep is the guarantee. @@ -235,7 +235,7 @@ Arming comes from memory. When the front-matter cache (LEAD) sees a `deferrals/* Item edits move or disarm the triggers: a new `due` re-arms; `status` other than `open` or a set `invalid_at` disables them. A due time already past at save fires at once, marked late; no "pick a future time" error. A missing `on_due` means `remind`, so silence never starts work. -The cache hook is `MemorySyncHooks` in [`Memory.ts`](../../../apps/server/src/kin/memory/Memory.ts): each new head goes once to `KinTriggerService.syncOrg`, which also disarms armed deferrals missing from the snapshot or sharing a duplicate id. A failing item is logged without blocking the others, and the head retries on the next two loads, then waits for the next head. `trigger_reconcile` (below) also re-syncs the whole org every 15 minutes and at startup, so a head that gave up is retried there. A due time whose fires would land before 1970 never arms; `kin_defer` accepts only years 2000–2099. A commit to the default branch moves the cache to the new head and reloads it, so a `kin_defer` item arms when its wake-up commits, not at the next wake-up; shadow (L0) commits never arm. Trigger ids are `deferral::heads_up|due`. Re-syncing an unchanged `due` writes nothing, so a fired `once` is never re-armed. The run row in `kin_deferral_runs` (migration K016) holds the snapshot and the flow state (below). The snapshot holds the item half (`recorded_at`, `superseded_by`, links) and the GitHub half (state, assignees and linked PRs of each `github.com///issues|pull/` or `/#` link), read at arm time through `KinDeferralGithubReader` in [`deferralGithub.ts`](../../../apps/server/src/kin/triggers/deferralGithub.ts). The GitHub half is `null` when any link couldn't be read. That reader is a narrow read-only seam (one issue or PR per call) for #44's REST client to back; until then its default fails with `unavailable`. +The cache hook is `MemorySyncHooks` in [`Memory.ts`](../../../apps/server/src/kin/memory/Memory.ts): each new head goes once to `KinTriggerService.syncOrg`, which also disarms armed deferrals missing from the snapshot or sharing a duplicate id. A failing item is logged without blocking the others, and the head retries on the next two loads, then waits for the next head. `trigger_reconcile` (below) also re-syncs the whole org every 15 minutes and at startup, so a head that gave up is retried there. A due time whose fires would land before 1970 never arms; `kin_defer` accepts only years 2000–2099. A commit to the default branch moves the cache to the new head and reloads it, so a `kin_defer` item arms when its wake-up commits, not at the next wake-up; shadow (L0) commits never arm. Trigger ids are `deferral::heads_up|due`. Re-syncing an unchanged `due` writes nothing, so a fired `once` is never re-armed. The run row in `kin_deferral_runs` (migration K016) holds the snapshot and the flow state (below). The snapshot holds the item half (`recorded_at`, `superseded_by`, links) and the GitHub half (state, assignees and linked PRs of each `github.com///issues|pull/` or `/#` link), read at arm time through `KinDeferralGithubReader` in [`deferralGithub.ts`](../../../apps/server/src/kin/triggers/deferralGithub.ts). The GitHub half is `null` when any link couldn't be read. That reader is a narrow read-only seam (one issue or PR per call), bounded to 10 s because it runs under the org memory lock. A link GitHub doesn't have is `not_found`; anything else that fails is `unavailable`. Both readers are backed by [`githubReaders.ts`](../../../apps/server/src/kin/triggers/githubReaders.ts) when the Kin App is configured, and otherwise keep their defaults (below). **Confirm.** When the lead calls `kin_defer` (LEAD), Kin writes the item, arms the triggers, and the lead posts a `KinDeferralCard` widget (CHAT envelope): "Deferred to Fri 13 Nov, 9:00. I'll start then." [Change] [Just remind me]. [Just remind me] sets `on_due: remind`, and the triggers re-arm. [Change] writes nothing: it hands the item to the lead, which asks the owner what to change, marks the old item `cancelled` and calls `kin_defer` for the new one. @@ -331,7 +331,8 @@ All are plain code in [`watchdogs.ts`](../../../apps/server/src/kin/triggers/wat | `index_freshness` | 1 h | `INDEX.md` "As of" missing, in the future or 36 h old: `Memory.rebuildIndex` (code, below), the same check wake-up preparation runs. | | `trigger_reconcile` | 15 min | Re-syncs deferral triggers from the whole memory snapshot (`KinTriggerService.syncOrg`). Checks every `KinWebhookSource` task exists, is enabled and isn't `blockedByPolicy`, records what it saw in `status` and tells the owner when it changes; a task whose secret doesn't match the bound hash gets one notice per binding. It never re-enables a task, re-keys or rebinds. Clears consumed inbox rows older than 7 days and fire-ledger rows older than 30 days. | -- **GitHub reads.** Through `KinWatchdogGithubReader` only: one read-only, batched call per repo for all the numbers the sweep needs, each bounded to 30 s. Only repos in `KIN_GITHUB_REPOS` are read. Its default is `null` because no credentialed GitHub client exists until APPR-8, so `github_recheck`, `stale_review` and `objective_at_risk` do nothing and `github_recheck` rows stay pending; nothing is lost while it waits. A repo that fails or times out skips the items that need it and keeps the rows pending. +- **GitHub reads.** Through `KinWatchdogGithubReader` only: one read-only, batched call per repo for all the numbers the sweep needs, each bounded to 30 s. Only repos in `KIN_GITHUB_REPOS` are read. A repo that fails, times out or is rate-limited skips the items that need it and keeps the rows pending; nothing retries before the next sweep. +- **The real readers.** With `KIN_GITHUB_APP_ID` set and the App's key in the keychain, Kin's server layer provides both readers over `KinGitHubApp` (approvals spec section 10). Each repo gets one `server`-holder installation token narrowed to `issues: read` and `pull_requests: read`, reused until five minutes before it expires and dropped on a 401. Reads go through GitHub GraphQL, up to 50 issues or PRs per query. A repo outside `KIN_GITHUB_REPOS`, or one the App can't mint a token for, fails closed without a GitHub call. A 429, a rate-limit 403 or a GraphQL `RATE_LIMITED` fails the repo for this read. The readers make no writes. Without the App, or if it can't be built, startup logs one warning and the defaults stay: the deferral reader fails `unavailable` (deferrals ask) and the watchdog reader is `null`, so `github_recheck`, `stale_review` and `objective_at_risk` do nothing and `github_recheck` rows stay pending; nothing is lost while it waits. - **Conditions from state.** A PR gives `pull_request` with action `opened` or `closed` and `merged`; an issue gives `issues` with `opened` or `closed`. `matchConditions` then applies as for events, so a condition on another event or action matches only through webhooks. A match queues a `watchdog` event whose finding starts "GitHub state shows condition met (read from GitHub by code)". - **Repeats.** Findings dedupe on `kin_events.dedupe_key` (item, links and state), so a later sweep queues nothing new. A chase is skipped while the person's ping for the item is active, or when it was stopped after what prompted the chase last changed (the commitment's due date, or the review request's time), so an answered chase stays answered until the due date moves. - **Source status is advisory.** The sink does not refuse on `status`: the task's shape and bound secret are the gate, and a delivery after the owner turns the task back on must not wait 15 minutes for reconcile. A hash mismatch has no status value; the sink already refuses it, and `setSecret` or setup is the way out. Routine rows are kept by the routines tick's own 15-minute reconcile and dream rows by `DreamRunner`, so `trigger_reconcile` doesn't repeat them. From fdaf08d39cb80993d1e19c6c8bcb48d49e064537 Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Mon, 9 Nov 2026 06:12:27 +0530 Subject: [PATCH 2/2] fix(kin): bound deferral GitHub reads per sync and refuse partial lists syncOrg now reads every pending deferral's links in one batched call per repo, all inside one 10 s budget, before writing any row. Links not read in time count as unavailable, so those deferrals ask. The readers drop closing PRs from other repos, fail an item whose assignees, closing PRs, review requests or review-request events run past one page, and drop the cached token on an integration 403 as well as a 401. fetchItems only sends positive safe integers. The App config decodes through KinGitHubAppConfig.layer and startup decodes the keychain item. The watchdog's 30 s bound stays only in watchdogs.ts. Co-Authored-By: Claude Opus 5.5 --- .../kin/triggers/KinTriggerService.test.ts | 44 ++++- .../src/kin/triggers/KinTriggerService.ts | 22 ++- .../server/src/kin/triggers/deferralGithub.ts | 108 +++++++++--- .../server/src/kin/triggers/deferrals.test.ts | 21 ++- .../src/kin/triggers/githubReaders.test.ts | 93 +++++++--- apps/server/src/kin/triggers/githubReaders.ts | 159 ++++++++++-------- docs/kin/specs/triggers.md | 4 +- 7 files changed, 310 insertions(+), 141 deletions(-) diff --git a/apps/server/src/kin/triggers/KinTriggerService.test.ts b/apps/server/src/kin/triggers/KinTriggerService.test.ts index f75ecc9f8da2e..cb3ba756db7fa 100644 --- a/apps/server/src/kin/triggers/KinTriggerService.test.ts +++ b/apps/server/src/kin/triggers/KinTriggerService.test.ts @@ -9,6 +9,7 @@ import { type MemoryItemHead, } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Schema from "effect/Schema"; import * as SqlClient from "effect/sql/SqlClient"; @@ -82,10 +83,15 @@ const card = Effect.fn("test.deferralCard")(function* (org: string) { it.effect("arms from front matter, re-arms on edits, and re-syncs without writes", () => { let reads = 0; const reader = { - read: () => + readRepo: (_org: KinOrgId, _repo: string, numbers: ReadonlyArray) => Effect.sync(() => { reads++; - return { state: "open" as const, assignees: [], linkedPrs: [] }; + return new Map( + numbers.map((number) => [ + number, + { state: "open" as const, assignees: [], linkedPrs: [] }, + ]), + ); }), }; const linked = (overrides: Partial = {}) => @@ -311,6 +317,40 @@ it.effect("syncs each head once, retries failures, and isolates failing items", }).pipe(Effect.provide(layer)), ); +it.effect("reads GitHub once per repo within one budget and asks for what it couldn't read", () => { + const calls: Array]> = []; + const reader = { + readRepo: (_org: KinOrgId, repo: string, numbers: ReadonlyArray) => + Effect.suspend(() => { + calls.push([repo, numbers]); + return Effect.never; + }), + }; + const items = Array.from({ length: 20 }, (_, index) => + deferral({ + id: KinMemoryItemId.make(`def-${index + 1}`), + path: `deferrals/def-${index + 1}.md`, + links: [`https://github.com/acme/api/issues/${index + 1}`], + }), + ); + return Effect.gen(function* () { + const { sql, service } = yield* setup(); + const sync = yield* service.syncOrg(orgId, items).pipe(Effect.forkChild); + yield* TestClock.adjust("10 seconds"); + assert.isDefined(sync.pollUnsafe(), "the whole sync fits in one 10 s budget"); + yield* Fiber.join(sync); + assert.deepEqual(calls, [["acme/api", Array.from({ length: 20 }, (_, index) => index + 1)]]); + const runs = yield* sql<{ state: string; snapshot: string }>` + SELECT state, snapshot_json AS snapshot FROM kin_deferral_runs + WHERE org_id = ${orgId} ORDER BY item_id`; + assert.lengthOf(runs, 20); + for (const run of runs) { + assert.equal(run.state, "armed"); + assert.include(run.snapshot, '"github":null'); + } + }).pipe(Effect.provideService(KinDeferralGithubReader, reader), Effect.provide(layer)); +}); + it.effect("refuses deferral cards from the model and taps until the dispatcher lands", () => Effect.gen(function* () { const { service } = yield* setup(); diff --git a/apps/server/src/kin/triggers/KinTriggerService.ts b/apps/server/src/kin/triggers/KinTriggerService.ts index 94012874f50d6..2d6dcdae2a30f 100644 --- a/apps/server/src/kin/triggers/KinTriggerService.ts +++ b/apps/server/src/kin/triggers/KinTriggerService.ts @@ -34,7 +34,7 @@ import { type DeferralPhase, type DeferralPlan, } from "./deferrals.ts"; -import { type GithubLinkState, readGithubLinks } from "./deferralGithub.ts"; +import { type GithubLinkState, readGithubLinks, readGithubLinkSets } from "./deferralGithub.ts"; export class KinTriggerSyncError extends Schema.TaggedError()( "KinTriggerSyncError", @@ -300,14 +300,22 @@ const make = Effect.gen(function* () { if (pending.length === 0) return; const timeZone = yield* cache.ownerTimeZone(orgId); const now = yield* Clock.currentTimeMillis; + // Every read happens before any write, so the memory lock waits on GitHub at most once. + const github = yield* readGithubLinkSets( + orgId, + pending.map(([, item, plan]) => (plan.armed && item !== undefined ? item.links : [])), + ); const failures: Array = []; - for (const [itemId, item, plan] of pending) - yield* ( - plan.armed && item !== undefined - ? readGithubLinks(orgId, item.links) - : Effect.succeed(null) + for (const [index, [itemId, item, plan]] of pending.entries()) + yield* apply( + orgId, + itemId, + item, + plan, + timeZone, + now, + plan.armed && item !== undefined ? (github[index] ?? null) : null, ).pipe( - Effect.flatMap((github) => apply(orgId, itemId, item, plan, timeZone, now, github)), wrap(orgId), Effect.catch((error) => { failures.push(error); diff --git a/apps/server/src/kin/triggers/deferralGithub.ts b/apps/server/src/kin/triggers/deferralGithub.ts index 21fdc58b16ec3..e1c458dd7ddc3 100644 --- a/apps/server/src/kin/triggers/deferralGithub.ts +++ b/apps/server/src/kin/triggers/deferralGithub.ts @@ -1,6 +1,7 @@ import { KinOrgId } from "@t3tools/contracts"; import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; export const GithubLinkState = Schema.Struct({ @@ -23,13 +24,14 @@ export class KinDeferralGithubError extends Schema.TaggedError Effect.Effect, KinDeferralGithubError>; + repo: string, + numbers: ReadonlyArray, + ) => Effect.Effect>, KinDeferralGithubError>; } // githubReaders.ts replaces this when the Kin App is configured. Without it every read fails, @@ -38,11 +40,16 @@ export const KinDeferralGithubReader = Context.Reference ({ - read: (orgId) => Effect.fail(new KinDeferralGithubError({ orgId, reason: "unavailable" })), + readRepo: (orgId) => + Effect.fail(new KinDeferralGithubError({ orgId, reason: "unavailable" })), }), }, ); +// Callers hold the org memory lock, so every GitHub read for one sync shares this budget. +const READ_BUDGET = "10 seconds"; +const READ_CONCURRENCY = 4; + const urlRef = /^https:\/\/github\.com\/([\w.-]+\/[\w.-]+)\/(?:issues|pull)\/(\d+)\/?$/; const shortRef = /^([\w.-]+\/[\w.-]+)#(\d+)$/; @@ -53,24 +60,79 @@ export function githubRef(link: string): GithubRef | undefined { return Number.isSafeInteger(number) && number > 0 ? { repo: match[1]!, number } : undefined; } -/** Reads every GitHub link; `null` means at least one couldn't be read. */ -export const readGithubLinks = Effect.fn("KinDeferralGithub.readLinks")(function* ( +const refKey = (repo: string, number: number) => `${repo.toLowerCase()}#${number}`; + +/** + * Reads the GitHub links of each list with one call per repo, all within one budget. A list is + * `null` when any of its links couldn't be read in time. + */ +export const readGithubLinkSets = Effect.fn("KinDeferralGithub.readLinkSets")(function* ( orgId: KinOrgId, - links: ReadonlyArray, + linkSets: ReadonlyArray>, ) { const reader = yield* KinDeferralGithubReader; - const states: Array = []; - for (const link of links) { - const ref = githubRef(link); - if (ref === undefined) continue; - const state = yield* reader.read(orgId, ref).pipe(Effect.option); - if (state._tag === "None") return null; - states.push({ - link, - state: state.value.state, - assignees: state.value.assignees.toSorted(), - linkedPrs: state.value.linkedPrs.toSorted((a, b) => a - b), - }); - } - return states; + const byRepo = new Map }>(); + for (const links of linkSets) + for (const link of links) { + const ref = githubRef(link); + if (ref === undefined) continue; + const key = ref.repo.toLowerCase(); + const entry = byRepo.get(key) ?? { repo: ref.repo, numbers: new Set() }; + entry.numbers.add(ref.number); + byRepo.set(key, entry); + } + const read = new Map>(); + yield* Effect.forEach( + byRepo.values(), + ({ repo, numbers }) => + reader + .readRepo( + orgId, + repo, + [...numbers].toSorted((a, b) => a - b), + ) + .pipe( + Effect.tap((states) => + Effect.sync(() => { + for (const [number, state] of states) + if (numbers.has(number)) read.set(refKey(repo, number), state); + }), + ), + Effect.catch((error) => + Effect.logWarning("Kin deferral GitHub read failed", { + orgId, + repo, + reason: error.reason, + }), + ), + ), + { concurrency: READ_CONCURRENCY, discard: true }, + ).pipe( + Effect.timeoutOption(READ_BUDGET), + Effect.tap((done) => + Option.isNone(done) + ? Effect.logWarning("Kin deferral GitHub reads ran out of time", { orgId }) + : Effect.void, + ), + ); + return linkSets.map((links): ReadonlyArray | null => { + const states: Array = []; + for (const link of links) { + const ref = githubRef(link); + if (ref === undefined) continue; + const state = read.get(refKey(ref.repo, ref.number)); + if (state === undefined) return null; + states.push({ + link, + state: state.state, + assignees: state.assignees.toSorted(), + linkedPrs: state.linkedPrs.toSorted((a, b) => a - b), + }); + } + return states; + }); }); + +/** Reads every GitHub link; `null` means at least one couldn't be read. */ +export const readGithubLinks = (orgId: KinOrgId, links: ReadonlyArray) => + readGithubLinkSets(orgId, [links]).pipe(Effect.map(([states]) => states ?? null)); diff --git a/apps/server/src/kin/triggers/deferrals.test.ts b/apps/server/src/kin/triggers/deferrals.test.ts index 4e97fa0d66697..395716fdbf402 100644 --- a/apps/server/src/kin/triggers/deferrals.test.ts +++ b/apps/server/src/kin/triggers/deferrals.test.ts @@ -34,11 +34,7 @@ import * as Memory from "../memory/Memory.ts"; import * as MemoryConfig from "../memory/MemoryConfig.ts"; import * as MemoryWriter from "../memory/MemoryWriter.ts"; import * as DeferralFlow from "./deferralFlow.ts"; -import { - KinDeferralGithubError, - KinDeferralGithubReader, - type GithubLinkState, -} from "./deferralGithub.ts"; +import { KinDeferralGithubReader, type GithubLinkState } from "./deferralGithub.ts"; import * as Deferrals from "./deferrals.ts"; import * as TriggerScheduler from "./KinTriggerScheduler.ts"; import * as TriggerService from "./KinTriggerService.ts"; @@ -75,12 +71,15 @@ const layerFor = ( ); const fakeGithub = (states: Map>) => ({ - read: (org: typeof orgId, ref: { readonly repo: string; readonly number: number }) => { - const state = states.get(`${ref.repo}#${ref.number}`); - return state === undefined - ? Effect.fail(new KinDeferralGithubError({ orgId: org, reason: "not_found" })) - : Effect.succeed(state); - }, + readRepo: (_org: typeof orgId, repo: string, numbers: ReadonlyArray) => + Effect.succeed( + new Map( + numbers.flatMap((number) => { + const state = states.get(`${repo}#${number}`); + return state === undefined ? [] : [[number, state] as const]; + }), + ), + ), }); // Stands in for LEAD-11, the worker release gate and #44's GitHub client. const openGates = diff --git a/apps/server/src/kin/triggers/githubReaders.test.ts b/apps/server/src/kin/triggers/githubReaders.test.ts index c6dbeb95b9eb3..6ddc295fd6d8f 100644 --- a/apps/server/src/kin/triggers/githubReaders.test.ts +++ b/apps/server/src/kin/triggers/githubReaders.test.ts @@ -17,7 +17,7 @@ import { buildSchema, parse, validate } from "graphql"; import * as KinGitHubApp from "../github/KinGitHubApp.ts"; import { fakeGitHub, type FakeAnswer, type RecordedRequest } from "../github/testFakes.ts"; -import { KinDeferralGithubReader, readGithubLinks } from "./deferralGithub.ts"; +import { KinDeferralGithubReader, readGithubLinks, readGithubLinkSets } from "./deferralGithub.ts"; import { KinGithubRepoScope } from "./githubEvents.ts"; import * as GithubReaders from "./githubReaders.ts"; import { KinWatchdogGithubReader } from "./watchdogs.ts"; @@ -27,40 +27,69 @@ const REPO = "GLDRoger/t3code"; const TOKEN = "ghs_reader_token_never_shown"; const NOW = Date.UTC(2026, 10, 8, 12); +const more = { pageInfo: { hasNextPage: false } }; +const pr = (number: number, repo = REPO) => ({ number, repository: { nameWithOwner: repo } }); const closedIssue = { __typename: "Issue", number: 7, state: "CLOSED", - assignees: { nodes: [{ login: "zoe" }, { login: "amir" }] }, - closedByPullRequestsReferences: { nodes: [{ number: 12 }, { number: 9 }] }, + assignees: { ...more, nodes: [{ login: "zoe" }, { login: "amir" }] }, + closedByPullRequestsReferences: { + ...more, + nodes: [pr(12), pr(9), pr(30, "someone/fork"), pr(31, REPO.toLowerCase())], + }, }; const mergedPr = { __typename: "PullRequest", number: 9, pullState: "MERGED", - assignees: { nodes: [] }, - reviewRequests: { nodes: [] }, - timelineItems: { nodes: [] }, + assignees: { ...more, nodes: [] }, + reviewRequests: { ...more, nodes: [] }, + timelineItems: { pageInfo: { hasPreviousPage: false }, nodes: [] }, }; const openPr = { __typename: "PullRequest", number: 11, pullState: "OPEN", - assignees: { nodes: [{ login: "amir" }] }, + assignees: { ...more, nodes: [{ login: "amir" }] }, reviewRequests: { + ...more, nodes: [{ requestedReviewer: { login: "Zoe" } }, { requestedReviewer: {} }], }, timelineItems: { + pageInfo: { hasPreviousPage: false }, nodes: [ { createdAt: "2026-11-01T00:00:00Z", requestedReviewer: { login: "zoe" } }, { createdAt: "2026-11-05T00:00:00Z", requestedReviewer: { login: "zoe" } }, ], }, }; +const truncated = { pageInfo: { hasNextPage: true } }; const fixtures = new Map([ [7, closedIssue], [9, mergedPr], [11, openPr], + [20, { ...closedIssue, number: 20, assignees: { ...closedIssue.assignees, ...truncated } }], + [ + 21, + { + ...closedIssue, + number: 21, + closedByPullRequestsReferences: { + ...closedIssue.closedByPullRequestsReferences, + ...truncated, + }, + }, + ], + [22, { ...openPr, number: 22, reviewRequests: { ...openPr.reviewRequests, ...truncated } }], + [ + 23, + { + ...openPr, + number: 23, + timelineItems: { ...openPr.timelineItems, pageInfo: { hasPreviousPage: true } }, + }, + ], ]); /** GraphQL answers from `fixtures`; numbers it lacks come back null with NOT_FOUND, as GitHub does. */ @@ -160,15 +189,32 @@ describe("Kin GitHub readers", () => { link: "https://github.com/GLDRoger/t3code/issues/7", state: "closed", assignees: ["amir", "zoe"], - linkedPrs: [9, 12], + linkedPrs: [9, 12, 31], }, ], ); assert.isNull(yield* readGithubLinks(orgId, ["GLDRoger/t3code#404"])); + // A list GitHub cut short fails the item rather than reporting part of it. + assert.deepStrictEqual( + yield* readGithubLinkSets(orgId, [ + ["GLDRoger/t3code#20"], + ["GLDRoger/t3code#21"], + ["GLDRoger/t3code#22"], + ]), + [ + null, + null, + [{ link: "GLDRoger/t3code#22", state: "open", assignees: ["amir"], linkedPrs: [] }], + ], + ); + assert.deepStrictEqual( + (yield* watchdog.readRepo(orgId, REPO, [20, 22, 23])).map(({ number }) => number), + [20], + ); assert.deepStrictEqual( requests.map(({ method, path }) => `${method} ${path}`), - ["POST /graphql", "POST /graphql", "POST /graphql"], + Array.from({ length: 5 }, () => "POST /graphql"), ); assert.equal(requests[0]?.authorization, `Bearer ${TOKEN}`); assert.deepInclude(requests[0]?.body, { variables: { owner: "GLDRoger", name: "t3code" } }); @@ -188,7 +234,7 @@ describe("Kin GitHub readers", () => { const { deferral, watchdog } = yield* readers; const read = yield* watchdog.readRepo(orgId, "other/repo", [7]).pipe(Effect.flip); assert.equal(read.reason, "unavailable"); - const one = yield* deferral.read(orgId, { repo: "other/repo", number: 7 }).pipe(Effect.flip); + const one = yield* deferral.readRepo(orgId, "other/repo", [7]).pipe(Effect.flip); assert.equal(one.reason, "unavailable"); assert.lengthOf(requests, 0); assert.lengthOf(mints, 0); @@ -219,7 +265,7 @@ describe("Kin GitHub readers", () => { "unavailable", ); assert.equal( - (yield* deferral.read(orgId, { repo: REPO, number: 7 }).pipe(Effect.flip)).reason, + (yield* deferral.readRepo(orgId, REPO, [7]).pipe(Effect.flip)).reason, "unavailable", ); assert.lengthOf(requests, 2); @@ -227,22 +273,19 @@ describe("Kin GitHub readers", () => { }, ); - it.effect("bounds each read in time", () => { + it.effect("gives every deferral link one shared budget", () => { const hanging = Layer.succeed( HttpClient.HttpClient, HttpClient.make(() => Effect.never), ); const { layer } = setup(graphql, hanging); return Effect.gen(function* () { - const { deferral, watchdog } = yield* readers; - const one = yield* deferral - .read(orgId, { repo: REPO, number: 7 }) - .pipe(Effect.flip, Effect.forkChild); - const batch = yield* watchdog.readRepo(orgId, REPO, [7]).pipe(Effect.flip, Effect.forkChild); + const read = yield* readGithubLinkSets(orgId, [ + ["GLDRoger/t3code#7"], + ["GLDRoger/t3code#9", "https://example.com/not-github"], + ]).pipe(Effect.forkChild); yield* TestClock.adjust("10 seconds"); - assert.equal((yield* Fiber.join(one)).reason, "unavailable"); - yield* TestClock.adjust("20 seconds"); - assert.equal((yield* Fiber.join(batch)).reason, "timeout"); + assert.deepStrictEqual(yield* Fiber.join(read), [null, null]); }).pipe(Effect.provide(layer)); }); @@ -256,17 +299,21 @@ describe("Kin GitHub readers", () => { const answers: ReadonlyArray = [ { status: 401, body: { message: "Bad credentials" } }, { status: 429, headers: { "retry-after": "60" }, body: { message: "slow down" } }, + { status: 403, body: { message: "Resource not accessible by integration" } }, { status: 500, body: { message: "boom" } }, ]; - const { requests, layer } = setup(() => answers[call++ % answers.length]!); + const { mints, requests, layer } = setup(() => answers[call++ % answers.length]!); return Effect.gen(function* () { yield* TestClock.setTime(NOW); const { deferral, watchdog } = yield* readers; const errors = [ yield* watchdog.readRepo(orgId, REPO, [7]).pipe(Effect.flip), yield* watchdog.readRepo(orgId, REPO, [7]).pipe(Effect.flip), - yield* deferral.read(orgId, { repo: REPO, number: 7 }).pipe(Effect.flip), + yield* deferral.readRepo(orgId, REPO, [7]).pipe(Effect.flip), + yield* deferral.readRepo(orgId, REPO, [7]).pipe(Effect.flip), ]; + // The 401 and the integration 403 drop the cached token, so the next read mints; a 429 keeps it. + assert.lengthOf(mints, 3); assert.isTrue(requests.every((request) => request.authorization === `Bearer ${TOKEN}`)); assert.isNotEmpty(lines); for (const text of [ @@ -297,7 +344,7 @@ describe("Kin GitHub readers", () => { return Effect.gen(function* () { assert.isNull(yield* KinWatchdogGithubReader); const read = yield* (yield* KinDeferralGithubReader) - .read(orgId, { repo: REPO, number: 7 }) + .readRepo(orgId, REPO, [7]) .pipe(Effect.flip); assert.equal(read.reason, "unavailable"); assert.lengthOf( diff --git a/apps/server/src/kin/triggers/githubReaders.ts b/apps/server/src/kin/triggers/githubReaders.ts index b47cab0415389..803245d88a30e 100644 --- a/apps/server/src/kin/triggers/githubReaders.ts +++ b/apps/server/src/kin/triggers/githubReaders.ts @@ -6,7 +6,7 @@ import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; -import type * as Redacted from "effect/Redacted"; +import * as Redacted from "effect/Redacted"; import * as Schema from "effect/Schema"; import { FetchHttpClient, HttpClient } from "effect/http"; @@ -16,7 +16,7 @@ import * as KinGitHubApp from "../github/KinGitHubApp.ts"; import type { KinGitHubPermissionSet } from "../github/KinGitHubPermissions.ts"; import { KinGitHubScope } from "../github/KinGitHubScope.ts"; import { - type GithubRef, + type GithubLinkState, KinDeferralGithubError, KinDeferralGithubReader, type KinDeferralGithubReaderShape, @@ -29,9 +29,6 @@ import { type KinWatchdogGithubReaderShape, } from "./watchdogs.ts"; -// The deferral read runs under the org memory lock, so it gets far less than a watchdog sweep. -const DEFERRAL_READ_TIMEOUT = "10 seconds"; -const WATCHDOG_READ_TIMEOUT = "30 seconds"; const ITEMS_PER_QUERY = 50; const TOKEN_REUSE_MARGIN_MS = 5 * 60_000; const READ_PERMISSIONS = { @@ -44,25 +41,33 @@ const ITEM_FRAGMENT = `fragment KinItem on IssueOrPullRequest { __typename ... on Issue { number state - assignees(first: 50) { nodes { login } } - closedByPullRequestsReferences(first: 50, includeClosedPrs: true) { nodes { number } } + assignees(first: 50) { pageInfo { hasNextPage } nodes { login } } + closedByPullRequestsReferences(first: 50, includeClosedPrs: true) { + pageInfo { hasNextPage } nodes { number repository { nameWithOwner } } + } } ... on PullRequest { number pullState: state - assignees(first: 50) { nodes { login } } - reviewRequests(first: 20) { nodes { requestedReviewer { ... on User { login } } } } + assignees(first: 50) { pageInfo { hasNextPage } nodes { login } } + reviewRequests(first: 20) { + pageInfo { hasNextPage } nodes { requestedReviewer { ... on User { login } } } + } timelineItems(last: 50, itemTypes: [REVIEW_REQUESTED_EVENT]) { + pageInfo { hasPreviousPage } nodes { ... on ReviewRequestedEvent { createdAt requestedReviewer { ... on User { login } } } } } } }`; -// Numbers are positive safe integers, so the aliases can't carry anything else into the query. +const isItemNumber = (number: number) => Number.isSafeInteger(number) && number > 0; +// Only positive safe integers reach the aliases, so they can't carry anything else into the query. export const itemsQuery = (numbers: ReadonlyArray) => `query($owner: String!, $name: String!) { repository(owner: $owner, name: $name) { ${numbers .map((number) => `i${number}: issueOrPullRequest(number: ${number}) { ...KinItem }`) .join(" ")} } }\n${ITEM_FRAGMENT}`; +const NextPage = Schema.Struct({ hasNextPage: Schema.Boolean }); const Logins = Schema.Struct({ + pageInfo: NextPage, nodes: Schema.Array(Schema.NullOr(Schema.Struct({ login: Schema.String }))), }); const Reviewer = Schema.NullOr(Schema.Struct({ login: Schema.optional(Schema.String) })); @@ -72,7 +77,15 @@ const Issue = Schema.Struct({ state: Schema.Literals(["OPEN", "CLOSED"]), assignees: Logins, closedByPullRequestsReferences: Schema.Struct({ - nodes: Schema.Array(Schema.NullOr(Schema.Struct({ number: Schema.Int }))), + pageInfo: NextPage, + nodes: Schema.Array( + Schema.NullOr( + Schema.Struct({ + number: Schema.Int, + repository: Schema.Struct({ nameWithOwner: Schema.String }), + }), + ), + ), }), }); const Pull = Schema.Struct({ @@ -81,9 +94,11 @@ const Pull = Schema.Struct({ pullState: Schema.Literals(["OPEN", "CLOSED", "MERGED"]), assignees: Logins, reviewRequests: Schema.Struct({ + pageInfo: NextPage, nodes: Schema.Array(Schema.NullOr(Schema.Struct({ requestedReviewer: Reviewer }))), }), timelineItems: Schema.Struct({ + pageInfo: Schema.Struct({ hasPreviousPage: Schema.Boolean }), nodes: Schema.Array( Schema.NullOr( Schema.Struct({ @@ -111,7 +126,7 @@ const isRepoSlug = Schema.is(RepoSlug); class KinGithubReadFailed extends Schema.TaggedError()("KinGithubReadFailed", { repo: Schema.String, - reason: Schema.Literals(["out_of_scope", "token", "rate_limited", "unavailable", "timeout"]), + reason: Schema.Literals(["out_of_scope", "token", "rate_limited", "unavailable"]), }) {} const lowerState = (item: Item): GithubItemState["state"] => { @@ -119,6 +134,8 @@ const lowerState = (item: Item): GithubItemState["state"] => { return state === "OPEN" ? "open" : state === "MERGED" ? "merged" : "closed"; }; +const sameRepo = (a: string, b: string) => a.toLowerCase() === b.toLowerCase(); + /** Pending user review requests, each dated by its latest request event; undated ones are left out. */ function reviewRequestsOf(pull: typeof Pull.Type): GithubItemState["reviewRequests"] { const requestedAt = new Map(); @@ -170,10 +187,12 @@ const make = Effect.gen(function* () { ) { if (!isRepoSlug(repo) || !inRepoScope(scope.repos, repo)) return yield* new KinGithubReadFailed({ repo, reason: "out_of_scope" }); + const wanted = [...new Set(numbers.filter(isItemNumber))]; + const items = new Map(); + if (wanted.length === 0) return items; const token = yield* tokenFor(repo); const [owner, name] = repo.split("/"); - const items = new Map(); - for (let start = 0; start < numbers.length; start += ITEMS_PER_QUERY) { + for (let start = 0; start < wanted.length; start += ITEMS_PER_QUERY) { const answer = yield* gitHubAnswer({ client, operation: "graphql.items", @@ -181,11 +200,15 @@ const make = Effect.gen(function* () { path: "/graphql", token, body: { - query: itemsQuery(numbers.slice(start, start + ITEMS_PER_QUERY)), + query: itemsQuery(wanted.slice(start, start + ITEMS_PER_QUERY)), variables: { owner, name }, }, }).pipe(Effect.mapError(() => new KinGithubReadFailed({ repo, reason: "unavailable" }))); - if (answer.status === 401) tokens.delete(repo.toLowerCase()); + if ( + answer.status === 401 || + (answer.status === 403 && answer.message === "Resource not accessible by integration") + ) + tokens.delete(repo.toLowerCase()); const body = decodeAnswer(answer.body); const errors = Option.isSome(body) ? (body.value.errors ?? []) : []; if (isGitHubRateLimit(answer) || errors.some((error) => error.type === "RATE_LIMITED")) { @@ -209,35 +232,25 @@ const make = Effect.gen(function* () { }); const deferral: KinDeferralGithubReaderShape = { - read: Effect.fn("KinGithubReaders.deferral")( - function* (_orgId: KinOrgId, ref: GithubRef) { - const item = (yield* fetchItems(ref.repo, [ref.number])).get(ref.number); - if (item === undefined) return Option.none(); - return Option.some({ - state: lowerState(item), - assignees: item.assignees.nodes.flatMap((node) => (node === null ? [] : [node.login])), - linkedPrs: - item.__typename === "Issue" - ? item.closedByPullRequestsReferences.nodes.flatMap((node) => - node === null ? [] : [node.number], - ) - : [], - }); + readRepo: Effect.fn("KinGithubReaders.deferral")( + function* (_orgId: KinOrgId, repo: string, numbers: ReadonlyArray) { + const states = new Map>(); + for (const item of (yield* fetchItems(repo, numbers)).values()) { + const linked = item.__typename === "Issue" ? item.closedByPullRequestsReferences : null; + if (item.assignees.pageInfo.hasNextPage || linked?.pageInfo.hasNextPage) continue; + states.set(item.number, { + state: lowerState(item), + assignees: item.assignees.nodes.flatMap((node) => (node === null ? [] : [node.login])), + linkedPrs: (linked?.nodes ?? []).flatMap((node) => + node === null || !sameRepo(node.repository.nameWithOwner, repo) ? [] : [node.number], + ), + }); + } + return states; }, - (effect, orgId, ref) => + (effect, orgId) => effect.pipe( - Effect.timeoutOrElse({ - duration: DEFERRAL_READ_TIMEOUT, - orElse: () => - Effect.fail(new KinGithubReadFailed({ repo: ref.repo, reason: "timeout" })), - }), Effect.mapError(() => new KinDeferralGithubError({ orgId, reason: "unavailable" })), - Effect.flatMap( - Option.match({ - onNone: () => Effect.fail(new KinDeferralGithubError({ orgId, reason: "not_found" })), - onSome: Effect.succeed, - }), - ), ), ), }; @@ -248,7 +261,13 @@ const make = Effect.gen(function* () { const items = yield* fetchItems(repo, numbers); return numbers.flatMap((number): ReadonlyArray => { const item = items.get(number); - if (item === undefined) return []; + if ( + item === undefined || + (item.__typename === "PullRequest" && + (item.reviewRequests.pageInfo.hasNextPage || + item.timelineItems.pageInfo.hasPreviousPage)) + ) + return []; const kind = item.__typename === "Issue" ? "issue" : "pull"; return [ { @@ -262,18 +281,7 @@ const make = Effect.gen(function* () { }, (effect, orgId, repo) => effect.pipe( - Effect.timeoutOrElse({ - duration: WATCHDOG_READ_TIMEOUT, - orElse: () => Effect.fail(new KinGithubReadFailed({ repo, reason: "timeout" })), - }), - Effect.mapError( - (failure) => - new KinWatchdogGithubError({ - orgId, - repo, - reason: failure.reason === "timeout" ? "timeout" : "unavailable", - }), - ), + Effect.mapError(() => new KinWatchdogGithubError({ orgId, repo, reason: "unavailable" })), ), ), }; @@ -286,24 +294,29 @@ const make = Effect.gen(function* () { /** Both readers over `KinGitHubApp`; needs the App id and its key in the keychain. */ export const readersLayer = Layer.effectContext(make); -const appLayer = (appId: string) => - Layer.effectDiscard( - Effect.gen(function* () { - const keychain = yield* Keychain.KinKeychain; - if (Option.isNone(yield* keychain.get("kin-github-app", appId))) - return yield* new KinGitHubApp.KinGitHubAppNotConfiguredError({ appId }); - }), - ).pipe( - Layer.provideMerge(KinGitHubApp.layer), - Layer.provideMerge( - Layer.mergeAll( - Layer.succeed(KinGitHubApp.KinGitHubAppConfig, { appId }), - KinGitHubScope.layer, - FetchHttpClient.layer, - Keychain.layer, - ), +const decodeSecrets = Schema.decodeUnknownOption( + Schema.fromJsonString(KinGitHubApp.KinGitHubAppSecrets), +); + +const appLayer = Layer.effectDiscard( + Effect.gen(function* () { + const { appId } = yield* KinGitHubApp.KinGitHubAppConfig; + const keychain = yield* Keychain.KinKeychain; + const stored = yield* keychain.get("kin-github-app", appId); + if (Option.isNone(Option.flatMap(stored, (value) => decodeSecrets(Redacted.value(value))))) + return yield* new KinGitHubApp.KinGitHubAppNotConfiguredError({ appId }); + }), +).pipe( + Layer.provideMerge(KinGitHubApp.layer), + Layer.provideMerge( + Layer.mergeAll( + KinGitHubApp.KinGitHubAppConfig.layer, + KinGitHubScope.layer, + FetchHttpClient.layer, + Keychain.layer, ), - ); + ), +); /** * The real readers when the Kin App is configured. Otherwise, or if the App can't be built, one @@ -319,7 +332,7 @@ export const layer = Layer.unwrap( return Layer.empty; } return readersLayer.pipe( - Layer.provide(Layer.mergeAll(appLayer(appId.value), repoScopeLayer)), + Layer.provide(Layer.mergeAll(appLayer, repoScopeLayer)), Layer.catchCause((cause) => Layer.effectDiscard( Effect.logWarning( diff --git a/docs/kin/specs/triggers.md b/docs/kin/specs/triggers.md index f01ddfb525109..a7c43474fd443 100644 --- a/docs/kin/specs/triggers.md +++ b/docs/kin/specs/triggers.md @@ -235,7 +235,7 @@ Arming comes from memory. When the front-matter cache (LEAD) sees a `deferrals/* Item edits move or disarm the triggers: a new `due` re-arms; `status` other than `open` or a set `invalid_at` disables them. A due time already past at save fires at once, marked late; no "pick a future time" error. A missing `on_due` means `remind`, so silence never starts work. -The cache hook is `MemorySyncHooks` in [`Memory.ts`](../../../apps/server/src/kin/memory/Memory.ts): each new head goes once to `KinTriggerService.syncOrg`, which also disarms armed deferrals missing from the snapshot or sharing a duplicate id. A failing item is logged without blocking the others, and the head retries on the next two loads, then waits for the next head. `trigger_reconcile` (below) also re-syncs the whole org every 15 minutes and at startup, so a head that gave up is retried there. A due time whose fires would land before 1970 never arms; `kin_defer` accepts only years 2000–2099. A commit to the default branch moves the cache to the new head and reloads it, so a `kin_defer` item arms when its wake-up commits, not at the next wake-up; shadow (L0) commits never arm. Trigger ids are `deferral::heads_up|due`. Re-syncing an unchanged `due` writes nothing, so a fired `once` is never re-armed. The run row in `kin_deferral_runs` (migration K016) holds the snapshot and the flow state (below). The snapshot holds the item half (`recorded_at`, `superseded_by`, links) and the GitHub half (state, assignees and linked PRs of each `github.com///issues|pull/` or `/#` link), read at arm time through `KinDeferralGithubReader` in [`deferralGithub.ts`](../../../apps/server/src/kin/triggers/deferralGithub.ts). The GitHub half is `null` when any link couldn't be read. That reader is a narrow read-only seam (one issue or PR per call), bounded to 10 s because it runs under the org memory lock. A link GitHub doesn't have is `not_found`; anything else that fails is `unavailable`. Both readers are backed by [`githubReaders.ts`](../../../apps/server/src/kin/triggers/githubReaders.ts) when the Kin App is configured, and otherwise keep their defaults (below). +The cache hook is `MemorySyncHooks` in [`Memory.ts`](../../../apps/server/src/kin/memory/Memory.ts): each new head goes once to `KinTriggerService.syncOrg`, which also disarms armed deferrals missing from the snapshot or sharing a duplicate id. A failing item is logged without blocking the others, and the head retries on the next two loads, then waits for the next head. `trigger_reconcile` (below) also re-syncs the whole org every 15 minutes and at startup, so a head that gave up is retried there. A due time whose fires would land before 1970 never arms; `kin_defer` accepts only years 2000–2099. A commit to the default branch moves the cache to the new head and reloads it, so a `kin_defer` item arms when its wake-up commits, not at the next wake-up; shadow (L0) commits never arm. Trigger ids are `deferral::heads_up|due`. Re-syncing an unchanged `due` writes nothing, so a fired `once` is never re-armed. The run row in `kin_deferral_runs` (migration K016) holds the snapshot and the flow state (below). The snapshot holds the item half (`recorded_at`, `superseded_by`, links) and the GitHub half (state, assignees and linked PRs of each `github.com///issues|pull/` or `/#` link), read at arm time through `KinDeferralGithubReader` in [`deferralGithub.ts`](../../../apps/server/src/kin/triggers/deferralGithub.ts). The GitHub half is `null` when any link couldn't be read. That reader is a narrow read-only seam: one batched call per repo for all of a sync's numbers. Because `syncOrg` runs under the org memory lock, every GitHub read in one sync shares a single 10 s budget, and all reads finish before any row is written. A link GitHub doesn't have, that fails, or that isn't read within the budget makes its item's GitHub half `null`, so the deferral asks. Both readers are backed by [`githubReaders.ts`](../../../apps/server/src/kin/triggers/githubReaders.ts) when the Kin App is configured, and otherwise keep their defaults (below). **Confirm.** When the lead calls `kin_defer` (LEAD), Kin writes the item, arms the triggers, and the lead posts a `KinDeferralCard` widget (CHAT envelope): "Deferred to Fri 13 Nov, 9:00. I'll start then." [Change] [Just remind me]. [Just remind me] sets `on_due: remind`, and the triggers re-arm. [Change] writes nothing: it hands the item to the lead, which asks the owner what to change, marks the old item `cancelled` and calls `kin_defer` for the new one. @@ -332,7 +332,7 @@ All are plain code in [`watchdogs.ts`](../../../apps/server/src/kin/triggers/wat | `trigger_reconcile` | 15 min | Re-syncs deferral triggers from the whole memory snapshot (`KinTriggerService.syncOrg`). Checks every `KinWebhookSource` task exists, is enabled and isn't `blockedByPolicy`, records what it saw in `status` and tells the owner when it changes; a task whose secret doesn't match the bound hash gets one notice per binding. It never re-enables a task, re-keys or rebinds. Clears consumed inbox rows older than 7 days and fire-ledger rows older than 30 days. | - **GitHub reads.** Through `KinWatchdogGithubReader` only: one read-only, batched call per repo for all the numbers the sweep needs, each bounded to 30 s. Only repos in `KIN_GITHUB_REPOS` are read. A repo that fails, times out or is rate-limited skips the items that need it and keeps the rows pending; nothing retries before the next sweep. -- **The real readers.** With `KIN_GITHUB_APP_ID` set and the App's key in the keychain, Kin's server layer provides both readers over `KinGitHubApp` (approvals spec section 10). Each repo gets one `server`-holder installation token narrowed to `issues: read` and `pull_requests: read`, reused until five minutes before it expires and dropped on a 401. Reads go through GitHub GraphQL, up to 50 issues or PRs per query. A repo outside `KIN_GITHUB_REPOS`, or one the App can't mint a token for, fails closed without a GitHub call. A 429, a rate-limit 403 or a GraphQL `RATE_LIMITED` fails the repo for this read. The readers make no writes. Without the App, or if it can't be built, startup logs one warning and the defaults stay: the deferral reader fails `unavailable` (deferrals ask) and the watchdog reader is `null`, so `github_recheck`, `stale_review` and `objective_at_risk` do nothing and `github_recheck` rows stay pending; nothing is lost while it waits. +- **The real readers.** With `KIN_GITHUB_APP_ID` set and the App's key in the keychain, Kin's server layer provides both readers over `KinGitHubApp` (approvals spec section 10). Each repo gets one `server`-holder installation token narrowed to `issues: read` and `pull_requests: read`, reused until five minutes before it expires and dropped on a 401 or a 403 "Resource not accessible by integration". Reads go through GitHub GraphQL, up to 50 issues or PRs per query. An item whose assignees, closing PRs, review requests or review-request events run past one page is left out (read as unavailable) rather than reported in part, and closing PRs from another repo are dropped. Time bounds belong to the callers. A repo outside `KIN_GITHUB_REPOS`, or one the App can't mint a token for, fails closed without a GitHub call. A 429, a rate-limit 403 or a GraphQL `RATE_LIMITED` fails the repo for this read. The readers make no writes. Without the App, or if it can't be built, startup logs one warning and the defaults stay: the deferral reader fails `unavailable` (deferrals ask) and the watchdog reader is `null`, so `github_recheck`, `stale_review` and `objective_at_risk` do nothing and `github_recheck` rows stay pending; nothing is lost while it waits. - **Conditions from state.** A PR gives `pull_request` with action `opened` or `closed` and `merged`; an issue gives `issues` with `opened` or `closed`. `matchConditions` then applies as for events, so a condition on another event or action matches only through webhooks. A match queues a `watchdog` event whose finding starts "GitHub state shows condition met (read from GitHub by code)". - **Repeats.** Findings dedupe on `kin_events.dedupe_key` (item, links and state), so a later sweep queues nothing new. A chase is skipped while the person's ping for the item is active, or when it was stopped after what prompted the chase last changed (the commitment's due date, or the review request's time), so an answered chase stays answered until the due date moves. - **Source status is advisory.** The sink does not refuse on `status`: the task's shape and bound secret are the gate, and a delivery after the owner turns the task back on must not wait 15 minutes for reconcile. A hash mismatch has no status value; the sink already refuses it, and `setSecret` or setup is the way out. Routine rows are kept by the routines tick's own 15-minute reconcile and dream rows by `DreamRunner`, so `trigger_reconcile` doesn't repeat them.