From 55e331e13ccd6fbb14ec68646f2c7e38b3e2ca6b Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Mon, 9 Nov 2026 14:48:55 +0530 Subject: [PATCH 1/8] feat(kin): memory writes to main go through verified PRs Co-Authored-By: Claude Opus 5.5 --- apps/server/src/kin/github/testFakes.ts | 20 +- apps/server/src/kin/lead/LeadMemory.test.ts | 1 + apps/server/src/kin/lead/layer.ts | 27 ++ apps/server/src/kin/memory/MemoryConfig.ts | 28 +- .../src/kin/memory/MemoryPullRequests.test.ts | 418 ++++++++++++++++++ .../src/kin/memory/MemoryPullRequests.ts | 292 ++++++++++++ .../src/kin/memory/MemoryWriter.test.ts | 1 + apps/server/src/kin/memory/MemoryWriter.ts | 218 ++++++++- apps/server/src/kin/triggers/githubReaders.ts | 3 +- 9 files changed, 982 insertions(+), 26 deletions(-) create mode 100644 apps/server/src/kin/memory/MemoryPullRequests.test.ts create mode 100644 apps/server/src/kin/memory/MemoryPullRequests.ts diff --git a/apps/server/src/kin/github/testFakes.ts b/apps/server/src/kin/github/testFakes.ts index 15759da41cdf..f29450c1a0d2 100644 --- a/apps/server/src/kin/github/testFakes.ts +++ b/apps/server/src/kin/github/testFakes.ts @@ -60,7 +60,9 @@ export interface FakeAnswer { } /** A GitHub that answers from `route`; nothing leaves the process. */ -export const fakeGitHub = (route: (request: RecordedRequest) => FakeAnswer) => { +export const fakeGitHub = ( + route: (request: RecordedRequest) => FakeAnswer | Effect.Effect, +) => { const requests: Array = []; const layer = Layer.succeed( HttpClient.HttpClient, @@ -77,13 +79,15 @@ export const fakeGitHub = (route: (request: RecordedRequest) => FakeAnswer) => { }; requests.push(recorded); const answer = route(recorded); - return Effect.succeed( - HttpClientResponse.fromWeb( - request, - new Response(answer.body === undefined ? null : JSON.stringify(answer.body), { - status: answer.status, - headers: { "content-type": "application/json", ...answer.headers }, - }), + return (Effect.isEffect(answer) ? answer : Effect.succeed(answer)).pipe( + Effect.map((answer) => + HttpClientResponse.fromWeb( + request, + new Response(answer.body === undefined ? null : JSON.stringify(answer.body), { + status: answer.status, + headers: { "content-type": "application/json", ...answer.headers }, + }), + ), ), ); }), diff --git a/apps/server/src/kin/lead/LeadMemory.test.ts b/apps/server/src/kin/lead/LeadMemory.test.ts index 27f077031cab..a9ef385ec175 100644 --- a/apps/server/src/kin/lead/LeadMemory.test.ts +++ b/apps/server/src/kin/lead/LeadMemory.test.ts @@ -239,6 +239,7 @@ it.live( Layer.provide( Layer.succeed(MemoryConfig, { baseDirectory: `${root}/state`, + directMainPush: true, repositories: [ { orgId, path: "a", repoUrl: remote }, { orgId: other, path: "b", repoUrl: remoteB }, diff --git a/apps/server/src/kin/lead/layer.ts b/apps/server/src/kin/lead/layer.ts index d304179a0028..3549d90b85e9 100644 --- a/apps/server/src/kin/lead/layer.ts +++ b/apps/server/src/kin/lead/layer.ts @@ -27,6 +27,8 @@ import * as Memory from "../memory/Memory.ts"; import * as MemoryConfig from "../memory/MemoryConfig.ts"; import * as KinPersistence from "../persistence/KinPersistence.ts"; import * as Writer from "../memory/MemoryWriter.ts"; +import * as MemoryPullRequests from "../memory/MemoryPullRequests.ts"; +import * as Cause from "effect/Cause"; import * as Scan from "../memory/secretScan.ts"; import * as LeadMemory from "./LeadMemory.ts"; import * as Cache from "../memory/MemoryCache.ts"; @@ -174,9 +176,34 @@ const withServices = (entry: Layer.Layer) => Layer.provide(GithubReaders.layer), ); +// TRIG-15: without the Kin App, memory writes to the default branch are refused unless +// KIN_MEMORY_DIRECT_PUSH=true, because the memory repo's ruleset rejects a direct push. +const memoryPullRequests = Layer.unwrap( + Effect.gen(function* () { + const off = + "memory writes to the default branch are refused unless KIN_MEMORY_DIRECT_PUSH=true"; + if (Option.isNone(yield* Config.option(Config.String("KIN_GITHUB_APP_ID")))) { + yield* Effect.logWarning(`Kin memory PRs are off: KIN_GITHUB_APP_ID is unset, so ${off}`); + return Layer.empty; + } + return MemoryPullRequests.layer.pipe( + Layer.provide(GithubReaders.appLayer), + Layer.catchCause((cause) => + Layer.effectDiscard( + Effect.logWarning( + `Kin memory PRs are off: the Kin GitHub App is unavailable, so ${off}`, + Cause.pretty(cause), + ), + ), + ), + ); + }), +); + const registration = McpHttpServer.toolkitRegistration(KinLeadToolkit, handlers); const toolkit = LeadMemory.layer.pipe( Layer.provideMerge(Writer.layer), + Layer.provide(memoryPullRequests), Layer.provide(LeadMemory.authority), Layer.provide(Cache.layer), Layer.provide(Scan.layer), diff --git a/apps/server/src/kin/memory/MemoryConfig.ts b/apps/server/src/kin/memory/MemoryConfig.ts index c9167ccf13de..f837b114990f 100644 --- a/apps/server/src/kin/memory/MemoryConfig.ts +++ b/apps/server/src/kin/memory/MemoryConfig.ts @@ -19,10 +19,17 @@ export class MemoryConfig extends Context.Service< readonly credentialHelper?: string; readonly sshCommand?: string; readonly botEmail?: string; + /** + * Without the Kin GitHub App, writes to the default branch are refused unless this is set; + * then they push directly, which a TRIG-15 ruleset on the memory repo rejects. + */ + readonly directMainPush?: boolean; readonly repositories: ReadonlyArray<{ readonly orgId: KinOrgId; readonly repoUrl: string; readonly path: string; + /** `owner/name` when `repoUrl` is on github.com; memory PRs go there. */ + readonly githubRepo?: string; }>; } >()("t3/kin/memory/MemoryConfig") {} @@ -38,6 +45,11 @@ export class MemoryCredentialConfigError extends Schema.TaggedError GITHUB_URL.exec(repoUrl)?.[1]; + const credential = Effect.fn("MemoryConfig.credential")(function* ( variable: MemoryCredentialConfigError["variable"], valid: (value: string) => boolean, @@ -62,14 +74,22 @@ export const layer = ( isCredentialHelperReference, ); const sshCommand = yield* credential("KIN_MEMORY_SSH_COMMAND", isSshCommandReference); + const directMainPush = yield* Config.Boolean("KIN_MEMORY_DIRECT_PUSH").pipe( + Config.withDefault(false), + ); return MemoryConfig.of({ baseDirectory: path.join(config.stateDir, "kin"), ...(credentialHelper === undefined ? {} : { credentialHelper }), ...(sshCommand === undefined ? {} : { sshCommand }), - repositories: repositories.map((repo) => ({ - ...repo, - path: encodeURIComponent(repo.orgId), - })), + directMainPush, + repositories: repositories.map((repo) => { + const githubRepo = githubRepoOf(repo.repoUrl); + return { + ...repo, + path: encodeURIComponent(repo.orgId), + ...(githubRepo === undefined ? {} : { githubRepo }), + }; + }), }); }), ); diff --git a/apps/server/src/kin/memory/MemoryPullRequests.test.ts b/apps/server/src/kin/memory/MemoryPullRequests.test.ts new file mode 100644 index 000000000000..f806bd5df212 --- /dev/null +++ b/apps/server/src/kin/memory/MemoryPullRequests.test.ts @@ -0,0 +1,418 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { expect, it } from "@effect/vitest"; +import { KinAgentId, KinMemoryItemId, KinOrgId } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Redacted from "effect/Redacted"; + +import * as Sqlite from "../../persistence/Sqlite.ts"; +import * as Vcs from "../../vcs/VcsProcess.ts"; +import { KinGitHubApp } from "../github/KinGitHubApp.ts"; +import { fakeGitHub, type FakeAnswer, type RecordedRequest } from "../github/testFakes.ts"; +import * as Persistence from "../persistence/KinPersistence.ts"; +import { fixtureItem } from "./__fixtures__/items.ts"; +import * as Memory from "./Memory.ts"; +import { githubRepoOf, MemoryConfig } from "./MemoryConfig.ts"; +import * as PullRequests from "./MemoryPullRequests.ts"; +import * as Writer from "./MemoryWriter.ts"; +import * as Scan from "./secretScan.ts"; + +const orgId = KinOrgId.make("fixture-org"); +const agentId = KinAgentId.make("fixture-lead"); +const repo = "GLDRoger/kin-memory-fixture"; +const gitLayer = Vcs.layer.pipe(Layer.provide(NodeServices.layer)); + +it.each([ + ["git@github.com:GLDRoger/kin-memory.git", "GLDRoger/kin-memory"], + ["https://github.com/GLDRoger/kin-memory", "GLDRoger/kin-memory"], + ["ssh://git@github.com/GLDRoger/kin.memory.git", "GLDRoger/kin.memory"], + ["git@gitlab.com:GLDRoger/kin-memory.git", undefined], + ["https://github.com/GLDRoger/kin-memory/extra", undefined], + ["/srv/memory.git", undefined], +])("finds the GitHub repo of %s", (url, slug) => { + expect(githubRepoOf(url)).toBe(slug); +}); + +type Scenario = + | "merged" + | "foreign-check" + | "moved-head" + | "check-failed" + | "check-pending" + | "merge-rejected" + | "owner-path" + | "keep" + | "keep-moved-head" + | "no-app"; + +it.live.each([ + "merged", + "foreign-check", + "moved-head", + "check-failed", + "check-pending", + "merge-rejected", + "owner-path", + "keep", + "keep-moved-head", + "no-app", +] satisfies ReadonlyArray)("memory writes through PRs: %s", (scenario) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const vcs = yield* Vcs.VcsProcess; + const root = yield* fs.realPath(yield* fs.makeTempDirectoryScoped()); + const source = `${root}/seed`; + const remote = `${root}/remote.git`; + const git = (cwd: string, args: ReadonlyArray) => + vcs + .run({ operation: "test.memoryPullRequests", command: "git", cwd, args }) + .pipe(Effect.map((result) => result.stdout.trim())); + const bare = (...args: ReadonlyArray) => git(root, [`--git-dir=${remote}`, ...args]); + const identity = ["-c", "user.name=Fixture", "-c", "user.email=fixture@example.invalid"]; + yield* fs.makeDirectory(source); + yield* git(source, ["init", "-b", "main"]); + yield* fs.writeFileString(`${source}/rules.md`, "Invented rules\n"); + yield* fs.writeFileString(`${source}/INDEX.md`, "As of 2026-11-08T00:00:00.000Z\n"); + yield* git(source, ["add", "--", "."]); + yield* git(source, [...identity, "commit", "-m", "fixture"]); + yield* git(root, ["clone", "--bare", "--", source, remote]); + const original = yield* bare("rev-parse", "refs/heads/main"); + + // A GitHub over the bare remote: it squash-merges there, as the real one would. + const pulls = new Map(); + const merges: Array<{ number: number; body: unknown }> = []; + const tokens: Array = []; + const branchHead = (branch: string) => bare("rev-parse", `refs/heads/${branch}`); + const pullBody = (number: number) => + Effect.gen(function* () { + const pull = pulls.get(number)!; + return { + number, + html_url: `https://github.com/${repo}/pull/${number}`, + state: pull.state, + head: { + sha: yield* branchHead(pull.branch), + ref: pull.branch, + repo: { full_name: repo }, + }, + base: { ref: "main" }, + }; + }); + const route = (request: RecordedRequest): Effect.Effect => + Effect.gen(function* () { + const path = request.path.slice(`/repos/${repo}`.length); + const body = request.body as Record | undefined; + if (request.method === "POST" && path === "/pulls") { + const number = pulls.size + 1; + pulls.set(number, { branch: body!.head!, state: "open" }); + return { status: 201, body: yield* pullBody(number) }; + } + const pull = /^\/pulls\/(\d+)(\/merge)?$/.exec(path); + if (pull) { + const number = Number(pull[1]); + const state = pulls.get(number)!; + if (request.method === "PATCH") { + state.state = body!.state!; + return { status: 200, body: yield* pullBody(number) }; + } + if (request.method === "GET") { + if (scenario === "moved-head" || scenario === "keep-moved-head") + yield* bare("update-ref", `refs/heads/${state.branch}`, original); + return { status: 200, body: yield* pullBody(number) }; + } + merges.push({ number, body }); + const head = yield* branchHead(state.branch); + if (scenario === "merge-rejected") + return { status: 405, body: { message: "Required status check is expected." } }; + if (body!.sha !== head) + return { status: 409, body: { message: "Head branch was modified." } }; + const parent = yield* bare("rev-parse", "refs/heads/main"); + const squash = yield* bare( + ...identity, + "commit-tree", + `${head}^{tree}`, + "-p", + parent, + "-m", + body!.commit_title!, + "-m", + body!.commit_message!, + ); + yield* bare("update-ref", "refs/heads/main", squash, parent); + state.state = "closed"; + return { status: 200, body: { merged: true, sha: squash } }; + } + const runs = /^\/actions\/runs\?head_sha=([a-f0-9]+)&event=pull_request/.exec(path); + if (runs) + return { + status: 200, + body: { + workflow_runs: [ + { + path: ".github/workflows/memory-check.yml", + event: "pull_request", + head_sha: runs[1], + check_suite_id: 1, + }, + // Any workflow in the repo can name a job `memory-check`. + { + path: ".github/workflows/other.yml", + event: "pull_request", + head_sha: runs[1], + check_suite_id: 2, + }, + ], + }, + }; + const checks = /^\/commits\/([a-f0-9]+)\/check-runs/.exec(path); + if (checks) { + const check = (suite: number, status: string, conclusion: string | null) => ({ + name: "memory-check", + head_sha: checks[1], + status, + conclusion, + app: { id: 15368 }, + check_suite: { id: suite }, + }); + return { + status: 200, + body: { + check_runs: + scenario === "foreign-check" + ? [check(2, "completed", "success")] + : scenario === "check-pending" + ? [check(1, "in_progress", null)] + : [check(1, "completed", scenario === "check-failed" ? "failure" : "success")], + }, + }; + } + if (request.method === "POST" && /^\/issues\/\d+\/comments$/.test(path)) + return { status: 201, body: {} }; + return { status: 404, body: { message: "Not Found" } }; + }).pipe(Effect.orDie); + const github = fakeGitHub(route); + const app = Layer.succeed( + KinGitHubApp, + KinGitHubApp.of({ + app: Effect.die("unused"), + installations: Effect.die("unused"), + installationToken: (input) => + Effect.sync(() => { + tokens.push(input); + return { + token: Redacted.make("fixture-installation-token"), + repo: input.repo, + expiresAt: "2026-11-09T01:00:00Z", + permissions: {}, + }; + }), + }), + ); + const pullRequests = Layer.effect( + PullRequests.MemoryPullRequests, + PullRequests.make({ pollInterval: 0, polls: 2 }), + ).pipe(Layer.provide(Layer.mergeAll(app, github.layer))); + + const notices: Array<{ branch: string; reason: string }> = []; + const recovery = Layer.succeed(Memory.MemoryRecoveryHooks, { + notify: (notice) => Effect.sync(() => void notices.push(notice)), + }); + const config = Layer.succeed(MemoryConfig, { + baseDirectory: root, + repositories: [{ orgId, repoUrl: remote, path: "state", githubRepo: repo }], + }); + const keeping = scenario === "keep" || scenario === "keep-moved-head"; + const auth = Layer.succeed(Writer.MemoryWriterAuthority, { + resolve: (_org, wakeupId) => + Effect.succeed({ + orgId, + wakeupId, + agentId, + level: keeping ? "L0" : "L1", + timeZone: "UTC", + startedAt: "2026-11-08T00:00:00.000Z", + eventIds: [], + sources: new Map([ + [ + "owner", + { saidBy: "fixture-owner", trust: "owner" as const, source: "fixture-owner-message" }, + ], + ]), + }), + }); + const memoryLayer = Memory.layer.pipe(Layer.provide(config), Layer.provide(recovery)); + const writerLayer = Writer.layer.pipe( + Layer.provideMerge(memoryLayer), + Layer.provide(scenario === "no-app" ? Layer.empty : pullRequests), + Layer.provide(recovery), + Layer.provide(config), + Layer.provide(Layer.succeed(Writer.MemoryWriterHooks, { notify: () => Effect.void })), + Layer.provide(auth), + Layer.provide(Scan.layer), + ); + + yield* Effect.gen(function* () { + const memory = yield* Memory.Memory; + const writer = yield* Writer.MemoryWriter; + const paths = yield* memory.ensureCheckouts(orgId); + yield* memory.prepareWakeup(orgId); + const localMain = () => git(paths.writer, ["rev-parse", "refs/heads/main"]); + const remoteFiles = (ref: string) => bare("ls-tree", "-r", "--name-only", ref); + const refused = (effect: Effect.Effect) => + effect.pipe( + Effect.flip, + Effect.map((error) => { + expect(error).toMatchObject({ _tag: "MemorySyncConflict" }); + return error; + }), + ); + // A refused write leaves the default branch where it was, locally and on GitHub. + const unchanged = Effect.gen(function* () { + expect(yield* bare("rev-parse", "refs/heads/main")).toBe(original); + expect(yield* localMain()).toBe(original); + expect(yield* memory.anchor(orgId, "main")).toBe(original); + expect((yield* memory.prepareWakeup(orgId)).head).toBe(original); + }); + const claim = { path: "claims/new.md", content: fixtureItem({ id: "new", kind: "claim" }) }; + + if (keeping) { + yield* writer.stage(orgId, "wake1", { + path: "deferrals/def-001.md", + content: fixtureItem(), + sourceEventId: "owner", + }); + const shadowed = yield* writer.commitWakeup(orgId, "wake1", "completed"); + const { paths: shown } = yield* writer.settleShadow(orgId, agentId, { _tag: "preview" }); + const keep = writer.settleShadow(orgId, agentId, { _tag: "keep", paths: shown }); + if (scenario === "keep-moved-head") { + yield* refused(keep); + yield* unchanged; + expect(merges).toEqual([]); + expect(yield* branchHead(`kin/shadow/${agentId}`)).toBe(shadowed.commit); + expect(pulls.get(1)?.state).toBe("closed"); + return; + } + expect(yield* keep).toEqual({ kept: 2, paths: shown }); + const branch = `kin/write/${agentId}/keep-${shadowed.commit.slice(0, 12)}`; + expect(pulls.get(1)?.branch).toBe(branch); + const main = yield* bare("rev-parse", "refs/heads/main"); + expect(merges).toEqual([ + { number: 1, body: expect.objectContaining({ sha: expect.any(String) }) }, + ]); + expect(yield* localMain()).toBe(main); + expect(yield* remoteFiles(main)).toContain("deferrals/def-001.md"); + expect(yield* branchHead(`kin/shadow/${agentId}`)).toBe(main); + return; + } + + yield* writer.stage(orgId, "wake1", { ...claim, sourceEventId: "owner" }); + if (scenario === "owner-path") + yield* writer.stage(orgId, "wake1", { + path: "decisions/dec-001.md", + content: fixtureItem({ id: "dec-001", kind: "decision", review_by: null }), + sourceEventId: "owner", + }); + const committing = writer.commitWakeup(orgId, "wake1", "completed"); + + if (scenario === "no-app") { + yield* refused(committing); + yield* unchanged; + expect(yield* bare("for-each-ref", "--format=%(refname)")).toBe("refs/heads/main"); + expect(notices.map((notice) => notice.reason)).toContainEqual( + expect.stringContaining("Kin GitHub App"), + ); + return; + } + if (scenario !== "merged" && scenario !== "owner-path") { + const error = yield* refused(committing); + yield* unchanged; + expect(merges.length).toBe(scenario === "merge-rejected" ? 1 : 0); + expect(pulls.get(1)?.state).toBe("closed"); + expect(github.requests).toContainEqual( + expect.objectContaining({ method: "POST", path: `/repos/${repo}/issues/1/comments` }), + ); + expect(notices.map((notice) => notice.reason)).toContainEqual( + expect.stringContaining(`https://github.com/${repo}/pull/1`), + ); + expect(error).toMatchObject({ + reason: expect.stringContaining( + { + "foreign-check": "did not come from .github/workflows/memory-check.yml", + "moved-head": "head is not the commit Kin pushed", + "check-failed": "memory-check failed", + "check-pending": "did not finish in time", + "merge-rejected": "GitHub refused the merge (HTTP 405", + }[scenario], + ), + }); + return; + } + + const result = yield* committing; + const main = yield* bare("rev-parse", "refs/heads/main"); + expect(result.commit).toBe(main); + expect(yield* localMain()).toBe(main); + expect(yield* memory.anchor(orgId, "main")).toBe(main); + expect(yield* bare("rev-parse", `${main}^`)).toBe(original); + expect(yield* bare("log", "-1", "--format=%s", main)).toBe(`kin(${agentId}): wake-up wake1`); + expect(yield* remoteFiles(main)).toContain("claims/new.md"); + // The merged branch is deleted; the merge was pinned to the commit Kin pushed. + expect(yield* bare("for-each-ref", "--format=%(refname)", "refs/heads/kin/write/")).toBe( + scenario === "owner-path" ? `refs/heads/kin/write/${agentId}/wake1-review` : "", + ); + const auto = [...pulls].find(([, pull]) => pull.branch === `kin/write/${agentId}/wake1`)!; + const autoHead = yield* git(paths.writer, ["rev-parse", `${main}^{tree}`]); + expect(merges).toEqual([ + { + number: auto[0], + body: expect.objectContaining({ merge_method: "squash", sha: expect.any(String) }), + }, + ]); + const pinned = (merges[0]!.body as { sha: string }).sha; + expect(yield* git(paths.writer, ["rev-parse", `${pinned}^{tree}`])).toBe(autoHead); + expect(tokens).toEqual([ + { + repo, + holder: "server", + permissions: { + contents: "write", + pull_requests: "write", + checks: "read", + actions: "read", + }, + }, + ...(scenario === "owner-path" ? [expect.objectContaining({ holder: "server" })] : []), + ]); + + if (scenario === "owner-path") { + const reviewBranch = `kin/write/${agentId}/wake1-review`; + const review = [...pulls].find(([, pull]) => pull.branch === reviewBranch)!; + // Kin never merges the owner's paths; the PR stays open for their review. + expect(review[1].state).toBe("open"); + expect(merges.map((merge) => merge.number)).not.toContain(review[0]); + expect(yield* remoteFiles(main)).not.toContain("decisions/dec-001.md"); + expect(yield* bare("show", `${main}:INDEX.md`)).not.toContain("dec-001"); + expect(yield* remoteFiles(reviewBranch)).toContain("decisions/dec-001.md"); + expect(yield* remoteFiles(reviewBranch)).not.toContain("claims/new.md"); + expect(notices.map((notice) => notice.reason)).toContainEqual( + `Memory changes on paths you review wait in https://github.com/${repo}/pull/${review[0]}`, + ); + // Reads follow the default branch while the review PR is open. + expect((yield* memory.prepareWakeup(orgId)).head).toBe(main); + expect( + (yield* memory.readItem(orgId, KinMemoryItemId.make("dec-001")).pipe(Effect.flip))._tag, + ).toBe("MemoryNotFound"); + } + }).pipe(Effect.provide(writerLayer)); + }).pipe( + Effect.provide( + Layer.mergeAll( + gitLayer, + NodeServices.layer, + Persistence.layer.pipe(Layer.provideMerge(Sqlite.layerMemory)), + ), + ), + Effect.scoped, + ), +); diff --git a/apps/server/src/kin/memory/MemoryPullRequests.ts b/apps/server/src/kin/memory/MemoryPullRequests.ts new file mode 100644 index 000000000000..213b2ffa1f65 --- /dev/null +++ b/apps/server/src/kin/memory/MemoryPullRequests.ts @@ -0,0 +1,292 @@ +import type { KinOrgId, RepoSlug } from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import type * as Redacted from "effect/Redacted"; +import * as Schema from "effect/Schema"; +import { HttpClient } from "effect/http"; + +import { gitHubAnswer, repoPath } from "../github/githubRest.ts"; +import { KinGitHubApp } from "../github/KinGitHubApp.ts"; +import { GITHUB_ACTIONS_INTEGRATION_ID } from "../github/KinRulesets.ts"; + +const MEMORY_CHECK_NAME = "memory-check"; +const MEMORY_CHECK_WORKFLOW = ".github/workflows/memory-check.yml"; + +const Pull = Schema.Struct({ + number: Schema.Int, + html_url: Schema.String, + state: Schema.String, + merged: Schema.optional(Schema.NullOr(Schema.Boolean)), + merge_commit_sha: Schema.optional(Schema.NullOr(Schema.String)), + head: Schema.Struct({ + sha: Schema.String, + ref: Schema.String, + repo: Schema.NullOr(Schema.Struct({ full_name: Schema.String })), + }), + base: Schema.Struct({ ref: Schema.String }), +}); +const decodePull = Schema.decodeUnknownOption(Pull); +const decodePulls = Schema.decodeUnknownOption(Schema.Array(Pull)); +const WorkflowRuns = Schema.Struct({ + workflow_runs: Schema.Array( + Schema.Struct({ + path: Schema.String, + event: Schema.String, + head_sha: Schema.String, + check_suite_id: Schema.optional(Schema.NullOr(Schema.Int)), + }), + ), +}); +const decodeRuns = Schema.decodeUnknownOption(WorkflowRuns); +const CheckRuns = Schema.Struct({ + check_runs: Schema.Array( + Schema.Struct({ + name: Schema.String, + head_sha: Schema.String, + status: Schema.String, + conclusion: Schema.NullOr(Schema.String), + app: Schema.NullOr(Schema.Struct({ id: Schema.Int })), + check_suite: Schema.NullOr(Schema.Struct({ id: Schema.Int })), + }), + ), +}); +const decodeChecks = Schema.decodeUnknownOption(CheckRuns); +const decodeMerge = Schema.decodeUnknownOption( + Schema.Struct({ merged: Schema.Boolean, sha: Schema.String }), +); + +export class MemoryPullRequestRefused extends Schema.TaggedError()( + "MemoryPullRequestRefused", + { + orgId: Schema.String, + // Built from fixed text, GitHub's status and error message; never a token. + reason: Schema.String, + url: Schema.optional(Schema.String), + }, +) {} + +type MemoryPullRequestOutcome = + | { readonly _tag: "merged"; readonly url: string; readonly commit: string } + | { readonly _tag: "awaiting_owner"; readonly url: string }; + +interface MemoryPullRequestInput { + readonly orgId: KinOrgId; + readonly repo: RepoSlug; + readonly branch: string; + readonly base: string; + /** The commit Kin pushed to `branch`; nothing else is ever merged. */ + readonly sha: string; + readonly title: string; + readonly message: string; + /** Touches paths CODEOWNERS gives the owner: opened for review, never merged by Kin. */ + readonly ownerReview: boolean; +} + +/** + * Lands Kin's own memory commits on the protected default branch (TRIG-15). Server-only: never + * reachable from a lead, worker or MCP tool. + */ +export class MemoryPullRequests extends Context.Service< + MemoryPullRequests, + { + readonly land: ( + input: MemoryPullRequestInput, + ) => Effect.Effect; + } +>()("t3/kin/memory/MemoryPullRequests") {} + +type Verdict = + | { readonly _tag: "pass" } + | { readonly _tag: "pending" } + | { readonly _tag: "fail"; readonly reason: string }; + +/** + * `memory-check` is pinned to the Actions App, which any workflow in the repo can post as, so only + * a check run from a `pull_request` run of the memory-check workflow on `sha` counts. + */ +const memoryCheckVerdict = ( + sha: string, + runs: typeof WorkflowRuns.Type, + checks: typeof CheckRuns.Type, +): Verdict => { + const suites = new Set( + runs.workflow_runs + .filter( + (run) => + run.path === MEMORY_CHECK_WORKFLOW && + run.event === "pull_request" && + run.head_sha === sha, + ) + .flatMap((run) => (run.check_suite_id == null ? [] : [run.check_suite_id])), + ); + const named = checks.check_runs.filter((check) => check.name === MEMORY_CHECK_NAME); + const foreign = named.some( + (check) => + check.app?.id !== GITHUB_ACTIONS_INTEGRATION_ID || + check.head_sha !== sha || + check.check_suite === null || + !suites.has(check.check_suite.id), + ); + if (foreign) + return { + _tag: "fail", + reason: `a ${MEMORY_CHECK_NAME} check that did not come from ${MEMORY_CHECK_WORKFLOW} ran on this commit`, + }; + if (named.length === 0 || named.some((check) => check.status !== "completed")) + return { _tag: "pending" }; + if (named.some((check) => check.conclusion !== "success")) + return { _tag: "fail", reason: `${MEMORY_CHECK_NAME} failed` }; + return { _tag: "pass" }; +}; + +export const make = Effect.fn("MemoryPullRequests.make")(function* (options: { + readonly pollInterval: Duration.Input; + readonly polls: number; +}) { + const app = yield* KinGitHubApp; + const client = yield* HttpClient.HttpClient; + + const land = Effect.fn("MemoryPullRequests.land")(function* (input: MemoryPullRequestInput) { + yield* Effect.annotateCurrentSpan({ "kin.github.repo": input.repo }); + const refuse = (reason: string, url?: string) => + new MemoryPullRequestRefused({ + orgId: input.orgId, + reason, + ...(url === undefined ? {} : { url }), + }); + const minted = yield* app + .installationToken({ + repo: input.repo, + permissions: { + contents: "write", + pull_requests: "write", + checks: "read", + actions: "read", + }, + holder: "server", + }) + .pipe( + Effect.mapError((error) => + refuse(`the Kin GitHub App gave no token for ${input.repo} (${error._tag})`), + ), + ); + const token: Redacted.Redacted = minted.token; + const base = repoPath(input.repo); + const call = ( + operation: string, + method: "GET" | "POST" | "PUT" | "PATCH", + path: string, + body?: unknown, + ) => + gitHubAnswer({ + client, + operation, + method, + path: `${base}${path}`, + token, + ...(body === undefined ? {} : { body }), + }).pipe(Effect.mapError(() => refuse(`could not reach GitHub for ${operation}`))); + const describe = (answer: { status: number; message: string }) => + `HTTP ${answer.status}${answer.message ? `: ${answer.message}` : ""}`; + + const opened = yield* call("pulls.create", "POST", "/pulls", { + title: input.title, + head: input.branch, + base: input.base, + body: input.ownerReview + ? "Kin's memory changes on paths you review. Kin will not merge this." + : `Kin merges this once \`${MEMORY_CHECK_NAME}\` passes on the commit it pushed.`, + maintainer_can_modify: false, + }); + let pull = opened.status === 201 ? Option.getOrUndefined(decodePull(opened.body)) : undefined; + // A retry of the same write finds the PR it opened before. + if (pull === undefined && opened.status === 422) { + const owner = input.repo.split("/")[0] ?? ""; + const listed = yield* call( + "pulls.list", + "GET", + `/pulls?state=open&head=${encodeURIComponent(`${owner}:${input.branch}`)}&base=${encodeURIComponent(input.base)}`, + ); + pull = Option.getOrUndefined(decodePulls(listed.body))?.[0]; + } + if (pull === undefined) return yield* refuse(`GitHub refused the PR (${describe(opened)})`); + const { number, html_url: url } = pull; + + const close = Effect.fn("MemoryPullRequests.close")( + function* (reason: string) { + yield* call("issues.comment", "POST", `/issues/${number}/comments`, { + body: `Kin closed this memory PR: ${reason}.`, + }); + yield* call("pulls.close", "PATCH", `/pulls/${number}`, { state: "closed" }); + }, + Effect.catch(() => + Effect.logWarning("Could not close a refused memory PR", { repo: input.repo, number }), + ), + ); + const refuseAndClose = (reason: string) => + close(reason).pipe(Effect.andThen(Effect.fail(refuse(reason, url)))); + + const headIsOurs = Effect.gen(function* () { + const answer = yield* call("pulls.get", "GET", `/pulls/${number}`); + const current = answer.status === 200 ? decodePull(answer.body) : Option.none(); + return ( + Option.isSome(current) && + current.value.state === "open" && + current.value.head.sha === input.sha && + current.value.head.ref === input.branch && + current.value.head.repo?.full_name.toLowerCase() === input.repo.toLowerCase() && + current.value.base.ref === input.base + ); + }); + if (!(yield* headIsOurs)) + return yield* refuseAndClose("the PR's head is not the commit Kin pushed"); + if (input.ownerReview) return { _tag: "awaiting_owner", url } as const; + + let verdict: Verdict = { _tag: "pending" }; + for (let poll = 0; poll < options.polls && verdict._tag === "pending"; poll++) { + if (poll > 0) yield* Effect.sleep(options.pollInterval); + const runs = yield* call( + "actions.runs", + "GET", + `/actions/runs?head_sha=${input.sha}&event=pull_request&per_page=100`, + ); + const checks = yield* call( + "checks.list", + "GET", + `/commits/${input.sha}/check-runs?check_name=${MEMORY_CHECK_NAME}&per_page=100`, + ); + const decodedRuns = runs.status === 200 ? decodeRuns(runs.body) : Option.none(); + const decodedChecks = checks.status === 200 ? decodeChecks(checks.body) : Option.none(); + if (Option.isSome(decodedRuns) && Option.isSome(decodedChecks)) + verdict = memoryCheckVerdict(input.sha, decodedRuns.value, decodedChecks.value); + } + if (verdict._tag === "pending") + return yield* refuseAndClose(`${MEMORY_CHECK_NAME} did not finish in time`); + if (verdict._tag === "fail") return yield* refuseAndClose(verdict.reason); + if (!(yield* headIsOurs)) + return yield* refuseAndClose("the PR's head moved after the check passed"); + + const [title, ...rest] = input.message.split("\n"); + const merge = yield* call("pulls.merge", "PUT", `/pulls/${number}/merge`, { + merge_method: "squash", + sha: input.sha, + commit_title: title, + commit_message: rest.join("\n").trim(), + }); + const merged = merge.status === 200 ? decodeMerge(merge.body) : Option.none(); + if (Option.isSome(merged) && merged.value.merged) + return { _tag: "merged", url, commit: merged.value.sha } as const; + return yield* refuseAndClose(`GitHub refused the merge (${describe(merge)})`); + }); + + return MemoryPullRequests.of({ land }); +}); + +/** Polls `memory-check` every 15 seconds for up to the job's 15-minute timeout. */ +export const layer = Layer.effect( + MemoryPullRequests, + make({ pollInterval: Duration.seconds(15), polls: 60 }), +); diff --git a/apps/server/src/kin/memory/MemoryWriter.test.ts b/apps/server/src/kin/memory/MemoryWriter.test.ts index 9da9751c3ea0..e13744291b45 100644 --- a/apps/server/src/kin/memory/MemoryWriter.test.ts +++ b/apps/server/src/kin/memory/MemoryWriter.test.ts @@ -224,6 +224,7 @@ it.live.each([ baseDirectory: root, credentialHelper: "fixture-server-helper", sshCommand: "ssh -i /fixture/kin-memory-key", + directMainPush: true, repositories: [{ orgId, repoUrl: remote, path: "state" }], }); const synced: Array> = []; diff --git a/apps/server/src/kin/memory/MemoryWriter.ts b/apps/server/src/kin/memory/MemoryWriter.ts index 67e13052bce4..738fadd3b724 100644 --- a/apps/server/src/kin/memory/MemoryWriter.ts +++ b/apps/server/src/kin/memory/MemoryWriter.ts @@ -6,7 +6,7 @@ import { readFiles, readCommit, readTree, objectHash, verifyTree } from "./objec import { buildIndex } from "./buildIndex.ts"; import { isCheckoutPath } from "./checkoutPaths.ts"; import * as MemoryCache from "./MemoryCache.ts"; -import type { KinAgentId, KinOrgId, MemoryTrust } from "@t3tools/contracts"; +import { RepoSlug, type KinAgentId, type KinOrgId, type MemoryTrust } from "@t3tools/contracts"; import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; @@ -33,6 +33,7 @@ import { journalEntry, MemoryJournalError, type JournalContext } from "./journal import { mergeMemoryFiles } from "./merge.ts"; import { isOwnerReviewedPath, kinShapeFailure } from "./lint.ts"; import { KinSecretScan, KinSecretScanError } from "./secretScan.ts"; +import { MemoryPullRequests } from "./MemoryPullRequests.ts"; export const LEAD_ENDED_WITHOUT_KIN_END = "Lead ended without calling kin_end."; @@ -181,6 +182,7 @@ export class MemoryWriter extends Context.Service< ) => Effect.Effect<{ readonly kept: number; readonly paths: ReadonlyArray }, Failure>; } >()("t3/kin/memory/MemoryWriter") {} +const isRepoSlug = Schema.is(RepoSlug); const dreamBranch = /^dream\/\d{4}-\d{2}-\d{2}$/; const folders = ["objectives", "commitments", "deferrals", "decisions", "people", "claims"]; const render = (fields: object, body: string) => `---\n${stringify(fields)}---\n${body}`; @@ -200,6 +202,12 @@ const make = Effect.gen(function* () { ); const config = yield* MemoryConfig; const cache = yield* MemoryCache.MemoryCache; + const pullRequests = yield* Effect.serviceOption(MemoryPullRequests); + const mainWrites = Option.isSome(pullRequests) + ? "pull_request" + : config.directMainPush === true + ? "direct" + : "refused"; const filesAt = (orgId: KinOrgId, writer: string, commit: string) => readFiles(orgId, writer, commit).pipe( Effect.provideService(FileSystem.FileSystem, fs), @@ -542,6 +550,126 @@ const make = Effect.gen(function* () { return yield* new MemoryConfigError({ orgId, operation: "writer", reason: "Unknown org" }); return repo; }); + const refuseMainWrite = Effect.fn("MemoryWriter.refuseMainWrite")(function* (orgId: KinOrgId) { + const reason = + "Memory writes to the default branch need the Kin GitHub App (TRIG-15); this one was not committed"; + yield* recovery.notify({ orgId, branch: config.defaultBranch ?? "main", reason }); + return yield* new MemorySyncConflict({ orgId, operation: "push", reason }); + }); + /** + * Lands Kin's commits through PRs (TRIG-15). The local default branch moves only after GitHub + * merged, and then to the remote head, so a refused PR leaves it where it was. + */ + const landOnMain = Effect.fn("MemoryWriter.landOnMain")(function* ( + orgId: KinOrgId, + writer: string, + input: { + readonly agentId: string; + readonly id: string; + readonly base: string; + readonly commit: string; + readonly message: string; + readonly review?: { readonly commit: string; readonly message: string }; + }, + ) { + if (Option.isNone(pullRequests)) return yield* refuseMainWrite(orgId); + const repo = yield* repoFor(orgId); + const main = config.defaultBranch ?? "main"; + const conflict = (reason: string) => + new MemorySyncConflict({ orgId, operation: "push", reason }); + if (repo.githubRepo === undefined || !isRepoSlug(repo.githubRepo)) + return yield* conflict("The memory repo is not on GitHub, so Kin cannot open its PR"); + const github = repo.githubRepo; + const open = Effect.fn("MemoryWriter.openPullRequest")(function* ( + branch: string, + commit: string, + message: string, + ownerReview: boolean, + ) { + yield* git(orgId, writer, [ + "push", + "--", + repo.repoUrl, + `+${commit}:refs/heads/${branch}`, + ]).pipe(Effect.mapError(() => conflict("Memory PR branch push rejected"))); + return yield* pullRequests.value.land({ + orgId, + repo: github, + branch, + base: main, + sha: commit, + title: message.split("\n")[0] ?? message, + message, + ownerReview, + }); + }); + const branch = `kin/write/${input.agentId}/${input.id}`; + if (input.review !== undefined) { + const reviewBranch = `${branch}-review`; + yield* open(reviewBranch, input.review.commit, input.review.message, true).pipe( + Effect.flatMap((opened) => + recovery.notify({ + orgId, + branch: reviewBranch, + reason: `Memory changes on paths you review wait in ${opened.url}`, + }), + ), + Effect.catch((error) => + recovery.notify({ + orgId, + branch: reviewBranch, + reason: `Memory changes on paths you review were not opened as a PR: ${error.reason}`, + }), + ), + ); + } + const landed = yield* open(branch, input.commit, input.message, false).pipe( + Effect.tapError((error) => + recovery.notify({ + orgId, + branch, + reason: `Memory write not merged: ${error.reason}${error._tag === "MemoryPullRequestRefused" && error.url ? ` (${error.url})` : ""}`, + }), + ), + Effect.mapError((error) => + error._tag === "MemoryPullRequestRefused" ? conflict(error.reason) : error, + ), + ); + if (landed._tag !== "merged") return yield* conflict("The memory PR was not merged"); + const advertised = yield* git(orgId, writer, [ + "ls-remote", + "--", + repo.repoUrl, + `refs/heads/${main}`, + ]); + const [head, name] = advertised.split(/\s+/); + if (!head || !/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(head) || name !== `refs/heads/${main}`) + return yield* conflict("The default branch is missing after the merge"); + yield* git(orgId, writer, ["fetch", "--", repo.repoUrl, head]); + const ancestor = (older: string, newer: string) => + git(orgId, writer, ["merge-base", "--is-ancestor", older, newer]).pipe( + Effect.as(true), + Effect.catchTags({ MemoryError: () => Effect.succeed(false) }), + ); + if (!(yield* ancestor(landed.commit, head)) || !(yield* ancestor(input.base, landed.commit))) + return yield* conflict("The merged memory commit is not on the default branch"); + const [parent, mergedTree, ownTree] = [ + yield* git(orgId, writer, ["rev-parse", `${landed.commit}^`]), + yield* git(orgId, writer, ["rev-parse", `${landed.commit}^{tree}`]), + yield* git(orgId, writer, ["rev-parse", `${input.commit}^{tree}`]), + ]; + if (parent === input.base && mergedTree !== ownTree) + return yield* conflict("GitHub merged a different tree than Kin pushed"); + yield* filesAt(orgId, writer, head); + yield* git(orgId, writer, ["update-ref", `refs/heads/${main}`, head, input.base]); + yield* memory.record(orgId, main, input.base, head); + yield* git(orgId, writer, ["push", "--", repo.repoUrl, `:refs/heads/${branch}`]).pipe( + Effect.catch(() => + Effect.logWarning("Could not delete a merged memory PR branch", { orgId }), + ), + ); + return head; + }); return MemoryWriter.of({ stage: Effect.fn("MemoryWriter.stage")((orgId, wakeupId, change) => report( @@ -819,6 +947,9 @@ const make = Effect.gen(function* () { }); const branch = ctx.level === "L0" ? `kin/shadow/${ctx.agentId}` : (config.defaultBranch ?? "main"); + const onMain = branch === (config.defaultBranch ?? "main"); + if (onMain && mainWrites === "refused") return yield* refuseMainWrite(orgId); + const viaPullRequest = onMain && mainWrites === "pull_request"; const recorded = yield* branchHead(orgId, writer, branch); const push = (sha: string) => git(orgId, writer, ["push", "--", repo.repoUrl, `${sha}:refs/heads/${branch}`]).pipe( @@ -853,7 +984,8 @@ const make = Effect.gen(function* () { !body.includes(`Kin-Request: ${fingerprint}`)) ) return yield* new MemoryWriteRejected({ reason: "Different wake-up retry" }); - yield* push(recorded); + // Through PRs the default branch only ever holds what GitHub already merged. + if (!viaPullRequest) yield* push(recorded); return { commit: recorded }; } if (outcome === "failed") state.files.clear(); @@ -871,7 +1003,9 @@ const make = Effect.gen(function* () { .filter(([path]) => regular.has(path)) .map(([path, bytes]) => [path, bytes.toString("utf8")]), ); - for (const [path, text] of state.files) combined.set(path, text); + // Paths the owner reviews reach the default branch only through their own PR. + const reviewed = (path: string) => viaPullRequest && isOwnerReviewedPath(path); + for (const [path, text] of state.files) if (!reviewed(path)) combined.set(path, text); state.files.set("INDEX.md", yield* indexFor(orgId, combined)); const findings = yield* scanner.scan({ orgId, @@ -888,6 +1022,49 @@ const make = Effect.gen(function* () { ]), ); const message = `${subject}\n\nEvents: ${ctx.eventIds.join(", ")}\nKin-Outcome: ${outcome}\nKin-Request: ${fingerprint}\n`; + if (viaPullRequest) { + const review = new Map([...changes].filter(([file]) => reviewed(file))); + const commit = yield* writeCommit( + orgId, + writer, + recorded, + recordedFiles, + new Map([...changes].filter(([file]) => !reviewed(file))), + message, + ); + const reviewMessage = `kin(${ctx.agentId}): review wake-up ${wakeupId}\n\nEvents: ${ctx.eventIds.join(", ")}\n`; + const head = yield* landOnMain(orgId, writer, { + agentId: ctx.agentId, + id: wakeupId, + base: recorded, + commit, + message, + ...(review.size === 0 + ? {} + : { + review: { + commit: yield* writeCommit( + orgId, + writer, + recorded, + recordedFiles, + review, + reviewMessage, + ), + message: reviewMessage, + }, + }), + }); + yield* memory.rebuildCache(orgId).pipe( + Effect.catch((error) => + Effect.logWarning("Could not refresh memory cache after commit", { + orgId, + errorTag: error._tag, + }), + ), + ); + return { commit: head }; + } const commit = yield* writeCommit( orgId, writer, @@ -1095,6 +1272,7 @@ const make = Effect.gen(function* () { `It changes paths you review in a pull request (${reviewed.slice(0, 5).join(", ")}${reviewed.length > 5 ? ", …" : ""}); start clean instead`, ); if (changes.size > 0) { + if (mainWrites === "refused") return yield* refuseMainWrite(orgId); kept = changes.size; const regular = yield* regularPaths(orgId, writer, mainHead); const combined = new Map( @@ -1108,22 +1286,37 @@ const make = Effect.gen(function* () { files: [...changes].map(([path, content]) => ({ path, content })), }); if (findings.length) return yield* new MemorySecretBlocked({ orgId, findings }); + const message = `kin(${agentId}): keep shadow memory\n\nKin-Shadow: ${shadowHead}\n`; const commit = yield* writeCommit( orgId, writer, mainHead, mainFiles, changes, - `kin(${agentId}): keep shadow memory\n\nKin-Shadow: ${shadowHead}\n`, + message, ); - yield* git(orgId, writer, ["update-ref", `refs/heads/${main}`, commit, mainHead]); - yield* memory.record(orgId, main, mainHead, commit); - yield* git(orgId, writer, [ - "push", - "--", - repo.repoUrl, - `${commit}:refs/heads/${main}`, - ]).pipe(Effect.mapError(() => conflict("Push rejected; retry or reconcile remote"))); + if (mainWrites === "pull_request") { + head = yield* landOnMain(orgId, writer, { + agentId, + id: `keep-${shadowHead.slice(0, 12)}`, + base: mainHead, + commit, + message, + }); + } else { + // Push first, so a rejected push leaves the local default branch where it was. + yield* git(orgId, writer, [ + "push", + "--", + repo.repoUrl, + `${commit}:refs/heads/${main}`, + ]).pipe( + Effect.mapError(() => conflict("Push rejected; retry or reconcile remote")), + ); + yield* git(orgId, writer, ["update-ref", `refs/heads/${main}`, commit, mainHead]); + yield* memory.record(orgId, main, mainHead, commit); + head = commit; + } yield* memory.rebuildCache(orgId).pipe( Effect.catch((error) => Effect.logWarning("Could not refresh memory cache after commit", { @@ -1132,7 +1325,6 @@ const make = Effect.gen(function* () { }), ), ); - head = commit; } } // A later trial at L0 starts from what the lead reads at L1, not the old shadow week. diff --git a/apps/server/src/kin/triggers/githubReaders.ts b/apps/server/src/kin/triggers/githubReaders.ts index 03ec4a7831ca..502797c55057 100644 --- a/apps/server/src/kin/triggers/githubReaders.ts +++ b/apps/server/src/kin/triggers/githubReaders.ts @@ -442,7 +442,8 @@ const decodeSecrets = Schema.decodeUnknownOption( Schema.fromJsonString(KinGitHubApp.KinGitHubAppSecrets), ); -const appLayer = Layer.effectDiscard( +/** The Kin App with its id, key check, scope and HTTP client; fails when the key is missing. */ +export const appLayer = Layer.effectDiscard( Effect.gen(function* () { const { appId } = yield* KinGitHubApp.KinGitHubAppConfig; const keychain = yield* Keychain.KinKeychain; From 8b66aaaf8845843e995514be5c876f42af0ad772 Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Mon, 9 Nov 2026 14:52:08 +0530 Subject: [PATCH 2/8] docs(kin): TRIG-15 memory writes land through verified PRs Co-Authored-By: Claude Opus 5.5 --- docs/kin/specs/lead.md | 9 ++++++--- docs/kin/specs/triggers.md | 2 +- ops/kin/host/README.md | 2 +- ops/kin/memory-repo/README.md | 4 ++-- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/docs/kin/specs/lead.md b/docs/kin/specs/lead.md index cfd94110efe9..5ce91d7393e8 100644 --- a/docs/kin/specs/lead.md +++ b/docs/kin/specs/lead.md @@ -417,9 +417,11 @@ Writer rules, enforced in code on `stage`: Staged bytes live only in server memory. Discard requires no checkout validation and works after restart. Completed commits contain only validated bytes, a journal appended to the recorded tree, and an optional generated index. Failed wake-ups discard item changes and commit the failure journal. A private index and `hash-object --no-filters`, `read-tree`, `update-index`, `write-tree`, `commit-tree`, and compare-and-swap `update-ref` build commits on the SQLite-recorded parent, never HEAD. Author and committer identity come from server configuration (default Kin Memory Bot ), never repository configuration. -Branches: at L1 and above the writer commits to the default branch. At L0 it commits to `kin/shadow/` (see Levels). Dreaming (triggers spec) commits to `dream/` branches and takes the same per-organization semaphore while it writes, so it never runs during a wake-up. +Branches: at L1 and above the writer commits to the default branch, through a pull request (below). At L0 it commits to `kin/shadow/` (see Levels). Dreaming (triggers spec) commits to `dream/` branches and takes the same per-organization semaphore while it writes, so it never runs during a wake-up. -Pushes and fetches use the configured URL explicitly, never a local remote name or pushurl. A retry of an already recorded wake-up pushes again before returning success. Synchronization uses verified ancestry and plumbing-only fast-forwards. Divergent trees merge disjoint edits and append-only journals in process, then use `commit-tree` with both parents. Content conflicts preserve the local head on a conflict branch, reset to the remote anchor and notify the owner. Missing or ambiguous merge bases preserve and notify, then fail closed without resetting; the organization stays blocked until an operator runs `Memory.recover`, and repeated attempts reuse the conflict branch already holding that head. Failed fetches retain the SQLite anchor. Existing writers with mismatched or missing anchors fail closed as `MemoryCheckoutInvalid`. +Pushes and fetches use the configured URL explicitly, never a local remote name or pushurl. A retry of an already recorded wake-up returns that commit (with `KIN_MEMORY_DIRECT_PUSH`, after pushing it again). Synchronization uses verified ancestry and plumbing-only fast-forwards. Divergent trees merge disjoint edits and append-only journals in process, then use `commit-tree` with both parents. Content conflicts preserve the local head on a conflict branch, reset to the remote anchor and notify the owner. Missing or ambiguous merge bases preserve and notify, then fail closed without resetting; the organization stays blocked until an operator runs `Memory.recover`, and repeated attempts reuse the conflict branch already holding that head. Failed fetches retain the SQLite anchor. Existing writers with mismatched or missing anchors fail closed as `MemoryCheckoutInvalid`. + +**Writes to the default branch go through PRs (TRIG-15).** The memory repo's `main` takes no direct push, so the writer pushes its commit to `kin/write//` (`keep-` for Keep) and `MemoryPullRequests` opens a PR with a `server` App token (`contents` and `pull_requests` write, `checks` and `actions` read). Before merging, Kin checks that the PR head is exactly the commit it pushed and that every `memory-check` check run on that commit belongs to a `pull_request` run of `.github/workflows/memory-check.yml` on it, then squash-merges through REST with `sha` pinned to that commit. GitHub's auto-merge is not used: it trusts any Actions workflow's `memory-check`. The local default branch and its anchor move only after the merge, to the fetched remote head, which must descend from the merged commit; when the merge parent is the old head, its tree must match Kin's. Changes to paths the owner reviews (`decisions/`, `people/`, `objectives/`, anything outside Kin's own folders) are split into their own PR on `-review`, opened on the old head without `INDEX.md` or the journal and never merged by Kin; the owner gets a notice with its link, and memory reads see them only once the owner merges. A failed or foreign check, a moved head, a check still running after 15 minutes or a refused merge closes the PR with a comment, posts the reason and link to the owner's direct chat, and fails the commit with `MemorySyncConflict`, leaving the default branch untouched. Without the Kin App, writes to the default branch are refused and the owner is told; `KIN_MEMORY_DIRECT_PUSH=true` restores direct pushes for a memory repo without the TRIG-15 ruleset. The writer holds the per-organization lock while it waits for the check, so the next wake-up starts from the merged head. Front matter matches the kin-memory README, including snake_case keys. Date-only values mean UTC midnight; date-times require an explicit UTC offset, and invalid dates make an item malformed. `links` values containing `#` must be quoted. `owner` and `said_by` hold handles; `orgId` and `path` belong only to the server-side `MemoryItemHead`. @@ -479,7 +481,7 @@ Memory at L0 goes to the shadow branch so the lead can keep track of its own def 3. When eligible, the owner's direct chat gets one `promotion` card (`KinPromotionCard`, one per lead and target level): "tech-lead is ready for L1: 34 of 39 reviewed would have gone out unchanged, none harmful, none unreviewed." [Raise to L1] [Another week]. Another week (`kin.levels.deferPromotion`) puts the card off for 7 days; Raise still works from settings meanwhile. A Harmful verdict, a new unreviewed entry or a lower level closes the card, and it reopens once the lead is eligible again; meanwhile its title carries the trial's counts, unreviewed included. 4. Raising a level is a permission change, so it is a privileged action. `kin.levels.raise` (the card button, or settings) creates a `KinPendingAction { kind: "set_level" }` with the reserved requester `kin-promotion`; `KinPendingActions.create` refuses `set_level` from any other requester, and `request_action` refuses it outright, so the lead cannot request it. The change applies only when the executor claims the action under the owner's signed grant: the `set_level` handler (`approvals/handlers/setLevel.ts`) calls `KinLevels.raiseByGrant`, whose write itself requires that claimed execution and the consumed grant of the lead's own owner. A lead goes up one level at a time and must still be eligible; a chat's cap goes back up to at most the lead's level (at the lead's level its row is deleted, so the chat follows the lead again). Until APPR-9 registers the handler, an approved raise lapses unapplied. 5. Lowering a level needs no signature and applies at once (`kin.levels.lower`, owner only, from settings or chat), for the lead or one chat. Lowering withdraws the open `set_level` raises it overtook: unapproved ones are cancelled and approved ones voided, for the lead and every chat above a new lead level, or for that chat above a new chat cap. An executing raise is refused by its handler, which rechecks the level. A Harmful verdict on an entry the lead recorded at L1 or above lowers the lead one level automatically and tells the owner why in their direct chat. Live actions have no review card yet; until one exists, the shadow entries a lead records at L1 and above stand in for them. -6. After a raise from L0 the promotion card offers Keep shadow memory or Start clean (`kin.levels.shadowMemory`, owner only, refused at L0). The card first lists the paths Keep would commit. Keep commits exactly that list onto the default branch as one commit through the memory writer, refusing conflicts, deletions, secrets, a list that no longer matches the shadow branch, and any path outside Kin's own folders (`commitments/`, `deferrals/`, `claims/`, `journal/`), since TRIG-15's CODEOWNERS reserves every other path, `decisions/`, `people/`, `objectives/` and `rules.md` included, for the owner's review in a PR. There is no memory-repo PR client yet, so for those only Start clean is available; for Kin's own folders the owner's tap is the review. Either way `kin/shadow/` then restarts from the default branch, so a later trial at L0 doesn't resume the old shadow week. +6. After a raise from L0 the promotion card offers Keep shadow memory or Start clean (`kin.levels.shadowMemory`, owner only, refused at L0). The card first lists the paths Keep would commit. Keep commits exactly that list as one commit and lands it through the writer's verified PR (Memory writes), refusing conflicts, deletions, secrets, a list that no longer matches the shadow branch, and any path outside Kin's own folders (`commitments/`, `deferrals/`, `claims/`, `journal/`), since TRIG-15's CODEOWNERS reserves every other path, `decisions/`, `people/`, `objectives/` and `rules.md` included, for the owner's review in a PR. For those only Start clean is available; for Kin's own folders the owner's tap is the review. A refused PR leaves the default branch and the shadow branch as they were. Either way `kin/shadow/` then restarts from the default branch, so a later trial at L0 doesn't resume the old shadow week. `kin.levels.get` (owner only) shows the lead's level, each chat's cap, the trial's counts and any open `set_level` raise. Thresholds live in code (`KinLevels.promotionRule`) and are shown on the cards. Scopes: `get` needs `orchestration:read`, `review` and `deferPromotion` `orchestration:operate`, and `lower`, `raise` and `shadowMemory` `settings:write`; every method then checks that the caller owns the lead. @@ -560,6 +562,7 @@ Evening summary: | Lead loops or runs long | 15 min hard timeout, then interrupt. | | Worker never reports | Synthesized `blocked` report. A worker running over 4 hours raises a `watchdog` event. | | Memory push fails | Retry; local commits stay; the next wake-up rebases. | +| Memory PR not merged | Kin closes it with a comment and tells the owner; the default branch stays as it was and the wake-up fails with `MemorySyncConflict`. | | Owner edited memory on GitHub | `pull --rebase` at wake-up start picks it up. A conflict aborts the rebase and fails the wake-up; an unavailable remote falls back to local HEAD. | | Duplicate event (replayed webhook, double timer) | Dropped by the `(org_id, dedupe_key)` index. | diff --git a/docs/kin/specs/triggers.md b/docs/kin/specs/triggers.md index 08f010941132..facd3f7e2c2b 100644 --- a/docs/kin/specs/triggers.md +++ b/docs/kin/specs/triggers.md @@ -399,7 +399,7 @@ The builder takes item heads (optionally marked `misleading` by the caller), las GitHub then blocks a bad auto-merge even if Kin's classifier is wrong. -**Daytime lead writes (decided).** Because `main` requires PRs, LEAD's daytime memory writes land as PRs that Kin auto-merges once `memory-check` passes; a write touching an owner path waits for the owner's review like any other PR. APPR-8 (#44) and APPR-11 (#46) have merged; the TRIG-15 follow-up is the `MemoryWriter` PR path and adding the memory repo to APPR-11's `KIN_REPOS` with `kind: "memory"` (approvals spec section 10, memory repo mode; lane A and lane C coordinate on #12). +**Daytime lead writes (decided).** Because `main` requires PRs, LEAD's daytime memory writes and the owner's Keep land as PRs that Kin merges itself once `memory-check` passes; changes to an owner path go to a separate PR that waits for the owner's review and that Kin never merges. A passing `memory-check` alone isn't trusted (see the memory repo README, Workflow permissions): Kin merges only a PR whose head is the commit it pushed, with REST `sha` pinned to it, after every `memory-check` check run on that commit traced back to a `pull_request` run of `.github/workflows/memory-check.yml` on it. GitHub's auto-merge would accept any workflow's check, so it isn't used. The writer side is in the lead spec (Memory writes). `MemoryPullRequests` needs the Kin App and the memory repo in `KIN_GITHUB_REPOS`; without the App, writes to `main` are refused unless `KIN_MEMORY_DIRECT_PUSH=true`, which suits only a memory repo without this ruleset. The ruleset itself comes from `KIN_REPOS` with `kind: "memory"` (#85). ### Secret scan before memory commits (gap 5) diff --git a/ops/kin/host/README.md b/ops/kin/host/README.md index db8f8b678675..2f3221f2f6ff 100644 --- a/ops/kin/host/README.md +++ b/ops/kin/host/README.md @@ -60,7 +60,7 @@ KIN_MEMORY_SSH_COMMAND="ssh -i /home/kin/.config/kin/kin-memory-deploy-key -o Id Point `KIN_MEMORY_REPOSITORIES` at the SSH URL (`git@github.com:/.git`), make sure `~/.ssh/known_hosts` holds GitHub's host key, and restart `kin-server`. The key stays out of `~/.ssh` because the approvals credential check tries every key there against GitHub, and a deploy key's greeting names a repo instead of the bot, so the check fails with `non_bot_identity` and privileged actions stop. **The key is not contained.** Agents run as `kin` and can read the file, so a prompt-injected lead or worker can push to the memory repo directly, around Kin's memory writer and its rules (owner-only `rules.md`, server-set trust, secret scanning, L0 writes only on `kin/shadow/*`). Until Kin runs memory git as a separate OS user, contain it on GitHub: - While every lead is at L0, protect `main` in the memory repo with a ruleset that requires a pull request and the `memory-check` workflow, with no bypass for this deploy key. The key can then update every ref except `main`, so a branch it pushes is never trusted on its own (see `ops/kin/memory-repo/README.md`, Workflow permissions). -- Don't raise any lead to L1 (which writes `main` directly) until memory git runs as a separate OS user whose key agents can't read. This is the same gap as the approval key (approvals spec, section 12) and has the same fix. +- Don't raise any lead to L1 (whose memory writes Kin merges into `main`) until memory git runs as a separate OS user whose key agents can't read. This is the same gap as the approval key (approvals spec, section 12) and has the same fix. **HTTPS.** Set `KIN_MEMORY_CREDENTIAL_HELPER` to a git credential helper's name or absolute path, with an `https://` repo URL. There is no GitHub App installation-token helper yet, so this route has nothing to point at until one is written. diff --git a/ops/kin/memory-repo/README.md b/ops/kin/memory-repo/README.md index 97c9a37d6d49..f77581bef346 100644 --- a/ops/kin/memory-repo/README.md +++ b/ops/kin/memory-repo/README.md @@ -16,7 +16,7 @@ Templates for the private memory repo (`kin-memory`). This public repo holds onl To pick up new lint or scanner rules, bump `T3CODE_REF` in a reviewed PR. The lint code and the gitleaks config always come from that commit, never from the PR's own files, which the job reads only as data. -The ruleset on `main` (PRs required, code-owner review, `memory-check` required, no bypass for Kin's bot) comes in the TRIG-15 follow-up with the `MemoryWriter` PR path. `scripts/kin-rulesets.ts` applies it once the memory repo is in `KIN_REPOS` with `kind: "memory"`, which accepts this file's owner-less Kin lines. +`scripts/kin-rulesets.ts` applies the ruleset on `main` (PRs required, code-owner review, `memory-check` required, no bypass for Kin's bot) from `KIN_REPOS`, where the memory repo has `kind: "memory"`, which accepts this file's owner-less Kin lines. Kin's own writes then reach `main` as PRs from `kin/write//` branches that it merges itself, and changes to owner paths as `…-review` PRs it leaves for you. That needs the Kin App installed on this repo and the repo in `KIN_GITHUB_REPOS`. Without the App, Kin refuses to write `main`; `KIN_MEMORY_DIRECT_PUSH=true` makes it push directly, which this ruleset rejects. ### The owner's own changes @@ -30,7 +30,7 @@ Nothing automates this yet: Kin has no code that adds the same-tree commit or op ### Workflow permissions -`memory-check` is pinned to the GitHub Actions App (15368), which trusts any Actions workflow in the repo, not just `memory-check.yml`. A workflow pushed to any branch could post a passing `memory-check` on another PR. The repo's read-only workflow-permissions setting doesn't stop this: anyone with write access, which includes the memory deploy key that agents can read, can raise a workflow's own `permissions:`. A ruleset `workflows` rule would close the gap, but it exists only in organization rulesets, and `GLDRoger` is a user account. So a passing `memory-check` alone is not trusted: before merging a PR, Kin checks that it merges the SHA it pushed itself, and that the passing check came from a `pull_request` run of `.github/workflows/memory-check.yml` on that SHA (TRIG-15 writer PR path). Never grant the Kin App Workflows permission on this repo. +`memory-check` is pinned to the GitHub Actions App (15368), which trusts any Actions workflow in the repo, not just `memory-check.yml`. A workflow pushed to any branch could post a passing `memory-check` on another PR. The repo's read-only workflow-permissions setting doesn't stop this: anyone with write access, which includes the memory deploy key that agents can read, can raise a workflow's own `permissions:`. A ruleset `workflows` rule would close the gap, but it exists only in organization rulesets, and `GLDRoger` is a user account. So a passing `memory-check` alone is not trusted: before merging a PR, Kin checks that its head is the SHA Kin pushed and merges with `sha` pinned to it, and refuses unless every `memory-check` check run on that SHA belongs to a `pull_request` run of `.github/workflows/memory-check.yml` on it. A `memory-check` job in any other workflow closes the PR. For the same reason Kin doesn't use GitHub's auto-merge. Never grant the Kin App Workflows permission on this repo. ### Before switching it on From 1122b89c92179092670dc6b6781f46fe995cd22a Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Mon, 9 Nov 2026 15:01:48 +0530 Subject: [PATCH 3/8] test(kin): shadow dry run pushes its mirror's main directly Co-Authored-By: Claude Opus 5.5 --- apps/server/scripts/kin-shadow-dry-run.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/server/scripts/kin-shadow-dry-run.ts b/apps/server/scripts/kin-shadow-dry-run.ts index 4e551d907088..8730c8a1c0dc 100644 --- a/apps/server/scripts/kin-shadow-dry-run.ts +++ b/apps/server/scripts/kin-shadow-dry-run.ts @@ -438,6 +438,8 @@ export const runDryRun = Effect.fn("KinDryRun.run")( ); const memoryConfig = MemoryConfig.of({ baseDirectory: path.join(stateDir, "kin"), + // Writes reach only the local mirror, so a write to main shows up in the ref check below. + directMainPush: true, repositories: [{ orgId, repoUrl: memoryUrl, path: encodeURIComponent(orgId) }], }); const refsBefore = yield* listRefs(stateDir, memoryUrl); From 4fd9b32a2b1229fb7865f7517758b1a074086b6c Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Mon, 9 Nov 2026 16:26:12 +0530 Subject: [PATCH 4/8] fix(kin): trace memory-check to the PR's own jobs and sweep orphaned memory PRs - Every memory-check check run on the commit (filter=all) must be a job of a pull_request run of memory-check.yml for this PR, branch, base and commit; those jobs must pass. A forged run in the real suite no longer counts. More than one page of check runs, runs or jobs fails closed. - Review PRs open only after the main PR merges, carry the pushed SHA in their body and the owner's notice, and their head is recorded in kin_memory_review_pulls. - A sweep on the Kin scheduler (startup, then every 5 minutes) closes any open kin/write/* PR no write waits on, and review PRs whose head moved, telling the owner. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/kin/lead/WakeupRunner.test.ts | 2 + apps/server/src/kin/lead/layer.ts | 32 +++ .../src/kin/memory/MemoryPullRequests.test.ts | 184 +++++++++--- .../src/kin/memory/MemoryPullRequests.ts | 268 ++++++++++++++---- apps/server/src/kin/memory/MemoryWriter.ts | 69 +++-- .../src/kin/persistence/KinMigrations.test.ts | 9 +- .../src/kin/persistence/KinMigrations.ts | 2 + .../migrations/K024_MemoryReviewPulls.ts | 13 + docs/kin/specs/lead.md | 2 +- docs/kin/specs/triggers.md | 2 +- ops/kin/memory-repo/README.md | 4 +- 11 files changed, 469 insertions(+), 118 deletions(-) create mode 100644 apps/server/src/kin/persistence/migrations/K024_MemoryReviewPulls.ts diff --git a/apps/server/src/kin/lead/WakeupRunner.test.ts b/apps/server/src/kin/lead/WakeupRunner.test.ts index 99c860ee274c..1e8a10d81502 100644 --- a/apps/server/src/kin/lead/WakeupRunner.test.ts +++ b/apps/server/src/kin/lead/WakeupRunner.test.ts @@ -132,6 +132,7 @@ describe("WakeupRunner", () => { commitDream: () => Effect.die("unused"), deleteDreamBranch: () => Effect.die("unused"), settleShadow: () => Effect.die("unused"), + sweepPullRequests: () => Effect.die("unused"), stage: (_org, wakeup) => Effect.sync(() => { assert.isTrue(owner === undefined || owner === wakeup); @@ -223,6 +224,7 @@ describe("WakeupRunner", () => { commitDream: () => Effect.die("unused"), deleteDreamBranch: () => Effect.die("unused"), settleShadow: () => Effect.die("unused"), + sweepPullRequests: () => Effect.die("unused"), discard: () => Effect.void, appendJournal: (_org, _wakeup, lines) => Effect.sync(() => diff --git a/apps/server/src/kin/lead/layer.ts b/apps/server/src/kin/lead/layer.ts index 3549d90b85e9..8660542740a4 100644 --- a/apps/server/src/kin/lead/layer.ts +++ b/apps/server/src/kin/lead/layer.ts @@ -29,6 +29,7 @@ import * as KinPersistence from "../persistence/KinPersistence.ts"; import * as Writer from "../memory/MemoryWriter.ts"; import * as MemoryPullRequests from "../memory/MemoryPullRequests.ts"; import * as Cause from "effect/Cause"; +import * as Clock from "effect/Clock"; import * as Scan from "../memory/secretScan.ts"; import * as LeadMemory from "./LeadMemory.ts"; import * as Cache from "../memory/MemoryCache.ts"; @@ -85,6 +86,32 @@ const dreamRuntime = DreamRunner.layer.pipe( Layer.provide(Cache.layer), ); +// TRIG-15: at startup and every 5 minutes, close memory PRs no write waits on. +const memoryPullRequestSweep = Effect.fn("KinMemoryPullRequests.sweepDue")(function* ( + writer: Writer.MemoryWriter["Service"], +) { + const { repositories } = yield* MemoryConfig.MemoryConfig; + let due = 0; + return Effect.gen(function* () { + const now = yield* Clock.currentTimeMillis; + if (now < due) return; + due = now + 5 * 60_000; + yield* Effect.forEach( + repositories, + ({ orgId }) => + writer.sweepPullRequests(orgId).pipe( + Effect.catchCause((cause) => + Effect.logWarning("Kin memory PR sweep failed", { + orgId, + cause: Cause.pretty(cause), + }), + ), + ), + { discard: true }, + ); + }); +}); + const runner = Layer.effectDiscard( Effect.gen(function* () { const runner = yield* WakeupRunner.WakeupRunner; @@ -96,6 +123,11 @@ const runner = Layer.effectDiscard( yield* promotion.value.attachShadowMemory(writer.value.settleShadow); yield* Effect.gen(function* () { const context = yield* Layer.build(routineRuntime); + if (Option.isSome(writer)) + yield* Context.get(context, Scheduler.Scheduler).register( + "kin-memory-pull-requests", + yield* memoryPullRequestSweep(writer.value), + ); yield* Layer.build(dreamRuntime.pipe(Layer.provide(Layer.succeedContext(context)))).pipe( Effect.catchCause((cause) => Effect.logError("Kin dreams unavailable", { cause })), ); diff --git a/apps/server/src/kin/memory/MemoryPullRequests.test.ts b/apps/server/src/kin/memory/MemoryPullRequests.test.ts index f806bd5df212..ec62edfdcc41 100644 --- a/apps/server/src/kin/memory/MemoryPullRequests.test.ts +++ b/apps/server/src/kin/memory/MemoryPullRequests.test.ts @@ -5,6 +5,7 @@ import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Redacted from "effect/Redacted"; +import * as SqlClient from "effect/sql/SqlClient"; import * as Sqlite from "../../persistence/Sqlite.ts"; import * as Vcs from "../../vcs/VcsProcess.ts"; @@ -34,30 +35,28 @@ it.each([ expect(githubRepoOf(url)).toBe(slug); }); -type Scenario = - | "merged" - | "foreign-check" - | "moved-head" - | "check-failed" - | "check-pending" - | "merge-rejected" - | "owner-path" - | "keep" - | "keep-moved-head" - | "no-app"; - -it.live.each([ +const scenarios = [ "merged", "foreign-check", + "forged-in-suite", + "run-wrong-sha", + "run-wrong-event", + "run-wrong-branch", + "run-wrong-pr", + "run-wrong-base", + "truncated", "moved-head", "check-failed", "check-pending", "merge-rejected", "owner-path", + "owner-path-refused", "keep", "keep-moved-head", "no-app", -] satisfies ReadonlyArray)("memory writes through PRs: %s", (scenario) => +] as const; + +it.live.each(scenarios)("memory writes through PRs: %s", (scenario) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const vcs = yield* Vcs.VcsProcess; @@ -143,47 +142,105 @@ it.live.each([ state.state = "closed"; return { status: 200, body: { merged: true, sha: squash } }; } + if (request.method === "GET" && /^\/pulls\?state=open&per_page=/.test(path)) { + const open = [...pulls].filter(([, pull]) => pull.state === "open"); + return { + status: 200, + body: yield* Effect.forEach(open, ([number]) => pullBody(number)), + }; + } + const pullOf = (sha: string) => + Effect.gen(function* () { + for (const [number, pull] of pulls) + if ((yield* branchHead(pull.branch)) === sha) return { number, branch: pull.branch }; + return yield* Effect.die(`no PR has ${sha}`); + }); const runs = /^\/actions\/runs\?head_sha=([a-f0-9]+)&event=pull_request/.exec(path); - if (runs) + if (runs) { + const sha = runs[1]!; + const pull = yield* pullOf(sha); + const run = (id: number, workflow: string) => ({ + id, + path: `.github/workflows/${workflow}`, + event: "pull_request", + head_sha: sha, + head_branch: pull.branch, + pull_requests: [{ number: pull.number, base: { ref: "main" } }], + }); + const ours = run(10, "memory-check.yml"); return { status: 200, body: { + total_count: 2, workflow_runs: [ { - path: ".github/workflows/memory-check.yml", - event: "pull_request", - head_sha: runs[1], - check_suite_id: 1, + ...ours, + ...(scenario === "run-wrong-sha" ? { head_sha: original } : {}), + ...(scenario === "run-wrong-event" ? { event: "push" } : {}), + ...(scenario === "run-wrong-branch" ? { head_branch: "kin/write/other" } : {}), + ...(scenario === "run-wrong-pr" + ? { pull_requests: [{ number: 99, base: { ref: "main" } }] } + : {}), + ...(scenario === "run-wrong-base" + ? { pull_requests: [{ number: pull.number, base: { ref: "dev" } }] } + : {}), }, // Any workflow in the repo can name a job `memory-check`. + run(20, "other.yml"), + ], + }, + }; + } + const jobs = /^\/actions\/runs\/(\d+)\/jobs\?/.exec(path); + if (jobs) { + const pending = scenario === "check-pending"; + const failed = scenario === "check-failed" || scenario === "owner-path-refused"; + return { + status: 200, + body: { + total_count: 1, + jobs: [ { - path: ".github/workflows/other.yml", - event: "pull_request", - head_sha: runs[1], - check_suite_id: 2, + // An Actions job's id is its check run's id. + id: jobs[1] === "10" ? 100 : 200, + name: "memory-check", + head_sha: yield* branchHead(pulls.get(1)!.branch), + status: pending ? "in_progress" : "completed", + conclusion: pending ? null : failed ? "failure" : "success", }, ], }, }; - const checks = /^\/commits\/([a-f0-9]+)\/check-runs/.exec(path); + } + const checks = /^\/commits\/([a-f0-9]+)\/check-runs\?(.*)$/.exec(path); if (checks) { - const check = (suite: number, status: string, conclusion: string | null) => ({ + const check = (id: number, suite: number) => ({ + id, name: "memory-check", head_sha: checks[1], - status, - conclusion, app: { id: 15368 }, check_suite: { id: suite }, }); + const all = [ + // Posted by another branch's workflow through the checks API, into our run's suite. + ...(scenario === "forged-in-suite" ? [check(300, 1)] : []), + check(100, 1), + ...(scenario === "foreign-check" ? [check(200, 2)] : []), + ]; + // GitHub's default `filter=latest` keeps only the newest run per name in each suite. + const listed = checks[2]!.split("&").includes("filter=all") + ? all + : all.filter( + (run, index) => + !all + .slice(index + 1) + .some((later) => later.check_suite.id === run.check_suite.id), + ); return { status: 200, body: { - check_runs: - scenario === "foreign-check" - ? [check(2, "completed", "success")] - : scenario === "check-pending" - ? [check(1, "in_progress", null)] - : [check(1, "completed", scenario === "check-failed" ? "failure" : "success")], + total_count: listed.length + (scenario === "truncated" ? 1 : 0), + check_runs: listed, }, }; } @@ -197,6 +254,7 @@ it.live.each([ KinGitHubApp.of({ app: Effect.die("unused"), installations: Effect.die("unused"), + patchHookConfig: () => Effect.die("unused"), installationToken: (input) => Effect.sync(() => { tokens.push(input); @@ -307,7 +365,8 @@ it.live.each([ } yield* writer.stage(orgId, "wake1", { ...claim, sourceEventId: "owner" }); - if (scenario === "owner-path") + const ownerPath = scenario === "owner-path" || scenario === "owner-path-refused"; + if (ownerPath) yield* writer.stage(orgId, "wake1", { path: "decisions/dec-001.md", content: fixtureItem({ id: "dec-001", kind: "decision", review_by: null }), @@ -325,10 +384,15 @@ it.live.each([ return; } if (scenario !== "merged" && scenario !== "owner-path") { + const foreign = "did not come from .github/workflows/memory-check.yml"; const error = yield* refused(committing); yield* unchanged; expect(merges.length).toBe(scenario === "merge-rejected" ? 1 : 0); expect(pulls.get(1)?.state).toBe("closed"); + // A refused write opens no review PR, so none is left waiting alone. + expect([...pulls.values()].map((pull) => pull.branch)).toEqual([ + `kin/write/${agentId}/wake1`, + ]); expect(github.requests).toContainEqual( expect.objectContaining({ method: "POST", path: `/repos/${repo}/issues/1/comments` }), ); @@ -338,9 +402,17 @@ it.live.each([ expect(error).toMatchObject({ reason: expect.stringContaining( { - "foreign-check": "did not come from .github/workflows/memory-check.yml", + "foreign-check": foreign, + "forged-in-suite": foreign, + "run-wrong-sha": foreign, + "run-wrong-event": foreign, + "run-wrong-branch": foreign, + "run-wrong-pr": foreign, + "run-wrong-base": foreign, + truncated: "GitHub listed more check runs than one page holds", "moved-head": "head is not the commit Kin pushed", "check-failed": "memory-check failed", + "owner-path-refused": "memory-check failed", "check-pending": "did not finish in time", "merge-rejected": "GitHub refused the merge (HTTP 405", }[scenario], @@ -395,14 +467,54 @@ it.live.each([ expect(yield* bare("show", `${main}:INDEX.md`)).not.toContain("dec-001"); expect(yield* remoteFiles(reviewBranch)).toContain("decisions/dec-001.md"); expect(yield* remoteFiles(reviewBranch)).not.toContain("claims/new.md"); + const reviewHead = yield* branchHead(reviewBranch); expect(notices.map((notice) => notice.reason)).toContainEqual( - `Memory changes on paths you review wait in https://github.com/${repo}/pull/${review[0]}`, + `Memory changes on paths you review wait in https://github.com/${repo}/pull/${review[0]}. Kin pushed \`${reviewHead}\`; if the PR's head differs, close it`, + ); + expect(github.requests).toContainEqual( + expect.objectContaining({ + method: "POST", + path: `/repos/${repo}/pulls`, + body: expect.objectContaining({ + head: reviewBranch, + body: expect.stringContaining(`Kin pushed \`${reviewHead}\``), + }), + }), ); // Reads follow the default branch while the review PR is open. expect((yield* memory.prepareWakeup(orgId)).head).toBe(main); expect( (yield* memory.readItem(orgId, KinMemoryItemId.make("dec-001")).pipe(Effect.flip))._tag, ).toBe("MemoryNotFound"); + + // A crashed write or an agent's own push leaves a `kin/write/*` PR nobody waits on. + const orphanBranch = "kin/write/other/orphan"; + yield* bare("update-ref", `refs/heads/${orphanBranch}`, original); + const orphan = pulls.size + 1; + pulls.set(orphan, { branch: orphanBranch, state: "open" }); + yield* writer.sweepPullRequests(orgId); + expect(pulls.get(orphan)?.state).toBe("closed"); + expect(github.requests).toContainEqual( + expect.objectContaining({ + method: "POST", + path: `/repos/${repo}/issues/${orphan}/comments`, + body: { body: "Kin closed this memory PR: no Kin memory write is waiting on it." }, + }), + ); + expect(review[1].state).toBe("open"); + const sql = yield* SqlClient.SqlClient; + const records = sql`SELECT branch, head FROM kin_memory_review_pulls`; + expect(yield* records).toEqual([{ branch: reviewBranch, head: reviewHead }]); + + // An agent with the deploy key swaps what the owner would merge. + yield* bare("update-ref", `refs/heads/${reviewBranch}`, original); + yield* writer.sweepPullRequests(orgId); + expect(review[1].state).toBe("closed"); + expect(notices.map((notice) => notice.reason)).toContainEqual( + `Kin closed https://github.com/${repo}/pull/${review[0]}, the memory PR for paths you review: its head is no longer \`${reviewHead}\`, the commit Kin pushed`, + ); + expect(merges.map((merge) => merge.number)).toEqual([auto[0]]); + expect(yield* records).toEqual([]); } }).pipe(Effect.provide(writerLayer)); }).pipe( diff --git a/apps/server/src/kin/memory/MemoryPullRequests.ts b/apps/server/src/kin/memory/MemoryPullRequests.ts index 213b2ffa1f65..62ebca64b26b 100644 --- a/apps/server/src/kin/memory/MemoryPullRequests.ts +++ b/apps/server/src/kin/memory/MemoryPullRequests.ts @@ -7,6 +7,7 @@ import * as Option from "effect/Option"; import type * as Redacted from "effect/Redacted"; import * as Schema from "effect/Schema"; import { HttpClient } from "effect/http"; +import * as SqlClient from "effect/sql/SqlClient"; import { gitHubAnswer, repoPath } from "../github/githubRest.ts"; import { KinGitHubApp } from "../github/KinGitHubApp.ts"; @@ -31,25 +32,42 @@ const Pull = Schema.Struct({ const decodePull = Schema.decodeUnknownOption(Pull); const decodePulls = Schema.decodeUnknownOption(Schema.Array(Pull)); const WorkflowRuns = Schema.Struct({ + total_count: Schema.Int, workflow_runs: Schema.Array( Schema.Struct({ + id: Schema.Int, path: Schema.String, event: Schema.String, head_sha: Schema.String, - check_suite_id: Schema.optional(Schema.NullOr(Schema.Int)), + head_branch: Schema.NullOr(Schema.String), + pull_requests: Schema.Array( + Schema.Struct({ number: Schema.Int, base: Schema.Struct({ ref: Schema.String }) }), + ), }), ), }); const decodeRuns = Schema.decodeUnknownOption(WorkflowRuns); -const CheckRuns = Schema.Struct({ - check_runs: Schema.Array( +const Jobs = Schema.Struct({ + total_count: Schema.Int, + jobs: Schema.Array( Schema.Struct({ + id: Schema.Int, name: Schema.String, head_sha: Schema.String, status: Schema.String, conclusion: Schema.NullOr(Schema.String), + }), + ), +}); +const decodeJobs = Schema.decodeUnknownOption(Jobs); +const CheckRuns = Schema.Struct({ + total_count: Schema.Int, + check_runs: Schema.Array( + Schema.Struct({ + id: Schema.Int, + name: Schema.String, + head_sha: Schema.String, app: Schema.NullOr(Schema.Struct({ id: Schema.Int })), - check_suite: Schema.NullOr(Schema.Struct({ id: Schema.Int })), }), ), }); @@ -85,6 +103,16 @@ interface MemoryPullRequestInput { readonly ownerReview: boolean; } +interface ClosedPullRequest { + readonly url: string; + readonly branch: string; + readonly reason: string; + readonly ownerReview: boolean; +} + +export const WRITE_BRANCH_PREFIX = "kin/write/"; +export const REVIEW_BRANCH_SUFFIX = "-review"; + /** * Lands Kin's own memory commits on the protected default branch (TRIG-15). Server-only: never * reachable from a lead, worker or MCP tool. @@ -95,6 +123,15 @@ export class MemoryPullRequests extends Context.Service< readonly land: ( input: MemoryPullRequestInput, ) => Effect.Effect; + /** + * Closes every open `kin/write/*` PR that no write is waiting on: review PRs whose head is not + * the commit Kin recorded, and all others. Run it under the writer lock, so no `land` is open. + */ + readonly sweep: (input: { + readonly orgId: KinOrgId; + readonly repo: RepoSlug; + readonly base: string; + }) => Effect.Effect, MemoryPullRequestRefused>; } >()("t3/kin/memory/MemoryPullRequests") {} @@ -103,63 +140,77 @@ type Verdict = | { readonly _tag: "pending" } | { readonly _tag: "fail"; readonly reason: string }; +const truncated = (reason: string): Verdict => ({ + _tag: "fail", + reason: `GitHub listed more ${reason} than one page holds`, +}); + +/** A `pull_request` run of the memory-check workflow for exactly this PR, branch and commit. */ +const isOurRun = ( + input: MemoryPullRequestInput, + number: number, + run: (typeof WorkflowRuns.Type)["workflow_runs"][number], +) => + run.path === MEMORY_CHECK_WORKFLOW && + run.event === "pull_request" && + run.head_sha === input.sha && + run.head_branch === input.branch && + run.pull_requests.some((pull) => pull.number === number && pull.base.ref === input.base); + /** - * `memory-check` is pinned to the Actions App, which any workflow in the repo can post as, so only - * a check run from a `pull_request` run of the memory-check workflow on `sha` counts. + * `memory-check` is pinned to the Actions App, which any workflow in the repo can post as, even + * into another run's check suite. An Actions job's id is its check run's id, so every + * `memory-check` check run on `sha` must be a job of one of our runs. */ const memoryCheckVerdict = ( sha: string, - runs: typeof WorkflowRuns.Type, checks: typeof CheckRuns.Type, + jobs: ReadonlyArray, ): Verdict => { - const suites = new Set( - runs.workflow_runs - .filter( - (run) => - run.path === MEMORY_CHECK_WORKFLOW && - run.event === "pull_request" && - run.head_sha === sha, - ) - .flatMap((run) => (run.check_suite_id == null ? [] : [run.check_suite_id])), - ); - const named = checks.check_runs.filter((check) => check.name === MEMORY_CHECK_NAME); - const foreign = named.some( + if (checks.total_count > checks.check_runs.length) return truncated("check runs"); + if (jobs.some((page) => page.total_count > page.jobs.length)) return truncated("jobs"); + const ours = jobs.flatMap((page) => page.jobs); + const ids = new Set(ours.map((job) => job.id)); + const foreign = checks.check_runs.some( (check) => - check.app?.id !== GITHUB_ACTIONS_INTEGRATION_ID || - check.head_sha !== sha || - check.check_suite === null || - !suites.has(check.check_suite.id), + check.name === MEMORY_CHECK_NAME && + (check.app?.id !== GITHUB_ACTIONS_INTEGRATION_ID || + check.head_sha !== sha || + !ids.has(check.id)), ); if (foreign) return { _tag: "fail", reason: `a ${MEMORY_CHECK_NAME} check that did not come from ${MEMORY_CHECK_WORKFLOW} ran on this commit`, }; - if (named.length === 0 || named.some((check) => check.status !== "completed")) + const named = ours.filter((job) => job.name === MEMORY_CHECK_NAME && job.head_sha === sha); + if (named.length === 0 || named.some((job) => job.status !== "completed")) return { _tag: "pending" }; - if (named.some((check) => check.conclusion !== "success")) + if (named.some((job) => job.conclusion !== "success")) return { _tag: "fail", reason: `${MEMORY_CHECK_NAME} failed` }; return { _tag: "pass" }; }; +const PAGE = 100; +const MAX_RUNS = 10; + export const make = Effect.fn("MemoryPullRequests.make")(function* (options: { readonly pollInterval: Duration.Input; readonly polls: number; }) { const app = yield* KinGitHubApp; const client = yield* HttpClient.HttpClient; + const sql = yield* SqlClient.SqlClient; - const land = Effect.fn("MemoryPullRequests.land")(function* (input: MemoryPullRequestInput) { - yield* Effect.annotateCurrentSpan({ "kin.github.repo": input.repo }); + const connect = Effect.fn("MemoryPullRequests.connect")(function* ( + orgId: KinOrgId, + repo: RepoSlug, + ) { const refuse = (reason: string, url?: string) => - new MemoryPullRequestRefused({ - orgId: input.orgId, - reason, - ...(url === undefined ? {} : { url }), - }); + new MemoryPullRequestRefused({ orgId, reason, ...(url === undefined ? {} : { url }) }); const minted = yield* app .installationToken({ - repo: input.repo, + repo, permissions: { contents: "write", pull_requests: "write", @@ -170,11 +221,11 @@ export const make = Effect.fn("MemoryPullRequests.make")(function* (options: { }) .pipe( Effect.mapError((error) => - refuse(`the Kin GitHub App gave no token for ${input.repo} (${error._tag})`), + refuse(`the Kin GitHub App gave no token for ${repo} (${error._tag})`), ), ); const token: Redacted.Redacted = minted.token; - const base = repoPath(input.repo); + const base = repoPath(repo); const call = ( operation: string, method: "GET" | "POST" | "PUT" | "PATCH", @@ -189,6 +240,25 @@ export const make = Effect.fn("MemoryPullRequests.make")(function* (options: { token, ...(body === undefined ? {} : { body }), }).pipe(Effect.mapError(() => refuse(`could not reach GitHub for ${operation}`))); + const close = Effect.fn("MemoryPullRequests.close")(function* (number: number, reason: string) { + yield* call("issues.comment", "POST", `/issues/${number}/comments`, { + body: `Kin closed this memory PR: ${reason}.`, + }); + const closed = yield* call("pulls.close", "PATCH", `/pulls/${number}`, { state: "closed" }); + if (closed.status !== 200) return yield* refuse(`GitHub did not close PR #${number}`); + }); + const records = sql<{ + branch: string; + head: string; + }>`SELECT branch, head FROM kin_memory_review_pulls WHERE org_id = ${orgId}`.pipe( + Effect.mapError(() => refuse("Kin could not read its record of review PRs")), + ); + return { refuse, call, close, records }; + }); + + const land = Effect.fn("MemoryPullRequests.land")(function* (input: MemoryPullRequestInput) { + yield* Effect.annotateCurrentSpan({ "kin.github.repo": input.repo }); + const { refuse, call, close } = yield* connect(input.orgId, input.repo); const describe = (answer: { status: number; message: string }) => `HTTP ${answer.status}${answer.message ? `: ${answer.message}` : ""}`; @@ -197,7 +267,7 @@ export const make = Effect.fn("MemoryPullRequests.make")(function* (options: { head: input.branch, base: input.base, body: input.ownerReview - ? "Kin's memory changes on paths you review. Kin will not merge this." + ? `Kin's memory changes on paths you review. Kin pushed \`${input.sha}\`; if this PR's head differs, close it. Kin will not merge this.` : `Kin merges this once \`${MEMORY_CHECK_NAME}\` passes on the commit it pushed.`, maintainer_can_modify: false, }); @@ -215,19 +285,13 @@ export const make = Effect.fn("MemoryPullRequests.make")(function* (options: { if (pull === undefined) return yield* refuse(`GitHub refused the PR (${describe(opened)})`); const { number, html_url: url } = pull; - const close = Effect.fn("MemoryPullRequests.close")( - function* (reason: string) { - yield* call("issues.comment", "POST", `/issues/${number}/comments`, { - body: `Kin closed this memory PR: ${reason}.`, - }); - yield* call("pulls.close", "PATCH", `/pulls/${number}`, { state: "closed" }); - }, - Effect.catch(() => - Effect.logWarning("Could not close a refused memory PR", { repo: input.repo, number }), - ), - ); const refuseAndClose = (reason: string) => - close(reason).pipe(Effect.andThen(Effect.fail(refuse(reason, url)))); + close(number, reason).pipe( + Effect.catch(() => + Effect.logWarning("Could not close a refused memory PR", { repo: input.repo, number }), + ), + Effect.andThen(Effect.fail(refuse(reason, url))), + ); const headIsOurs = Effect.gen(function* () { const answer = yield* call("pulls.get", "GET", `/pulls/${number}`); @@ -243,25 +307,53 @@ export const make = Effect.fn("MemoryPullRequests.make")(function* (options: { }); if (!(yield* headIsOurs)) return yield* refuseAndClose("the PR's head is not the commit Kin pushed"); - if (input.ownerReview) return { _tag: "awaiting_owner", url } as const; + if (input.ownerReview) { + // The sweep closes this PR if its head ever stops being the commit recorded here. + const recorded = yield* sql`INSERT INTO kin_memory_review_pulls (org_id, branch, head) + VALUES (${input.orgId}, ${input.branch}, ${input.sha}) + ON CONFLICT (org_id, branch) DO UPDATE SET head = excluded.head`.pipe(Effect.result); + if (recorded._tag === "Failure") + return yield* refuseAndClose("Kin could not record the commit it pushed"); + return { _tag: "awaiting_owner", url } as const; + } let verdict: Verdict = { _tag: "pending" }; for (let poll = 0; poll < options.polls && verdict._tag === "pending"; poll++) { if (poll > 0) yield* Effect.sleep(options.pollInterval); - const runs = yield* call( - "actions.runs", - "GET", - `/actions/runs?head_sha=${input.sha}&event=pull_request&per_page=100`, - ); + // Check runs first: a real job listed there is already listed by its run. const checks = yield* call( "checks.list", "GET", - `/commits/${input.sha}/check-runs?check_name=${MEMORY_CHECK_NAME}&per_page=100`, + `/commits/${input.sha}/check-runs?check_name=${MEMORY_CHECK_NAME}&filter=all&per_page=${PAGE}`, + ); + const runs = yield* call( + "actions.runs", + "GET", + `/actions/runs?head_sha=${input.sha}&event=pull_request&per_page=${PAGE}`, ); - const decodedRuns = runs.status === 200 ? decodeRuns(runs.body) : Option.none(); const decodedChecks = checks.status === 200 ? decodeChecks(checks.body) : Option.none(); - if (Option.isSome(decodedRuns) && Option.isSome(decodedChecks)) - verdict = memoryCheckVerdict(input.sha, decodedRuns.value, decodedChecks.value); + const decodedRuns = runs.status === 200 ? decodeRuns(runs.body) : Option.none(); + if (Option.isNone(decodedChecks) || Option.isNone(decodedRuns)) continue; + if (decodedRuns.value.total_count > decodedRuns.value.workflow_runs.length) { + verdict = truncated("workflow runs"); + break; + } + const ours = decodedRuns.value.workflow_runs.filter((run) => isOurRun(input, number, run)); + if (ours.length > MAX_RUNS) { + verdict = truncated("memory-check runs"); + break; + } + const jobs = yield* Effect.forEach(ours, (run) => + call( + "actions.jobs", + "GET", + `/actions/runs/${run.id}/jobs?filter=all&per_page=${PAGE}`, + ).pipe( + Effect.map((answer) => (answer.status === 200 ? decodeJobs(answer.body) : Option.none())), + ), + ); + if (jobs.some(Option.isNone)) continue; + verdict = memoryCheckVerdict(input.sha, decodedChecks.value, jobs.flatMap(Option.toArray)); } if (verdict._tag === "pending") return yield* refuseAndClose(`${MEMORY_CHECK_NAME} did not finish in time`); @@ -282,7 +374,65 @@ export const make = Effect.fn("MemoryPullRequests.make")(function* (options: { return yield* refuseAndClose(`GitHub refused the merge (${describe(merge)})`); }); - return MemoryPullRequests.of({ land }); + const sweep = Effect.fn("MemoryPullRequests.sweep")(function* (input: { + readonly orgId: KinOrgId; + readonly repo: RepoSlug; + readonly base: string; + }) { + yield* Effect.annotateCurrentSpan({ "kin.github.repo": input.repo }); + const { refuse, call, close, records } = yield* connect(input.orgId, input.repo); + const listed = yield* call("pulls.list", "GET", `/pulls?state=open&per_page=${PAGE}`); + const pulls = listed.status === 200 ? decodePulls(listed.body) : Option.none(); + if (Option.isNone(pulls)) return yield* refuse("GitHub did not list the open PRs"); + const recorded = new Map((yield* records).map((row) => [row.branch, row.head])); + const kept = new Set(); + const closed: Array = []; + for (const pull of pulls.value) { + const branch = pull.head.ref; + if (!branch.startsWith(WRITE_BRANCH_PREFIX)) continue; + const ownerReview = branch.endsWith(REVIEW_BRANCH_SUFFIX); + const pushed = recorded.get(branch); + if ( + ownerReview && + pushed === pull.head.sha && + pull.head.repo?.full_name.toLowerCase() === input.repo.toLowerCase() && + pull.base.ref === input.base + ) { + kept.add(branch); + continue; + } + const reason = !ownerReview + ? "no Kin memory write is waiting on it" + : pushed === undefined + ? "Kin did not open it" + : `its head is no longer \`${pushed}\`, the commit Kin pushed`; + const done = yield* close(pull.number, reason).pipe(Effect.result); + if (done._tag === "Failure") { + yield* Effect.logWarning("Could not close an orphaned memory PR", { + repo: input.repo, + number: pull.number, + }); + kept.add(branch); + continue; + } + closed.push({ url: pull.html_url, branch, reason, ownerReview }); + } + // Only a complete list shows which recorded review PRs are gone. + if (pulls.value.length >= PAGE) { + yield* Effect.logWarning("More open memory PRs than one sweep reads", { repo: input.repo }); + return closed; + } + for (const branch of recorded.keys()) { + if (kept.has(branch)) continue; + yield* sql`DELETE FROM kin_memory_review_pulls + WHERE org_id = ${input.orgId} AND branch = ${branch}`.pipe( + Effect.mapError(() => refuse("Kin could not update its record of review PRs")), + ); + } + return closed; + }); + + return MemoryPullRequests.of({ land, sweep }); }); /** Polls `memory-check` every 15 seconds for up to the job's 15-minute timeout. */ diff --git a/apps/server/src/kin/memory/MemoryWriter.ts b/apps/server/src/kin/memory/MemoryWriter.ts index 738fadd3b724..44b481cab2c0 100644 --- a/apps/server/src/kin/memory/MemoryWriter.ts +++ b/apps/server/src/kin/memory/MemoryWriter.ts @@ -33,7 +33,11 @@ import { journalEntry, MemoryJournalError, type JournalContext } from "./journal import { mergeMemoryFiles } from "./merge.ts"; import { isOwnerReviewedPath, kinShapeFailure } from "./lint.ts"; import { KinSecretScan, KinSecretScanError } from "./secretScan.ts"; -import { MemoryPullRequests } from "./MemoryPullRequests.ts"; +import { + MemoryPullRequests, + REVIEW_BRANCH_SUFFIX, + WRITE_BRANCH_PREFIX, +} from "./MemoryPullRequests.ts"; export const LEAD_ENDED_WITHOUT_KIN_END = "Lead ended without calling kin_end."; @@ -180,6 +184,8 @@ export class MemoryWriter extends Context.Service< agentId: KinAgentId, choice: ShadowChoice, ) => Effect.Effect<{ readonly kept: number; readonly paths: ReadonlyArray }, Failure>; + /** Closes `kin/write/*` PRs no write waits on (TRIG-15) and tells the owner about review PRs. */ + readonly sweepPullRequests: (orgId: KinOrgId) => Effect.Effect; } >()("t3/kin/memory/MemoryWriter") {} const isRepoSlug = Schema.is(RepoSlug); @@ -603,15 +609,30 @@ const make = Effect.gen(function* () { ownerReview, }); }); - const branch = `kin/write/${input.agentId}/${input.id}`; + const branch = `${WRITE_BRANCH_PREFIX}${input.agentId}/${input.id}`; + const landed = yield* open(branch, input.commit, input.message, false).pipe( + Effect.tapError((error) => + recovery.notify({ + orgId, + branch, + reason: `Memory write not merged: ${error.reason}${error._tag === "MemoryPullRequestRefused" && error.url ? ` (${error.url})` : ""}`, + }), + ), + Effect.mapError((error) => + error._tag === "MemoryPullRequestRefused" ? conflict(error.reason) : error, + ), + ); + if (landed._tag !== "merged") return yield* conflict("The memory PR was not merged"); + // Opened only once the rest merged, so a refused write never leaves a lone review PR. if (input.review !== undefined) { - const reviewBranch = `${branch}-review`; - yield* open(reviewBranch, input.review.commit, input.review.message, true).pipe( + const review = input.review; + const reviewBranch = `${branch}${REVIEW_BRANCH_SUFFIX}`; + yield* open(reviewBranch, review.commit, review.message, true).pipe( Effect.flatMap((opened) => recovery.notify({ orgId, branch: reviewBranch, - reason: `Memory changes on paths you review wait in ${opened.url}`, + reason: `Memory changes on paths you review wait in ${opened.url}. Kin pushed \`${review.commit}\`; if the PR's head differs, close it`, }), ), Effect.catch((error) => @@ -623,19 +644,6 @@ const make = Effect.gen(function* () { ), ); } - const landed = yield* open(branch, input.commit, input.message, false).pipe( - Effect.tapError((error) => - recovery.notify({ - orgId, - branch, - reason: `Memory write not merged: ${error.reason}${error._tag === "MemoryPullRequestRefused" && error.url ? ` (${error.url})` : ""}`, - }), - ), - Effect.mapError((error) => - error._tag === "MemoryPullRequestRefused" ? conflict(error.reason) : error, - ), - ); - if (landed._tag !== "merged") return yield* conflict("The memory PR was not merged"); const advertised = yield* git(orgId, writer, [ "ls-remote", "--", @@ -1342,6 +1350,31 @@ const make = Effect.gen(function* () { }), ), ), + sweepPullRequests: Effect.fn("MemoryWriter.sweepPullRequests")((orgId) => + locked( + orgId, + Effect.gen(function* () { + if (Option.isNone(pullRequests)) return; + const repo = yield* repoFor(orgId); + if (repo.githubRepo === undefined || !isRepoSlug(repo.githubRepo)) return; + const closed = yield* pullRequests.value + .sweep({ orgId, repo: repo.githubRepo, base: config.defaultBranch ?? "main" }) + .pipe( + Effect.mapError( + (error) => + new MemorySyncConflict({ orgId, operation: "sweep", reason: error.reason }), + ), + ); + for (const pull of closed) + if (pull.ownerReview) + yield* recovery.notify({ + orgId, + branch: pull.branch, + reason: `Kin closed ${pull.url}, the memory PR for paths you review: ${pull.reason}`, + }); + }), + ), + ), }); }); export const layer = Layer.effect(MemoryWriter, make).pipe(Layer.provide(MemoryCache.layer)); diff --git a/apps/server/src/kin/persistence/KinMigrations.test.ts b/apps/server/src/kin/persistence/KinMigrations.test.ts index 8123f6f59800..783937fe653c 100644 --- a/apps/server/src/kin/persistence/KinMigrations.test.ts +++ b/apps/server/src/kin/persistence/KinMigrations.test.ts @@ -37,6 +37,7 @@ const tableNames = [ "kin_leads", "kin_memory_heads", "kin_memory_reads", + "kin_memory_review_pulls", "kin_messages", "kin_org_state", "kin_orgs", @@ -126,6 +127,7 @@ describe("Kin migrations", () => { { migration_id: 21, name: "PushDevices" }, { migration_id: 22, name: "ChatLevels" }, { migration_id: 23, name: "ExecutionWriteStarted" }, + { migration_id: 24, name: "MemoryReviewPulls" }, ]); assert.deepStrictEqual( upstreamBefore.map((row) => [row.migration_id, row.name] as const), @@ -201,6 +203,7 @@ describe("Kin migrations", () => { { migration_id: 21, name: "PushDevices" }, { migration_id: 22, name: "ChatLevels" }, { migration_id: 23, name: "ExecutionWriteStarted" }, + { migration_id: 24, name: "MemoryReviewPulls" }, ]); assert.deepStrictEqual(yield* upstreamHistory, upstreamBefore); assert.deepStrictEqual(yield* sql`SELECT * FROM kin_orgs`, [ @@ -301,6 +304,7 @@ describe("Kin migrations", () => { [21, "PushDevices"], [22, "ChatLevels"], [23, "ExecutionWriteStarted"], + [24, "MemoryReviewPulls"], ]); const before = yield* kinSchema; assert.deepStrictEqual( @@ -355,6 +359,7 @@ describe("Kin migrations", () => { yield* sql`DROP TABLE kin_budget_charges`; yield* sql`ALTER TABLE kin_events DROP COLUMN budget_deferred`; for (const table of [ + "kin_memory_review_pulls", "kin_chat_levels", "kin_push_devices", "kin_executions", @@ -389,6 +394,7 @@ describe("Kin migrations", () => { [21, "PushDevices"], [22, "ChatLevels"], [23, "ExecutionWriteStarted"], + [24, "MemoryReviewPulls"], ]); assert.deepStrictEqual( yield* sql`SELECT member_id FROM kin_chat_members WHERE org_id = 'org' ORDER BY member_id`, @@ -437,6 +443,7 @@ describe("Kin migrations", () => { { migration_id: 21, name: "PushDevices" }, { migration_id: 22, name: "ChatLevels" }, { migration_id: 23, name: "ExecutionWriteStarted" }, + { migration_id: 24, name: "MemoryReviewPulls" }, ]); assert.deepStrictEqual(yield* runKinMigrations(), []); assert.deepStrictEqual(yield* runMigrations(), []); @@ -454,7 +461,7 @@ describe("Kin migrations", () => { const exit = yield* Effect.exit(runKinMigrations()); assert.isTrue(Exit.isFailure(exit)); assert.include(String(exit), "5_OneChatOwner"); - assert.include(String(exit), "below the latest applied migration 23"); + assert.include(String(exit), "below the latest applied migration 24"); assert.deepStrictEqual(yield* kinHistory, history); }).pipe(Effect.provide(NodeSqliteClient.layer({ filename: ":memory:" }))), ); diff --git a/apps/server/src/kin/persistence/KinMigrations.ts b/apps/server/src/kin/persistence/KinMigrations.ts index 13ab093c2132..5287e465c69c 100644 --- a/apps/server/src/kin/persistence/KinMigrations.ts +++ b/apps/server/src/kin/persistence/KinMigrations.ts @@ -25,6 +25,7 @@ import Approvals from "./migrations/K020_Approvals.ts"; import PushDevices from "./migrations/K021_PushDevices.ts"; import ChatLevels from "./migrations/K022_ChatLevels.ts"; import ExecutionWriteStarted from "./migrations/K023_ExecutionWriteStarted.ts"; +import MemoryReviewPulls from "./migrations/K024_MemoryReviewPulls.ts"; const table = "kin_sql_migrations"; const run = Migrator.make({}); @@ -52,6 +53,7 @@ const loader = Migrator.fromRecord({ "21_PushDevices": PushDevices, "22_ChatLevels": ChatLevels, "23_ExecutionWriteStarted": ExecutionWriteStarted, + "24_MemoryReviewPulls": MemoryReviewPulls, }); // The Migrator skips every id at or below the latest applied one, so a migration that diff --git a/apps/server/src/kin/persistence/migrations/K024_MemoryReviewPulls.ts b/apps/server/src/kin/persistence/migrations/K024_MemoryReviewPulls.ts new file mode 100644 index 000000000000..44f3b643537a --- /dev/null +++ b/apps/server/src/kin/persistence/migrations/K024_MemoryReviewPulls.ts @@ -0,0 +1,13 @@ +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/sql/SqlClient"; + +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + // The commit Kin pushed to each `kin/write/*-review` branch it left open for the owner. + yield* sql`CREATE TABLE kin_memory_review_pulls ( + org_id TEXT NOT NULL, + branch TEXT NOT NULL, + head TEXT NOT NULL, + PRIMARY KEY (org_id, branch) + )`; +}); diff --git a/docs/kin/specs/lead.md b/docs/kin/specs/lead.md index 5ce91d7393e8..cb091769095f 100644 --- a/docs/kin/specs/lead.md +++ b/docs/kin/specs/lead.md @@ -421,7 +421,7 @@ Branches: at L1 and above the writer commits to the default branch, through a pu Pushes and fetches use the configured URL explicitly, never a local remote name or pushurl. A retry of an already recorded wake-up returns that commit (with `KIN_MEMORY_DIRECT_PUSH`, after pushing it again). Synchronization uses verified ancestry and plumbing-only fast-forwards. Divergent trees merge disjoint edits and append-only journals in process, then use `commit-tree` with both parents. Content conflicts preserve the local head on a conflict branch, reset to the remote anchor and notify the owner. Missing or ambiguous merge bases preserve and notify, then fail closed without resetting; the organization stays blocked until an operator runs `Memory.recover`, and repeated attempts reuse the conflict branch already holding that head. Failed fetches retain the SQLite anchor. Existing writers with mismatched or missing anchors fail closed as `MemoryCheckoutInvalid`. -**Writes to the default branch go through PRs (TRIG-15).** The memory repo's `main` takes no direct push, so the writer pushes its commit to `kin/write//` (`keep-` for Keep) and `MemoryPullRequests` opens a PR with a `server` App token (`contents` and `pull_requests` write, `checks` and `actions` read). Before merging, Kin checks that the PR head is exactly the commit it pushed and that every `memory-check` check run on that commit belongs to a `pull_request` run of `.github/workflows/memory-check.yml` on it, then squash-merges through REST with `sha` pinned to that commit. GitHub's auto-merge is not used: it trusts any Actions workflow's `memory-check`. The local default branch and its anchor move only after the merge, to the fetched remote head, which must descend from the merged commit; when the merge parent is the old head, its tree must match Kin's. Changes to paths the owner reviews (`decisions/`, `people/`, `objectives/`, anything outside Kin's own folders) are split into their own PR on `-review`, opened on the old head without `INDEX.md` or the journal and never merged by Kin; the owner gets a notice with its link, and memory reads see them only once the owner merges. A failed or foreign check, a moved head, a check still running after 15 minutes or a refused merge closes the PR with a comment, posts the reason and link to the owner's direct chat, and fails the commit with `MemorySyncConflict`, leaving the default branch untouched. Without the Kin App, writes to the default branch are refused and the owner is told; `KIN_MEMORY_DIRECT_PUSH=true` restores direct pushes for a memory repo without the TRIG-15 ruleset. The writer holds the per-organization lock while it waits for the check, so the next wake-up starts from the merged head. +**Writes to the default branch go through PRs (TRIG-15).** The memory repo's `main` takes no direct push, so the writer pushes its commit to `kin/write//` (`keep-` for Keep) and `MemoryPullRequests` opens a PR with a `server` App token (`contents` and `pull_requests` write, `checks` and `actions` read). Before merging, Kin checks that the PR head is exactly the commit it pushed and that `memory-check` passed in a run it can trace: a `pull_request` run of `.github/workflows/memory-check.yml` on that commit and branch, listed for this PR and base, whose `memory-check` jobs all succeeded. Any workflow can post a `memory-check` check run, even into that run's check suite, so every `memory-check` check run on the commit (`filter=all`) must have the id of one of those jobs, an Actions job's id being its check run's id. A list longer than one page fails the check. Then Kin squash-merges through REST with `sha` pinned to that commit. GitHub's auto-merge is not used: it trusts any Actions workflow's `memory-check`. The local default branch and its anchor move only after the merge, to the fetched remote head, which must descend from the merged commit; when the merge parent is the old head, its tree must match Kin's. Changes to paths the owner reviews (`decisions/`, `people/`, `objectives/`, anything outside Kin's own folders) are split into their own PR on `-review`, built on the old head without `INDEX.md` or the journal, opened only after the main PR merged and never merged by Kin. Its body and the owner's notice name the commit Kin pushed, so the owner can close it if the head differs; memory reads see those changes only once the owner merges. Kin records that commit in `kin_memory_review_pulls`, because the memory deploy key can push to the branch. A failed or foreign check, a moved head, a check still running after 15 minutes or a refused merge closes the PR with a comment, posts the reason and link to the owner's direct chat, and fails the commit with `MemorySyncConflict`, leaving the default branch untouched. Without the Kin App, writes to the default branch are refused and the owner is told; `KIN_MEMORY_DIRECT_PUSH=true` restores direct pushes for a memory repo without the TRIG-15 ruleset. At startup and every 5 minutes, a sweep on the Kin scheduler takes the writer lock and closes, with a comment, every open `kin/write/*` PR no write waits on: any non-review PR (its write already ended) and any review PR whose head isn't the commit Kin recorded, telling the owner about the latter. It reads one page of open PRs per run. The writer holds the per-organization lock while it waits for the check, so the next wake-up starts from the merged head; the known cost is that a slow check blocks that organization's wake-ups, Keep and the sweep for up to 15 minutes. Front matter matches the kin-memory README, including snake_case keys. Date-only values mean UTC midnight; date-times require an explicit UTC offset, and invalid dates make an item malformed. `links` values containing `#` must be quoted. `owner` and `said_by` hold handles; `orgId` and `path` belong only to the server-side `MemoryItemHead`. diff --git a/docs/kin/specs/triggers.md b/docs/kin/specs/triggers.md index facd3f7e2c2b..4da7d9ce4c84 100644 --- a/docs/kin/specs/triggers.md +++ b/docs/kin/specs/triggers.md @@ -399,7 +399,7 @@ The builder takes item heads (optionally marked `misleading` by the caller), las GitHub then blocks a bad auto-merge even if Kin's classifier is wrong. -**Daytime lead writes (decided).** Because `main` requires PRs, LEAD's daytime memory writes and the owner's Keep land as PRs that Kin merges itself once `memory-check` passes; changes to an owner path go to a separate PR that waits for the owner's review and that Kin never merges. A passing `memory-check` alone isn't trusted (see the memory repo README, Workflow permissions): Kin merges only a PR whose head is the commit it pushed, with REST `sha` pinned to it, after every `memory-check` check run on that commit traced back to a `pull_request` run of `.github/workflows/memory-check.yml` on it. GitHub's auto-merge would accept any workflow's check, so it isn't used. The writer side is in the lead spec (Memory writes). `MemoryPullRequests` needs the Kin App and the memory repo in `KIN_GITHUB_REPOS`; without the App, writes to `main` are refused unless `KIN_MEMORY_DIRECT_PUSH=true`, which suits only a memory repo without this ruleset. The ruleset itself comes from `KIN_REPOS` with `kind: "memory"` (#85). +**Daytime lead writes (decided).** Because `main` requires PRs, LEAD's daytime memory writes and the owner's Keep land as PRs that Kin merges itself once `memory-check` passes; changes to an owner path go to a separate PR that waits for the owner's review and that Kin never merges. A passing `memory-check` alone isn't trusted (see the memory repo README, Workflow permissions): Kin merges only a PR whose head is the commit it pushed, with REST `sha` pinned to it, after every `memory-check` check run on that commit turned out to be a job of a `pull_request` run of `.github/workflows/memory-check.yml` for that PR, branch and commit, and those jobs passed. Review PRs carry the pushed commit in their body and notice, and a periodic sweep closes them if their head moves, along with any other `kin/write/*` PR no write waits on. GitHub's auto-merge would accept any workflow's check, so it isn't used. The writer side is in the lead spec (Memory writes). `MemoryPullRequests` needs the Kin App and the memory repo in `KIN_GITHUB_REPOS`; without the App, writes to `main` are refused unless `KIN_MEMORY_DIRECT_PUSH=true`, which suits only a memory repo without this ruleset. The ruleset itself comes from `KIN_REPOS` with `kind: "memory"` (#85). ### Secret scan before memory commits (gap 5) diff --git a/ops/kin/memory-repo/README.md b/ops/kin/memory-repo/README.md index f77581bef346..d09851785cc8 100644 --- a/ops/kin/memory-repo/README.md +++ b/ops/kin/memory-repo/README.md @@ -16,7 +16,7 @@ Templates for the private memory repo (`kin-memory`). This public repo holds onl To pick up new lint or scanner rules, bump `T3CODE_REF` in a reviewed PR. The lint code and the gitleaks config always come from that commit, never from the PR's own files, which the job reads only as data. -`scripts/kin-rulesets.ts` applies the ruleset on `main` (PRs required, code-owner review, `memory-check` required, no bypass for Kin's bot) from `KIN_REPOS`, where the memory repo has `kind: "memory"`, which accepts this file's owner-less Kin lines. Kin's own writes then reach `main` as PRs from `kin/write//` branches that it merges itself, and changes to owner paths as `…-review` PRs it leaves for you. That needs the Kin App installed on this repo and the repo in `KIN_GITHUB_REPOS`. Without the App, Kin refuses to write `main`; `KIN_MEMORY_DIRECT_PUSH=true` makes it push directly, which this ruleset rejects. +`scripts/kin-rulesets.ts` applies the ruleset on `main` (PRs required, code-owner review, `memory-check` required, no bypass for Kin's bot) from `KIN_REPOS`, where the memory repo has `kind: "memory"`, which accepts this file's owner-less Kin lines. Kin's own writes then reach `main` as PRs from `kin/write//` branches that it merges itself, and changes to owner paths as `…-review` PRs it leaves for you. Each review PR's body and Kin's message name the commit Kin pushed; the deploy key can push to that branch, so close the PR if its head differs. Kin also closes it within a few minutes when its head moves, and closes any other `kin/write/*` PR no write is waiting on. That needs the Kin App installed on this repo and the repo in `KIN_GITHUB_REPOS`. Without the App, Kin refuses to write `main`; `KIN_MEMORY_DIRECT_PUSH=true` makes it push directly, which this ruleset rejects. ### The owner's own changes @@ -30,7 +30,7 @@ Nothing automates this yet: Kin has no code that adds the same-tree commit or op ### Workflow permissions -`memory-check` is pinned to the GitHub Actions App (15368), which trusts any Actions workflow in the repo, not just `memory-check.yml`. A workflow pushed to any branch could post a passing `memory-check` on another PR. The repo's read-only workflow-permissions setting doesn't stop this: anyone with write access, which includes the memory deploy key that agents can read, can raise a workflow's own `permissions:`. A ruleset `workflows` rule would close the gap, but it exists only in organization rulesets, and `GLDRoger` is a user account. So a passing `memory-check` alone is not trusted: before merging a PR, Kin checks that its head is the SHA Kin pushed and merges with `sha` pinned to it, and refuses unless every `memory-check` check run on that SHA belongs to a `pull_request` run of `.github/workflows/memory-check.yml` on it. A `memory-check` job in any other workflow closes the PR. For the same reason Kin doesn't use GitHub's auto-merge. Never grant the Kin App Workflows permission on this repo. +`memory-check` is pinned to the GitHub Actions App (15368), which trusts any Actions workflow in the repo, not just `memory-check.yml`. A workflow pushed to any branch could post a passing `memory-check` on another PR. The repo's read-only workflow-permissions setting doesn't stop this: anyone with write access, which includes the memory deploy key that agents can read, can raise a workflow's own `permissions:`. A ruleset `workflows` rule would close the gap, but it exists only in organization rulesets, and `GLDRoger` is a user account. So a passing `memory-check` alone is not trusted: before merging a PR, Kin checks that its head is the SHA Kin pushed and merges with `sha` pinned to it. It refuses unless a `pull_request` run of `.github/workflows/memory-check.yml` for that PR, branch and SHA passed its `memory-check` job, and every `memory-check` check run on the SHA, superseded ones included, is one of that run's jobs by id. Belonging to the run's check suite isn't enough, since a workflow can post a check run into another run's suite. A `memory-check` from anywhere else closes the PR. For the same reason Kin doesn't use GitHub's auto-merge. Never grant the Kin App Workflows permission on this repo. ### Before switching it on From f0ce9841f711cdaee2d275a2ad03b222d55e89f1 Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Mon, 9 Nov 2026 16:59:33 +0530 Subject: [PATCH 5/8] fix(kin): bind memory-check runs to Kin's own PR and harden the memory PR sweep - Before the verdict and again before the merge, require that Kin's branch has only ever had this PR (state=all) and that the PR's base was never changed, failing closed when either list could run past one page. - The sweep looks up each recorded review branch by head, so newer PRs can't hide it; the one-page scan is only for orphaned kin/write/* PRs. - GitHub calls made under the writer lock time out after 30 seconds. - A moved review PR that can't be closed is reported, and the owner is told it's still open. Only recorded review PRs reach the owner; unrecorded ones are closed with a comment. - K024 SQL errors go through toPersistenceSqlError and name the operation in the refusal. - Tests pin the jobs filter=all on a re-run. Co-Authored-By: Claude Opus 5.5 --- .../src/kin/memory/MemoryPullRequests.test.ts | 228 +++++++++++++++--- .../src/kin/memory/MemoryPullRequests.ts | 221 ++++++++++++----- apps/server/src/kin/memory/MemoryWriter.ts | 17 +- 3 files changed, 364 insertions(+), 102 deletions(-) diff --git a/apps/server/src/kin/memory/MemoryPullRequests.test.ts b/apps/server/src/kin/memory/MemoryPullRequests.test.ts index ec62edfdcc41..2ee551f4019b 100644 --- a/apps/server/src/kin/memory/MemoryPullRequests.test.ts +++ b/apps/server/src/kin/memory/MemoryPullRequests.test.ts @@ -1,11 +1,14 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { expect, it } from "@effect/vitest"; import { KinAgentId, KinMemoryItemId, KinOrgId } from "@t3tools/contracts"; +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Redacted from "effect/Redacted"; import * as SqlClient from "effect/sql/SqlClient"; +import * as TestClock from "effect/testing/TestClock"; import * as Sqlite from "../../persistence/Sqlite.ts"; import * as Vcs from "../../vcs/VcsProcess.ts"; @@ -37,6 +40,7 @@ it.each([ const scenarios = [ "merged", + "rerun", "foreign-check", "forged-in-suite", "run-wrong-sha", @@ -45,19 +49,29 @@ const scenarios = [ "run-wrong-pr", "run-wrong-base", "truncated", + "second-pr", + "pulls-truncated", + "base-changed", + "events-truncated", "moved-head", "check-failed", "check-pending", "merge-rejected", "owner-path", + "owner-path-close-fails", "owner-path-refused", + "records-broken", + "hang", + "stall-body", "keep", "keep-moved-head", "no-app", ] as const; -it.live.each(scenarios)("memory writes through PRs: %s", (scenario) => +// `it.effect`: real git, but GitHub's silences only end when the test clock moves. +it.effect.each(scenarios)("memory writes through PRs: %s", (scenario) => Effect.gen(function* () { + yield* TestClock.setTime(Date.UTC(2026, 10, 9)); const fs = yield* FileSystem.FileSystem; const vcs = yield* Vcs.VcsProcess; const root = yield* fs.realPath(yield* fs.makeTempDirectoryScoped()); @@ -79,9 +93,13 @@ it.live.each(scenarios)("memory writes through PRs: %s", (scenario) => const original = yield* bare("rev-parse", "refs/heads/main"); // A GitHub over the bare remote: it squash-merges there, as the real one would. - const pulls = new Map(); + const pulls = new Map(); const merges: Array<{ number: number; body: unknown }> = []; const tokens: Array = []; + // The request GitHub never answers in `hang` and `stall-body`. + const silenced = yield* Deferred.make(); + let jobsListed = false; + let unclosable: number | undefined; const branchHead = (branch: string) => bare("rev-parse", `refs/heads/${branch}`); const pullBody = (number: number) => Effect.gen(function* () { @@ -91,27 +109,64 @@ it.live.each(scenarios)("memory writes through PRs: %s", (scenario) => html_url: `https://github.com/${repo}/pull/${number}`, state: pull.state, head: { - sha: yield* branchHead(pull.branch), + sha: pull.sha ?? (yield* branchHead(pull.branch)), ref: pull.branch, repo: { full_name: repo }, }, - base: { ref: "main" }, + base: { ref: pull.base }, }; }); const route = (request: RecordedRequest): Effect.Effect => Effect.gen(function* () { const path = request.path.slice(`/repos/${repo}`.length); + const query = new URL(path, "https://github.invalid").searchParams; const body = request.body as Record | undefined; if (request.method === "POST" && path === "/pulls") { + if (scenario === "hang") { + yield* Deferred.succeed(silenced, undefined); + return { status: 0, hang: true }; + } const number = pulls.size + 1; - pulls.set(number, { branch: body!.head!, state: "open" }); + pulls.set(number, { branch: body!.head!, state: "open", base: body!.base! }); + // Another PR from Kin's branch, into a base with its own `memory-check.yml`. + if (scenario === "second-pr") + pulls.set(number + 1, { branch: body!.head!, state: "closed", base: "attacker" }); return { status: 201, body: yield* pullBody(number) }; } + if (request.method === "GET" && path.startsWith("/pulls?") && query.has("head")) { + const branch = query.get("head")!.slice(`${repo.split("/")[0]}:`.length); + const listed = [...pulls].filter( + ([, pull]) => + pull.branch === branch && (query.get("state") === "all" || pull.state === "open"), + ); + return { + status: 200, + body: yield* Effect.forEach(listed, ([number]) => pullBody(number)), + ...(scenario === "pulls-truncated" && query.get("state") === "all" + ? { headers: { link: `; rel="next"` } } + : {}), + }; + } + const events = /^\/issues\/(\d+)\/events\?/.exec(path); + if (events) { + const labeled = Array.from({ length: 100 }, () => ({ event: "labeled" })); + return { + status: 200, + body: + scenario === "events-truncated" + ? labeled + : // Moved to an attacker's base for a run, then back, while Kin polled. + scenario === "base-changed" && jobsListed + ? [{ event: "base_ref_changed" }, { event: "base_ref_changed" }] + : [], + }; + } const pull = /^\/pulls\/(\d+)(\/merge)?$/.exec(path); if (pull) { const number = Number(pull[1]); const state = pulls.get(number)!; if (request.method === "PATCH") { + if (number === unclosable) return { status: 500, body: { message: "Server Error" } }; state.state = body!.state!; return { status: 200, body: yield* pullBody(number) }; } @@ -143,7 +198,11 @@ it.live.each(scenarios)("memory writes through PRs: %s", (scenario) => return { status: 200, body: { merged: true, sha: squash } }; } if (request.method === "GET" && /^\/pulls\?state=open&per_page=/.test(path)) { - const open = [...pulls].filter(([, pull]) => pull.state === "open"); + // Newest first, one page. + const open = [...pulls] + .filter(([, pull]) => pull.state === "open") + .toSorted(([a], [b]) => b - a) + .slice(0, Number(query.get("per_page"))); return { status: 200, body: yield* Effect.forEach(open, ([number]) => pullBody(number)), @@ -193,27 +252,34 @@ it.live.each(scenarios)("memory writes through PRs: %s", (scenario) => } const jobs = /^\/actions\/runs\/(\d+)\/jobs\?/.exec(path); if (jobs) { + jobsListed = true; const pending = scenario === "check-pending"; const failed = scenario === "check-failed" || scenario === "owner-path-refused"; + const job = (id: number) => + Effect.gen(function* () { + return { + id, + name: "memory-check", + head_sha: yield* branchHead(pulls.get(1)!.branch), + status: pending ? "in_progress" : "completed", + conclusion: pending ? null : failed ? "failure" : "success", + }; + }); + // An Actions job's id is its check run's id. + const ids = jobs[1] !== "10" ? [200] : scenario === "rerun" ? [100, 101] : [100]; + // A re-run's earlier attempts are listed only with `filter=all`. + const listed = query.get("filter") === "all" ? ids : ids.slice(-1); return { status: 200, - body: { - total_count: 1, - jobs: [ - { - // An Actions job's id is its check run's id. - id: jobs[1] === "10" ? 100 : 200, - name: "memory-check", - head_sha: yield* branchHead(pulls.get(1)!.branch), - status: pending ? "in_progress" : "completed", - conclusion: pending ? null : failed ? "failure" : "success", - }, - ], - }, + body: { total_count: listed.length, jobs: yield* Effect.forEach(listed, job) }, }; } const checks = /^\/commits\/([a-f0-9]+)\/check-runs\?(.*)$/.exec(path); if (checks) { + if (scenario === "stall-body") { + yield* Deferred.succeed(silenced, undefined); + return { status: 200, stallBody: true }; + } const check = (id: number, suite: number) => ({ id, name: "memory-check", @@ -225,6 +291,7 @@ it.live.each(scenarios)("memory writes through PRs: %s", (scenario) => // Posted by another branch's workflow through the checks API, into our run's suite. ...(scenario === "forged-in-suite" ? [check(300, 1)] : []), check(100, 1), + ...(scenario === "rerun" ? [check(101, 1)] : []), ...(scenario === "foreign-check" ? [check(200, 2)] : []), ]; // GitHub's default `filter=latest` keeps only the newest run per name in each suite. @@ -365,7 +432,14 @@ it.live.each(scenarios)("memory writes through PRs: %s", (scenario) => } yield* writer.stage(orgId, "wake1", { ...claim, sourceEventId: "owner" }); - const ownerPath = scenario === "owner-path" || scenario === "owner-path-refused"; + const reviewed = scenario === "owner-path" || scenario === "owner-path-close-fails"; + const ownerPath = + reviewed || scenario === "owner-path-refused" || scenario === "records-broken"; + const sql = yield* SqlClient.SqlClient; + const records = sql`SELECT branch, head FROM kin_memory_review_pulls ORDER BY branch`; + if (scenario === "records-broken") + yield* sql`CREATE TRIGGER no_insert BEFORE INSERT ON kin_memory_review_pulls + BEGIN SELECT RAISE(ABORT, 'fixture'); END`; if (ownerPath) yield* writer.stage(orgId, "wake1", { path: "decisions/dec-001.md", @@ -383,16 +457,43 @@ it.live.each(scenarios)("memory writes through PRs: %s", (scenario) => ); return; } - if (scenario !== "merged" && scenario !== "owner-path") { + if (scenario === "hang" || scenario === "stall-body") { + const running = yield* Effect.forkChild(committing); + yield* Deferred.await(silenced); + yield* TestClock.adjust("30 seconds"); + const error = yield* refused(Fiber.join(running)); + expect(error).toMatchObject({ + reason: expect.stringContaining( + `could not reach GitHub for ${scenario === "hang" ? "pulls.create" : "checks.list"}`, + ), + }); + yield* unchanged; + // The writer lock is free again: the sweep takes it and closes the PR left open. + yield* writer.sweepPullRequests(orgId); + expect(merges).toEqual([]); + expect([...pulls.values()].map((pull) => pull.state)).toEqual( + scenario === "hang" ? [] : ["closed"], + ); + return; + } + if ( + !reviewed && + scenario !== "merged" && + scenario !== "rerun" && + scenario !== "records-broken" + ) { const foreign = "did not come from .github/workflows/memory-check.yml"; const error = yield* refused(committing); yield* unchanged; expect(merges.length).toBe(scenario === "merge-rejected" ? 1 : 0); expect(pulls.get(1)?.state).toBe("closed"); // A refused write opens no review PR, so none is left waiting alone. - expect([...pulls.values()].map((pull) => pull.branch)).toEqual([ - `kin/write/${agentId}/wake1`, - ]); + expect([...pulls.values()].filter((pull) => pull.branch.endsWith("-review"))).toEqual([]); + // A PR from Kin's branch that isn't this one fails before Kin reads any run. + if (scenario === "second-pr") + expect(github.requests.map((request) => request.path)).not.toContainEqual( + expect.stringContaining("/actions/runs"), + ); expect(github.requests).toContainEqual( expect.objectContaining({ method: "POST", path: `/repos/${repo}/issues/1/comments` }), ); @@ -410,6 +511,10 @@ it.live.each(scenarios)("memory writes through PRs: %s", (scenario) => "run-wrong-pr": foreign, "run-wrong-base": foreign, truncated: "GitHub listed more check runs than one page holds", + "second-pr": "another PR was opened from Kin's branch", + "pulls-truncated": "GitHub listed more PRs from Kin's branch than one page holds", + "base-changed": "the PR's base branch was changed", + "events-truncated": "GitHub listed more PR events than one page holds", "moved-head": "head is not the commit Kin pushed", "check-failed": "memory-check failed", "owner-path-refused": "memory-check failed", @@ -431,7 +536,7 @@ it.live.each(scenarios)("memory writes through PRs: %s", (scenario) => expect(yield* remoteFiles(main)).toContain("claims/new.md"); // The merged branch is deleted; the merge was pinned to the commit Kin pushed. expect(yield* bare("for-each-ref", "--format=%(refname)", "refs/heads/kin/write/")).toBe( - scenario === "owner-path" ? `refs/heads/kin/write/${agentId}/wake1-review` : "", + ownerPath ? `refs/heads/kin/write/${agentId}/wake1-review` : "", ); const auto = [...pulls].find(([, pull]) => pull.branch === `kin/write/${agentId}/wake1`)!; const autoHead = yield* git(paths.writer, ["rev-parse", `${main}^{tree}`]); @@ -454,10 +559,36 @@ it.live.each(scenarios)("memory writes through PRs: %s", (scenario) => actions: "read", }, }, - ...(scenario === "owner-path" ? [expect.objectContaining({ holder: "server" })] : []), + ...(ownerPath ? [expect.objectContaining({ holder: "server" })] : []), ]); - if (scenario === "owner-path") { + if (scenario === "records-broken") { + const reviewNumber = [...pulls].find(([, pull]) => pull.branch.endsWith("-review"))![0]; + expect(pulls.get(reviewNumber)?.state).toBe("closed"); + expect(notices.map((notice) => notice.reason)).toContainEqual( + "Memory changes on paths you review were not opened as a PR: Kin could not record the commit it pushed (MemoryPullRequests.recordReview)", + ); + const sweepRefusal = Effect.gen(function* () { + const error = yield* refused(writer.sweepPullRequests(orgId)); + return error.reason; + }); + yield* sql`DROP TRIGGER no_insert`; + yield* sql`INSERT INTO kin_memory_review_pulls (org_id, branch, head) + VALUES (${orgId}, 'kin/write/gone-review', ${original}) + ON CONFLICT (org_id, branch) DO NOTHING`; + yield* sql`CREATE TRIGGER no_delete BEFORE DELETE ON kin_memory_review_pulls + BEGIN SELECT RAISE(ABORT, 'fixture'); END`; + expect(yield* sweepRefusal).toBe( + "Kin could not update its record of review PRs (MemoryPullRequests.forgetReview)", + ); + yield* sql`ALTER TABLE kin_memory_review_pulls RENAME TO kin_memory_review_pulls_gone`; + expect(yield* sweepRefusal).toBe( + "Kin could not read its record of review PRs (MemoryPullRequests.records)", + ); + return; + } + + if (reviewed) { const reviewBranch = `kin/write/${agentId}/wake1-review`; const review = [...pulls].find(([, pull]) => pull.branch === reviewBranch)!; // Kin never merges the owner's paths; the PR stays open for their review. @@ -491,7 +622,7 @@ it.live.each(scenarios)("memory writes through PRs: %s", (scenario) => const orphanBranch = "kin/write/other/orphan"; yield* bare("update-ref", `refs/heads/${orphanBranch}`, original); const orphan = pulls.size + 1; - pulls.set(orphan, { branch: orphanBranch, state: "open" }); + pulls.set(orphan, { branch: orphanBranch, state: "open", base: "main" }); yield* writer.sweepPullRequests(orgId); expect(pulls.get(orphan)?.state).toBe("closed"); expect(github.requests).toContainEqual( @@ -502,18 +633,49 @@ it.live.each(scenarios)("memory writes through PRs: %s", (scenario) => }), ); expect(review[1].state).toBe("open"); - const sql = yield* SqlClient.SqlClient; - const records = sql`SELECT branch, head FROM kin_memory_review_pulls`; expect(yield* records).toEqual([{ branch: reviewBranch, head: reviewHead }]); + // Agents flood the repo with newer PRs, which must not hide the review PR from the sweep. + for (let flood = 0; flood < 120; flood++) + pulls.set(pulls.size + 1, { + branch: `feature/${flood}`, + state: "open", + base: "main", + sha: original, + }); + // A review PR Kin never recorded is closed with a comment, without telling the owner. + const fakeBranch = "kin/write/other/fake-review"; + const fake = pulls.size + 1; + pulls.set(fake, { branch: fakeBranch, state: "open", base: "main", sha: original }); + // An agent with the deploy key swaps what the owner would merge. yield* bare("update-ref", `refs/heads/${reviewBranch}`, original); + if (scenario === "owner-path-close-fails") unclosable = review[0]; + const before = notices.length; yield* writer.sweepPullRequests(orgId); - expect(review[1].state).toBe("closed"); - expect(notices.map((notice) => notice.reason)).toContainEqual( - `Kin closed https://github.com/${repo}/pull/${review[0]}, the memory PR for paths you review: its head is no longer \`${reviewHead}\`, the commit Kin pushed`, + const moved = `https://github.com/${repo}/pull/${review[0]}`; + const reason = `its head is no longer \`${reviewHead}\`, the commit Kin pushed`; + expect(pulls.get(fake)?.state).toBe("closed"); + expect(github.requests).toContainEqual( + expect.objectContaining({ + method: "POST", + path: `/repos/${repo}/issues/${fake}/comments`, + body: { body: "Kin closed this memory PR: Kin did not open it." }, + }), ); expect(merges.map((merge) => merge.number)).toEqual([auto[0]]); + if (scenario === "owner-path-close-fails") { + expect(review[1].state).toBe("open"); + expect(notices.slice(before).map((notice) => notice.reason)).toEqual([ + `Kin could not close ${moved}, the memory PR for paths you review, and it is still open. Close it without merging: ${reason}`, + ]); + expect(yield* records).toEqual([{ branch: reviewBranch, head: reviewHead }]); + return; + } + expect(review[1].state).toBe("closed"); + expect(notices.slice(before).map((notice) => notice.reason)).toEqual([ + `Kin closed ${moved}, the memory PR for paths you review: ${reason}`, + ]); expect(yield* records).toEqual([]); } }).pipe(Effect.provide(writerLayer)); diff --git a/apps/server/src/kin/memory/MemoryPullRequests.ts b/apps/server/src/kin/memory/MemoryPullRequests.ts index 62ebca64b26b..be8f49f3f220 100644 --- a/apps/server/src/kin/memory/MemoryPullRequests.ts +++ b/apps/server/src/kin/memory/MemoryPullRequests.ts @@ -9,7 +9,8 @@ import * as Schema from "effect/Schema"; import { HttpClient } from "effect/http"; import * as SqlClient from "effect/sql/SqlClient"; -import { gitHubAnswer, repoPath } from "../github/githubRest.ts"; +import { toPersistenceSqlError } from "../../persistence/Errors.ts"; +import { type GitHubAnswer, gitHubAnswer, repoPath } from "../github/githubRest.ts"; import { KinGitHubApp } from "../github/KinGitHubApp.ts"; import { GITHUB_ACTIONS_INTEGRATION_ID } from "../github/KinRulesets.ts"; @@ -75,6 +76,9 @@ const decodeChecks = Schema.decodeUnknownOption(CheckRuns); const decodeMerge = Schema.decodeUnknownOption( Schema.Struct({ merged: Schema.Boolean, sha: Schema.String }), ); +const decodeEvents = Schema.decodeUnknownOption( + Schema.Array(Schema.Struct({ event: Schema.String })), +); export class MemoryPullRequestRefused extends Schema.TaggedError()( "MemoryPullRequestRefused", @@ -103,11 +107,12 @@ interface MemoryPullRequestInput { readonly ownerReview: boolean; } -interface ClosedPullRequest { +/** A review PR Kin recorded whose head or base moved; `closed` is false when closing it failed. */ +interface MovedReviewPullRequest { readonly url: string; readonly branch: string; readonly reason: string; - readonly ownerReview: boolean; + readonly closed: boolean; } export const WRITE_BRANCH_PREFIX = "kin/write/"; @@ -125,13 +130,14 @@ export class MemoryPullRequests extends Context.Service< ) => Effect.Effect; /** * Closes every open `kin/write/*` PR that no write is waiting on: review PRs whose head is not - * the commit Kin recorded, and all others. Run it under the writer lock, so no `land` is open. + * the commit Kin recorded, and all others. Returns only the recorded review PRs it closed or + * failed to close. Run it under the writer lock, so no `land` is open. */ readonly sweep: (input: { readonly orgId: KinOrgId; readonly repo: RepoSlug; readonly base: string; - }) => Effect.Effect, MemoryPullRequestRefused>; + }) => Effect.Effect, MemoryPullRequestRefused>; } >()("t3/kin/memory/MemoryPullRequests") {} @@ -145,7 +151,12 @@ const truncated = (reason: string): Verdict => ({ reason: `GitHub listed more ${reason} than one page holds`, }); -/** A `pull_request` run of the memory-check workflow for exactly this PR, branch and commit. */ +/** + * A `pull_request` run of the memory-check workflow on this branch and commit. GitHub fills + * `pull_requests` from the PRs open with that head now, not the one that triggered the run, so + * `land` also checks that no other PR ever came from this branch and that this PR's base never + * changed. + */ const isOurRun = ( input: MemoryPullRequestInput, number: number, @@ -193,6 +204,12 @@ const memoryCheckVerdict = ( const PAGE = 100; const MAX_RUNS = 10; +// Every call runs under the writer lock; GitHub's own edge gives up after about 10 seconds. +const CALL_TIMEOUT: Duration.Input = "30 seconds"; + +/** A full page, or a `next` link, may hide more; lists that decide anything fail closed on it. */ +const pastOnePage = (answer: GitHubAnswer, length: number) => + length >= PAGE || /rel="next"/.test(answer.headers["link"] ?? ""); export const make = Effect.fn("MemoryPullRequests.make")(function* (options: { readonly pollInterval: Duration.Input; @@ -226,6 +243,8 @@ export const make = Effect.fn("MemoryPullRequests.make")(function* (options: { ); const token: Redacted.Redacted = minted.token; const base = repoPath(repo); + const owner = repo.split("/")[0] ?? ""; + const fromBranch = (branch: string) => encodeURIComponent(`${owner}:${branch}`); const call = ( operation: string, method: "GET" | "POST" | "PUT" | "PATCH", @@ -238,8 +257,16 @@ export const make = Effect.fn("MemoryPullRequests.make")(function* (options: { method, path: `${base}${path}`, token, + timeout: CALL_TIMEOUT, ...(body === undefined ? {} : { body }), }).pipe(Effect.mapError(() => refuse(`could not reach GitHub for ${operation}`))); + const sqlRefusal = + (operation: string, reason: string) => + (effect: Effect.Effect) => + effect.pipe( + Effect.mapError(toPersistenceSqlError(`MemoryPullRequests.${operation}`)), + Effect.mapError((error) => refuse(`${reason} (${error.operation})`)), + ); const close = Effect.fn("MemoryPullRequests.close")(function* (number: number, reason: string) { yield* call("issues.comment", "POST", `/issues/${number}/comments`, { body: `Kin closed this memory PR: ${reason}.`, @@ -250,15 +277,14 @@ export const make = Effect.fn("MemoryPullRequests.make")(function* (options: { const records = sql<{ branch: string; head: string; - }>`SELECT branch, head FROM kin_memory_review_pulls WHERE org_id = ${orgId}`.pipe( - Effect.mapError(() => refuse("Kin could not read its record of review PRs")), - ); - return { refuse, call, close, records }; + }>`SELECT branch, head FROM kin_memory_review_pulls WHERE org_id = ${orgId} + ORDER BY branch`.pipe(sqlRefusal("records", "Kin could not read its record of review PRs")); + return { refuse, call, close, records, fromBranch, sqlRefusal }; }); const land = Effect.fn("MemoryPullRequests.land")(function* (input: MemoryPullRequestInput) { yield* Effect.annotateCurrentSpan({ "kin.github.repo": input.repo }); - const { refuse, call, close } = yield* connect(input.orgId, input.repo); + const { refuse, call, close, fromBranch, sqlRefusal } = yield* connect(input.orgId, input.repo); const describe = (answer: { status: number; message: string }) => `HTTP ${answer.status}${answer.message ? `: ${answer.message}` : ""}`; @@ -274,11 +300,10 @@ export const make = Effect.fn("MemoryPullRequests.make")(function* (options: { let pull = opened.status === 201 ? Option.getOrUndefined(decodePull(opened.body)) : undefined; // A retry of the same write finds the PR it opened before. if (pull === undefined && opened.status === 422) { - const owner = input.repo.split("/")[0] ?? ""; const listed = yield* call( "pulls.list", "GET", - `/pulls?state=open&head=${encodeURIComponent(`${owner}:${input.branch}`)}&base=${encodeURIComponent(input.base)}`, + `/pulls?state=open&head=${fromBranch(input.branch)}&base=${encodeURIComponent(input.base)}`, ); pull = Option.getOrUndefined(decodePulls(listed.body))?.[0]; } @@ -305,17 +330,52 @@ export const make = Effect.fn("MemoryPullRequests.make")(function* (options: { current.value.base.ref === input.base ); }); + // Ties every run on Kin's branch to this PR into `input.base`: an agent could open another PR + // from the branch into a base with its own `memory-check.yml`, or move this PR's base there. + const unboundReason = Effect.gen(function* () { + const listed = yield* call( + "pulls.list", + "GET", + `/pulls?state=all&head=${fromBranch(input.branch)}&per_page=${PAGE}`, + ); + const pulls = listed.status === 200 ? decodePulls(listed.body) : Option.none(); + if (Option.isNone(pulls)) return "GitHub did not list the PRs from Kin's branch"; + if (pastOnePage(listed, pulls.value.length)) + return "GitHub listed more PRs from Kin's branch than one page holds"; + if (pulls.value.length !== 1 || pulls.value[0]?.number !== number) + return "another PR was opened from Kin's branch"; + const answer = yield* call( + "issues.events", + "GET", + `/issues/${number}/events?per_page=${PAGE}`, + ); + const events = answer.status === 200 ? decodeEvents(answer.body) : Option.none(); + if (Option.isNone(events)) return "GitHub did not list the PR's events"; + if (pastOnePage(answer, events.value.length)) + return "GitHub listed more PR events than one page holds"; + if (events.value.some((event) => event.event === "base_ref_changed")) + return "the PR's base branch was changed"; + return undefined; + }); + const refuseUnbound = Effect.gen(function* () { + const reason = yield* unboundReason; + if (reason !== undefined) return yield* refuseAndClose(reason); + }); + if (!(yield* headIsOurs)) return yield* refuseAndClose("the PR's head is not the commit Kin pushed"); if (input.ownerReview) { // The sweep closes this PR if its head ever stops being the commit recorded here. const recorded = yield* sql`INSERT INTO kin_memory_review_pulls (org_id, branch, head) VALUES (${input.orgId}, ${input.branch}, ${input.sha}) - ON CONFLICT (org_id, branch) DO UPDATE SET head = excluded.head`.pipe(Effect.result); - if (recorded._tag === "Failure") - return yield* refuseAndClose("Kin could not record the commit it pushed"); + ON CONFLICT (org_id, branch) DO UPDATE SET head = excluded.head`.pipe( + sqlRefusal("recordReview", "Kin could not record the commit it pushed"), + Effect.result, + ); + if (recorded._tag === "Failure") return yield* refuseAndClose(recorded.failure.reason); return { _tag: "awaiting_owner", url } as const; } + yield* refuseUnbound; let verdict: Verdict = { _tag: "pending" }; for (let poll = 0; poll < options.polls && verdict._tag === "pending"; poll++) { @@ -360,6 +420,7 @@ export const make = Effect.fn("MemoryPullRequests.make")(function* (options: { if (verdict._tag === "fail") return yield* refuseAndClose(verdict.reason); if (!(yield* headIsOurs)) return yield* refuseAndClose("the PR's head moved after the check passed"); + yield* refuseUnbound; const [title, ...rest] = input.message.split("\n"); const merge = yield* call("pulls.merge", "PUT", `/pulls/${number}/merge`, { @@ -380,56 +441,94 @@ export const make = Effect.fn("MemoryPullRequests.make")(function* (options: { readonly base: string; }) { yield* Effect.annotateCurrentSpan({ "kin.github.repo": input.repo }); - const { refuse, call, close, records } = yield* connect(input.orgId, input.repo); - const listed = yield* call("pulls.list", "GET", `/pulls?state=open&per_page=${PAGE}`); - const pulls = listed.status === 200 ? decodePulls(listed.body) : Option.none(); - if (Option.isNone(pulls)) return yield* refuse("GitHub did not list the open PRs"); - const recorded = new Map((yield* records).map((row) => [row.branch, row.head])); - const kept = new Set(); - const closed: Array = []; - for (const pull of pulls.value) { - const branch = pull.head.ref; - if (!branch.startsWith(WRITE_BRANCH_PREFIX)) continue; - const ownerReview = branch.endsWith(REVIEW_BRANCH_SUFFIX); - const pushed = recorded.get(branch); - if ( - ownerReview && - pushed === pull.head.sha && - pull.head.repo?.full_name.toLowerCase() === input.repo.toLowerCase() && - pull.base.ref === input.base - ) { - kept.add(branch); - continue; - } - const reason = !ownerReview - ? "no Kin memory write is waiting on it" - : pushed === undefined - ? "Kin did not open it" - : `its head is no longer \`${pushed}\`, the commit Kin pushed`; - const done = yield* close(pull.number, reason).pipe(Effect.result); - if (done._tag === "Failure") { - yield* Effect.logWarning("Could not close an orphaned memory PR", { - repo: input.repo, - number: pull.number, + const { refuse, call, close, records, fromBranch, sqlRefusal } = yield* connect( + input.orgId, + input.repo, + ); + const list = (path: string, what: string) => + call("pulls.list", "GET", path).pipe( + Effect.flatMap((listed) => { + const pulls = listed.status === 200 ? decodePulls(listed.body) : Option.none(); + return Option.isSome(pulls) + ? Effect.succeed({ pulls: pulls.value, full: pastOnePage(listed, pulls.value.length) }) + : Effect.fail(refuse(`GitHub did not list ${what}`)); + }), + ); + // Read everything before closing anything, so a failed read loses no owner notice. + // Each recorded review branch is looked up by head: newer PRs can't push it off a page. + const reviews = yield* Effect.forEach(yield* records, (record) => + list( + `/pulls?state=open&head=${fromBranch(record.branch)}&per_page=${PAGE}`, + `the open PRs from ${record.branch}`, + ).pipe( + Effect.flatMap(({ pulls, full }) => + full + ? Effect.fail( + refuse(`GitHub listed more PRs from ${record.branch} than one page holds`), + ) + : Effect.succeed({ ...record, pulls }), + ), + ), + ); + // Orphans are found on one page of open PRs, newest first; older ones wait for a later sweep. + const open = yield* list(`/pulls?state=open&per_page=${PAGE}`, "the open PRs"); + if (open.full) + yield* Effect.logWarning("More open PRs than one memory PR sweep reads", { + repo: input.repo, + }); + + const handled = new Set(); + const moved: Array = []; + for (const review of reviews) { + let kept = false; + for (const pull of review.pulls) { + handled.add(pull.number); + const sameRepo = pull.head.repo?.full_name.toLowerCase() === input.repo.toLowerCase(); + if (pull.head.sha === review.head && sameRepo && pull.base.ref === input.base) { + kept = true; + continue; + } + const reason = + pull.head.sha !== review.head || !sameRepo + ? `its head is no longer \`${review.head}\`, the commit Kin pushed` + : `its base is no longer \`${input.base}\``; + const done = yield* close(pull.number, reason).pipe(Effect.result); + if (done._tag === "Failure") { + yield* Effect.logWarning("Could not close a moved memory review PR", { + repo: input.repo, + number: pull.number, + }); + kept = true; + } + moved.push({ + url: pull.html_url, + branch: review.branch, + reason, + closed: done._tag === "Success", }); - kept.add(branch); - continue; } - closed.push({ url: pull.html_url, branch, reason, ownerReview }); - } - // Only a complete list shows which recorded review PRs are gone. - if (pulls.value.length >= PAGE) { - yield* Effect.logWarning("More open memory PRs than one sweep reads", { repo: input.repo }); - return closed; + if (!kept) + yield* sql`DELETE FROM kin_memory_review_pulls + WHERE org_id = ${input.orgId} AND branch = ${review.branch}`.pipe( + sqlRefusal("forgetReview", "Kin could not update its record of review PRs"), + ); } - for (const branch of recorded.keys()) { - if (kept.has(branch)) continue; - yield* sql`DELETE FROM kin_memory_review_pulls - WHERE org_id = ${input.orgId} AND branch = ${branch}`.pipe( - Effect.mapError(() => refuse("Kin could not update its record of review PRs")), + for (const pull of open.pulls) { + const branch = pull.head.ref; + if (!branch.startsWith(WRITE_BRANCH_PREFIX) || handled.has(pull.number)) continue; + const reason = branch.endsWith(REVIEW_BRANCH_SUFFIX) + ? "Kin did not open it" + : "no Kin memory write is waiting on it"; + yield* close(pull.number, reason).pipe( + Effect.catch(() => + Effect.logWarning("Could not close an orphaned memory PR", { + repo: input.repo, + number: pull.number, + }), + ), ); } - return closed; + return moved; }); return MemoryPullRequests.of({ land, sweep }); diff --git a/apps/server/src/kin/memory/MemoryWriter.ts b/apps/server/src/kin/memory/MemoryWriter.ts index 44b481cab2c0..0ce5a795c0d5 100644 --- a/apps/server/src/kin/memory/MemoryWriter.ts +++ b/apps/server/src/kin/memory/MemoryWriter.ts @@ -1357,7 +1357,7 @@ const make = Effect.gen(function* () { if (Option.isNone(pullRequests)) return; const repo = yield* repoFor(orgId); if (repo.githubRepo === undefined || !isRepoSlug(repo.githubRepo)) return; - const closed = yield* pullRequests.value + const moved = yield* pullRequests.value .sweep({ orgId, repo: repo.githubRepo, base: config.defaultBranch ?? "main" }) .pipe( Effect.mapError( @@ -1365,13 +1365,14 @@ const make = Effect.gen(function* () { new MemorySyncConflict({ orgId, operation: "sweep", reason: error.reason }), ), ); - for (const pull of closed) - if (pull.ownerReview) - yield* recovery.notify({ - orgId, - branch: pull.branch, - reason: `Kin closed ${pull.url}, the memory PR for paths you review: ${pull.reason}`, - }); + for (const pull of moved) + yield* recovery.notify({ + orgId, + branch: pull.branch, + reason: pull.closed + ? `Kin closed ${pull.url}, the memory PR for paths you review: ${pull.reason}` + : `Kin could not close ${pull.url}, the memory PR for paths you review, and it is still open. Close it without merging: ${pull.reason}`, + }); }), ), ), From 8491121338388b48aa14286091fc851174cc0ffb Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Mon, 9 Nov 2026 17:01:20 +0530 Subject: [PATCH 6/8] docs(kin): memory-check runs are bound by Kin's checks, not pull_requests Co-Authored-By: Claude Opus 5.5 --- docs/kin/specs/lead.md | 2 +- docs/kin/specs/triggers.md | 2 +- ops/kin/memory-repo/README.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/kin/specs/lead.md b/docs/kin/specs/lead.md index dde36215ac50..38a402cf621d 100644 --- a/docs/kin/specs/lead.md +++ b/docs/kin/specs/lead.md @@ -421,7 +421,7 @@ Branches: at L1 and above the writer commits to the default branch, through a pu Pushes and fetches use the configured URL explicitly, never a local remote name or pushurl. A retry of an already recorded wake-up returns that commit (with `KIN_MEMORY_DIRECT_PUSH`, after pushing it again). Synchronization uses verified ancestry and plumbing-only fast-forwards. Divergent trees merge disjoint edits and append-only journals in process, then use `commit-tree` with both parents. Content conflicts preserve the local head on a conflict branch, reset to the remote anchor and notify the owner. Missing or ambiguous merge bases preserve and notify, then fail closed without resetting; the organization stays blocked until an operator runs `Memory.recover`, and repeated attempts reuse the conflict branch already holding that head. Failed fetches retain the SQLite anchor. Existing writers with mismatched or missing anchors fail closed as `MemoryCheckoutInvalid`. -**Writes to the default branch go through PRs (TRIG-15).** The memory repo's `main` takes no direct push, so the writer pushes its commit to `kin/write//` (`keep-` for Keep) and `MemoryPullRequests` opens a PR with a `server` App token (`contents` and `pull_requests` write, `checks` and `actions` read). Before merging, Kin checks that the PR head is exactly the commit it pushed and that `memory-check` passed in a run it can trace: a `pull_request` run of `.github/workflows/memory-check.yml` on that commit and branch, listed for this PR and base, whose `memory-check` jobs all succeeded. Any workflow can post a `memory-check` check run, even into that run's check suite, so every `memory-check` check run on the commit (`filter=all`) must have the id of one of those jobs, an Actions job's id being its check run's id. A list longer than one page fails the check. Then Kin squash-merges through REST with `sha` pinned to that commit. GitHub's auto-merge is not used: it trusts any Actions workflow's `memory-check`. The local default branch and its anchor move only after the merge, to the fetched remote head, which must descend from the merged commit; when the merge parent is the old head, its tree must match Kin's. Changes to paths the owner reviews (`decisions/`, `people/`, `objectives/`, anything outside Kin's own folders) are split into their own PR on `-review`, built on the old head without `INDEX.md` or the journal, opened only after the main PR merged and never merged by Kin. Its body and the owner's notice name the commit Kin pushed, so the owner can close it if the head differs; memory reads see those changes only once the owner merges. Kin records that commit in `kin_memory_review_pulls`, because the memory deploy key can push to the branch. A failed or foreign check, a moved head, a check still running after 15 minutes or a refused merge closes the PR with a comment, posts the reason and link to the owner's direct chat, and fails the commit with `MemorySyncConflict`, leaving the default branch untouched. Without the Kin App, writes to the default branch are refused and the owner is told; `KIN_MEMORY_DIRECT_PUSH=true` restores direct pushes for a memory repo without the TRIG-15 ruleset. At startup and every 5 minutes, a sweep on the Kin scheduler takes the writer lock and closes, with a comment, every open `kin/write/*` PR no write waits on: any non-review PR (its write already ended) and any review PR whose head isn't the commit Kin recorded, telling the owner about the latter. It reads one page of open PRs per run. The writer holds the per-organization lock while it waits for the check, so the next wake-up starts from the merged head; the known cost is that a slow check blocks that organization's wake-ups, Keep and the sweep for up to 15 minutes. +**Writes to the default branch go through PRs (TRIG-15).** The memory repo's `main` takes no direct push, so the writer pushes its commit to `kin/write//` (`keep-` for Keep) and `MemoryPullRequests` opens a PR with a `server` App token (`contents` and `pull_requests` write, `checks` and `actions` read). Before merging, Kin checks that the PR head is exactly the commit it pushed and that `memory-check` passed in a run it can trace: a `pull_request` run of `.github/workflows/memory-check.yml` on that commit and branch whose `memory-check` jobs all succeeded. A run's `pull_requests` comes from the PRs open with that head now, not from the PR that triggered it, so it can't tie the run to this PR and base. Instead, before reading runs and again just before merging, Kin requires that the branch has only ever had this PR (`pulls?state=all&head=`) and that the PR's issue events hold no `base_ref_changed`; otherwise an agent could run its own `memory-check.yml` through a second PR from the branch, or by moving this PR's base and back. Any workflow can post a `memory-check` check run, even into that run's check suite, so every `memory-check` check run on the commit (`filter=all`) must have the id of one of those jobs, an Actions job's id being its check run's id. Any of these lists running past one page (a full page or a `next` link) fails the check. Then Kin squash-merges through REST with `sha` pinned to that commit. GitHub's auto-merge is not used: it trusts any Actions workflow's `memory-check`. The local default branch and its anchor move only after the merge, to the fetched remote head, which must descend from the merged commit; when the merge parent is the old head, its tree must match Kin's. Changes to paths the owner reviews (`decisions/`, `people/`, `objectives/`, anything outside Kin's own folders) are split into their own PR on `-review`, built on the old head without `INDEX.md` or the journal, opened only after the main PR merged and never merged by Kin. Its body and the owner's notice name the commit Kin pushed, so the owner can close it if the head differs; memory reads see those changes only once the owner merges. Kin records that commit in `kin_memory_review_pulls`, because the memory deploy key can push to the branch. A failed or foreign check, a moved head, another PR from the branch, a changed base, a check still running after 15 minutes or a refused merge closes the PR with a comment, posts the reason and link to the owner's direct chat, and fails the commit with `MemorySyncConflict`, leaving the default branch untouched. Without the Kin App, writes to the default branch are refused and the owner is told; `KIN_MEMORY_DIRECT_PUSH=true` restores direct pushes for a memory repo without the TRIG-15 ruleset. At startup and every 5 minutes, a sweep on the Kin scheduler takes the writer lock and closes, with a comment, every open `kin/write/*` PR no write waits on: any non-review PR (its write already ended) and any review PR whose head isn't the commit Kin recorded. It looks up each recorded review branch by head, so newer PRs can't hide one, and tells the owner about each it closed, or couldn't close and left open; a `-review` PR Kin never recorded is closed with only a comment. Orphans are found on one page of open PRs, newest first, so a flood delays them to a later sweep. GitHub calls under the lock time out after 30 seconds. The writer holds the per-organization lock while it waits for the check, so the next wake-up starts from the merged head; the known cost is that a slow check blocks that organization's wake-ups, Keep and the sweep for up to 15 minutes. Front matter matches the kin-memory README, including snake_case keys. Date-only values mean UTC midnight; date-times require an explicit UTC offset, and invalid dates make an item malformed. `links` values containing `#` must be quoted. `owner` and `said_by` hold handles; `orgId` and `path` belong only to the server-side `MemoryItemHead`. diff --git a/docs/kin/specs/triggers.md b/docs/kin/specs/triggers.md index 4da7d9ce4c84..883f0cd203e9 100644 --- a/docs/kin/specs/triggers.md +++ b/docs/kin/specs/triggers.md @@ -399,7 +399,7 @@ The builder takes item heads (optionally marked `misleading` by the caller), las GitHub then blocks a bad auto-merge even if Kin's classifier is wrong. -**Daytime lead writes (decided).** Because `main` requires PRs, LEAD's daytime memory writes and the owner's Keep land as PRs that Kin merges itself once `memory-check` passes; changes to an owner path go to a separate PR that waits for the owner's review and that Kin never merges. A passing `memory-check` alone isn't trusted (see the memory repo README, Workflow permissions): Kin merges only a PR whose head is the commit it pushed, with REST `sha` pinned to it, after every `memory-check` check run on that commit turned out to be a job of a `pull_request` run of `.github/workflows/memory-check.yml` for that PR, branch and commit, and those jobs passed. Review PRs carry the pushed commit in their body and notice, and a periodic sweep closes them if their head moves, along with any other `kin/write/*` PR no write waits on. GitHub's auto-merge would accept any workflow's check, so it isn't used. The writer side is in the lead spec (Memory writes). `MemoryPullRequests` needs the Kin App and the memory repo in `KIN_GITHUB_REPOS`; without the App, writes to `main` are refused unless `KIN_MEMORY_DIRECT_PUSH=true`, which suits only a memory repo without this ruleset. The ruleset itself comes from `KIN_REPOS` with `kind: "memory"` (#85). +**Daytime lead writes (decided).** Because `main` requires PRs, LEAD's daytime memory writes and the owner's Keep land as PRs that Kin merges itself once `memory-check` passes; changes to an owner path go to a separate PR that waits for the owner's review and that Kin never merges. A passing `memory-check` alone isn't trusted (see the memory repo README, Workflow permissions): Kin merges only a PR whose head is the commit it pushed, with REST `sha` pinned to it, after every `memory-check` check run on that commit turned out to be a job of a `pull_request` run of `.github/workflows/memory-check.yml` for that branch and commit, and those jobs passed. A run's `pull_requests` reflects the PRs open with its head now, so Kin also requires that its branch has only ever had this PR and that the PR's base never changed. Review PRs carry the pushed commit in their body and notice, and a periodic sweep, looking each recorded review branch up by head, closes them if their head moves, along with any other `kin/write/*` PR no write waits on. GitHub's auto-merge would accept any workflow's check, so it isn't used. The writer side is in the lead spec (Memory writes). `MemoryPullRequests` needs the Kin App and the memory repo in `KIN_GITHUB_REPOS`; without the App, writes to `main` are refused unless `KIN_MEMORY_DIRECT_PUSH=true`, which suits only a memory repo without this ruleset. The ruleset itself comes from `KIN_REPOS` with `kind: "memory"` (#85). ### Secret scan before memory commits (gap 5) diff --git a/ops/kin/memory-repo/README.md b/ops/kin/memory-repo/README.md index d09851785cc8..38f071266e2d 100644 --- a/ops/kin/memory-repo/README.md +++ b/ops/kin/memory-repo/README.md @@ -30,7 +30,7 @@ Nothing automates this yet: Kin has no code that adds the same-tree commit or op ### Workflow permissions -`memory-check` is pinned to the GitHub Actions App (15368), which trusts any Actions workflow in the repo, not just `memory-check.yml`. A workflow pushed to any branch could post a passing `memory-check` on another PR. The repo's read-only workflow-permissions setting doesn't stop this: anyone with write access, which includes the memory deploy key that agents can read, can raise a workflow's own `permissions:`. A ruleset `workflows` rule would close the gap, but it exists only in organization rulesets, and `GLDRoger` is a user account. So a passing `memory-check` alone is not trusted: before merging a PR, Kin checks that its head is the SHA Kin pushed and merges with `sha` pinned to it. It refuses unless a `pull_request` run of `.github/workflows/memory-check.yml` for that PR, branch and SHA passed its `memory-check` job, and every `memory-check` check run on the SHA, superseded ones included, is one of that run's jobs by id. Belonging to the run's check suite isn't enough, since a workflow can post a check run into another run's suite. A `memory-check` from anywhere else closes the PR. For the same reason Kin doesn't use GitHub's auto-merge. Never grant the Kin App Workflows permission on this repo. +`memory-check` is pinned to the GitHub Actions App (15368), which trusts any Actions workflow in the repo, not just `memory-check.yml`. A workflow pushed to any branch could post a passing `memory-check` on another PR. The repo's read-only workflow-permissions setting doesn't stop this: anyone with write access, which includes the memory deploy key that agents can read, can raise a workflow's own `permissions:`. A ruleset `workflows` rule would close the gap, but it exists only in organization rulesets, and `GLDRoger` is a user account. So a passing `memory-check` alone is not trusted: before merging a PR, Kin checks that its head is the SHA Kin pushed and merges with `sha` pinned to it. It refuses unless a `pull_request` run of `.github/workflows/memory-check.yml` for that branch and SHA passed its `memory-check` job, and every `memory-check` check run on the SHA, superseded ones included, is one of that run's jobs by id. A run's `pull_requests` lists the PRs open with its head now, not the PR that triggered it, so Kin also refuses if its branch ever had another PR or the PR's base was ever changed: either could have run a `memory-check.yml` from another base. Belonging to the run's check suite isn't enough, since a workflow can post a check run into another run's suite. A `memory-check` from anywhere else closes the PR. For the same reason Kin doesn't use GitHub's auto-merge. Never grant the Kin App Workflows permission on this repo. ### Before switching it on From 4e27b33968d40c9d7de939cbfb97cceb7f910e6b Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Mon, 9 Nov 2026 17:10:55 +0530 Subject: [PATCH 7/8] test(kin): type the memory PR fixture's silent answers Co-Authored-By: Claude Opus 5.5 --- .../src/kin/memory/MemoryPullRequests.test.ts | 21 ++++++++----------- 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/apps/server/src/kin/memory/MemoryPullRequests.test.ts b/apps/server/src/kin/memory/MemoryPullRequests.test.ts index 2ee551f4019b..72d8b9c27020 100644 --- a/apps/server/src/kin/memory/MemoryPullRequests.test.ts +++ b/apps/server/src/kin/memory/MemoryPullRequests.test.ts @@ -124,7 +124,7 @@ it.effect.each(scenarios)("memory writes through PRs: %s", (scenario) => if (request.method === "POST" && path === "/pulls") { if (scenario === "hang") { yield* Deferred.succeed(silenced, undefined); - return { status: 0, hang: true }; + return { status: 0, hang: true as const }; } const number = pulls.size + 1; pulls.set(number, { branch: body!.head!, state: "open", base: body!.base! }); @@ -278,7 +278,7 @@ it.effect.each(scenarios)("memory writes through PRs: %s", (scenario) => if (checks) { if (scenario === "stall-body") { yield* Deferred.succeed(silenced, undefined); - return { status: 200, stallBody: true }; + return { status: 200, stallBody: true as const }; } const check = (id: number, suite: number) => ({ id, @@ -568,23 +568,20 @@ it.effect.each(scenarios)("memory writes through PRs: %s", (scenario) => expect(notices.map((notice) => notice.reason)).toContainEqual( "Memory changes on paths you review were not opened as a PR: Kin could not record the commit it pushed (MemoryPullRequests.recordReview)", ); - const sweepRefusal = Effect.gen(function* () { - const error = yield* refused(writer.sweepPullRequests(orgId)); - return error.reason; - }); + const sweepRefusal = refused(writer.sweepPullRequests(orgId)); yield* sql`DROP TRIGGER no_insert`; yield* sql`INSERT INTO kin_memory_review_pulls (org_id, branch, head) VALUES (${orgId}, 'kin/write/gone-review', ${original}) ON CONFLICT (org_id, branch) DO NOTHING`; yield* sql`CREATE TRIGGER no_delete BEFORE DELETE ON kin_memory_review_pulls BEGIN SELECT RAISE(ABORT, 'fixture'); END`; - expect(yield* sweepRefusal).toBe( - "Kin could not update its record of review PRs (MemoryPullRequests.forgetReview)", - ); + expect(yield* sweepRefusal).toMatchObject({ + reason: "Kin could not update its record of review PRs (MemoryPullRequests.forgetReview)", + }); yield* sql`ALTER TABLE kin_memory_review_pulls RENAME TO kin_memory_review_pulls_gone`; - expect(yield* sweepRefusal).toBe( - "Kin could not read its record of review PRs (MemoryPullRequests.records)", - ); + expect(yield* sweepRefusal).toMatchObject({ + reason: "Kin could not read its record of review PRs (MemoryPullRequests.records)", + }); return; } From de6aefb4d049cb4454d76dd2b39fc4eeef424bbd Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Mon, 9 Nov 2026 18:06:35 +0530 Subject: [PATCH 8/8] fix(kin): memory write retries get their own branch Each attempt's branch carries its commit, so a retry after a closed PR isn't refused as a second PR. Fork runs no longer count, the App token mint times out under the writer lock, close comes before the comment, and an unclosable review PR is reported once per server run. Co-Authored-By: Claude Opus 5.5 --- .../src/kin/memory/MemoryPullRequests.test.ts | 38 +++++++++++++++---- .../src/kin/memory/MemoryPullRequests.ts | 9 +++-- apps/server/src/kin/memory/MemoryWriter.ts | 10 ++++- docs/kin/specs/lead.md | 2 +- ops/kin/memory-repo/README.md | 2 +- 5 files changed, 46 insertions(+), 15 deletions(-) diff --git a/apps/server/src/kin/memory/MemoryPullRequests.test.ts b/apps/server/src/kin/memory/MemoryPullRequests.test.ts index 72d8b9c27020..44dc0a0daa02 100644 --- a/apps/server/src/kin/memory/MemoryPullRequests.test.ts +++ b/apps/server/src/kin/memory/MemoryPullRequests.test.ts @@ -63,6 +63,7 @@ const scenarios = [ "records-broken", "hang", "stall-body", + "run-from-fork", "keep", "keep-moved-head", "no-app", @@ -98,6 +99,7 @@ it.effect.each(scenarios)("memory writes through PRs: %s", (scenario) => const tokens: Array = []; // The request GitHub never answers in `hang` and `stall-body`. const silenced = yield* Deferred.make(); + let stalls = 0; let jobsListed = false; let unclosable: number | undefined; const branchHead = (branch: string) => bare("rev-parse", `refs/heads/${branch}`); @@ -224,6 +226,7 @@ it.effect.each(scenarios)("memory writes through PRs: %s", (scenario) => event: "pull_request", head_sha: sha, head_branch: pull.branch, + head_repository: { full_name: repo }, pull_requests: [{ number: pull.number, base: { ref: "main" } }], }); const ours = run(10, "memory-check.yml"); @@ -237,6 +240,9 @@ it.effect.each(scenarios)("memory writes through PRs: %s", (scenario) => ...(scenario === "run-wrong-sha" ? { head_sha: original } : {}), ...(scenario === "run-wrong-event" ? { event: "push" } : {}), ...(scenario === "run-wrong-branch" ? { head_branch: "kin/write/other" } : {}), + ...(scenario === "run-from-fork" + ? { head_repository: { full_name: "attacker/memory" } } + : {}), ...(scenario === "run-wrong-pr" ? { pull_requests: [{ number: 99, base: { ref: "main" } }] } : {}), @@ -260,7 +266,8 @@ it.effect.each(scenarios)("memory writes through PRs: %s", (scenario) => return { id, name: "memory-check", - head_sha: yield* branchHead(pulls.get(1)!.branch), + // The newest PR is the attempt being checked (a retry opens PR 2). + head_sha: yield* branchHead(pulls.get(Math.max(...pulls.keys()))!.branch), status: pending ? "in_progress" : "completed", conclusion: pending ? null : failed ? "failure" : "success", }; @@ -276,7 +283,8 @@ it.effect.each(scenarios)("memory writes through PRs: %s", (scenario) => } const checks = /^\/commits\/([a-f0-9]+)\/check-runs\?(.*)$/.exec(path); if (checks) { - if (scenario === "stall-body") { + // Only the first attempt stalls, so the retry after the sweep can land. + if (scenario === "stall-body" && stalls++ === 0) { yield* Deferred.succeed(silenced, undefined); return { status: 200, stallBody: true as const }; } @@ -419,8 +427,9 @@ it.effect.each(scenarios)("memory writes through PRs: %s", (scenario) => return; } expect(yield* keep).toEqual({ kept: 2, paths: shown }); - const branch = `kin/write/${agentId}/keep-${shadowed.commit.slice(0, 12)}`; - expect(pulls.get(1)?.branch).toBe(branch); + expect(pulls.get(1)?.branch).toMatch( + new RegExp(`^kin/write/${agentId}/keep-${shadowed.commit.slice(0, 12)}-[0-9a-f]{12}$`), + ); const main = yield* bare("rev-parse", "refs/heads/main"); expect(merges).toEqual([ { number: 1, body: expect.objectContaining({ sha: expect.any(String) }) }, @@ -474,6 +483,11 @@ it.effect.each(scenarios)("memory writes through PRs: %s", (scenario) => expect([...pulls.values()].map((pull) => pull.state)).toEqual( scenario === "hang" ? [] : ["closed"], ); + // A retry gets a fresh commit and branch, so the closed PR doesn't block it. + if (scenario === "stall-body") { + yield* writer.commitWakeup(orgId, "wake1", "completed"); + expect(merges.length).toBe(1); + } return; } if ( @@ -508,6 +522,7 @@ it.effect.each(scenarios)("memory writes through PRs: %s", (scenario) => "run-wrong-sha": foreign, "run-wrong-event": foreign, "run-wrong-branch": foreign, + "run-from-fork": foreign, "run-wrong-pr": foreign, "run-wrong-base": foreign, truncated: "GitHub listed more check runs than one page holds", @@ -535,10 +550,11 @@ it.effect.each(scenarios)("memory writes through PRs: %s", (scenario) => expect(yield* bare("log", "-1", "--format=%s", main)).toBe(`kin(${agentId}): wake-up wake1`); expect(yield* remoteFiles(main)).toContain("claims/new.md"); // The merged branch is deleted; the merge was pinned to the commit Kin pushed. + const wakeBranch = new RegExp(`^kin/write/${agentId}/wake1-[0-9a-f]{12}$`); + const auto = [...pulls].find(([, pull]) => wakeBranch.test(pull.branch))!; expect(yield* bare("for-each-ref", "--format=%(refname)", "refs/heads/kin/write/")).toBe( - ownerPath ? `refs/heads/kin/write/${agentId}/wake1-review` : "", + ownerPath ? `refs/heads/${auto[1].branch}-review` : "", ); - const auto = [...pulls].find(([, pull]) => pull.branch === `kin/write/${agentId}/wake1`)!; const autoHead = yield* git(paths.writer, ["rev-parse", `${main}^{tree}`]); expect(merges).toEqual([ { @@ -586,8 +602,10 @@ it.effect.each(scenarios)("memory writes through PRs: %s", (scenario) => } if (reviewed) { - const reviewBranch = `kin/write/${agentId}/wake1-review`; - const review = [...pulls].find(([, pull]) => pull.branch === reviewBranch)!; + const review = [...pulls].find(([, pull]) => + new RegExp(`^kin/write/${agentId}/wake1-[0-9a-f]{12}-review$`).test(pull.branch), + )!; + const reviewBranch = review[1].branch; // Kin never merges the owner's paths; the PR stays open for their review. expect(review[1].state).toBe("open"); expect(merges.map((merge) => merge.number)).not.toContain(review[0]); @@ -667,6 +685,10 @@ it.effect.each(scenarios)("memory writes through PRs: %s", (scenario) => `Kin could not close ${moved}, the memory PR for paths you review, and it is still open. Close it without merging: ${reason}`, ]); expect(yield* records).toEqual([{ branch: reviewBranch, head: reviewHead }]); + // The next sweep still can't close it, and doesn't tell the owner again. + const again = notices.length; + yield* writer.sweepPullRequests(orgId); + expect(notices.length).toBe(again); return; } expect(review[1].state).toBe("closed"); diff --git a/apps/server/src/kin/memory/MemoryPullRequests.ts b/apps/server/src/kin/memory/MemoryPullRequests.ts index be8f49f3f220..fa35ed0da2e2 100644 --- a/apps/server/src/kin/memory/MemoryPullRequests.ts +++ b/apps/server/src/kin/memory/MemoryPullRequests.ts @@ -41,6 +41,7 @@ const WorkflowRuns = Schema.Struct({ event: Schema.String, head_sha: Schema.String, head_branch: Schema.NullOr(Schema.String), + head_repository: Schema.NullOr(Schema.Struct({ full_name: Schema.String })), pull_requests: Schema.Array( Schema.Struct({ number: Schema.Int, base: Schema.Struct({ ref: Schema.String }) }), ), @@ -166,6 +167,7 @@ const isOurRun = ( run.event === "pull_request" && run.head_sha === input.sha && run.head_branch === input.branch && + run.head_repository?.full_name === input.repo && run.pull_requests.some((pull) => pull.number === number && pull.base.ref === input.base); /** @@ -237,6 +239,7 @@ export const make = Effect.fn("MemoryPullRequests.make")(function* (options: { holder: "server", }) .pipe( + Effect.timeout(CALL_TIMEOUT), Effect.mapError((error) => refuse(`the Kin GitHub App gave no token for ${repo} (${error._tag})`), ), @@ -268,11 +271,11 @@ export const make = Effect.fn("MemoryPullRequests.make")(function* (options: { Effect.mapError((error) => refuse(`${reason} (${error.operation})`)), ); const close = Effect.fn("MemoryPullRequests.close")(function* (number: number, reason: string) { - yield* call("issues.comment", "POST", `/issues/${number}/comments`, { - body: `Kin closed this memory PR: ${reason}.`, - }); const closed = yield* call("pulls.close", "PATCH", `/pulls/${number}`, { state: "closed" }); if (closed.status !== 200) return yield* refuse(`GitHub did not close PR #${number}`); + yield* call("issues.comment", "POST", `/issues/${number}/comments`, { + body: `Kin closed this memory PR: ${reason}.`, + }).pipe(Effect.ignore); }); const records = sql<{ branch: string; diff --git a/apps/server/src/kin/memory/MemoryWriter.ts b/apps/server/src/kin/memory/MemoryWriter.ts index 0ce5a795c0d5..97e8cc11a584 100644 --- a/apps/server/src/kin/memory/MemoryWriter.ts +++ b/apps/server/src/kin/memory/MemoryWriter.ts @@ -195,6 +195,7 @@ const render = (fields: object, body: string) => `---\n${stringify(fields)}---\n const make = Effect.gen(function* () { const memory = yield* Memory; const recovery = yield* MemoryRecoveryHooks; + const reportedOpen = new Set(); const authority = yield* MemoryWriterAuthority; const hooks = yield* MemoryWriterHooks; const scanner = yield* KinSecretScan; @@ -609,7 +610,8 @@ const make = Effect.gen(function* () { ownerReview, }); }); - const branch = `${WRITE_BRANCH_PREFIX}${input.agentId}/${input.id}`; + // Each attempt's commit is fresh, so a retry gets its own branch; `land` refuses reused ones. + const branch = `${WRITE_BRANCH_PREFIX}${input.agentId}/${input.id}-${input.commit.slice(0, 12)}`; const landed = yield* open(branch, input.commit, input.message, false).pipe( Effect.tapError((error) => recovery.notify({ @@ -1365,7 +1367,10 @@ const make = Effect.gen(function* () { new MemorySyncConflict({ orgId, operation: "sweep", reason: error.reason }), ), ); - for (const pull of moved) + for (const pull of moved) { + // A PR GitHub keeps refusing to close is reported once per process, not every sweep. + if (!pull.closed && reportedOpen.has(pull.url)) continue; + if (!pull.closed) reportedOpen.add(pull.url); yield* recovery.notify({ orgId, branch: pull.branch, @@ -1373,6 +1378,7 @@ const make = Effect.gen(function* () { ? `Kin closed ${pull.url}, the memory PR for paths you review: ${pull.reason}` : `Kin could not close ${pull.url}, the memory PR for paths you review, and it is still open. Close it without merging: ${pull.reason}`, }); + } }), ), ), diff --git a/docs/kin/specs/lead.md b/docs/kin/specs/lead.md index 38a402cf621d..fcde86ab833a 100644 --- a/docs/kin/specs/lead.md +++ b/docs/kin/specs/lead.md @@ -421,7 +421,7 @@ Branches: at L1 and above the writer commits to the default branch, through a pu Pushes and fetches use the configured URL explicitly, never a local remote name or pushurl. A retry of an already recorded wake-up returns that commit (with `KIN_MEMORY_DIRECT_PUSH`, after pushing it again). Synchronization uses verified ancestry and plumbing-only fast-forwards. Divergent trees merge disjoint edits and append-only journals in process, then use `commit-tree` with both parents. Content conflicts preserve the local head on a conflict branch, reset to the remote anchor and notify the owner. Missing or ambiguous merge bases preserve and notify, then fail closed without resetting; the organization stays blocked until an operator runs `Memory.recover`, and repeated attempts reuse the conflict branch already holding that head. Failed fetches retain the SQLite anchor. Existing writers with mismatched or missing anchors fail closed as `MemoryCheckoutInvalid`. -**Writes to the default branch go through PRs (TRIG-15).** The memory repo's `main` takes no direct push, so the writer pushes its commit to `kin/write//` (`keep-` for Keep) and `MemoryPullRequests` opens a PR with a `server` App token (`contents` and `pull_requests` write, `checks` and `actions` read). Before merging, Kin checks that the PR head is exactly the commit it pushed and that `memory-check` passed in a run it can trace: a `pull_request` run of `.github/workflows/memory-check.yml` on that commit and branch whose `memory-check` jobs all succeeded. A run's `pull_requests` comes from the PRs open with that head now, not from the PR that triggered it, so it can't tie the run to this PR and base. Instead, before reading runs and again just before merging, Kin requires that the branch has only ever had this PR (`pulls?state=all&head=`) and that the PR's issue events hold no `base_ref_changed`; otherwise an agent could run its own `memory-check.yml` through a second PR from the branch, or by moving this PR's base and back. Any workflow can post a `memory-check` check run, even into that run's check suite, so every `memory-check` check run on the commit (`filter=all`) must have the id of one of those jobs, an Actions job's id being its check run's id. Any of these lists running past one page (a full page or a `next` link) fails the check. Then Kin squash-merges through REST with `sha` pinned to that commit. GitHub's auto-merge is not used: it trusts any Actions workflow's `memory-check`. The local default branch and its anchor move only after the merge, to the fetched remote head, which must descend from the merged commit; when the merge parent is the old head, its tree must match Kin's. Changes to paths the owner reviews (`decisions/`, `people/`, `objectives/`, anything outside Kin's own folders) are split into their own PR on `-review`, built on the old head without `INDEX.md` or the journal, opened only after the main PR merged and never merged by Kin. Its body and the owner's notice name the commit Kin pushed, so the owner can close it if the head differs; memory reads see those changes only once the owner merges. Kin records that commit in `kin_memory_review_pulls`, because the memory deploy key can push to the branch. A failed or foreign check, a moved head, another PR from the branch, a changed base, a check still running after 15 minutes or a refused merge closes the PR with a comment, posts the reason and link to the owner's direct chat, and fails the commit with `MemorySyncConflict`, leaving the default branch untouched. Without the Kin App, writes to the default branch are refused and the owner is told; `KIN_MEMORY_DIRECT_PUSH=true` restores direct pushes for a memory repo without the TRIG-15 ruleset. At startup and every 5 minutes, a sweep on the Kin scheduler takes the writer lock and closes, with a comment, every open `kin/write/*` PR no write waits on: any non-review PR (its write already ended) and any review PR whose head isn't the commit Kin recorded. It looks up each recorded review branch by head, so newer PRs can't hide one, and tells the owner about each it closed, or couldn't close and left open; a `-review` PR Kin never recorded is closed with only a comment. Orphans are found on one page of open PRs, newest first, so a flood delays them to a later sweep. GitHub calls under the lock time out after 30 seconds. The writer holds the per-organization lock while it waits for the check, so the next wake-up starts from the merged head; the known cost is that a slow check blocks that organization's wake-ups, Keep and the sweep for up to 15 minutes. +**Writes to the default branch go through PRs (TRIG-15).** The memory repo's `main` takes no direct push, so the writer pushes its commit to `kin/write//-` (`keep--` for Keep, a fresh commit and so a fresh branch on every attempt, since the branch check refuses a branch that ever had another PR) and `MemoryPullRequests` opens a PR with a `server` App token (`contents` and `pull_requests` write, `checks` and `actions` read). Before merging, Kin checks that the PR head is exactly the commit it pushed and that `memory-check` passed in a run it can trace: a `pull_request` run of `.github/workflows/memory-check.yml` on that commit and branch, from the memory repo itself rather than a fork, whose `memory-check` jobs all succeeded. A run's `pull_requests` comes from the PRs open with that head now, not from the PR that triggered it, so it can't tie the run to this PR and base. Instead, before reading runs and again just before merging, Kin requires that the branch has only ever had this PR (`pulls?state=all&head=`) and that the PR's issue events hold no `base_ref_changed`; otherwise an agent could run its own `memory-check.yml` through a second PR from the branch, or by moving this PR's base and back. Any workflow can post a `memory-check` check run, even into that run's check suite, so every `memory-check` check run on the commit (`filter=all`) must have the id of one of those jobs, an Actions job's id being its check run's id. Any of these lists running past one page (a full page or a `next` link) fails the check. Then Kin squash-merges through REST with `sha` pinned to that commit. GitHub's auto-merge is not used: it trusts any Actions workflow's `memory-check`. The local default branch and its anchor move only after the merge, to the fetched remote head, which must descend from the merged commit; when the merge parent is the old head, its tree must match Kin's. Changes to paths the owner reviews (`decisions/`, `people/`, `objectives/`, anything outside Kin's own folders) are split into their own PR on `-review`, built on the old head without `INDEX.md` or the journal, opened only after the main PR merged and never merged by Kin. Its body and the owner's notice name the commit Kin pushed, so the owner can close it if the head differs; memory reads see those changes only once the owner merges. Kin records that commit in `kin_memory_review_pulls`, because the memory deploy key can push to the branch. A failed or foreign check, a moved head, another PR from the branch, a changed base, a check still running after 15 minutes or a refused merge closes the PR with a comment, posts the reason and link to the owner's direct chat, and fails the commit with `MemorySyncConflict`, leaving the default branch untouched. Without the Kin App, writes to the default branch are refused and the owner is told; `KIN_MEMORY_DIRECT_PUSH=true` restores direct pushes for a memory repo without the TRIG-15 ruleset. At startup and every 5 minutes, a sweep on the Kin scheduler takes the writer lock and closes, with a comment, every open `kin/write/*` PR no write waits on: any non-review PR (its write already ended) and any review PR whose head isn't the commit Kin recorded. It looks up each recorded review branch by head, so newer PRs can't hide one, and tells the owner about each it closed, or (once per server run) couldn't close and left open; a `-review` PR Kin never recorded is closed with only a comment. Orphans are found on one page of open PRs, newest first, so a flood delays them to a later sweep. GitHub calls under the lock time out after 30 seconds. The writer holds the per-organization lock while it waits for the check, so the next wake-up starts from the merged head; the known cost is that a slow check blocks that organization's wake-ups, Keep and the sweep for up to 15 minutes. Front matter matches the kin-memory README, including snake_case keys. Date-only values mean UTC midnight; date-times require an explicit UTC offset, and invalid dates make an item malformed. `links` values containing `#` must be quoted. `owner` and `said_by` hold handles; `orgId` and `path` belong only to the server-side `MemoryItemHead`. diff --git a/ops/kin/memory-repo/README.md b/ops/kin/memory-repo/README.md index 38f071266e2d..80624430d08e 100644 --- a/ops/kin/memory-repo/README.md +++ b/ops/kin/memory-repo/README.md @@ -16,7 +16,7 @@ Templates for the private memory repo (`kin-memory`). This public repo holds onl To pick up new lint or scanner rules, bump `T3CODE_REF` in a reviewed PR. The lint code and the gitleaks config always come from that commit, never from the PR's own files, which the job reads only as data. -`scripts/kin-rulesets.ts` applies the ruleset on `main` (PRs required, code-owner review, `memory-check` required, no bypass for Kin's bot) from `KIN_REPOS`, where the memory repo has `kind: "memory"`, which accepts this file's owner-less Kin lines. Kin's own writes then reach `main` as PRs from `kin/write//` branches that it merges itself, and changes to owner paths as `…-review` PRs it leaves for you. Each review PR's body and Kin's message name the commit Kin pushed; the deploy key can push to that branch, so close the PR if its head differs. Kin also closes it within a few minutes when its head moves, and closes any other `kin/write/*` PR no write is waiting on. That needs the Kin App installed on this repo and the repo in `KIN_GITHUB_REPOS`. Without the App, Kin refuses to write `main`; `KIN_MEMORY_DIRECT_PUSH=true` makes it push directly, which this ruleset rejects. +`scripts/kin-rulesets.ts` applies the ruleset on `main` (PRs required, code-owner review, `memory-check` required, no bypass for Kin's bot) from `KIN_REPOS`, where the memory repo has `kind: "memory"`, which accepts this file's owner-less Kin lines. Kin's own writes then reach `main` as PRs from `kin/write//-` branches, one per attempt, that it merges itself, and changes to owner paths as `…-review` PRs it leaves for you. Each review PR's body and Kin's message name the commit Kin pushed; the deploy key can push to that branch, so close the PR if its head differs. Kin also closes it within a few minutes when its head moves, and closes any other `kin/write/*` PR no write is waiting on. That needs the Kin App installed on this repo and the repo in `KIN_GITHUB_REPOS`. Without the App, Kin refuses to write `main`; `KIN_MEMORY_DIRECT_PUSH=true` makes it push directly, which this ruleset rejects. ### The owner's own changes