diff --git a/apps/server/src/kin/memory/Memory.ts b/apps/server/src/kin/memory/Memory.ts index 718a3dec3118..93a32751b0eb 100644 --- a/apps/server/src/kin/memory/Memory.ts +++ b/apps/server/src/kin/memory/Memory.ts @@ -120,6 +120,11 @@ export interface MemoryPaths { export type LeadView = { readonly lead: KinAgentId }; export type MemoryView = LeadView | "triggers"; export const shadowBranch = (agentId: string) => `kin/shadow/${agentId}`; +/** + * The lander's record of its last merged batch: a commit whose parents are GitHub's squash and the + * local commit whose tree that squash carries. It lives in the writer and goes when a purge reclones. + */ +export const LANDED_REF = "refs/kin/landed"; export interface MemoryQuery { readonly kind?: MemoryItemKind; @@ -239,8 +244,9 @@ export class Memory extends Context.Service< ) => Effect.Effect<{ head: string; indexAsOf: DateTime.Utc; read: string }, MemoryFailure>; /** * The writer's integrity check (`git fsck`) and default-branch sync with the remote, off the - * reply path. Skips an organization with no clone yet, or whose last sync failed: its next - * wake-up re-runs the sync and refuses to start until it passes. + * reply path; it reads the remote before taking the writer lock. Skips an organization with no + * clone yet, or whose last sync failed: its next wake-up re-runs the sync and refuses to start + * until it passes. */ readonly syncCheckout: (orgId: KinOrgId) => Effect.Effect; readonly readPromptSnapshot: ( @@ -1408,17 +1414,74 @@ const make = Effect.gen(function* () { /** The remote's default-branch head, fetched with no lock held: fetching only adds objects. */ const fetchRemote = (orgId: KinOrgId, writer: string, url: string): Effect.Effect => advertisedHead(orgId, writer, url, mainBranch).pipe(Effect.result); + /** `fetchRemote` for a caller about to take the writer lock; nothing before the first clone. */ + const prefetch = Effect.fn("Memory.prefetch")(function* (orgId: KinOrgId) { + const { writer, repo } = yield* pathsFor(orgId); + if (!(yield* io(orgId, "exists", fs.exists(writer)))) return undefined; + const generation = historyGenerations.get(orgId) ?? 0; + return { generation, fetched: yield* fetchRemote(orgId, writer, repo.repoUrl) }; + }); + // A purge that ran since the fetch dropped the checkout its objects went to. + const stillFetched = ( + orgId: KinOrgId, + prefetched: Effect.Success>, + ) => + prefetched?.generation === (historyGenerations.get(orgId) ?? 0) + ? prefetched.fetched + : undefined; + + /** + * The local commit whose tree GitHub's newest squash of this server's memory carries, while + * that squash is not merged in here. GitHub squash-merges, so git's ancestry never links the + * two: this commit's tree is the merge base, and local commits after it are changes on top. + */ + const landedBase = Effect.fn("Memory.landedBase")(function* ( + orgId: KinOrgId, + writer: string, + local: ReadonlyMap>, + remote: ReadonlyMap>, + remoteHead: string, + ) { + const commitAt = (sha: string) => + readCommit(orgId, writer, sha).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(VcsProcess, vcs), + ); + const recorded = yield* git(orgId, writer, [ + "rev-parse", + "--verify", + "--quiet", + LANDED_REF, + ]).pipe( + Effect.flatMap((sha) => commitAt(sha.trim())), + Effect.result, + ); + if (recorded._tag === "Success") { + const [theirs, ours] = recorded.success.parents; + // Once the squash is merged in, git's own merge base is at least as new as this record. + if (theirs && ours && remote.has(theirs) && local.has(ours) && !local.has(theirs)) + return ours; + } + // The record is written after GitHub merges. If the server stopped in between, the squash + // still names its commit, and an identical tree proves the pair without trusting that line. + const squash = yield* commitAt(remoteHead); + const named = /^Kin-Local: ([a-f0-9]{40}(?:[a-f0-9]{24})?)$/m.exec(squash.message)?.[1]; + if (named !== undefined && local.has(named) && (yield* commitAt(named)).tree === squash.tree) + return named; + return undefined; + }); - // Callers have run `ensureCheckouts`. The lander passes a head it fetched before taking the - // lock and skips `fsck`, which the periodic sync still runs. + // Callers have run `ensureCheckouts`. A head fetched before the lock was taken is merged as it + // is; the lander skips `fsck`, which the periodic sync and a wake-up's own sync still run. const sync = Effect.fn("Memory.sync")(function* ( orgId: KinOrgId, - wakeupId?: string, - prefetched?: Fetched, + prefetched: Fetched | undefined, + options: { readonly wakeupId?: string; readonly fsck: boolean }, ) { + const { wakeupId } = options; const paths = yield* pathsFor(orgId); yield* validateWriter(orgId, paths); - if (prefetched === undefined) + if (options.fsck) yield* git(orgId, paths.writer, [ "fsck", "--full", @@ -1442,7 +1505,9 @@ const make = Effect.gen(function* () { const localAncestors = yield* walkAncestors(orgId, paths.writer, head); if (localAncestors.has(remote)) next = head; else { - const mergeBase = uniqueMergeBase(localAncestors, remoteAncestors); + const mergeBase = + (yield* landedBase(orgId, paths.writer, localAncestors, remoteAncestors, remote)) ?? + uniqueMergeBase(localAncestors, remoteAncestors); if (!mergeBase) { const conflictBranch = yield* preserve( orgId, @@ -1651,10 +1716,10 @@ const make = Effect.gen(function* () { } return undefined; }); - const checkedSync = (orgId: KinOrgId, wakeupId?: string, prefetched?: Fetched) => - sync(orgId, wakeupId, prefetched).pipe( - Effect.tap(() => Effect.sync(() => synced.set(orgId, true))), - Effect.tapError(() => Effect.sync(() => synced.set(orgId, false))), + const checkedSync = (...args: Parameters) => + sync(...args).pipe( + Effect.tap(() => Effect.sync(() => synced.set(args[0], true))), + Effect.tapError(() => Effect.sync(() => synced.set(args[0], false))), ); const isAncestor = (orgId: KinOrgId, writer: string, older: string, newer: string) => @@ -1687,7 +1752,7 @@ const make = Effect.gen(function* () { ) { yield* ensureCheckouts(orgId); const { writer } = yield* pathsFor(orgId); - const remote = yield* checkedSync(orgId, undefined, fetched); + const remote = yield* checkedSync(orgId, fetched, { fsck: false }); const local = yield* anchor(orgId, mainBranch); if (remote === undefined) return { writer, local, remote }; return { @@ -1705,12 +1770,17 @@ const make = Effect.gen(function* () { orgId: KinOrgId, wakeupId?: string, view?: LeadView, + prefetched?: Fetched, ) { const paths = yield* ensureCheckouts(orgId); if (synced.get(orgId) === true) { yield* validateWriter(orgId, yield* pathsFor(orgId)); yield* refuseUnfinishedOperation(orgId, paths.writer); - } else yield* checkedSync(orgId, wakeupId); + } else + yield* checkedSync(orgId, prefetched, { + fsck: true, + ...(wakeupId === undefined ? {} : { wakeupId }), + }); const branch = config.defaultBranch ?? "main"; const head = yield* anchor(orgId, branch); // The read checkout holds the waking lead's view: its shadow branch at L0. @@ -1919,11 +1989,23 @@ const make = Effect.gen(function* () { ensureCheckouts: Effect.fn("Memory.ensureCheckoutsLocked")((orgId) => locked(orgId, ensureCheckouts(orgId, true)), ), - prepareWakeup: Effect.fn("Memory.prepareWakeup")((orgId, wakeupId, view) => - withWriterLock(orgId, locked(orgId, prepare(orgId, wakeupId, view))), - ), - syncCheckout: Effect.fn("Memory.syncCheckout")((orgId) => - withWriterLock( + prepareWakeup: Effect.fn("Memory.prepareWakeup")(function* (orgId, wakeupId, view) { + // A wake-up that has to sync reads GitHub before it takes the writer lock, like the lander. + const prefetched = synced.get(orgId) === true ? undefined : yield* prefetch(orgId); + return yield* withWriterLock( + orgId, + locked( + orgId, + Effect.suspend(() => prepare(orgId, wakeupId, view, stillFetched(orgId, prefetched))), + ), + ); + }), + syncCheckout: Effect.fn("Memory.syncCheckout")(function* (orgId) { + if (synced.get(orgId) === false) return; + // The network part, outside the writer lock, so a slow GitHub never holds up a lead. + const prefetched = yield* prefetch(orgId); + if (prefetched === undefined) return; + yield* withWriterLock( orgId, locked( orgId, @@ -1932,11 +2014,11 @@ const make = Effect.gen(function* () { const { writer } = yield* pathsFor(orgId); if (!(yield* io(orgId, "exists", fs.exists(writer)))) return; yield* ensureCheckouts(orgId); - yield* checkedSync(orgId); + yield* checkedSync(orgId, stillFetched(orgId, prefetched), { fsck: true }); }), ), - ), - ), + ); + }), readPromptSnapshot: Effect.fn("Memory.readPromptSnapshot")((orgId, expectedHead, view) => locked( orgId, diff --git a/apps/server/src/kin/memory/MemoryLander.ts b/apps/server/src/kin/memory/MemoryLander.ts index 4224032a904e..4cd0ba79b565 100644 --- a/apps/server/src/kin/memory/MemoryLander.ts +++ b/apps/server/src/kin/memory/MemoryLander.ts @@ -11,6 +11,7 @@ import * as Schema from "effect/Schema"; import { VcsProcess } from "../../vcs/VcsProcess.ts"; import { memoryGit } from "./git.ts"; import { + LANDED_REF, Memory, type MemoryConfigError, MemoryError, @@ -37,6 +38,18 @@ export type LandingOutcome = /** Still stuck on the same local head, so nothing was tried. */ | { readonly _tag: "waiting" }; +/** What went wrong, from the error's own tag and reason: a cause can quote git output and URLs. */ +const unexpected = (cause: Cause.Cause) => { + const error: unknown = Cause.squash(cause); + const field = (name: string) => { + const value = + typeof error === "object" && error !== null ? Reflect.get(error, name) : undefined; + return typeof value === "string" ? value : undefined; + }; + const detail = [field("_tag"), field("reason") ?? field("operation")].filter(Boolean).join(": "); + return `Kin hit an error while landing memory${detail ? ` (${detail})` : ""}`; +}; + /** Told after every landing run, for tests and logs; never on the wake-up path. */ export class MemoryLanderHooks extends Context.Reference<{ readonly settled: (orgId: KinOrgId, outcome: LandingOutcome) => Effect.Effect; @@ -79,10 +92,17 @@ const make = Effect.gen(function* () { GIT_COMMITTER_NAME: config.botName ?? "Kin Memory Bot", GIT_COMMITTER_EMAIL: config.botEmail ?? "kin-memory@localhost", }; - // The local head each org's landing stopped on; a new head, or a restart, tries again. - const stuck = new Map(); + // The local head each org's landing stopped on; a new head, or a restart, tries again. With no + // head, the failure came before one was read, and every signal tries again. + const stuck = new Map< + KinOrgId, + { readonly local: string | undefined; readonly reason: string } + >(); + // Review commits whose PR could not be opened, by ref, with the reason the owner last heard. + const unpublished = new Map(); // Review PRs GitHub keeps refusing to close are reported once per process, not every sweep. const reportedOpen = new Set(); + // One server owns a state directory (`serverOwnership.ts`), so no second process lands this repo. const signals = new Map>(); const provide = (effect: Effect.Effect) => @@ -112,7 +132,7 @@ const make = Effect.gen(function* () { const refuse = Effect.fn("MemoryLander.refuse")(function* ( orgId: KinOrgId, - local: string, + local: string | undefined, reason: string, options: { readonly url?: string; readonly told?: boolean } = {}, ) { @@ -222,28 +242,41 @@ const make = Effect.gen(function* () { }); return { url: landed.url, commit }; }).pipe(Effect.result); + if (opened._tag === "Failure") { + // The commit stays under its ref for the next landing run; the owner hears once per reason. + const reason = + "reason" in opened.failure && typeof opened.failure.reason === "string" + ? opened.failure.reason + : opened.failure._tag; + if (unpublished.get(ref) !== reason) + yield* recovery.notify({ + orgId, + branch, + reason: `Memory changes on paths you review were not opened as a PR: ${reason}. Kin keeps them on this server and tries again with the next memory change or restart`, + }); + unpublished.set(ref, reason); + continue; + } + unpublished.delete(ref); yield* recovery.notify({ orgId, branch, - reason: - opened._tag === "Success" - ? `Memory changes on paths you review wait in ${opened.success.url}. Kin pushed \`${opened.success.commit}\`; if the PR's head differs, close it` - : `Memory changes on paths you review were not opened as a PR: ${"reason" in opened.failure && typeof opened.failure.reason === "string" ? opened.failure.reason : opened.failure._tag}`, + reason: `Memory changes on paths you review wait in ${opened.success.url}. Kin pushed \`${opened.success.commit}\`; if the PR's head differs, close it`, }); yield* git(orgId, writer, ["update-ref", "-d", ref, sha]); } }); - const landOnce = Effect.fn("MemoryLander.landOnce")(function* (orgId: KinOrgId) { - const target = githubRepo(orgId); - if (Option.isNone(pullRequests) || target === undefined) return { _tag: "idle" } as const; - // Under the writer lock: sync merges what GitHub merged, so `local` descends from it. - const state = yield* memory.landingState(orgId); - if (state === undefined) return { _tag: "idle" } as const; + const landOnce = Effect.fn("MemoryLander.landOnce")(function* ( + orgId: KinOrgId, + target: NonNullable>, + state: NonNullable>>, + ) { + if (Option.isNone(pullRequests)) return { _tag: "idle" } as const; const { writer, local, remote } = state; if (remote !== undefined && !remote.pending) { - stuck.delete(orgId); yield* openReviews(orgId, writer, remote.head, target); + stuck.delete(orgId); return { _tag: "idle" } as const; } if (stuck.get(orgId)?.local === local) return { _tag: "waiting" } as const; @@ -294,11 +327,6 @@ const make = Effect.gen(function* () { if (landed.success._tag !== "merged") return yield* refuse(orgId, local, "GitHub opened the memory PR for review"); const { url, commit } = landed.success; - yield* git(orgId, writer, ["push", "--", target.repoUrl, `:refs/heads/${branch}`]).pipe( - Effect.catch(() => - Effect.logWarning("Could not delete a merged memory PR branch", { orgId }), - ), - ); // With GitHub's head unmoved, the squash must hold exactly the tree Kin pushed. yield* git(orgId, writer, ["fetch", "--", target.repoUrl, commit]); const [parent, merged] = [ @@ -307,26 +335,66 @@ const make = Effect.gen(function* () { ]; if (parent === remote.head && merged !== tree) return yield* refuse(orgId, local, "GitHub merged a different tree than Kin pushed", { url }); + // One ref to one commit, so both halves move together: GitHub's squash, then the local + // commit whose tree it carries. The next sync merges the squash in from that tree. + const landedAs = yield* git( + orgId, + writer, + ["commit-tree", tree, "-p", commit, "-p", local], + identity, + `kin: landed ${local} as ${commit}\n`, + ); + const before = yield* git(orgId, writer, ["rev-parse", "--verify", "--quiet", LANDED_REF]).pipe( + Effect.option, + ); + yield* git(orgId, writer, ["update-ref", LANDED_REF, landedAs]); stuck.delete(orgId); + yield* git(orgId, writer, ["push", "--", target.repoUrl, `:refs/heads/${branch}`]).pipe( + Effect.catch(() => + Effect.logWarning("Could not delete a merged memory PR branch", { orgId }), + ), + ); + // Local commits since the batch before; with no record yet, every one GitHub has only squashed. + const commits = yield* git(orgId, writer, [ + "rev-list", + "--count", + "--no-merges", + local, + `^${remote.head}`, + ...(Option.isSome(before) ? [`^${before.value}^2`] : []), + ]).pipe(Effect.orElseSucceed(() => "unknown")); + yield* Effect.logInfo("Kin memory batch landed", { + orgId, + commits, + pullRequest: url.slice(url.lastIndexOf("/") + 1), + }); return { _tag: "merged", url, commit } as const; }); - const land = Effect.fn("MemoryLander.land")(function* (orgId: KinOrgId) { + const land = Effect.fn("MemoryLander.land")(function* (orgId: KinOrgId, atStart = false) { for (let batch = 0; batch < MAX_BATCHES_PER_RUN; batch++) { - const outcome = yield* landOnce(orgId).pipe( - Effect.catchCause((cause) => - Effect.gen(function* () { - yield* Effect.logWarning("Kin memory landing failed", { + // The head this attempt read: a failure is held against it, never against a later commit. + let attempted: string | undefined; + const outcome = yield* Effect.gen(function* () { + const target = githubRepo(orgId); + if (target === undefined) return { _tag: "idle" } as const; + // Under the writer lock: sync merges what GitHub merged, so `local` descends from it. + const state = yield* memory.landingState(orgId); + if (state === undefined) return { _tag: "idle" } as const; + attempted = state.local; + return yield* landOnce(orgId, target, state); + }).pipe( + Effect.catchCauseIf( + (cause) => !Cause.hasInterruptsOnly(cause), + (cause) => + Effect.logWarning("Kin memory landing failed", { orgId, cause: Cause.pretty(cause), - }); - const head = yield* memory.anchor(orgId, main).pipe(Effect.option); - if (Option.isSome(head)) - stuck.set(orgId, { local: head.value, reason: "landing failed" }); - return { _tag: "stuck", reason: "landing failed" } as const; - }), + }).pipe(Effect.andThen(refuse(orgId, attempted, unexpected(cause)))), ), ); + if (atStart && batch === 0 && outcome._tag === "idle") + yield* Effect.logInfo("Kin memory lander started with nothing to land", { orgId }); yield* hooks.settled(orgId, outcome); if (outcome._tag !== "merged") return; } @@ -370,19 +438,24 @@ const make = Effect.gen(function* () { cause: Cause.pretty(cause), }), ), - Effect.andThen(land(orgId)), + Effect.andThen(land(orgId, true)), ); - yield* memory - .withLandingLock(orgId, resume) - .pipe( - Effect.andThen( - Queue.take(queue).pipe( - Effect.andThen(memory.withLandingLock(orgId, land(orgId))), - Effect.forever, - ), + // A defect in one run must not end the loop: later signals would land nothing. + const run = (atStart: boolean) => + memory.withLandingLock(orgId, atStart ? resume : land(orgId)).pipe( + Effect.catchCauseIf( + (cause) => !Cause.hasInterruptsOnly(cause), + (cause) => + Effect.logError("Kin memory lander run ended abnormally", { + orgId, + cause: Cause.pretty(cause), + }), ), - Effect.forkScoped, ); + yield* run(true).pipe( + Effect.andThen(Queue.take(queue).pipe(Effect.andThen(run(false)), Effect.forever)), + Effect.forkScoped, + ); } return MemoryLander.of({ diff --git a/apps/server/src/kin/memory/MemoryPullRequests.test.ts b/apps/server/src/kin/memory/MemoryPullRequests.test.ts index b5fc9f4faa6f..6829de462e68 100644 --- a/apps/server/src/kin/memory/MemoryPullRequests.test.ts +++ b/apps/server/src/kin/memory/MemoryPullRequests.test.ts @@ -3,6 +3,7 @@ import { expect, it } from "@effect/vitest"; import { KinAgentId, KinMemoryItemId, KinOrgId } from "@t3tools/contracts"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; @@ -72,6 +73,13 @@ const scenarios = [ "no-app", "batched", "restart", + "repeat-edit", + "owner-edit", + "owner-conflict", + "first-run", + "land-fails", + "review-retry", + "sync-fetch", ] as const; // `it.effect`: real git, but GitHub's silences only end when the test clock moves. @@ -107,11 +115,55 @@ it.effect.each(scenarios)("memory writes through PRs: %s", (scenario) => let stalls = 0; let jobsListed = false; let unclosable: number | undefined; - // `batched`: memory-check stays pending on the first PR until the test releases it. + // memory-check stays pending on the first PR until the test releases it. + const heldInCi = ["batched", "repeat-edit", "owner-edit", "owner-conflict"].includes(scenario); const firstPoll = yield* Deferred.make(); let ciDone = false; - // `restart`: GitHub refuses new PRs until the server restarts. - let createFails = scenario === "restart"; + // GitHub refuses new PRs until the server restarts. + let createFails = scenario === "restart" || scenario === "first-run"; + let reviewFails = scenario === "review-retry"; + const journalPath = "journal/2026/11-08.md"; + // The owner changes memory on GitHub itself, as a PR of their own would once merged. + let ownerHead: string | undefined; + const ownerCommits = (files: Record string>) => + Effect.gen(function* () { + yield* git(source, ["fetch", "--", remote, "main"]); + yield* git(source, ["reset", "--hard", "FETCH_HEAD"]); + for (const [path, edit] of Object.entries(files)) { + const file = `${source}/${path}`; + yield* fs.makeDirectory(file.slice(0, file.lastIndexOf("/")), { recursive: true }); + const current = yield* fs.readFileString(file).pipe(Effect.orElseSucceed(() => "")); + yield* fs.writeFileString(file, edit(current)); + } + yield* git(source, ["add", "--", "."]); + yield* git(source, [...identity, "commit", "-m", "owner edit"]); + yield* git(source, ["push", "--", remote, "HEAD:main"]); + ownerHead = yield* git(source, ["rev-parse", "HEAD"]); + }); + // `sync-fetch` holds every read of GitHub's head; `land-fails` breaks the batch commit. + const fetching = yield* Deferred.make(); + const fetchReleased = yield* Deferred.make(); + let holdFetch = false; + const attempting = yield* Deferred.make(); + const attemptReleased = yield* Deferred.make(); + let failLanding = scenario === "land-fails"; + const gatedGit = Layer.succeed( + Vcs.VcsProcess, + Vcs.VcsProcess.of({ + run: Effect.fn("test.memoryPullRequests.gatedGit")(function* (input) { + if (holdFetch && input.args.includes("ls-remote")) { + yield* Deferred.succeed(fetching, undefined); + yield* Deferred.await(fetchReleased); + } + if (failLanding && input.stdin?.startsWith("kin: memory from this server")) { + yield* Deferred.succeed(attempting, undefined); + yield* Deferred.await(attemptReleased); + return yield* Effect.die("fixture: the disk is full"); + } + return yield* vcs.run(input); + }), + }), + ); const branchHead = (branch: string) => bare("rev-parse", `refs/heads/${branch}`); const pullBody = (number: number) => Effect.gen(function* () { @@ -134,7 +186,8 @@ it.effect.each(scenarios)("memory writes through PRs: %s", (scenario) => const query = new URL(path, "https://github.invalid").searchParams; const body = request.body as Record | undefined; if (request.method === "POST" && path === "/pulls") { - if (createFails) return { status: 500, body: { message: "Server Error" } }; + if (createFails || (reviewFails && body!.head!.endsWith("-review"))) + return { status: 500, body: { message: "Server Error" } }; if (scenario === "hang") { yield* Deferred.succeed(silenced, undefined); return { status: 0, hang: true as const }; @@ -208,6 +261,18 @@ it.effect.each(scenarios)("memory writes through PRs: %s", (scenario) => ); yield* bare("update-ref", "refs/heads/main", squash, parent); state.state = "closed"; + // Right after the first batch merges, before Kin reads GitHub again. + if (merges.length === 1 && scenario === "owner-edit") + yield* ownerCommits({ + "claims/owner.md": () => + fixtureItem({ id: "owner", kind: "claim", title: "Claim owner", links: [] }), + [journalPath]: (current) => `${current}\n## Owner note\n`, + }); + if (merges.length === 1 && scenario === "owner-conflict") + yield* ownerCommits({ + "claims/new.md": (current) => + current.replace("Claim new", "Claim the owner reworded"), + }); return { status: 200, body: { merged: true, sha: squash } }; } if (request.method === "GET" && /^\/pulls\?state=open&per_page=/.test(path)) { @@ -272,7 +337,7 @@ it.effect.each(scenarios)("memory writes through PRs: %s", (scenario) => const jobs = /^\/actions\/runs\/(\d+)\/jobs\?/.exec(path); if (jobs) { jobsListed = true; - const pending = scenario === "check-pending" || (scenario === "batched" && !ciDone); + const pending = scenario === "check-pending" || (heldInCi && !ciDone); // `check-failed` fails only the first PR, so a later commit's attempt can land. const failed = (scenario === "check-failed" && pulls.size === 1) || scenario === "owner-path-refused"; @@ -294,7 +359,7 @@ it.effect.each(scenarios)("memory writes through PRs: %s", (scenario) => const answered = yield* Effect.forEach(listed, job); // The poll's git reads are done, so moving the test clock times out no subprocess. if (pending) yield* Deferred.succeed(silenced, undefined); - if (pending && scenario === "batched") yield* Deferred.succeed(firstPoll, undefined); + if (pending && heldInCi) yield* Deferred.succeed(firstPoll, undefined); return { status: 200, body: { total_count: listed.length, jobs: answered } }; } const checks = /^\/commits\/([a-f0-9]+)\/check-runs\?(.*)$/.exec(path); @@ -361,8 +426,7 @@ it.effect.each(scenarios)("memory writes through PRs: %s", (scenario) => const pullRequests = Layer.effect( PullRequests.MemoryPullRequests, PullRequests.make({ - pollInterval: - scenario === "check-pending" ? "20 minutes" : scenario === "batched" ? "1 minute" : 0, + pollInterval: scenario === "check-pending" ? "20 minutes" : heldInCi ? "1 minute" : 0, }), ).pipe(Layer.provide(Layer.mergeAll(app, github.layer))); @@ -407,6 +471,7 @@ it.effect.each(scenarios)("memory writes through PRs: %s", (scenario) => Layer.provide(auth), Layer.provide(Scan.layer), Layer.provide(landerHooks), + Layer.provide(gatedGit), ); // The first landing run that did something; runs with nothing to land are skipped. const landing = Effect.gen(function* () { @@ -438,10 +503,21 @@ it.effect.each(scenarios)("memory writes through PRs: %s", (scenario) => const notMerged = () => notices.filter((notice) => notice.reason.startsWith("Memory write not merged")); + // The periodic sync is still waiting on GitHub when the lead stages and commits. + holdFetch = scenario === "sync-fetch"; + const sync = + scenario === "sync-fetch" + ? yield* memory.syncCheckout(orgId).pipe(Effect.forkChild) + : undefined; + if (sync !== undefined) yield* Deferred.await(fetching); + yield* writer.stage(orgId, "wake1", claim("new")); const reviewed = scenario === "owner-path" || scenario === "owner-path-close-fails"; const ownerPath = - reviewed || scenario === "owner-path-refused" || scenario === "records-broken"; + reviewed || + scenario === "owner-path-refused" || + scenario === "records-broken" || + scenario === "review-retry"; const sql = yield* SqlClient.SqlClient; const records = sql`SELECT branch, head FROM kin_memory_review_pulls ORDER BY branch`; if (scenario === "records-broken") @@ -471,6 +547,11 @@ it.effect.each(scenarios)("memory writes through PRs: %s", (scenario) => expect(yield* localMain()).toBe(committed.commit); expect(yield* memory.anchor(orgId, "main")).toBe(committed.commit); expect(yield* git(paths.writer, ["rev-parse", `${committed.commit}^`])).toBe(original); + if (sync !== undefined) { + holdFetch = false; + yield* Deferred.succeed(fetchReleased, undefined); + yield* Fiber.join(sync); + } // A refused landing undoes nothing: this server keeps the commit and leads read it. const kept = Effect.gen(function* () { @@ -520,6 +601,146 @@ it.effect.each(scenarios)("memory writes through PRs: %s", (scenario) => return; } + if (scenario === "first-run") { + expect(yield* landing).toMatchObject({ _tag: "stuck" }); + // As on a server from before the landed record: GitHub holds a squash of this server's + // commits, with the same tree, and nothing here says so. + const pushed = yield* bare( + "for-each-ref", + "--format=%(objectname)", + "refs/heads/kin/write/land/", + ); + const squash = yield* bare( + ...identity, + "commit-tree", + `${pushed}^{tree}`, + "-p", + original, + "-m", + "squash", + ); + yield* bare("update-ref", "refs/heads/main", squash, original); + // And as there, the writer's own index is stale, so git reports staged changes in it. + yield* fs.writeFileString(`${paths.writer}/INDEX.md`, "stale\n"); + yield* git(paths.writer, ["add", "--", "INDEX.md"]); + expect(yield* git(paths.writer, ["status", "--porcelain"])).toContain("D claims/new.md"); + createFails = false; + return; + } + + if (scenario === "land-fails") { + // The first attempt is under way on wake1's commit when wake2 commits. + yield* Deferred.await(attempting); + yield* writer.stage(orgId, "wake2", claim("two")); + yield* writer.commitWakeup(orgId, "wake2", "completed"); + yield* Deferred.succeed(attemptReleased, undefined); + expect(yield* landing).toMatchObject({ _tag: "stuck" }); + // wake2's commit was never tried, so its signal tries it instead of waiting on wake1's. + expect(yield* landing).toMatchObject({ _tag: "stuck" }); + expect(yield* bare("rev-parse", "refs/heads/main")).toBe(original); + // The owner hears once: memory is only on this server, and the same failure stays quiet. + expect(notMerged()).toEqual([ + expect.objectContaining({ + reason: expect.stringContaining("Memory is still saved on this server"), + }), + ]); + yield* lander.signal(orgId); + expect(yield* landing).toEqual({ _tag: "waiting" }); + failLanding = false; + yield* writer.stage(orgId, "wake3", claim("three")); + yield* writer.commitWakeup(orgId, "wake3", "completed"); + expect(yield* landing).toMatchObject({ _tag: "merged" }); + yield* settled; + for (const id of ["new", "two", "three"]) + expect(yield* remoteFiles("main")).toContain(`claims/${id}.md`); + expect(notMerged()).toHaveLength(1); + return; + } + + if ( + scenario === "repeat-edit" || + scenario === "owner-edit" || + scenario === "owner-conflict" + ) { + yield* Deferred.await(firstPoll); + // memory-check is still running on wake1's batch when wake2 commits. + yield* writer.stage( + orgId, + "wake2", + scenario === "owner-edit" + ? claim("two") + : { + ...claim("new"), + content: fixtureItem({ + id: "new", + kind: "claim", + title: "Claim new", + links: [], + status: "done", + }), + }, + ); + const second = yield* writer.commitWakeup(orgId, "wake2", "completed"); + ciDone = true; + yield* TestClock.adjust("1 minute"); + expect(yield* landing).toMatchObject({ _tag: "merged" }); + const firstBatch = (merges[0]!.body as { sha: string }).sha; + expect(yield* bare("show", `${firstBatch}:claims/new.md`)).toContain("status: open"); + + if (scenario === "owner-conflict") { + // The owner reworded the item on GitHub that wake2 closed here: a real conflict. + expect(yield* landing).toMatchObject({ + _tag: "stuck", + reason: expect.stringContaining("could not combine"), + }); + expect(yield* localMain()).toBe(second.commit); + expect(yield* bare("rev-parse", "refs/heads/main")).toBe(ownerHead); + expect(pulls.size).toBe(1); + const told = () => + notices.filter((notice) => notice.reason.includes("could not combine")); + expect(told()).toHaveLength(1); + yield* lander.signal(orgId); + expect(yield* landing).toEqual({ _tag: "waiting" }); + expect(told()).toHaveLength(1); + expect(notices).toHaveLength(1); + return; + } + + // wake2's commit is a later local change on top of what landed, not a second writer. + expect(yield* landing).toMatchObject({ _tag: "merged" }); + yield* settled; + expect(merges.map((merge) => merge.number)).toEqual([1, 2]); + expect(notices).toEqual([]); + const head = yield* localMain(); + expect(yield* memory.anchor(orgId, "main")).toBe(head); + // Both wake-ups' own commits are still in local history. + expect(yield* writer.commitWakeup(orgId, "wake1", "completed")).toEqual(committed); + expect(yield* writer.commitWakeup(orgId, "wake2", "completed")).toEqual(second); + const remoteJournal = yield* bare("show", `main:${journalPath}`); + expect(yield* git(paths.writer, ["show", `${head}:${journalPath}`])).toBe(remoteJournal); + if (scenario === "repeat-edit") { + expect(yield* bare("show", "main:claims/new.md")).toContain("status: done"); + expect(remoteJournal.match(/wake-up wake\d/g)).toEqual([ + "wake-up wake1", + "wake-up wake2", + ]); + return; + } + // The owner's item and journal note are on both sides, and wake2's entry follows them once. + expect(yield* remoteFiles("main")).toContain("claims/owner.md"); + expect(yield* remoteFiles("main")).toContain("claims/two.md"); + expect((yield* memory.prepareWakeup(orgId)).head).toBe(head); + expect((yield* memory.readItem(orgId, KinMemoryItemId.make("owner"))).head.id).toBe( + "owner", + ); + expect(remoteJournal.match(/wake-up wake\d|Owner note/g)).toEqual([ + "wake-up wake1", + "Owner note", + "wake-up wake2", + ]); + return; + } + if (scenario === "batched") { yield* Deferred.await(firstPoll); // memory-check is still running on wake1's batch: another lead prepares and commits, @@ -559,7 +780,9 @@ it.effect.each(scenarios)("memory writes through PRs: %s", (scenario) => !reviewed && scenario !== "merged" && scenario !== "rerun" && - scenario !== "records-broken" + scenario !== "records-broken" && + scenario !== "review-retry" && + scenario !== "sync-fetch" ) { const foreign = "did not come from .github/workflows/memory-check.yml"; if (scenario === "check-pending") { @@ -685,11 +908,36 @@ it.effect.each(scenarios)("memory writes through PRs: %s", (scenario) => return; } + if (scenario === "review-retry") { + const failed = () => + notices.filter((notice) => notice.reason.includes("were not opened as a PR")); + const waiting = () => + git(paths.writer, ["for-each-ref", "--format=%(refname)", "refs/kin/review/"]); + // GitHub refused the review PR: the owner hears once and the commit waits for a retry. + expect(failed()).toHaveLength(1); + expect(yield* waiting()).toMatch(new RegExp(`^refs/kin/review/${agentId}/wake1-`)); + yield* lander.signal(orgId); + yield* settled; + expect(failed()).toHaveLength(1); + expect(yield* waiting()).not.toBe(""); + reviewFails = false; + yield* lander.signal(orgId); + yield* settled; + const review = [...pulls].find(([, pull]) => reviewBranch.test(pull.branch))!; + expect(review[1].state).toBe("open"); + expect(notices.map((notice) => notice.reason)).toContainEqual( + expect.stringContaining(`wait in https://github.com/${repo}/pull/${review[0]}`), + ); + expect(yield* waiting()).toBe(""); + expect(yield* records).toHaveLength(1); + return; + } + if (scenario === "records-broken") { const reviewNumber = [...pulls].find(([, pull]) => pull.branch.endsWith("-review"))![0]; expect(pulls.get(reviewNumber)?.state).toBe("closed"); expect(notices.map((notice) => notice.reason)).toContainEqual( - "Memory changes on paths you review were not opened as a PR: Kin could not record the commit it pushed (MemoryPullRequests.recordReview)", + "Memory changes on paths you review were not opened as a PR: Kin could not record the commit it pushed (MemoryPullRequests.recordReview). Kin keeps them on this server and tries again with the next memory change or restart", ); const sweepRefusal = lander.sweep(orgId).pipe(Effect.flip); yield* sql`DROP TRIGGER no_insert`; @@ -804,6 +1052,35 @@ it.effect.each(scenarios)("memory writes through PRs: %s", (scenario) => expect(yield* records).toEqual([]); }).pipe(Effect.provide(writerLayer)); + if (scenario === "first-run") { + // With matching trees there is nothing to land: the squash is merged in and no PR opens. + yield* Effect.gen(function* () { + const memory = yield* Memory.Memory; + yield* settled; + expect(github.requests.filter((request) => request.path.endsWith("/pulls"))).toHaveLength( + 1, + ); + expect(merges).toEqual([]); + const squash = yield* bare("rev-parse", "refs/heads/main"); + const head = yield* memory.anchor(orgId, "main"); + expect( + (yield* git(`${root}/state/memory`, ["show", "-s", "--format=%P", head])).split(" ")[0], + ).toBe(squash); + expect(yield* bare("ls-tree", "-r", "--name-only", "main")).toContain("claims/new.md"); + // The next wake-up lands on top, and nothing staged in the writer's index goes with it. + const writer = yield* Writer.MemoryWriter; + yield* memory.prepareWakeup(orgId); + yield* writer.stage(orgId, "wake2", claim("later")); + yield* writer.commitWakeup(orgId, "wake2", "completed"); + expect(yield* landing).toMatchObject({ _tag: "merged" }); + yield* settled; + const landed = yield* bare("ls-tree", "-r", "--name-only", "main"); + expect(landed).toContain("claims/new.md"); + expect(landed).toContain("claims/later.md"); + expect(yield* bare("show", "main:INDEX.md")).not.toContain("stale"); + }).pipe(Effect.provide(writerLayer)); + } + if (scenario === "restart") { // The restart closes the PR the crash left open before it lands anything, then lands the // commits the last run could not. diff --git a/apps/server/src/kin/memory/git.ts b/apps/server/src/kin/memory/git.ts index 9f8eb679dce6..4d361564035a 100644 --- a/apps/server/src/kin/memory/git.ts +++ b/apps/server/src/kin/memory/git.ts @@ -13,7 +13,7 @@ export class MemoryGitMissingRepository extends Schema.TaggedError(); diff --git a/apps/server/src/kin/memory/merge.test.ts b/apps/server/src/kin/memory/merge.test.ts index 812106316286..ee481949ae88 100644 --- a/apps/server/src/kin/memory/merge.test.ts +++ b/apps/server/src/kin/memory/merge.test.ts @@ -59,6 +59,15 @@ it.each([ "original\nlanded\nowner\n", "original\nlanded\nowner\n", ], + // The landed entry is in both copies, and each side then added one of its own. + [ + "both grew since", + "journal/day.md", + "original\n\n## 10:00 landed\n\n## 10:05 later\n", + "original\n\n## 10:00 landed\n\n## 10:07 owner\n", + "original\n\n## 10:00 landed\n\n## 10:07 owner\n\n## 10:05 later\n", + ], + ["same journal rewrite", "journal/day.md", "rewritten\n", "rewritten\n", "rewritten\n"], ])("merges back a landed batch: %s", (_name, path, ours, theirs, expected) => { const base = new Map([ [path, Buffer.from(path.startsWith("journal/") ? "original\n" : "before\n")], diff --git a/apps/server/src/kin/memory/merge.ts b/apps/server/src/kin/memory/merge.ts index 46740dd54675..48063cd0137f 100644 --- a/apps/server/src/kin/memory/merge.ts +++ b/apps/server/src/kin/memory/merge.ts @@ -10,6 +10,15 @@ export const uniqueMergeBase = ( return candidates.size === 1 ? [...candidates][0] : undefined; }; +/** Where the first line that differs starts, with the blank line that opens its journal entry. */ +const unlandedFrom = (journal: Buffer, differs: number, floor: number) => { + let start = differs; + while (start > floor && journal[start - 1] !== 10) start--; + return start > floor && journal.subarray(start - 1, start + 3).toString("latin1") === "\n## " + ? start - 1 + : start; +}; + export const mergeMemoryFiles = ( base: ReadonlyMap, local: ReadonlyMap, @@ -30,10 +39,10 @@ export const mergeMemoryFiles = ( const original = base.get(file) ?? Buffer.alloc(0); const bytes = local.get(file); const journal = file.startsWith("journal/"); - if (journal && (!bytes || !bytes.subarray(0, original.length).equals(original))) - return undefined; // Both sides made this change, as when the remote squash-merged a batch of local commits. if (same(remote.get(file), bytes) && after?.mode === modes.get(file)) continue; + if (journal && (!bytes || !bytes.subarray(0, original.length).equals(original))) + return undefined; if (same(remote.get(file), base.get(file)) && before?.mode === modes.get(file)) { if (bytes) { files.set(file, bytes); @@ -50,10 +59,16 @@ export const mergeMemoryFiles = ( ) { const theirs = remote.get(file)!; const ours = bytes!; - // One side already holds the other's entries when a landed batch is merged back. - if (ours.subarray(0, theirs.length).equals(theirs)) files.set(file, ours); - else if (!theirs.subarray(0, ours.length).equals(ours)) - files.set(file, Buffer.concat([theirs, ours.subarray(original.length)])); + // Entries both sides hold are kept once, even when the base predates them: a landed batch + // comes back in GitHub's copy, and only what follows it here is still unlanded. + let shared = original.length; + while (shared < ours.length && ours[shared] === theirs[shared]) shared++; + if (shared === theirs.length) files.set(file, ours); + else if (shared < ours.length) + files.set( + file, + Buffer.concat([theirs, ours.subarray(unlandedFrom(ours, shared, original.length))]), + ); } else return undefined; } return { files, modes }; diff --git a/docs/kin/specs/lead.md b/docs/kin/specs/lead.md index 33139e1a3d86..8694a2f962ce 100644 --- a/docs/kin/specs/lead.md +++ b/docs/kin/specs/lead.md @@ -232,7 +232,7 @@ Each wake-up: 1. `KinStop.isPaused(orgId)` → leave events queued and return. Before claiming on an existing thread, defer while any provider turn remains unfinished. 2. `KinBudget.admit(orgId, batchPriority)` → if refused, leave lower-priority events `queued` with `not_before` = next budget reset and run what remains. 3. Pick the model: `models.strong` if the batch holds a P0 or P1 event, a contradiction question or a `routine`; otherwise `models.cheap`. -4. `Memory.prepareWakeup(orgId)`: validate the writer (its tip matches the SQLite anchor, no unfinished git operation), build that lead's read checkout from scratch beside the old one at the anchored head and swap it in with two renames (old aside, new in), so the directory is the old checkout, the new one or briefly absent, never a mix, and rebuild `INDEX.md` first if its "As of" line is older than 36 hours. The integrity check and the default-branch sync with the remote (`Memory.syncCheckout`: `fsck`, then `ls-remote`, fetch and merge as below) run off the reply path, at start and every five minutes. The first wake-up after a start, or after a failed sync, runs that sync inline first and refuses to start while it fails, so a bad checkout is refused exactly as before; only the timing of the check moved. Network git (`ls-remote`, `fetch`, `push`) times out after 30 s, since it holds the writer lock. +4. `Memory.prepareWakeup(orgId)`: validate the writer (its tip matches the SQLite anchor, no unfinished git operation), build that lead's read checkout from scratch beside the old one at the anchored head and swap it in with two renames (old aside, new in), so the directory is the old checkout, the new one or briefly absent, never a mix, and rebuild `INDEX.md` first if its "As of" line is older than 36 hours. The integrity check and the default-branch sync with the remote (`Memory.syncCheckout`: `fsck`, then `ls-remote`, fetch and merge as below) run off the reply path, at start and every five minutes. The first wake-up after a start, or after a failed sync, runs that sync inline first and refuses to start while it fails, so a bad checkout is refused exactly as before; only the timing of the check moved. Both syncs read GitHub's head (`ls-remote`, `fetch`) before taking the writer lock, as the lander does, and then validate the writer and merge that head under it, so a slow GitHub never holds up another lead's stage or commit. A purge that ran in between drops the fetched head and the sync fetches again under the lock. Network git (`ls-remote`, `fetch`, `push`) times out after 30 s, since some of it still holds the writer lock: a shadow branch's push, a conflict branch, the first clone. 5. Build the prompt (below). 6. First wake-up: `ThreadLaunchService.launch({ commandId: "kin-wakeup:", projectId, title: "", modelSelection, runtimeMode: "full-access", interactionMode: "default", workspaceStrategy: { type: "root" }, initialMessage: { text: prompt, attachments: [] }, createdBy: "system", creationSource: "server" })`, then store `threadId` on `KinLead`. Later wake-ups: `ThreadManagementService.sendToThread({ commandId, threadId, messageId: "kin-wakeup-msg:", text: prompt, mode: "queue", freshContext: true, modelSelection, createdBy: "system", creationSource: "server" })`. One thread holds the lead's history for people to read; every wake-up starts a fresh provider process with its cwd set to the read clone, as LEAD-3 requires. 7. Register the durable lead thread through `KinIdentity.registerAgentThread` before launching or sending; keep it registered between wake-ups and call `endAgentThread` on retirement. `KinIdentity.resolveAgent` and the `insertMessage` write transaction additionally require a persisted running wake-up matching the org, agent and server-resolved caller thread, whose active run belongs to the wake-up (including restart and usage-limit continuations), whose root is exactly `kin_wakeups.run_id` when recorded, otherwise the lowest-ordinal run on that thread with `kin-wakeup-msg:`. The wake-up also adopts later runs whose root message is `kin-steer-msg::` (a late steer) or `kin-rescue-msg:` (the reply rescue), with an ordinal above the recorded `run_id`; the helpers live in `KinIdentity.ts`. Its persisted message, including an adopted run's, must belong to that root and have `createdBy=system` and `creationSource=server`, and every continuation's message must have `createdBy=agent` and `creationSource=server`. Other paths can produce that provenance (server notification wakes, and client-written scheduled tasks, whose upsert accepts `createdBy`/`creationSource`), but none of them sets a restart or usage-limit continuation link: client dispatches that set one are forced to `createdBy=user`, so only the server's recovery workers produce both. The id itself is client-suppliable, and durable registration is not authority outside a wake-up. Webhook inbox threads (`kin_webhook_sources.inbox_thread_id`) cannot be registered, even across organizations. Register `threadId → { orgId, agentId, wakeupId, level }` in `KinLeadRegistry`, a scoped overlap guard, not a tool authority source. It is filled before the first launch or later send and cleared after the run ends, including interruption. Retired leads never launch: their queued events become `expired` and their durable thread registration is ended. @@ -464,9 +464,9 @@ Staged bytes live only in server memory. Discard requires no checkout validation Branches: in chats the owner placed the lead in, the writer commits to the default branch, through a pull request (below). A wake-up from any other chat commits to `kin/shadow/` (see Permissions). Dreaming (triggers spec) commits to `dream/` branches and takes the same per-organization semaphore while it writes, so it never runs during a wake-up. -Pushes and fetches use the configured URL explicitly, never a local remote name or pushurl. A retry of an already recorded wake-up returns that commit (with `KIN_MEMORY_DIRECT_PUSH`, after pushing it again). Synchronization uses verified ancestry and plumbing-only fast-forwards. Divergent trees merge disjoint edits and append-only journals in process, then use `commit-tree` with both parents. Content conflicts keep the local head, which may hold commits the remote never got, and tell the owner once per pair of heads; the lander then can't land until the two are reconciled. Sync never pushes the default branch. Missing or ambiguous merge bases preserve and notify, then fail closed without resetting; the organization stays blocked until an operator runs `Memory.recover`, and repeated attempts reuse the conflict branch already holding that head. Failed fetches retain the SQLite anchor. Existing writers with mismatched or missing anchors fail closed as `MemoryCheckoutInvalid`. +Pushes and fetches use the configured URL explicitly, never a local remote name or pushurl. A retry of an already recorded wake-up returns that commit (with `KIN_MEMORY_DIRECT_PUSH`, after pushing it again). Synchronization uses verified ancestry and plumbing-only fast-forwards. Divergent trees merge disjoint edits and append-only journals in process, then use `commit-tree` with both parents. The merge base is git's unique merge base, except while GitHub's newest squash of this server's commits isn't merged in here: git's ancestry doesn't link a squash to the commits it carries, so the base is then the local commit whose tree that squash holds, read from `refs/kin/landed` (below). Local commits after it are changes on top of what landed, not a second writer. Content conflicts keep the local head, which may hold commits the remote never got, and tell the owner once per pair of heads; the lander then can't land until the two are reconciled. Sync never pushes the default branch. Missing or ambiguous merge bases preserve and notify, then fail closed without resetting; the organization stays blocked until an operator runs `Memory.recover`, and repeated attempts reuse the conflict branch already holding that head. Failed fetches retain the SQLite anchor. Existing writers with mismatched or missing anchors fail closed as `MemoryCheckoutInvalid`. -**Writes to the default branch commit here and land on GitHub in the background (TRIG-15).** A wake-up's memory is done once `commitWakeup` (or the owner's Undo) commits it to the local default branch and moves the anchor; the writer then signals `MemoryLander` and returns, so nothing in a wake-up waits on GitHub and the writer lock covers only local git work. The memory repo's `main` takes no direct push, so the lander, one per organization and one batch at a time, takes everything the local branch holds that the remote's lacks as one PR: it fetches GitHub's head with no lock held, merges it in under the writer lock (the sync below, without `fsck`), then outside the lock builds one commit on the remote head with the local tree and pushes it to `kin/write/land/` (a fresh commit and branch on every attempt, since the branch check refuses a branch that ever had another PR). Commits made while a batch waits on CI ride in the next batch. `MemoryPullRequests` opens the PR with a `server` App token (`contents` and `pull_requests` write, `checks` and `actions` read). Before merging, Kin checks that the PR head is exactly the commit it pushed and that `memory-check` passed in a run it can trace: a `pull_request` run of `.github/workflows/memory-check.yml` on that commit and branch, from the memory repo itself rather than a fork, whose `memory-check` jobs all succeeded. A run's `pull_requests` comes from the PRs open with that head now, not from the PR that triggered it, so it can't tie the run to this PR and base. Instead, before reading runs and again just before merging, Kin requires that the branch has only ever had this PR (`pulls?state=all&head=`) and that the PR's issue events hold no `base_ref_changed`; otherwise an agent could run its own `memory-check.yml` through a second PR from the branch, or by moving this PR's base and back. Any workflow can post a `memory-check` check run, even into that run's check suite, so every `memory-check` check run on the commit (`filter=all`) must have the id of one of those jobs, an Actions job's id being its check run's id. Any of these lists running past one page (a full page or a `next` link) fails the check. Then Kin squash-merges through REST with `sha` pinned to that commit; when the merge parent is the remote head the batch was built on, the squash's tree must match it. GitHub's auto-merge is not used: it trusts any Actions workflow's `memory-check`. The memory repo allows only squash merges, so the next sync folds the squash commit back in with a local merge commit whose parents are GitHub's head and the local one: local history, and with it each wake-up's own commit, is never rewritten. Changes to paths the owner reviews (`decisions/`, `people/`, `objectives/`, anything outside Kin's own folders) stay off the local default branch: the writer keeps them as a commit under `refs/kin/review//-`, and once nothing else is left to land the lander rebuilds them on the remote head, without `INDEX.md` or the journal, opens their own PR on `kin/write//--review`, tells the owner and drops the ref. Kin never merges it. Its body and the owner's notice name the commit Kin pushed, so the owner can close it if the head differs; memory reads see those changes only once the owner merges. Kin records that commit in `kin_memory_review_pulls`, because the memory deploy key can push to the branch. A failed or foreign check, a moved head, another PR from the branch, a changed base, a check still running after 15 minutes, a refused merge or an unreachable GitHub closes the PR with a comment and undoes nothing: the local commits stay, leads keep reading and writing them, and the owner's direct chat gets one "Memory write not merged" notice with the reason and link, saying memory is still saved on this server. Landing for that organization then waits until a later local commit changes the batch, or the server restarts; a retry that fails the same way posts nothing new. Memory writes post nothing else to the chat. Without the Kin App, writes to the default branch are refused and the owner is told; `KIN_MEMORY_DIRECT_PUSH=true` restores direct pushes, inside `commitWakeup`, for a memory repo without the TRIG-15 ruleset. At startup the lander first sweeps, then lands what the last run left. The sweep, at startup and every 5 minutes on the Kin scheduler, takes the landing lock rather than the writer lock and closes, with a comment, every open `kin/write/*` PR no landing waits on: any non-review PR (so a batch PR a crash left open is closed, never duplicated) and any review PR whose head isn't the commit Kin recorded. It looks up each recorded review branch by head, so newer PRs can't hide one, and tells the owner about each it closed, or (once per server run) couldn't close and left open; a `-review` PR Kin never recorded is closed with only a comment. Orphans are found on one page of open PRs, newest first, so a flood delays them to a later sweep. Each GitHub call times out after 30 seconds. The landing lock is taken before the writer lock and never inside it, and history rewrites take it too, so no batch is mid-landing while a purge rewrites the remote. Dream branch pushes and deletes also run under the landing lock rather than the writer lock. +**Writes to the default branch commit here and land on GitHub in the background (TRIG-15).** A wake-up's memory is done once `commitWakeup` (or the owner's Undo) commits it to the local default branch and moves the anchor; the writer then signals `MemoryLander` and returns, so nothing in a wake-up waits on GitHub and the writer lock covers only local git work. The memory repo's `main` takes no direct push, so the lander, one per organization and one batch at a time, takes everything the local branch holds that the remote's lacks as one PR: it fetches GitHub's head with no lock held, merges it in under the writer lock (the sync below, without `fsck`), then outside the lock builds one commit on the remote head with the local tree and pushes it to `kin/write/land/` (a fresh commit and branch on every attempt, since the branch check refuses a branch that ever had another PR). Commits made while a batch waits on CI ride in the next batch. `MemoryPullRequests` opens the PR with a `server` App token (`contents` and `pull_requests` write, `checks` and `actions` read). Before merging, Kin checks that the PR head is exactly the commit it pushed and that `memory-check` passed in a run it can trace: a `pull_request` run of `.github/workflows/memory-check.yml` on that commit and branch, from the memory repo itself rather than a fork, whose `memory-check` jobs all succeeded. A run's `pull_requests` comes from the PRs open with that head now, not from the PR that triggered it, so it can't tie the run to this PR and base. Instead, before reading runs and again just before merging, Kin requires that the branch has only ever had this PR (`pulls?state=all&head=`) and that the PR's issue events hold no `base_ref_changed`; otherwise an agent could run its own `memory-check.yml` through a second PR from the branch, or by moving this PR's base and back. Any workflow can post a `memory-check` check run, even into that run's check suite, so every `memory-check` check run on the commit (`filter=all`) must have the id of one of those jobs, an Actions job's id being its check run's id. Any of these lists running past one page (a full page or a `next` link) fails the check. Then Kin squash-merges through REST with `sha` pinned to that commit; when the merge parent is the remote head the batch was built on, the squash's tree must match it. GitHub's auto-merge is not used: it trusts any Actions workflow's `memory-check`. Once the merge is confirmed the lander points `refs/kin/landed` in the writer at one commit whose parents are GitHub's squash and the local commit whose tree the batch carried, so both halves of the record move together. The memory repo allows only squash merges, so the next sync folds the squash commit back in with a local merge commit whose parents are GitHub's head and the local one, using the recorded local commit's tree as the merge base. A commit made while the batch waited on CI, even to the same item, is then a change on top of what landed and rides the next batch; only an edit on GitHub to the same item is a conflict. Local history, and with it each wake-up's own commit, is never rewritten, and the local head descends from GitHub's, which a dream branch cut from it needs. The record is stale once its squash is merged in, when git's own merge base is at least as new, and is ignored unless its two commits are in local and GitHub history. If the server stops between GitHub's merge and writing the record, the squash still names its local commit (`Kin-Local:`); the sync accepts that only when the two trees are identical, so redoing it is safe. A writer with no record merges from git's merge base, which is a no-op merge when the trees already match. Each landed batch logs its commit count and PR number, and a start with nothing to land logs that too. Changes to paths the owner reviews (`decisions/`, `people/`, `objectives/`, anything outside Kin's own folders) stay off the local default branch: the writer keeps them as a commit under `refs/kin/review//-`, and once nothing else is left to land the lander rebuilds them on the remote head, without `INDEX.md` or the journal, opens their own PR on `kin/write//--review`, tells the owner and, once the PR is open and recorded, drops the ref. If that fails the ref stays, the owner hears once per reason, and the next landing run publishes it again. Kin never merges it. Its body and the owner's notice name the commit Kin pushed, so the owner can close it if the head differs; memory reads see those changes only once the owner merges. Kin records that commit in `kin_memory_review_pulls`, because the memory deploy key can push to the branch. A failed or foreign check, a moved head, another PR from the branch, a changed base, a check still running after 15 minutes, a refused merge or an unreachable GitHub closes the PR with a comment and undoes nothing: the local commits stay, leads keep reading and writing them, and the owner's direct chat gets one "Memory write not merged" notice with the reason and link, saying memory is still saved on this server. Any other failure while landing (git, the filesystem, the local database) is reported through the same notice, once per distinct cause, and held against the local head that attempt read, so a commit made meanwhile is still tried. Landing for that organization then waits until a later local commit changes the batch, or the server restarts; a retry that fails the same way posts nothing new. Memory writes post nothing else to the chat. Without the Kin App, writes to the default branch are refused and the owner is told; `KIN_MEMORY_DIRECT_PUSH=true` restores direct pushes, inside `commitWakeup`, for a memory repo without the TRIG-15 ruleset. At startup the lander first sweeps, then lands what the last run left. The sweep, at startup and every 5 minutes on the Kin scheduler, takes the landing lock rather than the writer lock and closes, with a comment, every open `kin/write/*` PR no landing waits on: any non-review PR (so a batch PR a crash left open is closed, never duplicated) and any review PR whose head isn't the commit Kin recorded. It looks up each recorded review branch by head, so newer PRs can't hide one, and tells the owner about each it closed, or (once per server run) couldn't close and left open; a `-review` PR Kin never recorded is closed with only a comment. Orphans are found on one page of open PRs, newest first, so a flood delays them to a later sweep. Each GitHub call times out after 30 seconds. The landing lock is taken before the writer lock and never inside it, and history rewrites take it too, so no batch is mid-landing while a purge rewrites the remote. Dream branch pushes and deletes also run under the landing lock rather than the writer lock. Front matter matches the kin-memory README, including snake_case keys. Date-only values mean UTC midnight; date-times require an explicit UTC offset, and invalid dates make an item malformed. `links` values containing `#` must be quoted. `owner` and `said_by` hold handles; `orgId` and `path` belong only to the server-side `MemoryItemHead`. @@ -716,7 +716,7 @@ LEAD-14. Withdrawn (owner, 10 Nov 2026): there is no shadow week to rehearse. **Run leads as a separate OS user before production.** Same-user residual attacks include direct SQLite edits (including `kin_memory_heads`), deleting the writer and losing never-pushed commits, object/pack/ref races between check and use (for example a crafted pack fooling git ancestry commands), `~/.ssh/config` and credential-input tampering, read-clone races, reading other orgs' directories, and attacking the server process. Hash-verifying reads and SQLite anchors are defense in depth, not a same-user sandbox. This writer now walks verified commit parents itself rather than trusting `merge-base`; races at other git check/use boundaries still require OS-user separation. -`syncCheckout` checks git integrity once per run (`fsck --full --strict` from the anchored head). Every application object read independently checks its hash. Three-way merging walks verified commit parents to find a common ancestor, then reads base, local and remote bytes with hash verification. Paths changed on both sides conflict except append-only journals and the regenerated INDEX.md. Journal append conflicts retain remote bytes followed by local entries in order. Original modes, symlinks and gitlinks are preserved; writer changes create only regular files. Paths both sides changed the same way, as when the remote squash-merged a batch of local commits, merge cleanly, and a journal one side extends keeps the longer copy. Other conflicts keep the local head and notify via `MemoryRecoveryHooks`; the next wake-up proceeds on it. Missing or ambiguous merge bases preserve the tip under `kin/conflict//-` (or a server-generated recovery id when no wake-up id is supplied); existing conflict branches are never intentionally overwritten. +`syncCheckout` checks git integrity once per run (`fsck --full --strict` from the anchored head). Every application object read independently checks its hash. Three-way merging walks verified commit parents to find a common ancestor (or takes the landed record's local commit, see Memory writes), then reads base, local and remote bytes with hash verification. Paths changed on both sides conflict except append-only journals and the regenerated INDEX.md. Journal append conflicts retain remote bytes followed by the local entries the remote copy lacks, in order: what the two copies share is kept once, cut at the first line that differs, so a landed entry is never repeated even when the base predates it. Original modes, symlinks and gitlinks are preserved; writer changes create only regular files. Paths both sides changed the same way, as when the remote squash-merged a batch of local commits, merge cleanly, journals included, and a journal one side extends keeps the longer copy. Other conflicts keep the local head and notify via `MemoryRecoveryHooks`; the next wake-up proceeds on it. Missing or ambiguous merge bases preserve the tip under `kin/conflict//-` (or a server-generated recovery id when no wake-up id is supplied); existing conflict branches are never intentionally overwritten. Operators or owners can call server-only `Memory.recover(orgId)` after losing a writer or to reconcile its anchors. It fetches remote main and shadow heads, keeps an anchored default-branch head whose objects survive and that shares a merge base with the remote's (the lander lands it), keeps and pushes an anchored shadow head the remote lacks or that fast-forwards the remote's (a refused push leaves the lead's only copy here), preserves other surviving unpushed anchored work on a conflict branch, recreates a missing writer, records remote anchors and notifies. This method must never be registered as an agent tool. For writers predating K011 anchors, stop the runner and delete only that organization's writer directory, then bootstrap again. Do not delete anchored writers to recover unpushed work; call `recover` first. Recovery cannot restore commits lost with a deleted writer before they reached any remote.