diff --git a/apps/server/src/kin/lead/LeadPrompt.test.ts b/apps/server/src/kin/lead/LeadPrompt.test.ts index fbc72428b0691..e064bf6219cc4 100644 --- a/apps/server/src/kin/lead/LeadPrompt.test.ts +++ b/apps/server/src/kin/lead/LeadPrompt.test.ts @@ -63,6 +63,9 @@ const memory = Layer.succeed( anchor: () => Effect.succeed("fixture"), record: () => Effect.void, recover: () => Effect.void, + withHistoryRewrite: () => Effect.die("Unused"), + purgeInFlight: () => Effect.die("Unused"), + historyGeneration: () => Effect.die("Unused"), readPromptSnapshot: (org, head) => org !== orgId || !["head-one", "head-two"].includes(head) ? Effect.fail(new Memory.MemoryError({ orgId: org, operation: "promptSnapshot" })) diff --git a/apps/server/src/kin/lead/layer.test.ts b/apps/server/src/kin/lead/layer.test.ts index 4cf8917612a81..dd1c1f2318a5f 100644 --- a/apps/server/src/kin/lead/layer.test.ts +++ b/apps/server/src/kin/lead/layer.test.ts @@ -44,6 +44,9 @@ vi.mock("../memory/Memory.ts", async (importOriginal) => { anchor: () => Effect.die("unused"), record: () => Effect.die("unused"), recover: () => Effect.die("unused"), + withHistoryRewrite: () => Effect.die("unused"), + purgeInFlight: () => Effect.die("unused"), + historyGeneration: () => Effect.die("unused"), ensureCheckouts: () => Effect.succeed({ writer: "/private/tmp", read: "/private/tmp" }), prepareWakeup: () => Effect.sync(() => { diff --git a/apps/server/src/kin/memory/Memory.ts b/apps/server/src/kin/memory/Memory.ts index cc6ca4aedc0cc..037fc25fc2784 100644 --- a/apps/server/src/kin/memory/Memory.ts +++ b/apps/server/src/kin/memory/Memory.ts @@ -142,6 +142,13 @@ export class MemorySyncHooks extends Context.Reference<{ defaultValue: () => ({ syncItems: () => Effect.void }), }) {} +export interface MemoryHistoryRewrite { + /** Writes the durable marker, then deletes the writer and read checkouts that hold the old history. */ + readonly quarantine: (marker: string) => Effect.Effect; + /** Clones fresh, records the remote heads without preserving old anchors, rebuilds, then clears the marker. */ + readonly reclone: () => Effect.Effect; +} + export class Memory extends Context.Service< Memory, { @@ -162,6 +169,15 @@ export class Memory extends Context.Service< head: string, ) => Effect.Effect; readonly recover: (orgId: KinOrgId) => Effect.Effect; + /** Server-only, for `KinPurge`: runs `body` under the writer lock with the checkout operations a history rewrite needs. */ + readonly withHistoryRewrite: ( + orgId: KinOrgId, + body: (rewrite: MemoryHistoryRewrite) => Effect.Effect, + ) => Effect.Effect; + /** The durable marker of an unfinished history rewrite; while present, nothing pushes old anchors. */ + readonly purgeInFlight: (orgId: KinOrgId) => Effect.Effect; + /** Bumped whenever a history rewrite drops the checkouts; work staged under an older value is stale. */ + readonly historyGeneration: (orgId: KinOrgId) => Effect.Effect; readonly ensureCheckouts: (orgId: KinOrgId) => Effect.Effect; readonly prepareWakeup: ( orgId: KinOrgId, @@ -314,6 +330,7 @@ const make = Effect.gen(function* () { const cache = yield* MemoryCache.MemoryCache; const locks = new Map(); const initializedCheckouts = new Set(); + const historyGenerations = new Map(); const duplicateWarnings = new Map>(); const snapshots = new Map(); const blobCaches = new Map(); @@ -438,6 +455,37 @@ const make = Effect.gen(function* () { return { repo, root, writer, read }; }); + const markerPath = (root: string) => path.join(root, "purge-in-flight"); + /** Half-finished clones from a crash; after a purge they can hold the old history. */ + const removeStaleClones = Effect.fn("Memory.removeStaleClones")(function* ( + orgId: KinOrgId, + root: string, + ) { + for (const entry of yield* io(orgId, "clones", fs.readDirectory(root))) + if (/^memory\.clone-[A-Za-z0-9]+$/.test(entry)) + yield* io( + orgId, + "recoverClone", + fs.remove(path.join(root, entry), { recursive: true, force: true }), + ); + }); + const purgeInFlight = Effect.fn("Memory.purgeInFlight")(function* (orgId: KinOrgId) { + const { root } = yield* pathsFor(orgId); + if (!(yield* io(orgId, "purgeMarker", fs.exists(markerPath(root))))) return undefined; + return yield* io(orgId, "purgeMarker", fs.readFileString(markerPath(root))); + }); + const refuseDuringPurge = Effect.fn("Memory.refuseDuringPurge")(function* ( + orgId: KinOrgId, + operation: string, + ) { + if ((yield* purgeInFlight(orgId)) !== undefined) + return yield* new MemorySyncConflict({ + orgId, + operation, + reason: "A memory purge is in flight; old history must not be pushed", + }); + }); + const walkAncestors = (...args: Parameters) => ancestors(...args).pipe( Effect.provideService(FileSystem.FileSystem, fs), @@ -504,14 +552,9 @@ const make = Effect.gen(function* () { if (yield* io(orgId, "exists", fs.exists(paths.writer))) { yield* validateWriter(orgId, paths, false); } else { - for (const entry of yield* io(orgId, "clones", fs.readDirectory(paths.root))) { - if (/^memory\.clone-[A-Za-z0-9]+$/.test(entry)) - yield* io( - orgId, - "recoverClone", - fs.remove(path.join(paths.root, entry), { recursive: true, force: true }), - ); - } + // The remote can still hold the old history until the purge's push lands. + yield* refuseDuringPurge(orgId, "clone"); + yield* removeStaleClones(orgId, paths.root); yield* io( orgId, "clone", @@ -845,6 +888,7 @@ const make = Effect.gen(function* () { head: string, wakeupId?: string, ) { + yield* refuseDuringPurge(orgId, "preserve"); const now = yield* Clock.currentTimeMillis; const id = wakeupId ?? `recovery-${now}-${head.slice(0, 12)}`; if (!/^[a-zA-Z0-9_-]+$/.test(id) || !/^[a-zA-Z0-9_-]+$/.test(orgId)) @@ -883,10 +927,11 @@ const make = Effect.gen(function* () { return branch; }); - const recover = Effect.fn("Memory.recover")(function* (orgId: KinOrgId) { + const recover = Effect.fn("Memory.recover")(function* (orgId: KinOrgId, reclone = false) { const paths = yield* pathsFor(orgId); yield* io(orgId, "mkdir", fs.makeDirectory(paths.root, { recursive: true })); const exists = yield* io(orgId, "exists", fs.exists(paths.writer)); + if (!exists && !reclone) yield* refuseDuringPurge(orgId, "clone"); if (!exists) yield* git(orgId, paths.root, [ "clone", @@ -1242,13 +1287,15 @@ const make = Effect.gen(function* () { } } if (next !== head) { - if (next !== remote) + if (next !== remote) { + yield* refuseDuringPurge(orgId, "merge"); yield* git(orgId, paths.writer, [ "push", "--", repo.repoUrl, `${next}:refs/heads/${branch}`, ]); + } yield* git(orgId, paths.writer, ["update-ref", `refs/heads/${branch}`, next, head]); yield* record(orgId, branch, head, next); head = next; @@ -1289,8 +1336,62 @@ const make = Effect.gen(function* () { return { head, indexAsOf: DateTime.toUtc(indexAsOf) }; }); + const dropCheckouts = Effect.fn("Memory.dropCheckouts")(function* (orgId: KinOrgId) { + const paths = yield* pathsFor(orgId); + historyGenerations.set(orgId, (historyGenerations.get(orgId) ?? 0) + 1); + // `dreams/` holds plain copies of memory files for a running dream. + for (const directory of [paths.writer, paths.read, path.join(paths.root, "dreams")]) + yield* io(orgId, "quarantine", fs.remove(directory, { recursive: true, force: true })); + if (yield* io(orgId, "exists", fs.exists(paths.root))) + yield* removeStaleClones(orgId, paths.root); + initializedCheckouts.delete(orgId); + pinnedHeads.delete(orgId); + snapshots.delete(orgId); + blobCaches.delete(orgId); + return paths; + }); + const historyRewrite = (orgId: KinOrgId): MemoryHistoryRewrite => ({ + quarantine: Effect.fn("Memory.quarantine")((marker: string) => + locked( + orgId, + Effect.gen(function* () { + const { root } = yield* pathsFor(orgId); + yield* io(orgId, "mkdir", fs.makeDirectory(root, { recursive: true })); + yield* io(orgId, "purgeMarker", fs.writeFileString(`${markerPath(root)}.tmp`, marker)); + yield* io(orgId, "purgeMarker", fs.rename(`${markerPath(root)}.tmp`, markerPath(root))); + yield* dropCheckouts(orgId); + }), + ), + ), + reclone: Effect.fn("Memory.reclone")(() => + locked( + orgId, + Effect.gen(function* () { + const { root } = yield* dropCheckouts(orgId); + // With no writer checkout, recover clones fresh and records the rewritten heads. + yield* recover(orgId, true); + yield* prepare(orgId); + yield* rebuildIndex(orgId); + yield* io(orgId, "purgeMarker", fs.remove(markerPath(root), { force: true })); + }), + ), + ), + }); + + const withHistoryRewrite = ( + orgId: KinOrgId, + body: (rewrite: MemoryHistoryRewrite) => Effect.Effect, + ) => withWriterLock(orgId, body(historyRewrite(orgId))); + return Memory.of({ withWriterLock, + withHistoryRewrite, + purgeInFlight: Effect.fn("Memory.purgeInFlightLocked")((orgId) => + locked(orgId, purgeInFlight(orgId)), + ), + historyGeneration: Effect.fn("Memory.historyGeneration")((orgId) => + Effect.sync(() => historyGenerations.get(orgId) ?? 0), + ), recover: Effect.fn("Memory.recoverLocked")((orgId) => withWriterLock(orgId, locked(orgId, recover(orgId))), ), diff --git a/apps/server/src/kin/memory/MemoryWriter.ts b/apps/server/src/kin/memory/MemoryWriter.ts index 86a40b2ead561..1b69e0424e4ec 100644 --- a/apps/server/src/kin/memory/MemoryWriter.ts +++ b/apps/server/src/kin/memory/MemoryWriter.ts @@ -196,6 +196,7 @@ const make = Effect.gen(function* () { KinOrgId, { wakeupId: string; + generation: number; files: Map; lines: string[]; supersessions: Map; @@ -307,8 +308,19 @@ const make = Effect.gen(function* () { const previous = sessions.get(orgId); if (previous && previous.wakeupId !== wakeupId) return yield* new MemoryWriteRejected({ reason: "Another wake-up owns the writer" }); + const generation = yield* memory.historyGeneration(orgId); + // A purge rewrote history after this wake-up staged; committing it could restore purged content. + if (previous && previous.generation !== generation) { + sessions.delete(orgId); + return yield* new MemorySyncConflict({ + orgId, + operation: "push", + reason: "Memory history was rewritten during this wake-up; its staged changes were dropped", + }); + } const state = previous ?? { wakeupId, + generation, files: new Map(), lines: [], supersessions: new Map(), @@ -377,6 +389,17 @@ const make = Effect.gen(function* () { const paths = yield* memory.ensureCheckouts(orgId); return paths; }); + // Every caller holds the writer lock, and the marker only changes under it. + const refuseDuringPurge = Effect.fn("MemoryWriter.refuseDuringPurge")(function* ( + orgId: KinOrgId, + ) { + if ((yield* memory.purgeInFlight(orgId)) !== undefined) + return yield* new MemorySyncConflict({ + orgId, + operation: "push", + reason: "A memory purge is in flight; old history must not be pushed", + }); + }); const report = (orgId: KinOrgId, wakeupId: string, effect: Effect.Effect) => effect.pipe( Effect.catchTags({ @@ -770,6 +793,7 @@ const make = Effect.gen(function* () { Effect.gen(function* () { const { ctx, state } = yield* session(orgId, wakeupId); const { writer } = yield* validate(orgId); + yield* refuseDuringPurge(orgId); const repo = config.repositories.find((entry) => entry.orgId === orgId); if (!repo) return yield* new MemoryConfigError({ @@ -899,7 +923,20 @@ const make = Effect.gen(function* () { if (!dreamBranch.test(input.branch)) return yield* new MemoryWriteRejected({ reason: "Invalid dream branch" }); const { writer } = yield* validate(orgId); + yield* refuseDuringPurge(orgId); const repo = yield* repoFor(orgId); + // A purge's reclone drops the old history, so a dream started before it has no base here. + if ( + !/^[a-f0-9]{40}([a-f0-9]{24})?$/.test(input.base) || + (yield* git(orgId, writer, ["cat-file", "-e", `${input.base}^{commit}`]).pipe( + Effect.result, + ))._tag === "Failure" + ) + return yield* new MemorySyncConflict({ + orgId, + operation: "push", + reason: "The dream's base is no longer in memory history", + }); const baseFiles = yield* filesAt(orgId, writer, input.base); const regular = yield* regularPaths(orgId, writer, input.base); const combined = new Map( @@ -964,6 +1001,7 @@ const make = Effect.gen(function* () { if (!dreamBranch.test(branch)) return yield* new MemoryWriteRejected({ reason: "Invalid dream branch" }); const { writer } = yield* validate(orgId); + yield* refuseDuringPurge(orgId); const repo = yield* repoFor(orgId); const advertised = yield* git(orgId, writer, [ "ls-remote", diff --git a/apps/server/src/kin/memory/objects.ts b/apps/server/src/kin/memory/objects.ts index e7c4d33155168..1d285684a9fdf 100644 --- a/apps/server/src/kin/memory/objects.ts +++ b/apps/server/src/kin/memory/objects.ts @@ -9,7 +9,7 @@ import { MemoryCheckoutInvalid } from "./Memory.ts"; export const readObjects = Effect.fnUntraced(function* ( orgId: KinOrgId, cwd: string, - requests: ReadonlyArray<{ sha: string; type: "blob" | "tree" | "commit" }>, + requests: ReadonlyArray<{ sha: string; type: "blob" | "tree" | "commit" | "tag" }>, ) { const invalid = () => new MemoryCheckoutInvalid({ @@ -34,7 +34,7 @@ export const readObjects = Effect.fnUntraced(function* ( for (const { sha, type } of requests) { const end = buffer.indexOf(10, offset); const header = buffer.subarray(offset, end).toString("ascii"); - const match = /^(\w+) (blob|tree|commit) (\d+)$/.exec(header); + const match = /^(\w+) (blob|tree|commit|tag) (\d+)$/.exec(header); if (end < 0 || !match || match[1] !== sha || match[2] !== type) return yield* invalid(); const length = Number(match[3]); offset = end + 1; diff --git a/apps/server/src/kin/memory/purge.test.ts b/apps/server/src/kin/memory/purge.test.ts new file mode 100644 index 0000000000000..38ab8a9c6f42a --- /dev/null +++ b/apps/server/src/kin/memory/purge.test.ts @@ -0,0 +1,1076 @@ +// @effect-diagnostics-next-line nodeBuiltinImport:off -- A synchronous probe decides the platform gate before tests register. +import * as NodeChildProcess from "node:child_process"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { describe, expect, it } from "@effect/vitest"; +import { vi } from "vite-plus/test"; +import { + KinActionId, + KinAgentId, + KinMemoryItemId, + KinOrgId, + KinPersonId, + ThreadId, + type KinGrantPayload, + type KinPendingAction, +} from "@t3tools/contracts"; +import * as Cause from "effect/Cause"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import { ChildProcessSpawner } from "effect/process"; +import * as Sqlite from "../../persistence/Sqlite.ts"; +import * as Persistence from "../persistence/KinPersistence.ts"; +import * as Vcs from "../../vcs/VcsProcess.ts"; +import { fixtureItem } from "./__fixtures__/items.ts"; +import { parseFrontMatter } from "./frontMatter.ts"; +import * as Memory from "./Memory.ts"; +import { MemoryConfig } from "./MemoryConfig.ts"; +import * as Writer from "./MemoryWriter.ts"; +import * as Purge from "./purge.ts"; +import * as PurgePush from "./purgePush.ts"; +import * as Scan from "./secretScan.ts"; + +const orgId = KinOrgId.make("fixture-org"); +const token = ["gh", "p_"].join("") + "A1b2C3d4E5".repeat(3) + "abcdef"; +// TRIG-20 installs git-filter-repo on the Kin host; machines without it skip the real rewrite. +const filterRepoMissing = NodeChildProcess.spawnSync("git-filter-repo", ["--version"]).status !== 0; + +const actionId = KinActionId.make("act-1"); +const thumbprint = "T".repeat(43); +const targetDigest = "d".repeat(64); +const actionFor = (plan: Purge.PurgePlan): KinPendingAction => ({ + id: actionId, + orgId, + idempotencyKey: "e".repeat(64), + kind: "memory_purge", + target: Purge.purgePlanJson(plan).target, + resolved: { _tag: "none" }, + targetDigest, + nonce: "fixture-nonce", + revision: 1, + reason: "Invented purge", + requestedBy: { agentId: KinAgentId.make("fixture-lead"), threadId: ThreadId.make("thread-1") }, + provenance: { startedBy: "schedule", startedByRole: "none", chain: [] }, + requiredApprovers: [{ role: "owner" }], + state: "executing", + createdAt: DateTime.makeUnsafe("2026-11-08T00:00:00.000Z"), + updatedAt: DateTime.makeUnsafe("2026-11-08T00:00:00.000Z"), + expiresAt: DateTime.makeUnsafe("2026-11-09T00:00:00.000Z"), +}); +const grantFor = (plan: Purge.PurgePlan): KinGrantPayload => ({ + v: 1, + orgId, + actionId, + revision: 1, + nonce: "fixture-nonce", + jti: "jti-1", + kind: "memory_purge", + targetDigest, + target: Purge.purgePlanJson(plan).target, + approver: { humanId: KinPersonId.make("fixture-owner"), kid: thumbprint }, + decision: "approve", + iat: 0, + exp: 900, +}); +const gateOpen = Effect.provideService(Purge.PurgeExecutorGate, true); + +const services = Layer.mergeAll( + Vcs.layer.pipe(Layer.provide(NodeServices.layer)), + NodeServices.layer, + Persistence.layer.pipe(Layer.provideMerge(Sqlite.layerMemory)), +); + +interface PurgeOptions { + readonly push?: ( + real: PurgePush.KinPurgePush["Service"], + ) => Partial | undefined; + readonly scan?: Layer.Layer; + readonly filterRepo?: string; + readonly queue?: "default"; +} + +const fixture = Effect.fn(function* (tokenInMessage = false) { + const fs = yield* FileSystem.FileSystem; + const vcs = yield* Vcs.VcsProcess; + const root = yield* fs.realPath(yield* fs.makeTempDirectoryScoped()); + const source = `${root}/seed`; + const remote = `${root}/remote.git`; + // A file:// URL transfers reachable objects only, like a real remote; a bare path copies them all. + const remoteUrl = `file://${remote}`; + const git = (cwd: string, args: ReadonlyArray) => + vcs + .run({ + operation: "test.purge", + command: "git", + cwd, + args: cwd === remote ? [`--git-dir=${remote}`, ...args] : args, + maxOutputBytes: 64 * 1024 * 1024, + }) + .pipe(Effect.map((result) => result.stdout.trim())); + const commit = (message: string) => + git(source, [ + "-c", + "user.name=Fixture Bot", + "-c", + "user.email=fixture@example.invalid", + "commit", + "-q", + "-m", + message, + ]); + yield* fs.makeDirectory(`${source}/claims`, { recursive: true }); + yield* git(source, ["init", "-q", "-b", "main"]); + yield* fs.writeFileString(`${source}/rules.md`, "Invented rules\n"); + yield* fs.writeFileString(`${source}/INDEX.md`, "As of 2026-11-08T00:00:00.000Z\n"); + yield* fs.writeFileString( + `${source}/claims/leak.md`, + fixtureItem({ id: "claim-leak", kind: "claim", title: "Invented vendor note" }) + + `vendor key: ${token}\n`, + ); + yield* fs.writeFileString( + `${source}/claims/keep.md`, + fixtureItem({ id: "claim-keep", kind: "claim", title: "Invented keeper" }), + ); + yield* git(source, ["add", "--", "."]); + yield* commit(tokenInMessage ? `fixture ${token}` : "fixture"); + yield* git(source, ["switch", "-q", "-c", "side"]); + yield* fs.writeFileString(`${source}/claims/leak.md`, `side copy ${token}\n`); + yield* git(source, ["add", "--", "."]); + yield* commit("side"); + yield* git(source, ["switch", "-q", "main"]); + yield* fs.writeFileString(`${source}/rules.md`, "Invented rules, revised\n"); + yield* git(source, ["add", "--", "."]); + yield* commit("revise rules"); + yield* git(root, ["clone", "-q", "--bare", "--", source, remote]); + // GitHub's read-only pull refs: a purge can't rewrite them and must not wait on them. + yield* git(remote, ["update-ref", "refs/pull/1/head", "refs/heads/side"]); + const pullHead = yield* git(remote, ["rev-parse", "refs/pull/1/head"]); + const remoteHistory = git(remote, ["log", "--exclude=refs/pull/*", "--all", "-p"]); + + const events: string[] = []; + const config = Layer.succeed(MemoryConfig, { + baseDirectory: root, + repositories: [{ orgId, repoUrl: remoteUrl, path: "state" }], + }); + const purgeLayer = (options: PurgeOptions = {}) => { + const realPush = PurgePush.gitPushLayer({ credentialHelper: "fixture-helper" }); + const push = options.push + ? Layer.effect( + PurgePush.KinPurgePush, + Effect.gen(function* () { + const real = yield* PurgePush.KinPurgePush; + return PurgePush.KinPurgePush.of({ ...real, ...options.push?.(real) }); + }), + ).pipe(Layer.provide(realPush)) + : realPush; + return Purge.layer.pipe( + Layer.provideMerge( + Memory.layer.pipe( + Layer.provide(config), + Layer.provide(Layer.succeed(Memory.MemoryRecoveryHooks, { notify: () => Effect.void })), + ), + ), + Layer.provide(config), + Layer.provide(options.scan ?? Scan.layer), + Layer.provide(push), + Layer.provide( + Layer.succeed(Purge.GitFilterRepoCommand, options.filterRepo ?? "git-filter-repo"), + ), + Layer.provide( + options.queue === "default" + ? Layer.empty + : Layer.succeed(Purge.KinPurgeQueue, { + pause: () => Effect.sync(() => events.push("pause")), + resume: () => Effect.sync(() => events.push("resume")), + }), + ), + ); + }; + /** The purge plus the dream and wake-up writer, which push to the same remote. */ + const writerLayer = (options: PurgeOptions = {}) => + Writer.layer.pipe( + Layer.provideMerge(purgeLayer(options)), + Layer.provide(config), + Layer.provide(Scan.layer), + Layer.provide(Layer.succeed(Memory.MemoryRecoveryHooks, { notify: () => Effect.void })), + Layer.provide(Layer.succeed(Writer.MemoryWriterHooks, { notify: () => Effect.void })), + Layer.provide( + Layer.succeed(Writer.MemoryWriterAuthority, { + resolve: (_org, wakeupId) => + Effect.succeed({ + orgId, + wakeupId, + agentId: KinAgentId.make("fixture-lead"), + level: "L1", + timeZone: "UTC", + startedAt: "2026-11-08T00:00:00.000Z", + eventIds: [], + sources: new Map(), + }), + }), + ), + ); + const writerHistory = (writer: string) => + Effect.gen(function* () { + if (!(yield* fs.exists(`${writer}/.git`))) return ""; + return [ + yield* git(writer, ["log", "--all", "-p"]), + yield* git(writer, ["cat-file", "--batch-all-objects", "--batch"]), + ].join("\n"); + }); + /** Seeds anchors and checkouts the way a lead's first wake-up does. */ + const prepare = Effect.gen(function* () { + const memory = yield* Memory.Memory; + const paths = yield* memory.ensureCheckouts(orgId); + yield* memory.prepareWakeup(orgId); + return paths; + }); + return { + fs, + git, + root, + source, + remote, + remoteUrl, + events, + commit, + purgeLayer, + writerLayer, + writerHistory, + prepare, + pullHead, + remoteHistory, + }; +}); + +const dropPlan = (remoteUrl: string, paths = ["claims/leak.md"]): Purge.PurgePlan => ({ + orgId, + repo: remoteUrl, + mode: "drop_file", + paths, +}); +const runPurge = (plan: Purge.PurgePlan) => + Effect.gen(function* () { + const target = yield* (yield* Purge.KinPurge).storePlan(plan); + expect(target).toEqual(actionFor(plan).target); + return yield* Purge.memoryPurgeHandler + .run(actionFor(plan), grantFor(plan)) + .pipe(gateOpen, Effect.exit); + }); +const reconcile = (plan: Purge.PurgePlan) => + Purge.memoryPurgeHandler.reconcile(actionFor(plan)).pipe(gateOpen); +const failureOf = (exit: Exit.Exit) => + Exit.isFailure(exit) ? Cause.findErrorOption(exit.cause) : undefined; + +describe.skipIf(filterRepoMissing)("purge with git-filter-repo", () => { + it.live.each(["drop_file", "redact_text"] as const)("purges a fixture repo: %s", (mode) => + Effect.gen(function* () { + const { + fs, + git, + root, + remote, + remoteUrl, + events, + writerLayer, + writerHistory, + prepare, + pullHead, + remoteHistory, + } = yield* fixture(mode === "redact_text"); + yield* Effect.gen(function* () { + const memory = yield* Memory.Memory; + const writer = yield* Writer.MemoryWriter; + const paths = yield* prepare; + // A wake-up staged against the old history before the queue paused. + yield* writer.stage(orgId, "wake1", { + path: "claims/staged.md", + content: fixtureItem({ + id: "claim-staged", + kind: "claim", + title: "Invented staged", + links: [], + }), + }); + const oldMain = yield* git(remote, ["rev-parse", "refs/heads/main"]); + const oldSide = yield* git(remote, ["rev-parse", "refs/heads/side"]); + const plan: Purge.PurgePlan = + mode === "redact_text" + ? { orgId, repo: remoteUrl, mode, patterns: [token] } + : dropPlan(remoteUrl); + yield* fs.makeDirectory(`${root}/state/purge-scratch/mirror-stale`, { recursive: true }); + const exit = yield* runPurge(plan); + + expect(Exit.isSuccess(exit)).toBe(true); + expect(yield* fs.readDirectory(`${root}/state/purge-scratch`)).toEqual([]); + if (!Exit.isSuccess(exit)) return; + const result = exit.value; + const newMain = yield* git(remote, ["rev-parse", "refs/heads/main"]); + const newSide = yield* git(remote, ["rev-parse", "refs/heads/side"]); + expect(events).toEqual(["pause", "resume"]); + expect(yield* memory.purgeInFlight(orgId)).toBeUndefined(); + expect( + yield* writer.commitWakeup(orgId, "wake1", "completed").pipe(Effect.result), + ).toMatchObject({ failure: { _tag: "MemorySyncConflict" } }); + // A dream that read its base before the purge. + expect( + yield* writer + .commitDream(orgId, { + branch: "dream/2026-11-08", + base: oldMain, + files: new Map(), + message: "dream", + }) + .pipe(Effect.result), + ).toMatchObject({ failure: { _tag: "MemorySyncConflict" } }); + expect(yield* git(remote, ["rev-parse", "refs/heads/main"])).toBe(newMain); + expect(yield* reconcile(plan)).toBe("done"); + expect(yield* git(remote, ["rev-parse", "refs/pull/1/head"])).toBe(pullHead); + expect(result).toMatchObject({ oldHead: oldMain, newHead: newMain }); + expect(result.rewrittenRefs).toEqual([ + { ref: "refs/heads/main", old: oldMain, new: newMain }, + { ref: "refs/heads/side", old: oldSide, new: newSide }, + ]); + expect(result.audit).toMatchObject({ + orgId, + event: "execution_finished", + actionId: "act-1", + kind: "memory_purge", + grantJti: "jti-1", + keyThumbprint: thumbprint, + headSha: newMain, + }); + expect(JSON.parse(result.audit.result)).toMatchObject({ + oldHead: oldMain, + newHead: newMain, + }); + expect(result.audit.result).not.toContain(token); + + expect(yield* remoteHistory).not.toContain(token); + expect(yield* writerHistory(paths.writer)).not.toContain(token); + expect(yield* memory.anchor(orgId, "main")).toBe(newMain); + expect(yield* git(remote, ["show", `${newMain}:claims/keep.md`])).toContain("claim-keep"); + + const leak = yield* git(remote, ["show", `${newMain}:claims/leak.md`]); + if (mode === "redact_text") { + expect(result.tombstones).toEqual([]); + expect(leak).toContain("vendor key: ***REMOVED***"); + return; + } + expect(result.tombstones).toEqual(["claims/leak.md"]); + // Only this purge's exact tombstone counts as purged; a look-alike at the path is old content. + const later = `${root}/later`; + yield* git(root, ["clone", "-q", "--", remote, later]); + yield* fs.writeFileString(`${later}/claims/leak.md`, `${leak}\nInvented restored note\n`); + yield* git(later, [ + "-c", + "user.name=Fixture Bot", + "-c", + "user.email=fixture@example.invalid", + "commit", + "-qam", + "look-alike tombstone", + ]); + yield* git(later, ["push", "-q", "origin", "main"]); + expect(yield* reconcile(plan)).toBe("not_done"); + const tombstone = yield* parseFrontMatter(orgId, "claims/leak.md", leak + "\n"); + expect(tombstone.frontMatter).toMatchObject({ + id: "claim-leak", + kind: "claim", + status: "cancelled", + said_by: "fixture-owner", + source: "kin-action:act-1", + }); + expect(tombstone.frontMatter.invalid_at).not.toBeNull(); + expect(tombstone.body).toMatch( + /^Purged on \d{4}-\d{2}-\d{2} by the owner \(action act-1\)/, + ); + expect( + (yield* memory.readItemQuietly(orgId, KinMemoryItemId.make("claim-leak"))).head.status, + ).toBe("cancelled"); + expect(yield* fs.readFileString(`${paths.read}/INDEX.md`)).toContain("Purged item"); + }).pipe(Effect.provide(writerLayer())); + }).pipe(Effect.provide(services), Effect.scoped), + ); + + it.live.each([ + ["wrong pattern", { _tag: "KinPurgeFailed", stage: "verify", queueHeld: false }], + ["missing tool", { _tag: "KinPurgeFailed", stage: "rewrite", queueHeld: false }], + ["path typo", { _tag: "KinPurgeRejected" }], + ["interrupted", undefined], + ["no pause", { _tag: "KinPurgePauseUnavailable" }], + // `clone --bare` leaves notes and custom refs behind, and they would keep the old history. + ["notes ref", { _tag: "KinPurgeFailed", stage: "clone", queueHeld: false }], + // filter-repo leaves a tag on a blob or tree alone, and history checks walk commits only. + ["blob tag", { _tag: "KinPurgeFailed", stage: "clone", queueHeld: false }], + // filter-repo rewrites tag messages but not tagger idents. + ["tagger ident", { _tag: "KinPurgeFailed", stage: "verify", queueHeld: false }], + ] as const)( + "a purge that stops before the push resumes the queue and changes nothing: %s", + ([scenario, expected]) => + Effect.gen(function* () { + const { fs, git, root, remote, remoteUrl, events, purgeLayer, writerHistory, prepare } = + yield* fixture(); + const layer = purgeLayer({ + ...(scenario === "missing tool" ? { filterRepo: `${root}/missing/git-filter-repo` } : {}), + ...(scenario === "interrupted" + ? { scan: Layer.succeed(Scan.KinSecretScan, { scan: () => Effect.interrupt }) } + : {}), + ...(scenario === "no pause" ? { queue: "default" as const } : {}), + }); + yield* Effect.gen(function* () { + const memory = yield* Memory.Memory; + const paths = yield* prepare; + const oldMain = yield* git(remote, ["rev-parse", "refs/heads/main"]); + const plan: Purge.PurgePlan = + scenario === "wrong pattern" + ? { orgId, repo: remoteUrl, mode: "redact_text", patterns: ["Invented vendor note"] } + : scenario === "tagger ident" + ? { + orgId, + repo: remoteUrl, + mode: "redact_text", + patterns: [token], + } + : dropPlan(remoteUrl, scenario === "path typo" ? ["claims/leek.md"] : undefined); + if (scenario === "notes ref") + yield* git(remote, ["update-ref", "refs/notes/commits", "refs/heads/side"]); + if (scenario === "tagger ident") + yield* git(remote, [ + "-c", + "user.name=Tagger", + "-c", + `user.email=${token}`, + "tag", + "-a", + "v1", + "-m", + `release ${token}`, + "refs/heads/side", + ]); + if (scenario === "blob tag") + yield* git(remote, [ + "update-ref", + "refs/tags/blobtag", + "refs/heads/side:claims/leak.md", + ]); + const exit = yield* runPurge(plan); + + expect(Exit.isFailure(exit)).toBe(true); + if (!Exit.isFailure(exit)) return; + if (expected === undefined) expect(Cause.hasInterruptsOnly(exit.cause)).toBe(true); + else expect(failureOf(exit)).toMatchObject({ value: expected }); + if (scenario === "wrong pattern") + expect(failureOf(exit)).toMatchObject({ + value: { reason: expect.stringMatching(/Secret scan/) }, + }); + if (scenario === "path typo") + expect(failureOf(exit)).toMatchObject({ + value: { reason: expect.stringMatching(/not in history/) }, + }); + if (scenario === "notes ref" || scenario === "blob tag") { + expect(failureOf(exit)).toMatchObject({ + value: { + reason: expect.stringContaining( + scenario === "notes ref" ? "refs/notes/commits" : "refs/tags/blobtag", + ), + }, + }); + expect(yield* reconcile(plan)).toBe("unknown"); + } + expect(JSON.stringify(exit)).not.toContain(token); + expect(events).toEqual(scenario === "no pause" ? [] : ["pause", "resume"]); + expect(yield* git(remote, ["rev-parse", "refs/heads/main"])).toBe(oldMain); + expect(yield* memory.anchor(orgId, "main")).toBe(oldMain); + expect(yield* memory.purgeInFlight(orgId)).toBeUndefined(); + expect(yield* writerHistory(paths.writer)).toContain(token); + expect( + yield* fs + .readDirectory(`${root}/state/purge-scratch`) + .pipe(Effect.orElseSucceed(() => [])), + ).toEqual([]); + }).pipe(Effect.provide(layer)); + }).pipe(Effect.provide(services), Effect.scoped), + ); + + it.live.each(["error", "checkout during push", "crash", "interrupt"] as const)( + "a push that lands and then fails never republishes the purged history: %s", + (scenario) => + Effect.gen(function* () { + const { fs, git, root, remote, remoteUrl, events, writerLayer, writerHistory, prepare } = + yield* fixture(); + const plan = dropPlan(remoteUrl); + // A dream or wake-up asking for checkouts while the remote still holds the old history. + let beforePush: Effect.Effect = Effect.void; + const landing = writerLayer({ + push: (real) => ({ + push: (input) => + Effect.suspend(() => beforePush).pipe( + Effect.andThen(real.push(input)), + Effect.andThen( + scenario === "crash" + ? Effect.die("fixture crash after the push") + : scenario === "interrupt" + ? Effect.interrupt + : Effect.fail(new PurgePush.KinPurgePushError({ reason: "timeout" })), + ), + ), + }), + }); + const target = Purge.purgePlanJson(plan).target; + const paths = yield* Effect.gen(function* () { + const memory = yield* Memory.Memory; + const paths = yield* prepare; + const oldMain = yield* git(remote, ["rev-parse", "refs/heads/main"]); + let checkout: Exit.Exit | undefined; + if (scenario === "checkout during push") + beforePush = memory.ensureCheckouts(orgId).pipe( + Effect.exit, + Effect.map((exit) => { + checkout = exit; + }), + ); + const exit = yield* runPurge(plan); + if (scenario === "checkout during push") + expect(checkout && failureOf(checkout)).toMatchObject({ + value: { _tag: "MemorySyncConflict" }, + }); + expect(Exit.isFailure(exit)).toBe(true); + if (scenario === "interrupt") + expect(Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause)).toBe(true); + else + expect(failureOf(exit)).toMatchObject({ + value: { _tag: "KinPurgeFailed", stage: "push", queueHeld: true }, + }); + expect(events).toEqual(["pause"]); + expect(yield* memory.purgeInFlight(orgId)).toBeDefined(); + expect(yield* fs.exists(paths.writer)).toBe(false); + expect(yield* fs.exists(paths.read)).toBe(false); + + const writer = yield* Writer.MemoryWriter; + const dream = yield* writer + .commitDream(orgId, { + branch: "dream/2026-11-08", + base: oldMain, + files: new Map([ + [ + "claims/dream.md", + fixtureItem({ id: "claim-dream", kind: "claim", title: "Invented dream" }), + ], + ]), + message: "dream", + }) + .pipe(Effect.result); + expect(dream).toMatchObject({ failure: { _tag: "MemorySyncConflict" } }); + expect( + yield* writer.commitWakeup(orgId, "wake1", "completed").pipe(Effect.result), + ).toMatchObject({ failure: { _tag: "MemorySyncConflict" } }); + return paths; + }).pipe(Effect.provide(landing)); + + // A crash restarts the server: fresh Memory and purge services over the same database and files. + const after = scenario === "crash" ? writerLayer() : landing; + yield* Effect.gen(function* () { + const memory = yield* Memory.Memory; + const purge = yield* Purge.KinPurge; + yield* memory.prepareWakeup(orgId).pipe(Effect.result); + yield* memory.recover(orgId).pipe(Effect.result); + expect(yield* fs.exists(paths.writer)).toBe(false); + expect(yield* git(remote, ["log", "--exclude=refs/pull/*", "--all", "-p"])).not.toContain( + token, + ); + expect( + yield* git(remote, ["for-each-ref", "--format=%(refname)", "refs/heads/"]), + ).not.toMatch(/kin\/conflict|dream\//); + + // The marker keeps the plan, so a lost plan file can't strand the held queue. + expect(yield* purge.discardPlan(target).pipe(Effect.result)).toMatchObject({ + failure: { _tag: "KinPurgeRejected" }, + }); + yield* fs.remove(`${root}/${target.path}`); + expect(yield* reconcile(plan)).toBe("done"); + expect(events).toEqual(["pause", "resume"]); + expect(yield* memory.purgeInFlight(orgId)).toBeUndefined(); + const newMain = yield* git(remote, ["rev-parse", "refs/heads/main"]); + expect((yield* memory.prepareWakeup(orgId)).head).toBe(newMain); + expect(yield* writerHistory(paths.writer)).not.toContain(token); + expect(yield* git(remote, ["log", "--exclude=refs/pull/*", "--all", "-p"])).not.toContain( + token, + ); + yield* purge.discardPlan(target); + }).pipe(Effect.provide(after)); + }).pipe(Effect.provide(services), Effect.scoped), + ); + + const lateRef = "The remote may still hold purged history after the push; reconcile it"; + it.live.each([ + ["refused", "Force-push failed: remote_rejected", "not_done"], + ["lease rejected", "Force-push failed: lease_rejected", "unknown"], + // Created after the mirror clone, so the push's lease never covers them. + ["late branch", lateRef, "unknown"], + ["late tag", lateRef, "unknown"], + ["late notes ref", lateRef, "unknown"], + ] as const)( + "a push that does not land holds the queue until reconcile: %s", + ([scenario, reason, verdict]) => + Effect.gen(function* () { + const { + fs, + git, + source, + remote, + remoteUrl, + events, + commit, + purgeLayer, + writerHistory, + prepare, + } = yield* fixture(); + const plan = dropPlan(remoteUrl); + const layer = purgeLayer({ + push: (real) => ({ + push: (input) => + scenario === "refused" + ? Effect.fail(new PurgePush.KinPurgePushError({ reason: "remote_rejected" })) + : scenario !== "lease rejected" + ? git(remote, [ + "update-ref", + { + "late branch": "refs/heads/sneaky", + "late tag": "refs/tags/sneaky", + "late notes ref": "refs/notes/commits", + }[scenario], + "refs/heads/side", + ]).pipe(Effect.orDie, Effect.andThen(real.push(input))) + : Effect.gen(function* () { + // The remote moves after the purge's mirror clone. + yield* fs.writeFileString( + `${source}/claims/other.md`, + "Invented later note\n", + ); + yield* git(source, ["add", "--", "."]); + yield* commit("later"); + yield* git(source, ["push", "-q", remote, "main"]); + }).pipe(Effect.orDie, Effect.andThen(real.push(input))), + }), + }); + yield* Effect.gen(function* () { + const memory = yield* Memory.Memory; + const paths = yield* prepare; + const oldMain = yield* git(remote, ["rev-parse", "refs/heads/main"]); + const exit = yield* runPurge(plan); + expect(failureOf(exit)).toMatchObject({ + value: { + _tag: "KinPurgeFailed", + stage: "push", + queueHeld: true, + reason, + }, + }); + expect(events).toEqual(["pause"]); + const remoteMain = yield* git(remote, ["rev-parse", "refs/heads/main"]); + if (scenario === "refused") expect(remoteMain).toBe(oldMain); + else expect(remoteMain).not.toBe(oldMain); + expect(yield* git(remote, ["log", "--all", "-p"])).toContain(token); + + expect(yield* reconcile(plan)).toBe(verdict); + if (verdict === "unknown") { + expect(events).toEqual(["pause"]); + expect(yield* memory.purgeInFlight(orgId)).toBeDefined(); + return; + } + expect(events).toEqual(["pause", "resume"]); + expect(yield* memory.purgeInFlight(orgId)).toBeUndefined(); + expect(yield* memory.anchor(orgId, "main")).toBe(oldMain); + expect(yield* writerHistory(paths.writer)).toContain(token); + }).pipe(Effect.provide(layer)); + }).pipe(Effect.provide(services), Effect.scoped), + ); +}); + +it.live("a marker left before the checkouts dropped blocks every writer push until reconcile", () => + Effect.gen(function* () { + const { fs, git, root, remote, remoteUrl, events, writerLayer, prepare } = yield* fixture(); + const plan = dropPlan(remoteUrl); + yield* Effect.gen(function* () { + const memory = yield* Memory.Memory; + const writer = yield* Writer.MemoryWriter; + const paths = yield* prepare; + const oldMain = yield* git(remote, ["rev-parse", "refs/heads/main"]); + const { json, target } = Purge.purgePlanJson(plan); + yield* (yield* Purge.KinPurge).storePlan(plan); + // A crash after the marker rename and before `dropCheckouts`. + yield* fs.writeFileString( + `${root}/state/purge-in-flight`, + JSON.stringify({ + sha256: target.sha256, + actionId, + oldHead: oldMain, + plan: json, + tombstones: [], + }), + ); + expect(yield* fs.exists(paths.writer)).toBe(true); + const guarded = { + failure: { + _tag: "MemorySyncConflict", + operation: "push", + reason: expect.stringMatching(/purge is in flight/), + }, + }; + yield* writer.stage(orgId, "wake1", { + path: "claims/staged.md", + content: fixtureItem({ + id: "claim-staged", + kind: "claim", + title: "Invented staged", + links: [], + }), + }); + expect( + yield* writer.commitWakeup(orgId, "wake1", "completed").pipe(Effect.result), + ).toMatchObject(guarded); + expect( + yield* writer + .commitDream(orgId, { + branch: "dream/2026-11-08", + base: oldMain, + files: new Map(), + message: "dream", + }) + .pipe(Effect.result), + ).toMatchObject(guarded); + expect( + yield* writer.deleteDreamBranch(orgId, "dream/2026-11-08").pipe(Effect.result), + ).toMatchObject(guarded); + expect(yield* git(remote, ["rev-parse", "refs/heads/main"])).toBe(oldMain); + expect(yield* git(remote, ["for-each-ref", "--format=%(refname)", "refs/heads/dream/"])).toBe( + "", + ); + + expect(yield* reconcile(plan)).toBe("not_done"); + expect(events).toEqual(["resume"]); + expect(yield* memory.purgeInFlight(orgId)).toBeUndefined(); + }).pipe(Effect.provide(writerLayer())); + }).pipe(Effect.provide(services), Effect.scoped), +); + +const drop: Purge.PurgePlan = { + orgId, + repo: "fixture-remote", + mode: "drop_file", + paths: ["claims/leak.md"], +}; + +const unitLayer = (baseDirectory: string) => + Purge.layer.pipe( + Layer.provide(Layer.mock(Memory.Memory)({ purgeInFlight: () => Effect.undefined })), + Layer.provide( + Layer.succeed(MemoryConfig, { + baseDirectory, + repositories: [{ orgId, repoUrl: "fixture-remote", path: "state" }], + }), + ), + Layer.provide(Layer.mock(Scan.KinSecretScan)({})), + Layer.provide(Layer.mock(PurgePush.KinPurgePush)({})), + Layer.provide(Layer.mock(Vcs.VcsProcess)({})), + Layer.provide(NodeServices.layer), + ); + +it.effect("the executor gate refuses a purge until APPR-9, APPR-10 and LEAD-11 land", () => + Effect.gen(function* () { + const action = actionFor(drop); + for (const attempt of [ + Purge.memoryPurgeHandler.precheck(action), + Purge.memoryPurgeHandler.run(action, grantFor(drop)), + Purge.memoryPurgeHandler.reconcile(action), + Effect.gen(function* () { + return yield* (yield* Purge.KinPurge).purge(action, grantFor(drop)); + }), + ]) { + const result = yield* Effect.result(attempt); + expect(result).toMatchObject({ failure: { _tag: "KinPurgeExecutorNotReady" } }); + } + }).pipe(Effect.provide(unitLayer("/nonexistent"))), +); + +it.effect("a stored plan binds its file to the action digest until it is discarded", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped(); + yield* Effect.gen(function* () { + const purge = yield* Purge.KinPurge; + const action = actionFor(drop); + const target = yield* purge.storePlan(drop); + expect(target).toEqual(action.target); + expect(yield* fs.readFileString(`${root}/${target.path}`)).toBe( + Purge.purgePlanJson(drop).json, + ); + yield* purge.precheck(action).pipe(gateOpen); + + yield* fs.writeFileString( + `${root}/${target.path}`, + Purge.purgePlanJson({ ...drop, paths: ["claims/keep.md"] }).json, + ); + expect(yield* purge.precheck(action).pipe(gateOpen, Effect.result)).toMatchObject({ + failure: { _tag: "KinPurgeRejected", reason: "Plan does not match the approved digest" }, + }); + expect( + yield* purge + .precheck({ ...action, target: { ...target, path: "../escape.json" } }) + .pipe(gateOpen, Effect.result), + ).toMatchObject({ failure: { _tag: "KinPurgeRejected", reason: "Invalid plan path" } }); + + yield* purge.discardPlan(target); + expect(yield* fs.exists(`${root}/${target.path}`)).toBe(false); + expect(yield* purge.precheck(action).pipe(gateOpen, Effect.result)).toMatchObject({ + failure: { _tag: "KinPurgeRejected", reason: "Plan file is missing" }, + }); + expect( + yield* purge.storePlan({ ...drop, repo: "other-remote" }).pipe(Effect.result), + ).toMatchObject({ failure: { _tag: "KinPurgeRejected" } }); + }).pipe(Effect.provide(unitLayer(root))); + }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), +); + +const pushInput = { + orgId, + repoUrl: "https://example.invalid/memory.git", + gitDir: "/nonexistent", + updates: [{ ref: "refs/heads/main", old: "a".repeat(40), new: "b".repeat(40) }], +}; +const pushWith = ( + owner: Parameters[0], + output: { readonly code: number; readonly stdout?: string; readonly stderr?: string }, + repoUrl = pushInput.repoUrl, +) => { + const calls: Array = []; + return Effect.gen(function* () { + const result = yield* (yield* PurgePush.KinPurgePush) + .push({ ...pushInput, repoUrl }) + .pipe(Effect.result); + return { result, calls }; + }).pipe( + Effect.provide( + PurgePush.gitPushLayer(owner).pipe( + Layer.provide(NodeServices.layer), + Layer.provide( + Layer.mock(Vcs.VcsProcess)({ + run: (input) => + Effect.sync(() => { + calls.push(input); + return { + exitCode: ChildProcessSpawner.ExitCode(output.code), + stdout: output.stdout ?? "", + stderr: output.stderr ?? "", + stdoutTruncated: false, + stderrTruncated: false, + }; + }), + }), + ), + ), + ), + ); +}; + +it.effect.each([ + ["no credential", {}, pushInput.repoUrl, "credential_unavailable"], + [ + "inline helper script", + { credentialHelper: "!f() { echo password=hunter2; }; f" }, + pushInput.repoUrl, + "credential_unavailable", + ], + [ + "shell in ssh command", + { sshCommand: "ssh -i key; cat /etc/passwd" }, + pushInput.repoUrl, + "credential_unavailable", + ], + [ + "https without a helper", + { sshCommand: "ssh -i /keys/owner" }, + pushInput.repoUrl, + "credential_unavailable", + ], + [ + "ssh without a command", + { credentialHelper: "osxkeychain" }, + "git@example.invalid:memory.git", + "credential_unavailable", + ], + [ + "plain http", + { credentialHelper: "osxkeychain" }, + "http://example.invalid/memory.git", + "unsupported_remote", + ], + [ + "remote helper", + { credentialHelper: "osxkeychain", sshCommand: "ssh -i /keys/owner" }, + "ext::ssh -i /keys/owner example.invalid %S memory.git", + "unsupported_remote", + ], +] as const)( + "the owner push fails closed without a matching credential reference: %s", + ([, owner, repoUrl, reason]) => + Effect.gen(function* () { + const { result, calls } = yield* pushWith(owner, { code: 0 }, repoUrl); + expect(result).toMatchObject({ failure: { reason } }); + expect(calls).toEqual([]); + }), +); + +it.effect("the owner push passes only credential references and never prompts", () => + Effect.gen(function* () { + const { result, calls } = yield* Effect.acquireUseRelease( + Effect.sync(() => vi.stubEnv("GIT_CONFIG_PARAMETERS", "'core.hookspath'='/invented'")), + () => + pushWith( + { credentialHelper: "osxkeychain", sshCommand: "ssh -i /keys/owner" }, + { code: 0 }, + ), + () => Effect.sync(() => vi.unstubAllEnvs()), + ); + expect(result._tag).toBe("Success"); + const [call] = calls; + expect(call?.args).toContain("credential.helper=osxkeychain"); + expect(call?.env).toMatchObject({ + GIT_TERMINAL_PROMPT: "0", + GIT_ASKPASS: "false", + SSH_ASKPASS: "false", + SSH_ASKPASS_REQUIRE: "never", + GIT_SSH_COMMAND: + "ssh -i /keys/owner -F none -o BatchMode=yes -o IdentitiesOnly=yes -o IdentityAgent=none", + }); + expect(call?.env?.HOME).toMatch(/kin-push-home-/); + expect(call?.env).toHaveProperty("GIT_CONFIG_PARAMETERS", undefined); + expect(call?.env).toHaveProperty("SSH_AUTH_SOCK", undefined); + + const helperOnly = yield* pushWith({ credentialHelper: "osxkeychain" }, { code: 0 }); + expect(helperOnly.calls[0]?.env?.GIT_SSH_COMMAND).toBe("false"); + }), +); + +it.effect.each([ + [ + "stale lease", + { stdout: "To x\n!\trefs/heads/main:refs/heads/main\t[rejected] (stale info)\nDone\n" }, + "lease_rejected", + ], + [ + "ruleset", + { + stdout: + "To x\n!\trefs/heads/main:refs/heads/main\t[remote rejected] (protected branch hook declined)\n", + }, + "remote_rejected", + ], + [ + "auth", + { stderr: `fatal: Authentication failed for 'https://x:${token}@example.invalid/'\n` }, + "auth", + ], + [ + "network", + { stderr: "fatal: unable to access 'https://example.invalid/': Could not resolve host\n" }, + "network", + ], + ["other", { stderr: "fatal: something else\n" }, "unknown"], +] as const)("classifies a failed owner push: %s", ([, output, reason]) => + Effect.gen(function* () { + const { result } = yield* pushWith({ credentialHelper: "osxkeychain" }, { code: 1, ...output }); + expect(result).toMatchObject({ failure: { _tag: "KinPurgePushError", reason } }); + expect(JSON.stringify(result)).not.toContain(token); + }), +); + +it.each([ + ["parent escape", { paths: ["../claims/leak.md"] }], + ["inner escape", { paths: ["claims/../rules.md"] }], + ["absolute", { paths: ["/claims/leak.md"] }], + ["option", { paths: ["--force.md"] }], + ["option segment", { paths: ["claims/-x.md"] }], + ["git dir", { paths: [".git/config.md"] }], + ["git dir case", { paths: ["claims/.GIT./x.md"] }], + ["empty segment", { paths: ["claims//leak.md"] }], + ["dot segment", { paths: ["claims/./leak.md"] }], + ["backslash", { paths: ["claims\\leak.md"] }], + ["newline", { paths: ["claims/leak.md\nrules.md"] }], + ["decomposed", { paths: ["claims/café.md"] }], + ["not markdown", { paths: ["claims"] }], + ["rules", { paths: ["rules.md"] }], + ["no paths", { paths: [] }], + ["duplicate", { paths: ["claims/leak.md", "claims/leak.md"] }], +] as const)("rejects an unsafe drop plan: %s", (_name, overrides) => { + const plan = { ...drop, ...overrides }; + expect(Purge.purgeRejection(plan, actionFor(plan), grantFor(plan))).toBeDefined(); +}); + +it.each([ + ["no patterns", []], + ["short pattern", ["short"]], + ["duplicate", ["Invented text", "Invented text"]], + ["rule injection", ["Invented==>regex:.*"]], + ["extra rule", ["Invented text\nregex:.*"]], +] as const)("rejects an unsafe redact plan: %s", (_name, patterns) => { + const plan: Purge.PurgePlan = { orgId, repo: "fixture-remote", mode: "redact_text", patterns }; + expect(Purge.purgeRejection(plan, actionFor(plan), grantFor(plan))).toBeDefined(); +}); + +it.each([ + [ + "digest", + (a: KinPendingAction, g: KinGrantPayload) => + [ + { ...a, target: { ...Purge.purgePlanJson(drop).target, sha256: "0".repeat(64) } }, + g, + ] as const, + ], + [ + "plan path", + (a: KinPendingAction, g: KinGrantPayload) => + [ + { ...a, target: { ...Purge.purgePlanJson(drop).target, path: "purge-plans/other.json" } }, + g, + ] as const, + ], + [ + "action", + (a: KinPendingAction, g: KinGrantPayload) => + [a, { ...g, actionId: KinActionId.make("act-2") }] as const, + ], + ["revision", (a: KinPendingAction, g: KinGrantPayload) => [a, { ...g, revision: 2 }] as const], + [ + "grant target", + (a: KinPendingAction, g: KinGrantPayload) => + [ + a, + { ...g, target: Purge.purgePlanJson({ ...drop, paths: ["claims/keep.md"] }).target }, + ] as const, + ], + [ + "grant digest", + (a: KinPendingAction, g: KinGrantPayload) => + [a, { ...g, targetDigest: "0".repeat(64) }] as const, + ], + [ + "org", + (a: KinPendingAction, g: KinGrantPayload) => + [a, { ...g, orgId: KinOrgId.make("other-org") }] as const, + ], + [ + "action id", + (a: KinPendingAction, g: KinGrantPayload) => + [ + { ...a, id: KinActionId.make("act 1\nforged") }, + { ...g, actionId: KinActionId.make("act 1\nforged") }, + ] as const, + ], +] as const)("rejects authority that does not bind the plan: %s", (_name, tamper) => { + expect(Purge.purgeRejection(drop, actionFor(drop), grantFor(drop))).toBeUndefined(); + expect(Purge.purgeRejection(drop, ...tamper(actionFor(drop), grantFor(drop)))).toBeDefined(); +}); diff --git a/apps/server/src/kin/memory/purge.ts b/apps/server/src/kin/memory/purge.ts new file mode 100644 index 0000000000000..b84450f257f02 --- /dev/null +++ b/apps/server/src/kin/memory/purge.ts @@ -0,0 +1,1039 @@ +// @effect-diagnostics-next-line nodeBuiltinImport:off -- The plan digest must match the grant's sha256 content target. +import * as NodeCrypto from "node:crypto"; +import { + KinOrgId, + type KinActionTarget, + type KinGrantPayload, + type KinPendingAction, + type MemoryItemKind, +} from "@t3tools/contracts"; +import * as Cause from "effect/Cause"; +import * as Clock from "effect/Clock"; +import * as Context from "effect/Context"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; +import * as Schema from "effect/Schema"; +import { stringify } from "yaml"; + +import { VcsProcess } from "../../vcs/VcsProcess.ts"; +import { KinPurgePush, hardenedGitEnv, scrubbedGitEnv, type PurgeRefUpdate } from "./purgePush.ts"; +import { parseFrontMatter } from "./frontMatter.ts"; +import { memoryGit } from "./git.ts"; +import { Memory, type MemoryHistoryRewrite } from "./Memory.ts"; +import { MemoryConfig } from "./MemoryConfig.ts"; +import { readObjects } from "./objects.ts"; +import { KinSecretScan, type SecretFinding } from "./secretScan.ts"; + +/** `redact_text` rewrites every blob and commit message in history, so it takes no paths. */ +const PurgePlanSchema = Schema.Union([ + Schema.Struct({ + orgId: KinOrgId, + repo: Schema.String, + mode: Schema.Literal("drop_file"), + paths: Schema.Array(Schema.String), + }), + Schema.Struct({ + orgId: KinOrgId, + repo: Schema.String, + mode: Schema.Literal("redact_text"), + patterns: Schema.Array(Schema.String), + }), +]); +export type PurgePlan = typeof PurgePlanSchema.Type; +const decodePlan = Schema.decodeUnknownEffect(Schema.fromJsonString(PurgePlanSchema)); + +const Tombstones = Schema.Array(Schema.String); +/** Carries the plan so `reconcile` and re-runs still work if the plan file is lost. */ +const PurgeMarker = Schema.Struct({ + sha256: Schema.String, + actionId: Schema.String, + oldHead: Schema.String, + plan: Schema.String, + /** Blob ids of this purge's own tombstones; any other blob at a purged path is old content. */ + tombstones: Tombstones, +}); +const decodeMarker = Schema.decodeUnknownOption(Schema.fromJsonString(PurgeMarker)); +const decodeTombstones = Schema.decodeUnknownOption(Schema.fromJsonString(Tombstones)); +const encodeTombstones = Schema.encodeEffect(Schema.fromJsonString(Tombstones)); + +/** A `memory_purge` action's target: the stored plan file and its sha256. */ +export type PurgeTarget = Extract; +const purgeTarget = (action: KinPendingAction) => + action.kind === "memory_purge" && action.target._tag === "content" ? action.target : undefined; + +/** Lane B's `KinAuditAppendInput` (APPR-10) for the `execution_finished` entry. */ +export interface PurgeAuditInput { + readonly orgId: KinOrgId; + readonly event: "execution_finished"; + readonly actionId: string; + readonly kind: "memory_purge"; + readonly grantJti: string; + readonly keyThumbprint: string; + readonly headSha: string; + readonly result: string; +} + +export interface PurgeResult { + readonly mode: PurgePlan["mode"]; + readonly paths: ReadonlyArray; + readonly oldHead: string; + readonly newHead: string; + readonly rewrittenRefs: ReadonlyArray; + readonly tombstones: ReadonlyArray; + readonly audit: PurgeAuditInput; +} + +export class KinPurgeRejected extends Schema.TaggedError()("KinPurgeRejected", { + reason: Schema.String, +}) {} +export class KinPurgeExecutorNotReady extends Schema.TaggedError()( + "KinPurgeExecutorNotReady", + { actionId: Schema.String }, +) {} +export class KinPurgePauseUnavailable extends Schema.TaggedError()( + "KinPurgePauseUnavailable", + { orgId: Schema.String }, +) {} +export class KinPurgeFailed extends Schema.TaggedError()("KinPurgeFailed", { + orgId: Schema.String, + stage: Schema.Literals([ + "clone", + "rewrite", + "verify", + "tombstone", + "quarantine", + "push", + "rebuild", + ]), + reason: Schema.String, + /** Once the checkouts are quarantined for the push, the queue stays paused until `reconcile`. */ + queueHeld: Schema.Boolean, + cause: Schema.optional(Schema.Defect()), +}) {} + +/** + * LEAD-11 supplies the real org pause; until then the purge can't run. `pause` must not return while a + * wake-up or dream is still running for the org: their staged files and dream copies predate the rewrite. + */ +export class KinPurgeQueue extends Context.Reference<{ + readonly pause: (orgId: KinOrgId) => Effect.Effect; + readonly resume: (orgId: KinOrgId) => Effect.Effect; +}>("t3/kin/memory/purge/KinPurgeQueue", { + defaultValue: () => ({ + pause: (orgId) => Effect.fail(new KinPurgePauseUnavailable({ orgId })), + resume: () => Effect.void, + }), +}) {} + +export const GitFilterRepoCommand = Context.Reference( + "t3/kin/memory/purge/GitFilterRepoCommand", + { defaultValue: () => "git-filter-repo" }, +); + +const literalPathspecs = { GIT_LITERAL_PATHSPECS: "1" }; + +const itemKinds = new Map([ + ["objectives", "objective"], + ["commitments", "commitment"], + ["deferrals", "deferral"], + ["decisions", "decision"], + ["people", "person"], + ["claims", "claim"], +]); + +const isPurgePath = (path: string) => + path.length > 0 && + path.length <= 512 && + path.normalize("NFC") === path && + !/[\u0000-\u001f\u007f-\u009f\\]|\p{Default_Ignorable_Code_Point}/u.test(path) && + path.endsWith(".md") && + path !== "rules.md" && + path + .split("/") + .every( + (segment) => + segment.length > 0 && + segment !== "." && + segment !== ".." && + !segment.startsWith("-") && + segment.trim() === segment && + !/^\.git[. ]*$/i.test(segment), + ); + +const isPattern = (pattern: string) => + pattern.length >= 8 && + pattern.length <= 1024 && + !/[\u0000-\u001f\u007f]/.test(pattern) && + !pattern.includes("==>"); + +const sha256 = (text: string) => NodeCrypto.createHash("sha256").update(text).digest("hex"); +const planFilePath = /^purge-plans\/[a-f0-9]{64}\.json$/; + +/** The stored plan file and the action target that binds it, from one serialization. */ +export const purgePlanJson = (plan: PurgePlan) => { + const json = JSON.stringify( + plan.mode === "drop_file" + ? { orgId: plan.orgId, repo: plan.repo, mode: plan.mode, paths: plan.paths } + : { orgId: plan.orgId, repo: plan.repo, mode: plan.mode, patterns: plan.patterns }, + ); + const digest = sha256(json); + return { + json, + target: { + _tag: "content", + path: `purge-plans/${digest}.json`, + sha256: digest, + } satisfies PurgeTarget, + }; +}; + +const planRejection = (plan: PurgePlan) => { + if (plan.mode === "drop_file") { + if (plan.paths.length === 0 || plan.paths.length > 64) return "Expected 1-64 paths"; + if (new Set(plan.paths).size !== plan.paths.length) return "Duplicate paths"; + if (!plan.paths.every(isPurgePath)) return "Invalid path"; + return undefined; + } + if (plan.patterns.length === 0 || plan.patterns.length > 64) return "Expected 1-64 patterns"; + if (new Set(plan.patterns).size !== plan.patterns.length) return "Duplicate patterns"; + if (!plan.patterns.every(isPattern)) return "Invalid pattern"; + return undefined; +}; + +/** Returns why the plan, its action or its grant is unacceptable, or `undefined`. */ +export const purgeRejection = ( + plan: PurgePlan, + action: KinPendingAction, + grant?: KinGrantPayload, +) => { + const actionTarget = purgeTarget(action); + if (!actionTarget) return "Action is not a memory purge"; + if (action.orgId !== plan.orgId || (grant !== undefined && grant.orgId !== plan.orgId)) + return "Grant, action and plan organizations differ"; + if ( + grant !== undefined && + (grant.actionId !== action.id || + grant.revision !== action.revision || + grant.kind !== action.kind || + grant.targetDigest !== action.targetDigest || + grant.target._tag !== "content" || + grant.target.sha256 !== actionTarget.sha256) + ) + return "Grant does not bind this action revision"; + const { target } = purgePlanJson(plan); + if (actionTarget.sha256 !== target.sha256 || actionTarget.path !== target.path) + return "Plan does not match the approved digest"; + if (!/^[A-Za-z0-9_-]{1,128}$/.test(action.id)) return "Invalid action id"; + return planRejection(plan); +}; + +const isPurgeFailed = Schema.is(KinPurgeFailed); +const markerJson = (marker: typeof PurgeMarker.Type) => JSON.stringify(marker); +const auditResult = (result: { + readonly mode: PurgePlan["mode"]; + readonly paths: ReadonlyArray; + readonly oldHead: string; + readonly newHead: string; + readonly refs: ReadonlyArray; +}) => JSON.stringify(result); +const render = (fields: object, body: string) => `---\n${stringify(fields)}---\n${body}`; +const scanBatchFiles = 256; +const scanBatchBytes = 4 * 1024 * 1024; + +export class KinPurge extends Context.Service< + KinPurge, + { + /** Writes the plan file a `memory_purge` action targets; the proposer calls it before `request_action`. */ + readonly storePlan: ( + plan: PurgePlan, + ) => Effect.Effect; + /** Deletes the plan file once the action is terminal; it can hold the redacted secret. */ + readonly discardPlan: (target: PurgeTarget) => Effect.Effect; + readonly precheck: ( + action: KinPendingAction, + ) => Effect.Effect; + /** Server-only: reached through `memoryPurgeHandler`, never a model, MCP tool or RPC. */ + readonly purge: ( + action: KinPendingAction, + grant: KinGrantPayload, + ) => Effect.Effect< + PurgeResult, + KinPurgeExecutorNotReady | KinPurgeRejected | KinPurgePauseUnavailable | KinPurgeFailed + >; + readonly reconcile: ( + action: KinPendingAction, + ) => Effect.Effect< + "done" | "not_done" | "unknown", + KinPurgeExecutorNotReady | KinPurgeRejected | KinPurgeFailed + >; + } +>()("t3/kin/memory/purge/KinPurge") {} + +// Gated until APPR-9 (executor), APPR-10 (audit) and LEAD-11 (queue pause) land. +const purgeExecutorReady = false; +/** Test seam for the release gate; production always reads `purgeExecutorReady`. */ +export const PurgeExecutorGate = Context.Reference( + "t3/kin/memory/purge/PurgeExecutorGate", + { defaultValue: () => purgeExecutorReady }, +); + +const make = Effect.gen(function* () { + const memory = yield* Memory; + const config = yield* MemoryConfig; + const scanner = yield* KinSecretScan; + const pusher = yield* KinPurgePush; + const queue = yield* KinPurgeQueue; + const filterRepo = yield* GitFilterRepoCommand; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const vcs = yield* VcsProcess; + const provide = (effect: Effect.Effect) => + effect.pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(VcsProcess, vcs), + ); + const git = ( + work: string, + args: ReadonlyArray, + stdin?: string, + env: NodeJS.ProcessEnv = {}, + ) => + provide(memoryGit(work, args, stdin, { ...scrubbedGitEnv(), ...hardenedGitEnv, ...env })).pipe( + Effect.provideService(MemoryConfig, config), + Effect.map((stdout) => stdout.trim()), + ); + const failed = + (orgId: KinOrgId, stage: KinPurgeFailed["stage"], reason: string) => + (effect: Effect.Effect) => + effect.pipe( + Effect.mapError((cause) => + isPurgeFailed(cause) + ? cause + : new KinPurgeFailed({ orgId, stage, reason, queueHeld: false, cause }), + ), + ); + const fail = (orgId: KinOrgId, stage: KinPurgeFailed["stage"], reason: string) => + Effect.fail(new KinPurgeFailed({ orgId, stage, reason, queueHeld: false })); + const reject = (reason: string) => Effect.fail(new KinPurgeRejected({ reason })); + + const gate = Effect.fn("KinPurge.gate")(function* (action: KinPendingAction) { + if (!(yield* PurgeExecutorGate)) + return yield* new KinPurgeExecutorNotReady({ actionId: action.id }); + }); + + const repoFor = (plan: PurgePlan) => { + const repo = config.repositories.find((entry) => entry.orgId === plan.orgId); + return repo && repo.repoUrl === plan.repo ? repo : undefined; + }; + + const planFile = (target: PurgeTarget) => + planFilePath.test(target.path) && target.path === `purge-plans/${target.sha256}.json` + ? Effect.succeed(path.resolve(config.baseDirectory, target.path)) + : reject("Invalid plan path"); + /** The tombstone ids outlive the marker, so a `reconcile` after a finished purge still knows them. */ + const tombstonesFile = (planPath: string) => planPath.replace(/\.json$/, ".tombstones.json"); + + const markerFor = (orgId: KinOrgId) => + memory.purgeInFlight(orgId).pipe( + Effect.map((inFlight) => ({ + inFlight, + marker: inFlight === undefined ? undefined : Option.getOrUndefined(decodeMarker(inFlight)), + })), + ); + + const loadPlan = Effect.fn("KinPurge.loadPlan")(function* ( + action: KinPendingAction, + target: PurgeTarget, + ) { + const file = yield* planFile(target); + const stored = yield* fs.readFileString(file).pipe(Effect.option); + const json = Option.isSome(stored) + ? stored.value + : config.repositories.some((entry) => entry.orgId === action.orgId) + ? yield* markerFor(action.orgId).pipe( + Effect.map(({ marker }) => + marker?.sha256 === target.sha256 ? marker.plan : undefined, + ), + Effect.orElseSucceed(() => undefined), + ) + : undefined; + if (json === undefined) return yield* reject("Plan file is missing"); + const plan = yield* decodePlan(json).pipe( + Effect.mapError(() => new KinPurgeRejected({ reason: "Plan file is malformed" })), + ); + if (purgePlanJson(plan).json !== json) + return yield* reject("Plan does not match the approved digest"); + return { plan, file }; + }); + + /** Validates the action against its stored plan and the org's config, and returns any purge marker. */ + const checked = Effect.fn("KinPurge.checked")(function* ( + action: KinPendingAction, + grant?: KinGrantPayload, + ) { + yield* gate(action); + const target = purgeTarget(action); + if (!target) return yield* reject("Action is not a memory purge"); + const { plan, file } = yield* loadPlan(action, target); + const rejection = purgeRejection(plan, action, grant); + if (rejection !== undefined) return yield* reject(rejection); + const repo = repoFor(plan); + if (!repo) return yield* reject("Plan repo is not this organization's memory"); + const { inFlight, marker } = yield* markerFor(plan.orgId).pipe( + failed(plan.orgId, "clone", "Could not read the purge marker"), + ); + return { plan, file, target, repo, inFlight, marker }; + }); + + const branchRefs = Effect.fn("KinPurge.branchRefs")(function* (work: string) { + const lines = yield* git(work, [ + "for-each-ref", + "--format=%(refname) %(objectname)", + "refs/heads/", + "refs/tags/", + ]); + return new Map( + lines.split("\n").flatMap((line) => { + const [ref, sha] = line.split(" "); + return ref && sha ? [[ref, sha] as const] : []; + }), + ); + }); + + /** Every blob ever introduced, merge sides included, plus every commit and tag object unless `paths` narrows it. */ + const historyFiles = Effect.fn("KinPurge.historyFiles")(function* ( + orgId: KinOrgId, + work: string, + paths?: ReadonlyArray, + ) { + const raw = yield* git( + work, + [ + "log", + "--all", + "--raw", + "--no-abbrev", + "--no-renames", + "--diff-merges=separate", + "--format=", + "-z", + ...(paths ? ["--", ...paths] : []), + ], + undefined, + literalPathspecs, + ); + const blobs = new Map(); + const tokens = raw.split("\0"); + for (let index = 0; index < tokens.length; index++) { + const header = /^\n*:\d{6} (\d{6}) [a-f0-9]+ ([a-f0-9]+) [A-Z]\d*$/.exec(tokens[index] ?? ""); + if (!header) continue; + const path = tokens[++index] ?? ""; + const [, mode, sha] = header; + if (!sha || /^0+$/.test(sha) || mode === "160000") continue; + blobs.set(`${sha} ${path}`, { sha, path }); + } + const commits = paths + ? [] + : (yield* git(work, ["rev-list", "--all"])).split("\n").filter(Boolean); + // Tagger idents survive filter-repo's text and message rewrites; nested tags included. + const tags = paths + ? [] + : (yield* git(work, [ + "cat-file", + "--batch-all-objects", + "--batch-check=%(objectname) %(objecttype)", + ])) + .split("\n") + .flatMap((line) => { + const [sha, type] = line.split(" "); + return sha && type === "tag" ? [sha] : []; + }); + const requests = [ + ...[...blobs.values()].map(({ sha, path }) => ({ sha, type: "blob" as const, path })), + ...commits.map((sha) => ({ sha, type: "commit" as const, path: `commit ${sha}` })), + ...tags.map((sha) => ({ sha, type: "tag" as const, path: `tag ${sha}` })), + ]; + const files: Array<{ sha: string; path: string; content: Uint8Array }> = []; + for (let index = 0; index < requests.length; index += 32) { + const batch = requests.slice(index, index + 32); + const objects = yield* provide(readObjects(orgId, work, batch)); + for (const [position, request] of batch.entries()) { + const content = objects[position]; + if (content) files.push({ sha: request.sha, path: request.path, content }); + } + } + return files; + }); + + const containsPattern = ( + files: ReadonlyArray<{ readonly content: Uint8Array }>, + patterns: ReadonlyArray, + ) => { + const needles = patterns.map((pattern) => Buffer.from(pattern, "utf8")); + return files.some((file) => + needles.some((needle) => Buffer.from(file.content).includes(needle)), + ); + }; + + const pathInHistory = (work: string, file: string) => + git(work, ["rev-list", "--all", "-n", "1", "--", file], undefined, literalPathspecs).pipe( + Effect.map((commits) => commits.length > 0), + ); + + /** Whether any reachable commit still holds what the plan removes, besides this purge's `tombstones`. */ + const planInHistory = Effect.fn("KinPurge.planInHistory")(function* ( + plan: PurgePlan, + tombstones: ReadonlyArray, + work: string, + ) { + if (plan.mode === "drop_file") + return (yield* historyFiles(plan.orgId, work, plan.paths)).some( + (file) => !tombstones.includes(file.sha), + ); + return containsPattern(yield* historyFiles(plan.orgId, work), plan.patterns); + }); + + const scanAll = Effect.fn("KinPurge.scanAll")(function* ( + orgId: KinOrgId, + files: ReadonlyArray<{ path: string; content: Uint8Array }>, + ) { + const findings: SecretFinding[] = []; + let batch: Array<{ path: string; content: Uint8Array }> = []; + let bytes = 0; + const flush = Effect.gen(function* () { + if (batch.length) findings.push(...(yield* scanner.scan({ orgId, files: batch }))); + batch = []; + bytes = 0; + }); + for (const file of files) { + if (batch.length >= scanBatchFiles || bytes + file.content.length > scanBatchBytes) + yield* flush; + batch.push(file); + bytes += file.content.length; + } + yield* flush; + return findings; + }); + + const tombstone = Effect.fn("KinPurge.tombstone")(function* ( + orgId: KinOrgId, + work: string, + oldHead: string, + path: string, + action: KinPendingAction, + grant: KinGrantPayload, + now: number, + ) { + const folder = path.split("/")[0] ?? ""; + const kind = itemKinds.get(folder); + if (!kind) return undefined; + const blob = yield* git(work, ["rev-parse", "--verify", "--quiet", `${oldHead}:${path}`]).pipe( + Effect.option, + ); + if (blob._tag === "None") return undefined; + const original = yield* provide( + readObjects(orgId, work, [{ sha: blob.value, type: "blob" }]), + ).pipe( + Effect.flatMap((objects) => + parseFrontMatter(orgId, path, objects[0]?.toString("utf8") ?? ""), + ), + Effect.option, + ); + const iso = DateTime.formatIso(DateTime.makeUnsafe(now)); + const content = render( + { + id: + original._tag === "Some" + ? original.value.frontMatter.id + : (path.split("/").at(-1) ?? path).slice(0, -3), + kind, + title: "Purged item", + status: "cancelled", + due: null, + recorded_at: iso, + valid_from: iso, + invalid_at: iso, + superseded_by: null, + review_by: null, + source: `kin-action:${action.id}`, + said_by: grant.approver.humanId, + trust: "owner", + links: [], + }, + `Purged on ${iso.slice(0, 10)} by the owner (action ${action.id}).\n`, + ); + yield* parseFrontMatter(orgId, path, content); + return content; + }); + + const commitTombstones = Effect.fn("KinPurge.commitTombstones")(function* ( + work: string, + branch: string, + head: string, + tombstones: ReadonlyMap, + actionId: string, + now: number, + ) { + const date = `${Math.floor(now / 1000)} +0000`; + const env = { + GIT_INDEX_FILE: `${work}/tombstone-index`, + GIT_AUTHOR_NAME: config.botName ?? "Kin Memory Bot", + GIT_AUTHOR_EMAIL: config.botEmail ?? "kin-memory@localhost", + GIT_COMMITTER_NAME: config.botName ?? "Kin Memory Bot", + GIT_COMMITTER_EMAIL: config.botEmail ?? "kin-memory@localhost", + GIT_AUTHOR_DATE: date, + GIT_COMMITTER_DATE: date, + }; + yield* git(work, ["read-tree", head], undefined, env); + const blobs: string[] = []; + const entries: string[] = []; + for (const [path, content] of tombstones) { + const blob = yield* git(work, ["hash-object", "-w", "--stdin"], content); + blobs.push(blob); + entries.push(`100644 blob ${blob}\t${path}`); + } + yield* git(work, ["update-index", "--index-info"], entries.join("\n") + "\n", env); + const tree = yield* git(work, ["write-tree"], undefined, env); + const commit = yield* git( + work, + ["commit-tree", tree, "-p", head, "-m", `Purge tombstones (action ${actionId})`], + undefined, + env, + ); + yield* git(work, ["update-ref", `refs/heads/${branch}`, commit, head]); + return { commit, blobs }; + }); + + /** + * A fresh bare clone of `refs/heads/*` and `refs/tags/*` under the org's purge directory; stale ones + * from a crashed purge are swept first. GitHub's read-only `refs/pull/*` stay out of reach. Any other + * advertised ref (`refs/notes/*`, custom namespaces) would keep the old history, so it fails closed. + */ + const mirror = Effect.fn("KinPurge.mirror")(function* ( + orgId: KinOrgId, + repo: { readonly repoUrl: string; readonly path: string }, + ) { + const advertised = yield* pusher + .listRefs({ orgId, repoUrl: repo.repoUrl }) + .pipe(failed(orgId, "clone", "Could not list remote refs")); + const foreign = advertised.filter( + (ref) => ref !== "HEAD" && !/^refs\/(heads|tags|pull)\//.test(ref), + ); + if (foreign.length > 0) + return yield* fail( + orgId, + "clone", + `Remote has refs a purge can't rewrite; remove them first: ${foreign.slice(0, 10).join(", ")}`, + ); + const root = path.resolve(config.baseDirectory, repo.path, "purge-scratch"); + const work = yield* Effect.gen(function* () { + yield* fs.remove(root, { recursive: true, force: true }); + yield* fs.makeDirectory(root, { recursive: true }); + return yield* fs.makeTempDirectoryScoped({ directory: root, prefix: "mirror-" }); + }).pipe(failed(orgId, "clone", "Could not create a scratch directory")); + yield* git(work, [ + "clone", + "--bare", + "--no-local", + "--template=", + "--", + repo.repoUrl, + `${work}/.git`, + ]).pipe(failed(orgId, "clone", "Bare clone failed")); + // filter-repo rewrites commits only, and `historyFiles` walks commits only. + const nonCommitTags = (yield* git(work, [ + "for-each-ref", + "--format=%(refname) %(objecttype) %(*objecttype)", + "refs/tags/", + ]).pipe(failed(orgId, "clone", "Could not list tags"))) + .split("\n") + .flatMap((line) => { + const [ref, type, peeled] = line.split(" "); + return ref && type !== "commit" && !(type === "tag" && peeled === "commit") ? [ref] : []; + }); + if (nonCommitTags.length > 0) + return yield* fail( + orgId, + "clone", + `Remote has tags that don't point at a commit; remove them first: ${nonCommitTags.slice(0, 10).join(", ")}`, + ); + const before = yield* branchRefs(work).pipe(failed(orgId, "clone", "Could not list refs")); + const head = before.get(`refs/heads/${config.defaultBranch ?? "main"}`); + if (!head) return yield* fail(orgId, "clone", "Remote default branch is missing"); + return { work, before, head }; + }); + + const rewrite = Effect.fn("KinPurge.rewrite")(function* (input: { + readonly plan: PurgePlan; + readonly planFile: string; + readonly target: PurgeTarget; + readonly action: KinPendingAction; + readonly grant: KinGrantPayload; + readonly repo: { readonly repoUrl: string; readonly path: string }; + readonly inFlight: string | undefined; + readonly marker: typeof PurgeMarker.Type | undefined; + readonly history: MemoryHistoryRewrite; + readonly commit: () => void; + }) { + const { plan, target, action, grant, repo, history } = input; + const { orgId } = plan; + const branch = config.defaultBranch ?? "main"; + const now = yield* Clock.currentTimeMillis; + if ( + (yield* memory + .purgeInFlight(orgId) + .pipe(failed(orgId, "clone", "Could not read the purge marker"))) !== input.inFlight + ) { + input.commit(); + return yield* fail(orgId, "clone", "The purge marker changed while waiting for the lock"); + } + const { work, before, head: oldHead } = yield* mirror(orgId, repo); + + // A re-run after a push that landed: nothing to rewrite, but the checkouts still need a reclone. + if (input.inFlight !== undefined) { + const remaining = yield* planInHistory(plan, input.marker?.tombstones ?? [], work).pipe( + failed(orgId, "verify", "Could not check history"), + ); + if (!remaining) { + yield* history + .reclone() + .pipe(failed(orgId, "rebuild", "Could not rebuild local memory checkouts")); + return { + oldHead: input.marker?.oldHead ?? oldHead, + newHead: oldHead, + updates: [], + tombstones: [], + }; + } + } + + const tombstones = new Map(); + if (plan.mode === "drop_file") + for (const file of plan.paths) { + const present = yield* pathInHistory(work, file).pipe( + failed(orgId, "verify", "Could not check history"), + ); + if (!present) + return yield* reject( + "A purge path is not in history; check its spelling and Unicode normalization", + ); + const content = yield* tombstone(orgId, work, oldHead, file, action, grant, now).pipe( + failed(orgId, "tombstone", "Could not build a tombstone"), + ); + if (content !== undefined) tombstones.set(file, content); + } + + const rules = `${work}/rules.txt`; + if (plan.mode === "redact_text") + yield* fs + .writeFileString( + rules, + plan.patterns.map((pattern) => `literal:${pattern}==>***REMOVED***`).join("\n") + "\n", + ) + .pipe(failed(orgId, "rewrite", "Could not write replacement rules")); + yield* vcs + .run({ + operation: "KinPurge.filterRepo", + command: filterRepo, + cwd: `${work}/.git`, + args: [ + "--quiet", + ...(plan.mode === "drop_file" + ? ["--invert-paths", ...plan.paths.flatMap((file) => ["--path", file])] + : ["--replace-text", rules, "--replace-message", rules]), + ], + env: { + ...scrubbedGitEnv(), + ...hardenedGitEnv, + // filter-repo accepts only a fresh clone opened as GIT_DIR=. from its own directory. + GIT_DIR: ".", + GIT_CONFIG_COUNT: "3", + GIT_CONFIG_KEY_0: "core.hooksPath", + GIT_CONFIG_VALUE_0: "/dev/null", + GIT_CONFIG_KEY_1: "core.fsmonitor", + GIT_CONFIG_VALUE_1: "false", + GIT_CONFIG_KEY_2: "core.attributesFile", + GIT_CONFIG_VALUE_2: "/dev/null", + }, + timeoutMs: 600_000, + }) + .pipe(failed(orgId, "rewrite", "git filter-repo failed")); + + const rewritten = yield* branchRefs(work).pipe(failed(orgId, "rewrite", "Could not list refs")); + const rewrittenHead = rewritten.get(`refs/heads/${branch}`); + if (!rewrittenHead) return yield* fail(orgId, "rewrite", "Rewrite removed the default branch"); + if ([...before].every(([ref, sha]) => rewritten.get(ref) === sha)) + return yield* fail(orgId, "rewrite", "Nothing in history matched the plan"); + + if (plan.mode === "drop_file") { + const remaining = yield* planInHistory(plan, [], work).pipe( + failed(orgId, "verify", "Could not check rewritten history"), + ); + if (remaining) return yield* fail(orgId, "verify", "Purged paths remain in history"); + } + const { commit: newHead, blobs } = + tombstones.size > 0 + ? yield* commitTombstones(work, branch, rewrittenHead, tombstones, action.id, now).pipe( + failed(orgId, "tombstone", "Could not commit tombstones"), + ) + : { commit: rewrittenHead, blobs: [] }; + + const files = yield* historyFiles(orgId, work).pipe( + failed(orgId, "verify", "Could not read rewritten history"), + ); + if (plan.mode === "redact_text" && containsPattern(files, plan.patterns)) + return yield* fail(orgId, "verify", "Redacted text remains in history"); + const findings = yield* scanAll(orgId, files).pipe( + failed(orgId, "verify", "Secret scan of rewritten history failed"), + ); + if (findings.length > 0) + return yield* fail( + orgId, + "verify", + `Secret scan found ${findings.length} finding(s) in rewritten history: ${[ + ...new Set(findings.map((finding) => finding.ruleId)), + ].join(", ")}`, + ); + + const after = yield* branchRefs(work).pipe(failed(orgId, "verify", "Could not list refs")); + const updates: PurgeRefUpdate[] = [...before] + .filter(([ref, sha]) => after.get(ref) !== sha) + .map(([ref, old]) => ({ ref, old, new: after.get(ref) ?? null })); + + yield* Effect.gen(function* () { + const file = tombstonesFile(input.planFile); + yield* fs.writeFileString(`${file}.tmp`, yield* encodeTombstones(blobs)); + yield* fs.rename(`${file}.tmp`, file); + }).pipe(failed(orgId, "tombstone", "Could not record the tombstones")); + // From here a push may land at any moment, so local checkouts holding the old history must go + // first; the marker keeps preserve and merges from pushing it back until a reclone or reconcile. + input.commit(); + yield* history + .quarantine( + markerJson({ + sha256: target.sha256, + actionId: action.id, + oldHead: input.marker?.oldHead ?? oldHead, + plan: purgePlanJson(plan).json, + tombstones: blobs, + }), + ) + .pipe(failed(orgId, "quarantine", "Could not quarantine local memory checkouts")); + yield* pusher.push({ orgId, repoUrl: repo.repoUrl, gitDir: work, updates }).pipe( + Effect.mapError( + (error) => + new KinPurgeFailed({ + orgId, + stage: "push", + reason: `Force-push failed: ${error.reason}`, + queueHeld: true, + }), + ), + ); + // The lease covers only refs this purge saw, so one created since can still hold the old history. + const clean = yield* mirror(orgId, repo).pipe( + Effect.flatMap((fresh) => planInHistory(plan, blobs, fresh.work)), + Effect.map((remaining) => !remaining), + Effect.orElseSucceed(() => false), + ); + if (!clean) + return yield* new KinPurgeFailed({ + orgId, + stage: "push", + reason: "The remote may still hold purged history after the push; reconcile it", + queueHeld: true, + }); + yield* history + .reclone() + .pipe(failed(orgId, "rebuild", "Could not rebuild local memory checkouts")); + return { + oldHead: input.marker?.oldHead ?? oldHead, + newHead, + updates, + tombstones: [...tombstones.keys()], + }; + }); + + return KinPurge.of({ + storePlan: Effect.fn("KinPurge.storePlan")(function* (plan) { + const rejection = planRejection(plan) ?? (repoFor(plan) ? undefined : "Unknown memory repo"); + if (rejection !== undefined) return yield* reject(rejection); + const { json, target } = purgePlanJson(plan); + const file = yield* planFile(target); + yield* Effect.gen(function* () { + yield* fs.makeDirectory(path.dirname(file), { recursive: true }); + yield* fs.writeFileString(`${file}.tmp`, json); + yield* fs.rename(`${file}.tmp`, file); + }).pipe(failed(plan.orgId, "clone", "Could not store the plan")); + return target; + }), + + discardPlan: Effect.fn("KinPurge.discardPlan")(function* (target) { + const file = yield* planFile(target); + for (const { orgId } of config.repositories) { + const { inFlight, marker } = yield* markerFor(orgId).pipe( + Effect.mapError( + () => new KinPurgeRejected({ reason: "Could not read the purge marker" }), + ), + ); + if (inFlight !== undefined && (marker === undefined || marker.sha256 === target.sha256)) + return yield* reject("This plan's purge is in flight; reconcile it first"); + } + yield* Effect.all([ + fs.remove(file, { force: true }), + fs.remove(tombstonesFile(file), { force: true }), + ]).pipe(Effect.mapError(() => new KinPurgeRejected({ reason: "Could not delete the plan" }))); + }), + + precheck: Effect.fn("KinPurge.precheck")(function* (action) { + const { target, inFlight, marker } = yield* checked(action); + if (inFlight !== undefined && marker?.sha256 !== target.sha256) + return yield* reject("Another purge is in flight; reconcile it first"); + }), + + purge: Effect.fn("KinPurge.purge")(function* (action, grant) { + const { plan, file, target, repo, inFlight, marker } = yield* checked(action, grant); + if (inFlight !== undefined && marker?.sha256 !== target.sha256) + return yield* reject("Another purge is in flight; reconcile it first"); + const { orgId } = plan; + // An unresolved marker means the queue is already held for this plan; never resume it early. + let committed = inFlight !== undefined; + yield* queue.pause(orgId); + const { oldHead, newHead, updates, tombstones } = yield* memory + .withHistoryRewrite(orgId, (history) => + Effect.scoped( + rewrite({ + plan, + planFile: file, + target, + action, + grant, + repo, + inFlight, + marker, + history, + commit: () => { + committed = true; + }, + }), + ), + ) + .pipe( + Effect.onExit((exit) => + Exit.isSuccess(exit) || !committed ? queue.resume(orgId) : Effect.void, + ), + Effect.catchCause((cause) => { + if (!committed || Cause.hasInterruptsOnly(cause)) return Effect.failCause(cause); + const error = Option.getOrUndefined(Cause.findErrorOption(cause)); + return Effect.fail( + new KinPurgeFailed({ + orgId, + stage: isPurgeFailed(error) ? error.stage : "push", + reason: isPurgeFailed(error) + ? error.reason + : "Purge stopped after quarantining local checkouts", + queueHeld: true, + ...(isPurgeFailed(error) && error.cause !== undefined + ? { cause: error.cause } + : {}), + }), + ); + }), + ); + const paths = plan.mode === "drop_file" ? plan.paths : []; + return { + mode: plan.mode, + paths, + oldHead, + newHead, + rewrittenRefs: updates, + tombstones, + audit: { + orgId, + event: "execution_finished", + actionId: action.id, + kind: "memory_purge", + grantJti: grant.jti, + keyThumbprint: grant.approver.kid, + headSha: newHead, + result: auditResult({ mode: plan.mode, paths, oldHead, newHead, refs: updates }), + }, + } satisfies PurgeResult; + }), + + reconcile: Effect.fn("KinPurge.reconcile")(function* (action) { + const { plan, file, target, repo, inFlight, marker } = yield* checked(action); + if (inFlight !== undefined && marker?.sha256 !== target.sha256) return "unknown"; + const { orgId } = plan; + const tombstones = + marker?.tombstones ?? + Option.getOrElse( + Option.flatMap( + yield* fs.readFileString(tombstonesFile(file)).pipe(Effect.option), + decodeTombstones, + ), + () => [], + ); + return yield* memory.withHistoryRewrite(orgId, (history) => + Effect.scoped( + Effect.gen(function* () { + const observed = yield* mirror(orgId, repo).pipe( + Effect.flatMap(({ work, head }) => + planInHistory(plan, tombstones, work).pipe( + Effect.map((remaining) => ({ head, remaining })), + ), + ), + Effect.option, + ); + if (Option.isNone(observed)) return "unknown" as const; + const { head, remaining } = observed.value; + const verdict = !remaining + ? ("done" as const) + : marker === undefined || head === marker.oldHead + ? ("not_done" as const) + : ("unknown" as const); + if (verdict === "unknown") return verdict; + if (inFlight !== undefined) + yield* history.reclone().pipe( + Effect.mapError( + () => + new KinPurgeFailed({ + orgId, + stage: "rebuild", + reason: "Could not rebuild local memory checkouts", + queueHeld: true, + }), + ), + ); + yield* queue.resume(orgId); + return verdict; + }), + ), + ); + }), + }); +}); + +export const layer = Layer.effect(KinPurge, make); + +/** APPR-9's `KinActionHandler` for `memory_purge`; the executor calls it with a verified grant. */ +export const memoryPurgeHandler = { + kind: "memory_purge", + precheck: Effect.fn("KinPurge.handlerPrecheck")(function* (action: KinPendingAction) { + return yield* (yield* KinPurge).precheck(action); + }), + run: Effect.fn("KinPurge.handlerRun")(function* ( + action: KinPendingAction, + grant: KinGrantPayload, + ) { + return yield* (yield* KinPurge).purge(action, grant); + }), + reconcile: Effect.fn("KinPurge.handlerReconcile")(function* (action: KinPendingAction) { + return yield* (yield* KinPurge).reconcile(action); + }), +} as const; diff --git a/apps/server/src/kin/memory/purgePush.ts b/apps/server/src/kin/memory/purgePush.ts new file mode 100644 index 0000000000000..2c80689fad72e --- /dev/null +++ b/apps/server/src/kin/memory/purgePush.ts @@ -0,0 +1,216 @@ +import type { KinOrgId } from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; + +import { VcsProcess } from "../../vcs/VcsProcess.ts"; + +export interface PurgeRefUpdate { + readonly ref: string; + readonly old: string; + readonly new: string | null; +} + +export class KinPurgePushError extends Schema.TaggedError()( + "KinPurgePushError", + { + reason: Schema.Literals([ + "credential_unavailable", + "unsupported_remote", + "lease_rejected", + "remote_rejected", + "rejected", + "auth", + "network", + "timeout", + "unknown", + ]), + }, +) {} +export class KinPurgePush extends Context.Service< + KinPurgePush, + { + readonly push: (input: { + readonly orgId: KinOrgId; + readonly repoUrl: string; + readonly gitDir: string; + readonly updates: ReadonlyArray; + }) => Effect.Effect; + /** Every ref the remote advertises, read with the same owner credential as `push`. */ + readonly listRefs: (input: { + readonly orgId: KinOrgId; + readonly repoUrl: string; + }) => Effect.Effect, KinPurgePushError>; + } +>()("t3/kin/memory/purgePush/KinPurgePush") {} + +/** Inherited `GIT_*` variables can point git at another repository, config or object store. */ +export const scrubbedGitEnv = (): NodeJS.ProcessEnv => + Object.fromEntries( + Object.keys(process.env) + .filter((name) => /^GIT_/i.test(name)) + .map((name) => [name, undefined]), + ); +export const hardenedGitEnv = { + GIT_CONFIG_NOSYSTEM: "1", + GIT_CONFIG_GLOBAL: "/dev/null", + GIT_TERMINAL_PROMPT: "0", + GIT_ATTR_NOSYSTEM: "1", + GIT_NO_REPLACE_OBJECTS: "1", + GIT_ASKPASS: "", +}; +const pushFailure = (stdout: string, stderr: string): KinPurgePushError["reason"] => { + const rejected = stdout.split("\n").filter((line) => line.startsWith("!")); + if (rejected.some((line) => line.includes("(stale info)"))) return "lease_rejected"; + if (rejected.some((line) => line.includes("[remote rejected]"))) return "remote_rejected"; + if (rejected.length > 0) return "rejected"; + if ( + /authentication failed|permission denied|could not read (username|password)|terminal prompts disabled|returned error: 40[13]/i.test( + stderr, + ) + ) + return "auth"; + if ( + /could not resolve host|connection (refused|reset|timed out)|operation timed out|network is unreachable|unable to access|early eof/i.test( + stderr, + ) + ) + return "network"; + return "unknown"; +}; + +// A helper name or absolute path, never an inline `!` script that could carry the secret itself. +const isCredentialHelperReference = (helper: string) => + /^(?:[A-Za-z0-9][A-Za-z0-9._-]*|\/[A-Za-z0-9._/-]+)$/.test(helper); +const isSshCommandReference = (command: string) => /^[A-Za-z0-9 ._/=,:@+-]+$/.test(command); +/** `file://` carries no credential; https needs the helper and SSH the owner's command. */ +const remoteKind = (url: string) => + /^https:\/\//i.test(url) + ? "https" + : /^ssh:\/\//i.test(url) || /^[A-Za-z0-9._-]+@[A-Za-z0-9.-]+:(?!\/\/)/.test(url) + ? "ssh" + : /^file:\/\//i.test(url) + ? "file" + : undefined; + +/** + * Pushes and lists refs with a reference to the owner's credential: a git credential helper or an SSH command naming + * its key file. The secret never enters argv or env, and with neither configured the push fails closed. + */ +export const gitPushLayer = (owner: { + readonly credentialHelper?: string; + readonly sshCommand?: string; +}) => + Layer.effect( + KinPurgePush, + Effect.gen(function* () { + const vcs = yield* VcsProcess; + const fs = yield* FileSystem.FileSystem; + /** Runs git against the remote with only the owner's credential reference, in a scratch HOME. */ + const run = Effect.fn("KinPurgePush.run")(function* ( + repoUrl: string, + args: ReadonlyArray, + cwd?: string, + ) { + const { credentialHelper, sshCommand } = owner; + if ( + (credentialHelper === undefined && sshCommand === undefined) || + (credentialHelper !== undefined && !isCredentialHelperReference(credentialHelper)) || + (sshCommand !== undefined && !isSshCommandReference(sshCommand)) + ) + return yield* new KinPurgePushError({ reason: "credential_unavailable" }); + const kind = remoteKind(repoUrl); + if (kind === undefined) + return yield* new KinPurgePushError({ reason: "unsupported_remote" }); + if ( + (kind === "https" && credentialHelper === undefined) || + (kind === "ssh" && sshCommand === undefined) + ) + return yield* new KinPurgePushError({ reason: "credential_unavailable" }); + // An empty HOME keeps curl's `.netrc` out; `-F none` keeps `~/.ssh/config` out. + const home = yield* fs + .makeTempDirectoryScoped({ prefix: "kin-push-home-" }) + .pipe(Effect.mapError(() => new KinPurgePushError({ reason: "unknown" }))); + const result = yield* vcs + .run({ + operation: "KinPurgePush.git", + command: "git", + cwd: cwd ?? home, + args: [ + "--no-replace-objects", + ...(cwd ? [`--git-dir=${cwd}/.git`] : []), + "-c", + "core.hooksPath=/dev/null", + "-c", + "core.fsmonitor=false", + "-c", + "core.attributesFile=/dev/null", + "-c", + "protocol.file.allow=user", + "-c", + "credential.helper=", + ...(credentialHelper ? ["-c", `credential.helper=${credentialHelper}`] : []), + ...args, + ], + env: { + ...scrubbedGitEnv(), + ...hardenedGitEnv, + HOME: home, + XDG_CONFIG_HOME: home, + GIT_ASKPASS: "false", + SSH_ASKPASS: "false", + SSH_ASKPASS_REQUIRE: "never", + SSH_AUTH_SOCK: undefined, + GCM_INTERACTIVE: "never", + // Without an owner SSH command, refuse SSH instead of using the host's keys or agent. + GIT_SSH_COMMAND: sshCommand + ? `${sshCommand} -F none -o BatchMode=yes -o IdentitiesOnly=yes -o IdentityAgent=none` + : "false", + }, + allowNonZeroExit: true, + maxOutputBytes: 16 * 1024 * 1024, + timeoutMs: 600_000, + }) + .pipe( + // The cause can echo the remote URL or helper output; keep both out of errors and logs. + Effect.mapError( + (error) => + new KinPurgePushError({ + reason: error._tag === "VcsProcessTimeoutError" ? "timeout" : "unknown", + }), + ), + ); + if (Number(result.exitCode) !== 0) + return yield* new KinPurgePushError({ + reason: pushFailure(result.stdout, result.stderr), + }); + return result.stdout; + }); + return KinPurgePush.of({ + push: Effect.fn("KinPurgePush.git")(function* ({ repoUrl, gitDir, updates }) { + yield* run( + repoUrl, + [ + "push", + "--atomic", + "--porcelain", + ...updates.map((update) => `--force-with-lease=${update.ref}:${update.old}`), + "--", + repoUrl, + ...updates.map((update) => `${update.new ?? ""}:${update.ref}`), + ], + gitDir, + ); + }, Effect.scoped), + listRefs: Effect.fn("KinPurgePush.listRefs")(function* ({ repoUrl }) { + const stdout = yield* run(repoUrl, ["ls-remote", "--", repoUrl]); + return stdout.split("\n").flatMap((line) => { + const ref = line.split("\t")[1]; + return ref ? [ref] : []; + }); + }, Effect.scoped), + }); + }), + ); diff --git a/docs/kin/specs/triggers.md b/docs/kin/specs/triggers.md index 8b1cc15155874..9c9f0a30d6b10 100644 --- a/docs/kin/specs/triggers.md +++ b/docs/kin/specs/triggers.md @@ -426,17 +426,27 @@ Known gaps, accepted for phase 1: JWE (five-part) tokens and JWT headers longer ### Owner-signed purge (gap 5) -Pending action `memory.purge` with `{ orgId, repo, paths, mode: "drop_file" | "redact_text", patterns? }`. The owner opens it from an item's menu, or Kin proposes it when the weekly `dream_full` scans full history and finds a secret. Approval uses a native card and a signed grant (APPR). The executor then: +Pending action `memory_purge`. The plan is `{ orgId, repo, mode: "drop_file", paths }` or `{ orgId, repo, mode: "redact_text", patterns }`; `redact_text` rewrites every blob and commit message in history, so it takes no paths. `KinPurge.storePlan` writes `purgePlanJson(plan)` to `/purge-plans/.json` and returns the action's `content` target, whose path and sha256 come from that same serialization, so the grant binds the exact plan; `discardPlan` deletes it once the action is terminal, since it can hold the secret, and refuses while that plan's purge marker exists. The owner opens the action from an item's menu, or Kin proposes it when the weekly `dream_full` scans full history and finds a secret. Approval uses a native card and a signed grant (APPR). `memoryPurgeHandler` in `apps/server/src/kin/memory/purge.ts` is APPR-9's `KinActionHandler` (`precheck`, `run`, `reconcile`) and the only entry point; no model, MCP tool or RPC reaches `KinPurge`. Every `KinPurge` method refuses with `KinPurgeExecutorNotReady` until APPR-9, APPR-10 and LEAD-11 land. -1. Pauses the org's lead queue and takes the memory writer lock. -2. Makes a fresh mirror clone and runs `git filter-repo --invert-paths --path

` (`drop_file`) or `--replace-text ` (`redact_text`), then re-runs the scan over all history. -3. Force-pushes the rewritten branches with the owner's credential. This needs the owner as a bypass actor on the memory repo's ruleset. It closes open dream PRs that touch the paths. -4. For `drop_file`, commits a tombstone at the same path: `status: cancelled`, `invalid_at: now`, body "Purged on by the owner (audit )". Memory still shows that something was there. -5. Re-clones every local checkout and worktree of the memory repo, rebuilds the cache and `INDEX.md`, and resumes the queue. -6. Writes a `KinAuditEntry` (APPR) with the grant, device, paths, old head, new head and the refs rewritten. -7. Tells the owner what a purge can't reach: GitHub keeps `refs/pull/*` and cached PR diffs (removing those needs a GitHub Support request), forks, local backups (Time Machine), and the chat database. A leaked credential must be rotated whatever happens. +Before anything runs, the stored plan must re-serialize to the same bytes and match the target; the grant, action and plan must share the org; the grant must bind the action id, revision, kind, `targetDigest` and plan digest; and `repo` must equal the org's configured memory URL. Paths: 1–64, unique, NFC, relative `.md` files other than `rules.md`, with no `..`, `.`, empty, `-`-leading or `.git` segment, backslash or control character, and each must exist in history (literal pathspecs), so a typo or an NFD name fails instead of purging nothing. `redact_text` takes 1–64 literal patterns of 8–1024 characters without control characters or `==>`; each becomes one `literal:` rule, so a pattern can't add a rule or switch to regex. Every argument goes to git and `git-filter-repo` as an argv entry, never through a shell, with inherited `GIT_*` variables removed. -**Dependency:** `git-filter-repo` (git's recommended history rewriter; `brew install git-filter-repo`). Check its licence and release activity when installing (TRIG-17). +`run` then: + +1. Pauses the org's lead queue through `KinPurgeQueue`. The default fails `KinPurgePauseUnavailable` until LEAD-11 builds the pause, because a reclone deletes the read checkout a running lead uses as its workspace. `pause` must not return while a wake-up or dream is running: their staged files and dream copies predate the rewrite. As a backstop, every quarantine and reclone bumps `Memory.historyGeneration`, and a `MemoryWriter` session started under an older generation fails `MemorySyncConflict` and drops its staged changes instead of committing them onto the rewritten head. It takes the memory writer lock through `Memory.withHistoryRewrite`, which is the only way to reach the quarantine and reclone operations. +2. Lists the remote's refs with `git ls-remote` under the owner credential `KinPurgePush` uses, and fails closed at the clone stage if it advertises anything other than `HEAD`, `refs/heads/*`, `refs/tags/*` and `refs/pull/*`: `refs/notes/*` or a custom namespace such as `refs/custom/*` would keep the old history out of reach. It then bare-clones `refs/heads/*` and `refs/tags/*` into `//purge-scratch` (stale clones from a crashed purge are swept first) and runs `git filter-repo --invert-paths --path

` (`drop_file`) or `--replace-text --replace-message ` (`redact_text`). It also fails closed at the clone stage on any tag that doesn't peel to a commit, because filter-repo leaves tags on blobs and trees alone. A plan that changes no ref fails. +3. For `drop_file`, checks that no reachable commit touches the paths, then commits a tombstone on the default branch for each item path present at the old head: the original `id` and the folder's kind, `title: Purged item`, `status: cancelled`, `invalid_at: now`, `source: kin-action:`, `said_by` the approving owner, body "Purged on by the owner (action )". Memory still shows that something was there. Journal and other non-item paths get no tombstone. +4. Reads every blob ever introduced at every path, merge sides included, every commit object and every tag object (tagger idents survive `--replace-message`). It fails if a redacted pattern remains, then runs `KinSecretScan` over all of it and fails closed on **any** finding: findings carry no fingerprint, so the purge can't prove one unrelated, and a copy under an older path would otherwise survive. Allowlisted fingerprints don't count. +5. Records the tombstones' blob ids in `purge-plans/.tombstones.json`, writes the durable marker `//purge-in-flight` (plan digest, action id, old head, the plan itself and the tombstone ids), and deletes the writer and read checkouts, any half-finished `memory.clone-*` and the dream scratch `/dreams/`, before the push. A push can land and still report failure (a timeout, a crash, a restart), and those checkouts hold the old history: `Memory.preserve` would push it back as a `kin/conflict/*` branch, and a dream as `dream/`. While the marker exists, `ensureCheckouts` and `recover` won't clone (until the push lands the remote still has the old history), and `preserve`, the wake-up merge, `commitWakeup`, `commitDream` and `deleteDreamBranch` refuse to push. Only the purge's own reclone clones. +6. Force-pushes the changed `refs/heads/*` and `refs/tags/*` in one atomic push, each leased at its cloned value, through `KinPurgePush`. Failures are classified from `--porcelain` output (`lease_rejected`, `remote_rejected`, `auth`, `network`, `timeout`) without the cause, so neither the URL nor helper output is logged. This needs the owner as a bypass actor on the memory repo's ruleset. Closing open dream PRs that touch the paths waits on the GitHub client. +7. Re-lists and re-mirrors the remote and runs the same history check as `reconcile`; a ref created during the purge that still holds the content fails at the push stage with `queueHeld: true`. Then reclones: clones fresh, records the rewritten anchors without preserving the old ones, rebuilds the read checkout, cache and `INDEX.md`, clears the marker, and resumes the queue. A failure or interruption before step 5 resumes the queue with nothing changed; from step 5 on, any failure, defect or interruption leaves `KinPurgeFailed` with `queueHeld: true` and the queue paused. +8. Returns the `execution_finished` audit input (grant `jti`, key thumbprint, new head; APPR-10 adds the device label from the key) with the paths, old head, new head and rewritten refs; APPR-10 writes it as a `KinAuditEntry`. +9. Tells the owner what a purge can't reach: GitHub keeps `refs/pull/*` (read-only, so the clone leaves them out and `reconcile` doesn't wait on them) and cached PR diffs (removing those needs a GitHub Support request); forks, local backups (Time Machine) and the chat database keep their copies. It won't rewrite `refs/notes/*` or custom refs such as `refs/custom/*` either: their presence fails the purge in step 2 and makes `reconcile` answer `unknown`, so the owner deletes them on the remote first. A leaked credential must be rotated whatever happens. + +`reconcile` bare-clones the remote the same way and checks whether the plan's paths or patterns are still in history, counting only blobs whose ids match this purge's recorded tombstones as purged, so a look-alike tombstone counts as old content; it reads the plan from the marker if the plan file is gone. Gone is `done`; still there with the default branch at the marker's old head is `not_done`; anything else, an unreachable remote, or a remote advertising refs outside heads, tags and pulls is `unknown` and changes nothing. `done` and `not_done` reclone, clear the marker and resume the queue. A re-run of the same plan while its marker exists reclones instead of resuming against old checkouts, and reports success when the push had already landed. [Kin memory purge](../../operations/kin-memory-purge.md) covers clearing a stuck marker by hand. + +**Owner credential.** `gitPushLayer` takes a reference to the owner's credential, never the secret: a git credential helper name or absolute path (no inline `!` script), or an SSH command naming its key file (no shell metacharacters; T3 appends `-F none`, `BatchMode`, `IdentitiesOnly` and `IdentityAgent=none`). An `https://` remote needs the helper and an SSH remote the command; `file://` takes neither, and any other transport fails `unsupported_remote`. git runs with an empty scratch `HOME`, so `~/.netrc` and `~/.ssh/config` can't supply another credential. This is the one exception to APPR's rule that the owner credential never reaches a child process: the helper or key is read by git or ssh, not by T3. With neither configured the push fails `credential_unavailable`, SSH without an owner command is refused rather than using the host's keys or agent, and `GIT_TERMINAL_PROMPT=0`, `GIT_ASKPASS`, `SSH_ASKPASS` and `SSH_ASKPASS_REQUIRE=never` keep anything from prompting. + +**Dependency:** `git-filter-repo` (git's recommended history rewriter; MIT, also usable under git's GPL), checked at v2.47.0. TRIG-20 installs it on the Kin host; `GitFilterRepoCommand` overrides the binary path. The real-rewrite tests skip on machines without it. ### Upstream contract tests (gap 7) @@ -533,7 +543,7 @@ Kin runs on one always-on machine, the **Kin host**. The laptop and phone are cl - provides the `kin_defer` tool. - **CHAT:** `KinOrgId`; delivering messages and pings; `KinWidget` payloads `KinDeferralCard`, `KinHeadsUpCard`, `KinOwnerCheckCard`, `KinDreamReviewCard`; the chat message store for dream material; verified authors for trust checks. - **APPR:** - - action kinds `github.app_hook.configure`, `memory.dream_review`, `memory.purge`, `kin.secret_allow`; + - action kinds `github.app_hook.configure`, `memory.dream_review`, `memory_purge`, `kin.secret_allow`; - the standing `dream:auto` delegation; - the executor merging at a bound SHA; - `KinAuditEntry`; @@ -557,7 +567,7 @@ Kin runs on one always-on machine, the **Kin host**. The laptop and phone are cl - **Deferral:** unchanged recheck, then heads-up, then silence, then `started` at due. A linked issue closed before due leads to `asked`, and nothing starts. With the stop switch on, it asks. [Monday] re-arms to next Monday 09:00. A `due` fire consumed two hours late asks; a queued fire for a disarmed or moved deferral is dropped. - **Dream:** a dream touching `rules.md` fails, and `main` is unchanged. A dream with one journal fold and one `decisions/` change opens two PRs, and only the auto one merges. - **Secret scan:** a journal line holding `ghp_` plus 36 characters blocks the entire commit, and the owner gets one typed notification without the value. -- **Purge:** on a fixture repo, the file is gone from `git log --all -p`, a tombstone exists, and an audit row exists. +- **Purge:** on a fixture repo, the file is gone from `git log --all -p` and the recloned checkout, a tombstone exists and parses, and the returned audit input carries the old and new heads. A surviving secret fails closed before the push. A push that lands and then errors, crashes or is interrupted never republishes the old history through the next wake-up, a dream branch or a checkout cloned during the push, and `reconcile` reports `done` with a `refs/pull/*` ref still holding it. A `refs/notes/*` ref fails the purge at the clone and makes `reconcile` answer `unknown`; a wake-up that staged before the purge can't commit after it; a look-alike tombstone makes `reconcile` answer `not_done`. - **Contract tests:** `vp test run kin-contract` passes on today's upstream. Renaming `scheduledTaskId` in a scratch branch makes it fail. ## Tasks @@ -582,7 +592,7 @@ Kin runs on one always-on machine, the **Kin host**. The laptop and phone are cl | TRIG-14 | Dream split, auto-merge under delegation, review card, rejections; full weekly variant | `classifyDream.ts`, `DreamRunner.ts` | TRIG-13, APPR-4; auto-merge waits for APPR-12 (phase 3), and until then the owner approves both PRs | the two-PR test passes; a rejected item is not proposed again | backend | | TRIG-15 | Memory repo guard rails | `kin-memory`: `CODEOWNERS`, `memory-check.yml`, ruleset | APPR-11 | a PR touching `decisions/` can't merge without owner review; a planted token fails the check | backend | | TRIG-16 | Secret scanner, writer hook, allowlist | `secretScan.ts` | LEAD-3 | rule fixtures (one hit and one near-miss per rule) pass; commit refusal and typed owner notification tested | backend | -| TRIG-17 | Owner-signed purge | `purge.ts` | TRIG-16, APPR-9, APPR-10 | purge test passes on a fixture repo; `git-filter-repo` provenance noted in the PR | backend | +| TRIG-17 | Owner-signed purge | `purge.ts` | TRIG-16, APPR-9, APPR-10, LEAD-11 | purge test passes on a fixture repo; `git-filter-repo` provenance noted in the PR | backend | | TRIG-18 | Server contract tests: triggers, relay, `delegate_task`, managed policy | `apps/server/src/kin/contract/*` | TRIG-4 | `cd apps/server && vp test run kin-contract` passes | backend | | TRIG-19 | `McpAppWebView` contract test | `apps/mobile/src/features/chats/contract/*` | — | `cd apps/mobile && vp test run kin-contract` passes | UI | | TRIG-20 | Kin host setup: systemd unit, `kin` user, `setup.sh`, `backup.sh`, verify source-build `up --connect` and hold | `ops/kin/host/*` | — | after a reboot, the server is reachable from the phone over T3 Connect; a GitHub redelivery while it's stopped arrives on start; a restore drill from backup works | backend | diff --git a/docs/operations/kin-memory-purge.md b/docs/operations/kin-memory-purge.md new file mode 100644 index 0000000000000..d1f160b5ea11e --- /dev/null +++ b/docs/operations/kin-memory-purge.md @@ -0,0 +1,24 @@ +# Kin memory purge + +> For maintainers operating a Kin host. The design is in [the triggers spec](../kin/specs/triggers.md#owner-signed-purge-gap-5). + +## Refs a purge can't rewrite + +A purge rewrites only `refs/heads/*` and `refs/tags/*`. GitHub's read-only `refs/pull/*` are left alone. If the remote advertises any other ref, such as `refs/notes/commits` or a custom `refs/custom/*`, the purge fails at the clone stage, changes nothing and resumes the queue, and `reconcile` answers `unknown`. Those refs can still hold the purged content, so list them with `git ls-remote `, delete them on the remote (`git push --delete `), and run the purge again. A tag that points at a blob or tree fails the same way. A ref created on the remote before the purge's post-push check fails it at the push stage with the queue held, and `reconcile` answers `unknown` until you delete that ref. Anyone holding an old clone can still push the content back after that check. + +## A stuck purge marker + +A purge writes `/kin//purge-in-flight` before it force-pushes, and leaves it there when the push fails, crashes or is interrupted. While it exists, the org's lead queue stays paused, memory won't clone a checkout, and nothing pushes memory history. `reconcile` on the action clears it: it reads the plan from the marker, so it works even after the plan file is gone. Retry `reconcile` first; it answers `unknown` while the remote is unreachable, has moved, or advertises refs a purge can't rewrite. + +The marker and the `purge-plans/*.tombstones.json` files live beside the lead's read workspace, and `reconcile` trusts them, so a lead with full filesystem access could tamper with them. + +Delete the marker by hand only when `reconcile` keeps answering `unknown` and you have checked the remote yourself. First stop the T3 server, or confirm that no purge or `reconcile` for the org is running: both hold the memory writer lock and write these paths. Deleting the marker tells Kin that no local copy of the old history will be pushed again, so first delete everything under `` that can hold it: + +- `memory` and `memory-read` (the writer and read checkouts); +- `dreams` (dream scratch copies); +- any `memory.clone-*` (a clone a crash left half-finished); +- `purge-scratch` and any `purge-scratch/mirror-*` inside it (the purge's rewritten bare clone, which also holds old objects). + +Then confirm with `git log --branches --tags -p` on a fresh clone, and `git ls-remote` for refs outside heads, tags and pulls, whether the purged content is still on the remote. If it is, run the purge again from a new owner-approved action; if it's gone, the next wake-up clones the rewritten history. + +Last, resume the org's queue. Until LEAD-11 builds the org pause there is no operator path to resume it: the purge can't run then either, because `KinPurgeQueue.pause` fails by default, so a stuck marker can only come from a host running LEAD-11's pause.