From d2fe834dd7130b64bd62bd970b2257733da5bfcd Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Sun, 8 Nov 2026 21:44:19 +0530 Subject: [PATCH 1/7] feat(kin): owner-signed memory purge behind the executor gate (TRIG-17) Rewrites memory history with git-filter-repo in a fresh mirror clone, re-scans every blob and commit message, commits tombstones for dropped items, force-pushes with leases through an injectable push step, and reclones local checkouts. The memory_purge handler refuses until APPR-9 and APPR-10 land. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/kin/lead/LeadPrompt.test.ts | 1 + apps/server/src/kin/lead/layer.test.ts | 1 + apps/server/src/kin/memory/Memory.ts | 14 + apps/server/src/kin/memory/purge.test.ts | 320 +++++++++ apps/server/src/kin/memory/purge.ts | 692 ++++++++++++++++++++ docs/kin/specs/triggers.md | 27 +- 6 files changed, 1044 insertions(+), 11 deletions(-) create mode 100644 apps/server/src/kin/memory/purge.test.ts create mode 100644 apps/server/src/kin/memory/purge.ts diff --git a/apps/server/src/kin/lead/LeadPrompt.test.ts b/apps/server/src/kin/lead/LeadPrompt.test.ts index fbc72428b0691..e7b432db80150 100644 --- a/apps/server/src/kin/lead/LeadPrompt.test.ts +++ b/apps/server/src/kin/lead/LeadPrompt.test.ts @@ -63,6 +63,7 @@ const memory = Layer.succeed( anchor: () => Effect.succeed("fixture"), record: () => Effect.void, recover: () => Effect.void, + reclone: () => Effect.die("Unused"), readPromptSnapshot: (org, head) => org !== orgId || !["head-one", "head-two"].includes(head) ? Effect.fail(new Memory.MemoryError({ orgId: org, operation: "promptSnapshot" })) diff --git a/apps/server/src/kin/lead/layer.test.ts b/apps/server/src/kin/lead/layer.test.ts index 4cf8917612a81..def615b16aa21 100644 --- a/apps/server/src/kin/lead/layer.test.ts +++ b/apps/server/src/kin/lead/layer.test.ts @@ -44,6 +44,7 @@ vi.mock("../memory/Memory.ts", async (importOriginal) => { anchor: () => Effect.die("unused"), record: () => Effect.die("unused"), recover: () => Effect.die("unused"), + reclone: () => Effect.die("unused"), ensureCheckouts: () => Effect.succeed({ writer: "/private/tmp", read: "/private/tmp" }), prepareWakeup: () => Effect.sync(() => { diff --git a/apps/server/src/kin/memory/Memory.ts b/apps/server/src/kin/memory/Memory.ts index cc6ca4aedc0cc..f0dc60b358bad 100644 --- a/apps/server/src/kin/memory/Memory.ts +++ b/apps/server/src/kin/memory/Memory.ts @@ -162,6 +162,8 @@ export class Memory extends Context.Service< head: string, ) => Effect.Effect; readonly recover: (orgId: KinOrgId) => Effect.Effect; + /** Server-only, for history rewrites: the caller holds the writer lock. Never preserves old anchors. */ + readonly reclone: (orgId: KinOrgId) => Effect.Effect; readonly ensureCheckouts: (orgId: KinOrgId) => Effect.Effect; readonly prepareWakeup: ( orgId: KinOrgId, @@ -1289,8 +1291,20 @@ const make = Effect.gen(function* () { return { head, indexAsOf: DateTime.toUtc(indexAsOf) }; }); + const reclone = Effect.fn("Memory.reclone")(function* (orgId: KinOrgId) { + const paths = yield* pathsFor(orgId); + for (const directory of [paths.writer, paths.read]) + yield* io(orgId, "reclone", fs.remove(directory, { recursive: true, force: true })); + initializedCheckouts.delete(orgId); + // With no writer checkout, recover clones fresh and records the rewritten heads. + yield* recover(orgId); + yield* prepare(orgId); + yield* rebuildIndex(orgId); + }); + return Memory.of({ withWriterLock, + reclone: Effect.fn("Memory.recloneLocked")((orgId) => locked(orgId, reclone(orgId))), recover: Effect.fn("Memory.recoverLocked")((orgId) => withWriterLock(orgId, locked(orgId, recover(orgId))), ), diff --git a/apps/server/src/kin/memory/purge.test.ts b/apps/server/src/kin/memory/purge.test.ts new file mode 100644 index 0000000000000..464f0778889cd --- /dev/null +++ b/apps/server/src/kin/memory/purge.test.ts @@ -0,0 +1,320 @@ +// @effect-diagnostics-next-line nodeBuiltinImport:off -- A synchronous probe decides the platform gate before tests register. +import * as NodeChildProcess from "node:child_process"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { describe, expect, it } from "@effect/vitest"; +import { KinMemoryItemId, KinOrgId } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Sqlite from "../../persistence/Sqlite.ts"; +import * as Persistence from "../persistence/KinPersistence.ts"; +import * as Vcs from "../../vcs/VcsProcess.ts"; +import { fixtureItem } from "./__fixtures__/items.ts"; +import { parseFrontMatter } from "./frontMatter.ts"; +import * as Memory from "./Memory.ts"; +import { MemoryConfig } from "./MemoryConfig.ts"; +import * as Purge from "./purge.ts"; +import * as Scan from "./secretScan.ts"; + +const orgId = KinOrgId.make("fixture-org"); +const token = ["gh", "p_"].join("") + "A1b2C3d4E5".repeat(3) + "abcdef"; +// TRIG-20 installs git-filter-repo on the Kin host; machines without it skip the real rewrite. +const filterRepoMissing = NodeChildProcess.spawnSync("git-filter-repo", ["--version"]).status !== 0; + +const authorityFor = (plan: Purge.PurgePlan): Purge.PurgeAuthority => ({ + action: { + id: "act-1", + orgId, + kind: "memory_purge", + revision: 1, + target: { _tag: "content", path: "purge/act-1.json", sha256: Purge.purgePlanDigest(plan) }, + }, + grant: { + id: "grant-1", + orgId, + actionId: "act-1", + revision: 1, + jti: "jti-1", + approverHumanId: "fixture-owner", + keyThumbprint: "thumb-1", + }, + auditId: "audit-1", +}); + +describe.skipIf(filterRepoMissing)("purge with git-filter-repo", () => { + it.live.each([ + "drop_file", + "redact_text", + "wrong pattern", + "push fails", + "missing tool", + ] as const)("purge on a fixture repo: %s", (scenario) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const vcs = yield* Vcs.VcsProcess; + const root = yield* fs.realPath(yield* fs.makeTempDirectoryScoped()); + const source = `${root}/seed`; + const remote = `${root}/remote.git`; + // A file:// URL transfers reachable objects only, like a real remote; a bare path copies them all. + const remoteUrl = `file://${remote}`; + const git = (cwd: string, args: ReadonlyArray) => + vcs + .run({ + operation: "test.purge", + command: "git", + cwd, + args: cwd === remote ? [`--git-dir=${remote}`, ...args] : args, + maxOutputBytes: 64 * 1024 * 1024, + }) + .pipe(Effect.map((result) => result.stdout.trim())); + const commit = (message: string) => + git(source, [ + "-c", + "user.name=Fixture Bot", + "-c", + "user.email=fixture@example.invalid", + "commit", + "-q", + "-m", + message, + ]); + yield* fs.makeDirectory(`${source}/claims`, { recursive: true }); + yield* git(source, ["init", "-q", "-b", "main"]); + yield* fs.writeFileString(`${source}/rules.md`, "Invented rules\n"); + yield* fs.writeFileString(`${source}/INDEX.md`, "As of 2026-11-08T00:00:00.000Z\n"); + yield* fs.writeFileString( + `${source}/claims/leak.md`, + fixtureItem({ id: "claim-leak", kind: "claim", title: "Invented vendor note" }) + + `vendor key: ${token}\n`, + ); + yield* fs.writeFileString( + `${source}/claims/keep.md`, + fixtureItem({ id: "claim-keep", kind: "claim", title: "Invented keeper" }), + ); + yield* git(source, ["add", "--", "."]); + yield* commit(scenario === "redact_text" ? `fixture ${token}` : "fixture"); + yield* git(source, ["switch", "-q", "-c", "side"]); + yield* fs.writeFileString(`${source}/claims/leak.md`, `side copy ${token}\n`); + yield* git(source, ["add", "--", "."]); + yield* commit("side"); + yield* git(source, ["switch", "-q", "main"]); + yield* fs.writeFileString(`${source}/rules.md`, "Invented rules, revised\n"); + yield* git(source, ["add", "--", "."]); + yield* commit("revise rules"); + yield* git(root, ["clone", "-q", "--bare", "--", source, remote]); + + const events: string[] = []; + const config = Layer.succeed(MemoryConfig, { + baseDirectory: root, + repositories: [{ orgId, repoUrl: remoteUrl, path: "state" }], + }); + const memoryLayer = Memory.layer.pipe( + Layer.provide(config), + Layer.provide(Layer.succeed(Memory.MemoryRecoveryHooks, { notify: () => Effect.void })), + ); + const push = + scenario === "push fails" + ? Layer.succeed(Purge.KinPurgePush, { + push: () => Effect.fail(new Purge.KinPurgePushError({ reason: "fixture refusal" })), + }) + : Purge.gitPushLayer({}).pipe(Layer.provide(config)); + const layer = Purge.layer.pipe( + Layer.provideMerge(memoryLayer), + Layer.provide(config), + Layer.provide(Scan.layer), + Layer.provide(push), + Layer.provide( + Layer.succeed( + Purge.GitFilterRepoCommand, + scenario === "missing tool" ? `${root}/missing/git-filter-repo` : "git-filter-repo", + ), + ), + Layer.provide( + Layer.succeed(Purge.KinPurgeQueue, { + pause: () => Effect.sync(() => events.push("pause")), + resume: () => Effect.sync(() => events.push("resume")), + }), + ), + ); + + yield* Effect.gen(function* () { + const memory = yield* Memory.Memory; + const paths = yield* memory.ensureCheckouts(orgId); + yield* memory.prepareWakeup(orgId); + const oldMain = yield* git(remote, ["rev-parse", "refs/heads/main"]); + const oldSide = yield* git(remote, ["rev-parse", "refs/heads/side"]); + const plan: Purge.PurgePlan = + scenario === "redact_text" + ? { + orgId, + repo: remoteUrl, + mode: "redact_text", + paths: ["claims/leak.md"], + patterns: [token], + } + : scenario === "wrong pattern" + ? { + orgId, + repo: remoteUrl, + mode: "redact_text", + paths: ["claims/leak.md"], + patterns: ["Invented vendor note"], + } + : { orgId, repo: remoteUrl, mode: "drop_file", paths: ["claims/leak.md"] }; + const result = yield* Purge.memoryPurgeHandler + .run(plan, authorityFor(plan)) + .pipe(Effect.provideService(Purge.PurgeExecutorGate, true), Effect.result); + + if (scenario !== "drop_file" && scenario !== "redact_text") { + expect(result._tag).toBe("Failure"); + if (result._tag === "Failure") { + expect(result.failure).toMatchObject({ + _tag: "KinPurgeFailed", + ...{ + "wrong pattern": { stage: "verify", queueHeld: false }, + "push fails": { stage: "push", queueHeld: true }, + "missing tool": { stage: "rewrite", queueHeld: false }, + }[scenario], + }); + expect(JSON.stringify(result.failure)).not.toContain(token); + } + expect(events).toEqual(scenario === "push fails" ? ["pause"] : ["pause", "resume"]); + expect(yield* git(remote, ["rev-parse", "refs/heads/main"])).toBe(oldMain); + expect(yield* memory.anchor(orgId, "main")).toBe(oldMain); + return; + } + + expect(result._tag).toBe("Success"); + if (result._tag !== "Success") return; + const newMain = yield* git(remote, ["rev-parse", "refs/heads/main"]); + const newSide = yield* git(remote, ["rev-parse", "refs/heads/side"]); + expect(events).toEqual(["pause", "resume"]); + expect(result.success).toMatchObject({ oldHead: oldMain, newHead: newMain }); + expect(result.success.rewrittenRefs).toEqual([ + { ref: "refs/heads/main", old: oldMain, new: newMain }, + { ref: "refs/heads/side", old: oldSide, new: newSide }, + ]); + expect(result.success.audit).toMatchObject({ + orgId, + event: "execution_finished", + actionId: "act-1", + kind: "memory_purge", + grantJti: "jti-1", + keyThumbprint: "thumb-1", + headSha: newMain, + }); + expect(JSON.parse(result.success.audit.result)).toMatchObject({ + oldHead: oldMain, + newHead: newMain, + }); + + for (const history of [ + yield* git(remote, ["log", "--all", "-p"]), + yield* git(paths.writer, ["log", "--all", "-p"]), + yield* git(paths.writer, ["cat-file", "--batch-all-objects", "--batch"]), + ]) + expect(history).not.toContain(token); + expect(yield* memory.anchor(orgId, "main")).toBe(newMain); + expect(yield* git(remote, ["show", `${newMain}:claims/keep.md`])).toContain("claim-keep"); + + const leak = yield* git(remote, ["show", `${newMain}:claims/leak.md`]); + if (scenario === "redact_text") { + expect(result.success.tombstones).toEqual([]); + expect(leak).toContain("vendor key: ***REMOVED***"); + return; + } + expect(result.success.tombstones).toEqual(["claims/leak.md"]); + const tombstone = yield* parseFrontMatter(orgId, "claims/leak.md", leak + "\n"); + expect(tombstone.frontMatter).toMatchObject({ + id: "claim-leak", + kind: "claim", + status: "cancelled", + said_by: "fixture-owner", + }); + expect(tombstone.frontMatter.invalid_at).not.toBeNull(); + expect(tombstone.body).toMatch( + /^Purged on \d{4}-\d{2}-\d{2} by the owner \(audit audit-1\)/, + ); + expect( + (yield* memory.readItemQuietly(orgId, KinMemoryItemId.make("claim-leak"))).head.status, + ).toBe("cancelled"); + expect(yield* fs.readFileString(`${paths.read}/INDEX.md`)).toContain("Purged item"); + }).pipe(Effect.provide(layer)); + }).pipe( + Effect.provide( + Layer.mergeAll( + Vcs.layer.pipe(Layer.provide(NodeServices.layer)), + NodeServices.layer, + Persistence.layer.pipe(Layer.provideMerge(Sqlite.layerMemory)), + ), + ), + Effect.scoped, + ), + ); +}); + +const drop: Purge.PurgePlan = { + orgId, + repo: "fixture-remote", + mode: "drop_file", + paths: ["claims/leak.md"], +}; + +it.effect("the executor gate refuses a purge until APPR-9 and APPR-10 land", () => + Effect.gen(function* () { + const result = yield* Purge.memoryPurgeHandler + .run(drop, authorityFor(drop)) + .pipe(Effect.provide(Layer.mock(Purge.KinPurge)({})), Effect.result); + expect(result._tag).toBe("Failure"); + if (result._tag === "Failure") expect(result.failure._tag).toBe("KinPurgeExecutorNotReady"); + }), +); + +it.each([ + ["parent escape", { paths: ["../claims/leak.md"] }], + ["inner escape", { paths: ["claims/../rules.md"] }], + ["absolute", { paths: ["/claims/leak.md"] }], + ["option", { paths: ["--force.md"] }], + ["option segment", { paths: ["claims/-x.md"] }], + ["git dir", { paths: [".git/config.md"] }], + ["git dir case", { paths: ["claims/.GIT./x.md"] }], + ["empty segment", { paths: ["claims//leak.md"] }], + ["dot segment", { paths: ["claims/./leak.md"] }], + ["backslash", { paths: ["claims\\leak.md"] }], + ["newline", { paths: ["claims/leak.md\nrules.md"] }], + ["not markdown", { paths: ["claims"] }], + ["rules", { paths: ["rules.md"] }], + ["no paths", { paths: [] }], + ["duplicate", { paths: ["claims/leak.md", "claims/leak.md"] }], + ["drop with patterns", { patterns: ["Invented text"] }], + ["no patterns", { mode: "redact_text" }], + ["short pattern", { mode: "redact_text", patterns: ["short"] }], + ["rule injection", { mode: "redact_text", patterns: ["Invented==>regex:.*"] }], + ["extra rule", { mode: "redact_text", patterns: ["Invented text\nregex:.*"] }], +] as const)("rejects an unsafe plan: %s", (_name, overrides) => { + const plan = { ...drop, ...overrides }; + expect(Purge.purgeRejection(plan, authorityFor(plan))).toBeDefined(); +}); + +it.each([ + [ + "digest", + (a: Purge.PurgeAuthority) => ({ + ...a, + action: { ...a.action, target: { ...a.action.target, sha256: "0".repeat(64) } }, + }), + ], + ["action", (a: Purge.PurgeAuthority) => ({ ...a, grant: { ...a.grant, actionId: "act-2" } })], + ["revision", (a: Purge.PurgeAuthority) => ({ ...a, grant: { ...a.grant, revision: 2 } })], + [ + "org", + (a: Purge.PurgeAuthority) => ({ + ...a, + grant: { ...a.grant, orgId: KinOrgId.make("other-org") }, + }), + ], + ["audit id", (a: Purge.PurgeAuthority) => ({ ...a, auditId: "audit 1\nforged" })], +] as const)("rejects authority that does not bind the plan: %s", (_name, tamper) => { + expect(Purge.purgeRejection(drop, authorityFor(drop))).toBeUndefined(); + expect(Purge.purgeRejection(drop, tamper(authorityFor(drop)))).toBeDefined(); +}); diff --git a/apps/server/src/kin/memory/purge.ts b/apps/server/src/kin/memory/purge.ts new file mode 100644 index 0000000000000..1189d0286bb0f --- /dev/null +++ b/apps/server/src/kin/memory/purge.ts @@ -0,0 +1,692 @@ +// @effect-diagnostics-next-line nodeBuiltinImport:off -- The plan digest must match the grant's sha256 content target. +import * as NodeCrypto from "node:crypto"; +import type { KinOrgId, MemoryItemKind } from "@t3tools/contracts"; +import * as Clock from "effect/Clock"; +import * as Context from "effect/Context"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; +import { stringify } from "yaml"; + +import { VcsProcess } from "../../vcs/VcsProcess.ts"; +import { parseFrontMatter } from "./frontMatter.ts"; +import { memoryGit } from "./git.ts"; +import { Memory } from "./Memory.ts"; +import { MemoryConfig } from "./MemoryConfig.ts"; +import { readObjects } from "./objects.ts"; +import { KinSecretScan, type SecretFinding } from "./secretScan.ts"; + +export interface PurgePlan { + readonly orgId: KinOrgId; + readonly repo: string; + readonly paths: ReadonlyArray; + readonly mode: "drop_file" | "redact_text"; + readonly patterns?: ReadonlyArray; +} + +/** Lane B's `KinGrant` (APPR-4) fields the purge binds to. The executor verifies the signature. */ +export interface PurgeGrant { + readonly id: string; + readonly orgId: KinOrgId; + readonly actionId: string; + readonly revision: number; + readonly jti: string; + readonly approverHumanId: string; + readonly keyThumbprint: string; +} + +/** Lane B's `KinPendingAction` (APPR-2) for kind `memory_purge`; the target is the plan's digest. */ +export interface PurgeAction { + readonly id: string; + readonly orgId: KinOrgId; + readonly kind: "memory_purge"; + readonly revision: number; + readonly target: { readonly _tag: "content"; readonly path: string; readonly sha256: string }; +} + +export interface PurgeAuthority { + readonly action: PurgeAction; + readonly grant: PurgeGrant; + /** Id of the executor's `execution_started` audit entry, written before `run`. */ + readonly auditId: string; + readonly deviceLabel?: string; +} + +/** Lane B's `KinAuditAppendInput` (APPR-10) for the `execution_finished` entry. */ +export interface PurgeAuditInput { + readonly orgId: KinOrgId; + readonly event: "execution_finished"; + readonly actionId: string; + readonly kind: "memory_purge"; + readonly grantJti: string; + readonly keyThumbprint: string; + readonly deviceLabel?: string; + readonly headSha: string; + readonly result: string; +} + +export interface PurgeRefUpdate { + readonly ref: string; + readonly old: string; + readonly new: string | null; +} + +export interface PurgeResult { + readonly mode: PurgePlan["mode"]; + readonly paths: ReadonlyArray; + readonly oldHead: string; + readonly newHead: string; + readonly rewrittenRefs: ReadonlyArray; + readonly tombstones: ReadonlyArray; + readonly audit: PurgeAuditInput; +} + +export class KinPurgeRejected extends Schema.TaggedError()("KinPurgeRejected", { + reason: Schema.String, +}) {} +export class KinPurgeExecutorNotReady extends Schema.TaggedError()( + "KinPurgeExecutorNotReady", + { actionId: Schema.String }, +) {} +export class KinPurgePushError extends Schema.TaggedError()( + "KinPurgePushError", + { reason: Schema.String }, +) {} +export class KinPurgeFailed extends Schema.TaggedError()("KinPurgeFailed", { + orgId: Schema.String, + stage: Schema.Literals(["clone", "rewrite", "verify", "tombstone", "push", "rebuild"]), + reason: Schema.String, + /** Once the push starts, local checkouts may hold the old history, so the queue stays paused. */ + queueHeld: Schema.Boolean, + cause: Schema.optional(Schema.Defect()), +}) {} + +/** LEAD-11 supplies the real org pause; the default is a no-op that the purge still calls. */ +export class KinPurgeQueue extends Context.Reference<{ + readonly pause: (orgId: KinOrgId) => Effect.Effect; + readonly resume: (orgId: KinOrgId) => Effect.Effect; +}>("t3/kin/memory/purge/KinPurgeQueue", { + defaultValue: () => ({ pause: () => Effect.void, resume: () => Effect.void }), +}) {} + +export const GitFilterRepoCommand = Context.Reference( + "t3/kin/memory/purge/GitFilterRepoCommand", + { defaultValue: () => "git-filter-repo" }, +); + +export class KinPurgePush extends Context.Service< + KinPurgePush, + { + readonly push: (input: { + readonly orgId: KinOrgId; + readonly repoUrl: string; + readonly gitDir: string; + readonly updates: ReadonlyArray; + }) => Effect.Effect; + } +>()("t3/kin/memory/purge/KinPurgePush") {} + +/** Pushes with the owner's git credential helper or SSH command; T3 never reads the secret. */ +export const gitPushLayer = (owner: { + readonly credentialHelper?: string; + readonly sshCommand?: string; +}) => + Layer.effect( + KinPurgePush, + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const vcs = yield* VcsProcess; + const config = yield* MemoryConfig; + const ownerConfig = MemoryConfig.of({ + baseDirectory: config.baseDirectory, + repositories: config.repositories, + ...(owner.credentialHelper ? { credentialHelper: owner.credentialHelper } : {}), + ...(owner.sshCommand ? { sshCommand: owner.sshCommand } : {}), + }); + return KinPurgePush.of({ + push: Effect.fn("KinPurgePush.git")(function* ({ repoUrl, gitDir, updates }) { + yield* memoryGit(gitDir, [ + "push", + "--atomic", + "--porcelain", + ...updates.map((update) => `--force-with-lease=${update.ref}:${update.old}`), + "--", + repoUrl, + ...updates.map((update) => `${update.new ?? ""}:${update.ref}`), + ]).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(VcsProcess, vcs), + Effect.provideService(MemoryConfig, ownerConfig), + // The cause can echo the remote URL or helper output; keep both out of errors and logs. + Effect.mapError(() => new KinPurgePushError({ reason: "Force-push failed" })), + ); + }), + }); + }), + ); + +const itemKinds = new Map([ + ["objectives", "objective"], + ["commitments", "commitment"], + ["deferrals", "deferral"], + ["decisions", "decision"], + ["people", "person"], + ["claims", "claim"], +]); + +const isPurgePath = (path: string) => + path.length > 0 && + path.length <= 512 && + path.normalize("NFC") === path && + !/[\u0000-\u001f\u007f-\u009f\\]|\p{Default_Ignorable_Code_Point}/u.test(path) && + path.endsWith(".md") && + path !== "rules.md" && + path + .split("/") + .every( + (segment) => + segment.length > 0 && + segment !== "." && + segment !== ".." && + !segment.startsWith("-") && + segment.trim() === segment && + !/^\.git[. ]*$/i.test(segment), + ); + +const isPattern = (pattern: string) => + pattern.length >= 8 && + pattern.length <= 1024 && + !/[\u0000-\u001f\u007f]/.test(pattern) && + !pattern.includes("==>"); + +export const purgePlanDigest = (plan: PurgePlan) => + NodeCrypto.createHash("sha256") + .update( + JSON.stringify({ + orgId: plan.orgId, + repo: plan.repo, + mode: plan.mode, + paths: plan.paths, + patterns: plan.patterns ?? [], + }), + ) + .digest("hex"); + +/** Returns why the plan or its authority is unacceptable, or `undefined`. */ +export const purgeRejection = (plan: PurgePlan, authority: PurgeAuthority) => { + const { action, grant } = authority; + if (action.kind !== "memory_purge" || action.target._tag !== "content") + return "Action is not a memory purge"; + if (action.orgId !== plan.orgId || grant.orgId !== plan.orgId) + return "Grant, action and plan organizations differ"; + if (grant.actionId !== action.id || grant.revision !== action.revision) + return "Grant does not bind this action revision"; + if (action.target.sha256 !== purgePlanDigest(plan)) + return "Plan does not match the approved digest"; + if (!/^[A-Za-z0-9_-]{1,128}$/.test(authority.auditId)) return "Invalid audit id"; + if (plan.paths.length === 0 || plan.paths.length > 64) return "Expected 1-64 paths"; + if (new Set(plan.paths).size !== plan.paths.length) return "Duplicate paths"; + if (!plan.paths.every(isPurgePath)) return "Invalid path"; + const patterns = plan.patterns ?? []; + if (plan.mode === "drop_file" && patterns.length > 0) return "drop_file takes no patterns"; + if (plan.mode === "redact_text") { + if (patterns.length === 0 || patterns.length > 64) return "Expected 1-64 patterns"; + if (new Set(patterns).size !== patterns.length) return "Duplicate patterns"; + if (!patterns.every(isPattern)) return "Invalid pattern"; + } + return undefined; +}; + +const isPurgeFailed = Schema.is(KinPurgeFailed); +const auditResult = (result: { + readonly mode: PurgePlan["mode"]; + readonly paths: ReadonlyArray; + readonly oldHead: string; + readonly newHead: string; + readonly refs: ReadonlyArray; +}) => JSON.stringify(result); +const render = (fields: object, body: string) => `---\n${stringify(fields)}---\n${body}`; +const scanBatchFiles = 256; +const scanBatchBytes = 4 * 1024 * 1024; + +export class KinPurge extends Context.Service< + KinPurge, + { + /** Server-only: reached through `memoryPurgeHandler`, never a model, MCP tool or RPC. */ + readonly purge: ( + plan: PurgePlan, + authority: PurgeAuthority, + ) => Effect.Effect; + } +>()("t3/kin/memory/purge/KinPurge") {} + +const make = Effect.gen(function* () { + const memory = yield* Memory; + const config = yield* MemoryConfig; + const scanner = yield* KinSecretScan; + const pusher = yield* KinPurgePush; + const queue = yield* KinPurgeQueue; + const filterRepo = yield* GitFilterRepoCommand; + const fs = yield* FileSystem.FileSystem; + const vcs = yield* VcsProcess; + const provide = (effect: Effect.Effect) => + effect.pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(VcsProcess, vcs), + ); + const git = ( + work: string, + args: ReadonlyArray, + stdin?: string, + env: NodeJS.ProcessEnv = {}, + ) => + provide(memoryGit(work, args, stdin, env)).pipe( + Effect.provideService(MemoryConfig, config), + Effect.map((stdout) => stdout.trim()), + ); + const failed = + (orgId: KinOrgId, stage: KinPurgeFailed["stage"], reason: string) => + (effect: Effect.Effect) => + effect.pipe( + Effect.mapError((cause) => + isPurgeFailed(cause) + ? cause + : new KinPurgeFailed({ orgId, stage, reason, queueHeld: false, cause }), + ), + ); + const fail = (orgId: KinOrgId, stage: KinPurgeFailed["stage"], reason: string) => + Effect.fail(new KinPurgeFailed({ orgId, stage, reason, queueHeld: false })); + + const branchRefs = Effect.fn("KinPurge.branchRefs")(function* (work: string) { + const lines = yield* git(work, [ + "for-each-ref", + "--format=%(refname) %(objectname)", + "refs/heads/", + "refs/tags/", + ]); + return new Map( + lines.split("\n").flatMap((line) => { + const [ref, sha] = line.split(" "); + return ref && sha ? [[ref, sha] as const] : []; + }), + ); + }); + + const historyFiles = Effect.fn("KinPurge.historyFiles")(function* ( + orgId: KinOrgId, + work: string, + ) { + const raw = yield* git(work, [ + "log", + "--all", + "--raw", + "--no-abbrev", + "--no-renames", + "--diff-merges=separate", + "--format=", + "-z", + ]); + const blobs = new Map(); + const tokens = raw.split("\0"); + for (let index = 0; index < tokens.length; index++) { + const header = /^\n*:\d{6} (\d{6}) [a-f0-9]+ ([a-f0-9]+) [A-Z]\d*$/.exec(tokens[index] ?? ""); + if (!header) continue; + const path = tokens[++index] ?? ""; + const [, mode, sha] = header; + if (!sha || /^0+$/.test(sha) || mode === "160000") continue; + blobs.set(`${sha} ${path}`, { sha, path }); + } + const commits = (yield* git(work, ["rev-list", "--all"])).split("\n").filter(Boolean); + const requests = [ + ...[...blobs.values()].map(({ sha, path }) => ({ sha, type: "blob" as const, path })), + ...commits.map((sha) => ({ sha, type: "commit" as const, path: `commit ${sha}` })), + ]; + const files: Array<{ path: string; content: Uint8Array }> = []; + for (let index = 0; index < requests.length; index += 32) { + const batch = requests.slice(index, index + 32); + const objects = yield* provide(readObjects(orgId, work, batch)); + for (const [position, request] of batch.entries()) { + const content = objects[position]; + if (content) files.push({ path: request.path, content }); + } + } + return files; + }); + + const scanAll = Effect.fn("KinPurge.scanAll")(function* ( + orgId: KinOrgId, + files: ReadonlyArray<{ path: string; content: Uint8Array }>, + ) { + const findings: SecretFinding[] = []; + let batch: Array<{ path: string; content: Uint8Array }> = []; + let bytes = 0; + const flush = Effect.gen(function* () { + if (batch.length) findings.push(...(yield* scanner.scan({ orgId, files: batch }))); + batch = []; + bytes = 0; + }); + for (const file of files) { + if (batch.length >= scanBatchFiles || bytes + file.content.length > scanBatchBytes) + yield* flush; + batch.push(file); + bytes += file.content.length; + } + yield* flush; + return findings; + }); + + const tombstone = Effect.fn("KinPurge.tombstone")(function* ( + orgId: KinOrgId, + work: string, + oldHead: string, + path: string, + authority: PurgeAuthority, + now: number, + ) { + const folder = path.split("/")[0] ?? ""; + const kind = itemKinds.get(folder); + if (!kind) return undefined; + const blob = yield* git(work, ["rev-parse", "--verify", "--quiet", `${oldHead}:${path}`]).pipe( + Effect.option, + ); + if (blob._tag === "None") return undefined; + const original = yield* provide( + readObjects(orgId, work, [{ sha: blob.value, type: "blob" }]), + ).pipe( + Effect.flatMap((objects) => + parseFrontMatter(orgId, path, objects[0]?.toString("utf8") ?? ""), + ), + Effect.option, + ); + const iso = DateTime.formatIso(DateTime.makeUnsafe(now)); + const content = render( + { + id: + original._tag === "Some" + ? original.value.frontMatter.id + : (path.split("/").at(-1) ?? path).slice(0, -3), + kind, + title: "Purged item", + status: "cancelled", + due: null, + recorded_at: iso, + valid_from: iso, + invalid_at: iso, + superseded_by: null, + review_by: null, + source: `kin-audit:${authority.auditId}`, + said_by: authority.grant.approverHumanId, + trust: "owner", + links: [], + }, + `Purged on ${iso.slice(0, 10)} by the owner (audit ${authority.auditId}).\n`, + ); + yield* parseFrontMatter(orgId, path, content); + return content; + }); + + const commitTombstones = Effect.fn("KinPurge.commitTombstones")(function* ( + orgId: KinOrgId, + work: string, + branch: string, + head: string, + tombstones: ReadonlyMap, + auditId: string, + now: number, + ) { + const scratch = yield* fs.makeTempDirectoryScoped({ prefix: "kin-purge-index-" }); + const date = `${Math.floor(now / 1000)} +0000`; + const env = { + GIT_INDEX_FILE: `${scratch}/index`, + GIT_AUTHOR_NAME: config.botName ?? "Kin Memory Bot", + GIT_AUTHOR_EMAIL: config.botEmail ?? "kin-memory@localhost", + GIT_COMMITTER_NAME: config.botName ?? "Kin Memory Bot", + GIT_COMMITTER_EMAIL: config.botEmail ?? "kin-memory@localhost", + GIT_AUTHOR_DATE: date, + GIT_COMMITTER_DATE: date, + }; + yield* git(work, ["read-tree", head], undefined, env); + const entries: string[] = []; + for (const [path, content] of tombstones) { + const blob = yield* git(work, ["hash-object", "-w", "--stdin"], content); + entries.push(`100644 blob ${blob}\t${path}`); + } + yield* git(work, ["update-index", "--index-info"], entries.join("\n") + "\n", env); + const tree = yield* git(work, ["write-tree"], undefined, env); + const commit = yield* git( + work, + ["commit-tree", tree, "-p", head, "-m", `Purge tombstones (audit ${auditId})`], + undefined, + env, + ); + yield* git(work, ["update-ref", `refs/heads/${branch}`, commit, head]); + return commit; + }); + + const rewrite = Effect.fn("KinPurge.rewrite")(function* ( + plan: PurgePlan, + authority: PurgeAuthority, + repoUrl: string, + onPush: () => void, + ) { + const { orgId } = plan; + const branch = config.defaultBranch ?? "main"; + const now = yield* Clock.currentTimeMillis; + const work = yield* fs + .makeTempDirectoryScoped({ prefix: "kin-purge-" }) + .pipe(failed(orgId, "clone", "Could not create a scratch directory")); + yield* git(work, [ + "clone", + "--mirror", + "--no-local", + "--template=", + "--", + repoUrl, + `${work}/.git`, + ]).pipe(failed(orgId, "clone", "Mirror clone failed")); + const before = yield* branchRefs(work).pipe(failed(orgId, "clone", "Could not list refs")); + const oldHead = before.get(`refs/heads/${branch}`); + if (!oldHead) return yield* fail(orgId, "clone", "Remote default branch is missing"); + + const tombstones = new Map(); + if (plan.mode === "drop_file") + for (const path of plan.paths) { + const content = yield* tombstone(orgId, work, oldHead, path, authority, now).pipe( + failed(orgId, "tombstone", "Could not build a tombstone"), + ); + if (content !== undefined) tombstones.set(path, content); + } + + const scratch = yield* fs + .makeTempDirectoryScoped({ prefix: "kin-purge-rules-" }) + .pipe(failed(orgId, "rewrite", "Could not create a scratch directory")); + const rules = `${scratch}/rules.txt`; + if (plan.mode === "redact_text") + yield* fs + .writeFileString( + rules, + (plan.patterns ?? []).map((pattern) => `literal:${pattern}==>***REMOVED***`).join("\n") + + "\n", + ) + .pipe(failed(orgId, "rewrite", "Could not write replacement rules")); + yield* vcs + .run({ + operation: "KinPurge.filterRepo", + command: filterRepo, + cwd: `${work}/.git`, + args: [ + "--quiet", + ...(plan.mode === "drop_file" + ? ["--invert-paths", ...plan.paths.flatMap((path) => ["--path", path])] + : ["--replace-text", rules, "--replace-message", rules]), + ], + env: { + // filter-repo accepts only a fresh clone opened as GIT_DIR=. from its own directory. + GIT_DIR: ".", + GIT_CONFIG_NOSYSTEM: "1", + GIT_CONFIG_GLOBAL: "/dev/null", + GIT_TERMINAL_PROMPT: "0", + GIT_NO_REPLACE_OBJECTS: "1", + GIT_CONFIG_COUNT: "3", + GIT_CONFIG_KEY_0: "core.hooksPath", + GIT_CONFIG_VALUE_0: "/dev/null", + GIT_CONFIG_KEY_1: "core.fsmonitor", + GIT_CONFIG_VALUE_1: "false", + GIT_CONFIG_KEY_2: "core.attributesFile", + GIT_CONFIG_VALUE_2: "/dev/null", + }, + timeoutMs: 600_000, + }) + .pipe(failed(orgId, "rewrite", "git filter-repo failed")); + + const rewritten = yield* branchRefs(work).pipe(failed(orgId, "rewrite", "Could not list refs")); + const rewrittenHead = rewritten.get(`refs/heads/${branch}`); + if (!rewrittenHead) return yield* fail(orgId, "rewrite", "Rewrite removed the default branch"); + if ([...before].every(([ref, sha]) => rewritten.get(ref) === sha)) + return yield* fail(orgId, "rewrite", "Nothing in history matched the plan"); + + if (plan.mode === "drop_file") { + const remaining = yield* git(work, ["rev-list", "--all", "--", ...plan.paths]).pipe( + failed(orgId, "verify", "Could not check rewritten history"), + ); + if (remaining) return yield* fail(orgId, "verify", "Purged paths remain in history"); + } + const newHead = + tombstones.size > 0 + ? yield* commitTombstones( + orgId, + work, + branch, + rewrittenHead, + tombstones, + authority.auditId, + now, + ).pipe(failed(orgId, "tombstone", "Could not commit tombstones")) + : rewrittenHead; + + const files = yield* historyFiles(orgId, work).pipe( + failed(orgId, "verify", "Could not read rewritten history"), + ); + const leftovers = (plan.patterns ?? []).map((pattern) => Buffer.from(pattern, "utf8")); + if ( + files.some((file) => leftovers.some((pattern) => Buffer.from(file.content).includes(pattern))) + ) + return yield* fail(orgId, "verify", "Redacted text remains in history"); + const findings = yield* scanAll(orgId, files).pipe( + failed(orgId, "verify", "Secret scan of rewritten history failed"), + ); + if (findings.length > 0) + return yield* fail( + orgId, + "verify", + `Secret scan found ${findings.length} finding(s) in rewritten history: ${[ + ...new Set(findings.map((finding) => finding.ruleId)), + ].join(", ")}`, + ); + + const after = yield* branchRefs(work).pipe(failed(orgId, "verify", "Could not list refs")); + const updates: PurgeRefUpdate[] = [...before] + .filter(([ref, sha]) => after.get(ref) !== sha) + .map(([ref, old]) => ({ ref, old, new: after.get(ref) ?? null })); + onPush(); + yield* pusher + .push({ orgId, repoUrl, gitDir: work, updates }) + .pipe(failed(orgId, "push", "Force-push failed")); + yield* memory + .reclone(orgId) + .pipe(failed(orgId, "rebuild", "Could not rebuild local memory checkouts")); + return { oldHead, newHead, updates, tombstones: [...tombstones.keys()] }; + }); + + return KinPurge.of({ + purge: Effect.fn("KinPurge.purge")(function* (plan, authority) { + const rejection = purgeRejection(plan, authority); + if (rejection !== undefined) return yield* new KinPurgeRejected({ reason: rejection }); + const repo = config.repositories.find((entry) => entry.orgId === plan.orgId); + if (!repo || repo.repoUrl !== plan.repo) + return yield* new KinPurgeRejected({ + reason: "Plan repo is not this organization's memory", + }); + let pushStarted = false; + yield* queue.pause(plan.orgId); + const exit = yield* memory + .withWriterLock( + plan.orgId, + Effect.scoped( + rewrite(plan, authority, repo.repoUrl, () => { + pushStarted = true; + }), + ), + ) + .pipe(Effect.exit); + if (Exit.isFailure(exit)) { + if (!pushStarted) { + yield* queue.resume(plan.orgId); + return yield* Effect.failCause(exit.cause); + } + return yield* Effect.failCause(exit.cause).pipe( + Effect.mapError( + (error) => + new KinPurgeFailed({ + orgId: error.orgId, + stage: error.stage, + reason: error.reason, + queueHeld: true, + ...(error.cause === undefined ? {} : { cause: error.cause }), + }), + ), + ); + } + yield* queue.resume(plan.orgId); + const { oldHead, newHead, updates, tombstones } = exit.value; + return { + mode: plan.mode, + paths: plan.paths, + oldHead, + newHead, + rewrittenRefs: updates, + tombstones, + audit: { + orgId: plan.orgId, + event: "execution_finished", + actionId: authority.action.id, + kind: "memory_purge", + grantJti: authority.grant.jti, + keyThumbprint: authority.grant.keyThumbprint, + ...(authority.deviceLabel === undefined ? {} : { deviceLabel: authority.deviceLabel }), + headSha: newHead, + result: auditResult({ + mode: plan.mode, + paths: plan.paths, + oldHead, + newHead, + refs: updates, + }), + }, + } satisfies PurgeResult; + }), + }); +}); + +export const layer = Layer.effect(KinPurge, make); + +// Gated until APPR-9 (executor) and APPR-10 (audit) land. +const purgeExecutorReady = false; +/** Test seam for the release gate; production always reads `purgeExecutorReady`. */ +export const PurgeExecutorGate = Context.Reference( + "t3/kin/memory/purge/PurgeExecutorGate", + { defaultValue: () => purgeExecutorReady }, +); + +/** The only way to purge: APPR-9's executor calls `run` with a verified grant. */ +export const memoryPurgeHandler = { + kind: "memory_purge", + run: Effect.fn("KinPurge.handler")(function* (plan: PurgePlan, authority: PurgeAuthority) { + if (!(yield* PurgeExecutorGate)) + return yield* new KinPurgeExecutorNotReady({ actionId: authority.action.id }); + return yield* (yield* KinPurge).purge(plan, authority); + }), +} as const; diff --git a/docs/kin/specs/triggers.md b/docs/kin/specs/triggers.md index 8b1cc15155874..6ad54f2aef1c9 100644 --- a/docs/kin/specs/triggers.md +++ b/docs/kin/specs/triggers.md @@ -426,17 +426,22 @@ Known gaps, accepted for phase 1: JWE (five-part) tokens and JWT headers longer ### Owner-signed purge (gap 5) -Pending action `memory.purge` with `{ orgId, repo, paths, mode: "drop_file" | "redact_text", patterns? }`. The owner opens it from an item's menu, or Kin proposes it when the weekly `dream_full` scans full history and finds a secret. Approval uses a native card and a signed grant (APPR). The executor then: +Pending action `memory_purge`. Its `content` target holds the sha256 of the plan `{ orgId, repo, paths, mode: "drop_file" | "redact_text", patterns? }` (`purgePlanDigest`), so the grant binds the exact plan. The owner opens it from an item's menu, or Kin proposes it when the weekly `dream_full` scans full history and finds a secret. Approval uses a native card and a signed grant (APPR). `memoryPurgeHandler` in `apps/server/src/kin/memory/purge.ts` is the only entry point; no model, MCP tool or RPC reaches `KinPurge`. It refuses with `KinPurgeExecutorNotReady` until APPR-9 and APPR-10 land. -1. Pauses the org's lead queue and takes the memory writer lock. -2. Makes a fresh mirror clone and runs `git filter-repo --invert-paths --path

` (`drop_file`) or `--replace-text ` (`redact_text`), then re-runs the scan over all history. -3. Force-pushes the rewritten branches with the owner's credential. This needs the owner as a bypass actor on the memory repo's ruleset. It closes open dream PRs that touch the paths. -4. For `drop_file`, commits a tombstone at the same path: `status: cancelled`, `invalid_at: now`, body "Purged on by the owner (audit )". Memory still shows that something was there. -5. Re-clones every local checkout and worktree of the memory repo, rebuilds the cache and `INDEX.md`, and resumes the queue. -6. Writes a `KinAuditEntry` (APPR) with the grant, device, paths, old head, new head and the refs rewritten. -7. Tells the owner what a purge can't reach: GitHub keeps `refs/pull/*` and cached PR diffs (removing those needs a GitHub Support request), forks, local backups (Time Machine), and the chat database. A leaked credential must be rotated whatever happens. +Before anything runs, the grant, action and plan must share the org, the grant must bind the action id and revision, and `repo` must equal the org's configured memory URL. Paths: 1–64, unique, relative `.md` files other than `rules.md`, with no `..`, `.`, empty, `-`-leading or `.git` segment, backslash or control character. `redact_text` takes 1–64 literal patterns of 8–1024 characters without control characters or `==>`; each becomes one `literal:` rule, so a pattern can't add a rule or switch to regex. `drop_file` takes none. Every argument goes to git and `git-filter-repo` as an argv entry, never through a shell. -**Dependency:** `git-filter-repo` (git's recommended history rewriter; `brew install git-filter-repo`). Check its licence and release activity when installing (TRIG-17). +The executor then: + +1. Pauses the org's lead queue (`KinPurgeQueue`, a no-op until LEAD-11 builds the pause) and takes the memory writer lock. +2. Mirror-clones the remote into a scratch directory and runs `git filter-repo --invert-paths --path

` (`drop_file`) or `--replace-text --replace-message ` (`redact_text`). A plan that changes no ref fails. +3. For `drop_file`, checks that no reachable commit touches the paths, then commits a tombstone on the default branch for each item path present at the old head: the original `id` and the folder's kind, `title: Purged item`, `status: cancelled`, `invalid_at: now`, `said_by` the approving owner, body "Purged on by the owner (audit )", where `` is the executor's `execution_started` audit entry. Memory still shows that something was there. Journal and other non-item paths get no tombstone. +4. Reads every blob ever introduced at every path, merge sides included, and every commit object. It fails if a redacted pattern remains, then runs `KinSecretScan` over all of it and fails closed on **any** finding: findings carry no fingerprint, so the purge can't prove one unrelated, and a copy under an older path would otherwise survive. Allowlisted fingerprints don't count. Annotated tag messages are not scanned. +5. Force-pushes the changed `refs/heads/*` and `refs/tags/*` in one atomic push, each leased at its cloned value, through `KinPurgePush`. `gitPushLayer` runs git with the owner's credential helper or SSH command, so T3 never reads the credential, and push errors drop their cause so neither the URL nor helper output is logged. This needs the owner as a bypass actor on the memory repo's ruleset. Closing open dream PRs that touch the paths waits on the GitHub client. +6. `Memory.reclone` deletes the writer and read checkouts, clones fresh, records the rewritten anchors without preserving the old ones (a conflict branch would push the purged history back), and rebuilds the read checkout, cache and `INDEX.md`. The queue resumes. A failure before the push resumes the queue with nothing changed; from the push on, `KinPurgeFailed` carries `queueHeld: true` and the queue stays paused for owner recovery. +7. Returns the `execution_finished` audit input (grant `jti`, key thumbprint, device, new head) with the paths, old head, new head and rewritten refs; APPR-10 writes it as a `KinAuditEntry`. +8. Tells the owner what a purge can't reach: GitHub keeps `refs/pull/*` and cached PR diffs (removing those needs a GitHub Support request), forks, local backups (Time Machine), and the chat database. A leaked credential must be rotated whatever happens. + +**Dependency:** `git-filter-repo` (git's recommended history rewriter; MIT, also usable under git's GPL), checked at v2.47.0. TRIG-20 installs it on the Kin host; `GitFilterRepoCommand` overrides the binary path. The real-rewrite tests skip on machines without it. ### Upstream contract tests (gap 7) @@ -533,7 +538,7 @@ Kin runs on one always-on machine, the **Kin host**. The laptop and phone are cl - provides the `kin_defer` tool. - **CHAT:** `KinOrgId`; delivering messages and pings; `KinWidget` payloads `KinDeferralCard`, `KinHeadsUpCard`, `KinOwnerCheckCard`, `KinDreamReviewCard`; the chat message store for dream material; verified authors for trust checks. - **APPR:** - - action kinds `github.app_hook.configure`, `memory.dream_review`, `memory.purge`, `kin.secret_allow`; + - action kinds `github.app_hook.configure`, `memory.dream_review`, `memory_purge`, `kin.secret_allow`; - the standing `dream:auto` delegation; - the executor merging at a bound SHA; - `KinAuditEntry`; @@ -557,7 +562,7 @@ Kin runs on one always-on machine, the **Kin host**. The laptop and phone are cl - **Deferral:** unchanged recheck, then heads-up, then silence, then `started` at due. A linked issue closed before due leads to `asked`, and nothing starts. With the stop switch on, it asks. [Monday] re-arms to next Monday 09:00. A `due` fire consumed two hours late asks; a queued fire for a disarmed or moved deferral is dropped. - **Dream:** a dream touching `rules.md` fails, and `main` is unchanged. A dream with one journal fold and one `decisions/` change opens two PRs, and only the auto one merges. - **Secret scan:** a journal line holding `ghp_` plus 36 characters blocks the entire commit, and the owner gets one typed notification without the value. -- **Purge:** on a fixture repo, the file is gone from `git log --all -p`, a tombstone exists, and an audit row exists. +- **Purge:** on a fixture repo, the file is gone from `git log --all -p` and the recloned checkout, a tombstone exists and parses, and the returned audit input carries the old and new heads. A surviving secret fails closed before the push. - **Contract tests:** `vp test run kin-contract` passes on today's upstream. Renaming `scheduledTaskId` in a scratch branch makes it fail. ## Tasks From 6fb2fba887eed0ce0728a94173a31677beec56be Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Sun, 8 Nov 2026 23:09:46 +0530 Subject: [PATCH 2/7] fix(kin): purge never republishes rewritten history (TRIG-17) A push that lands but reports failure left the old history in the writer checkout, and the next wake-up's preserve pushed it back as a conflict branch. The purge now writes a durable marker and deletes the local checkouts before the push, and preserve and the wake-up merge refuse to push while the marker exists. - memoryPurgeHandler is APPR-9's KinActionHandler: precheck, run and reconcile, all behind the executor gate inside KinPurge itself. - The plan is stored as the file its action targets (purgePlanJson); redact_text takes no paths. - The queue decision runs on every exit; defects and interrupts after the quarantine hold the queue. The default queue fails pause_unavailable until LEAD-11. - The owner push takes only credential references, fails closed, never prompts, scrubs inherited GIT_* env and classifies failures. - Paths must exist in history (literal pathspecs); scratch lives under the org's purge directory and is swept on the next run. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/kin/lead/LeadPrompt.test.ts | 3 +- apps/server/src/kin/lead/layer.test.ts | 3 +- apps/server/src/kin/memory/Memory.ts | 88 ++- apps/server/src/kin/memory/purge.test.ts | 810 +++++++++++++++----- apps/server/src/kin/memory/purge.ts | 710 +++++++++++------ apps/server/src/kin/memory/purgePush.ts | 162 ++++ docs/kin/specs/triggers.md | 29 +- 7 files changed, 1333 insertions(+), 472 deletions(-) create mode 100644 apps/server/src/kin/memory/purgePush.ts diff --git a/apps/server/src/kin/lead/LeadPrompt.test.ts b/apps/server/src/kin/lead/LeadPrompt.test.ts index e7b432db80150..92b5771034382 100644 --- a/apps/server/src/kin/lead/LeadPrompt.test.ts +++ b/apps/server/src/kin/lead/LeadPrompt.test.ts @@ -63,7 +63,8 @@ const memory = Layer.succeed( anchor: () => Effect.succeed("fixture"), record: () => Effect.void, recover: () => Effect.void, - reclone: () => Effect.die("Unused"), + withHistoryRewrite: () => Effect.die("Unused"), + purgeInFlight: () => Effect.die("Unused"), readPromptSnapshot: (org, head) => org !== orgId || !["head-one", "head-two"].includes(head) ? Effect.fail(new Memory.MemoryError({ orgId: org, operation: "promptSnapshot" })) diff --git a/apps/server/src/kin/lead/layer.test.ts b/apps/server/src/kin/lead/layer.test.ts index def615b16aa21..f1d1e8a6722c8 100644 --- a/apps/server/src/kin/lead/layer.test.ts +++ b/apps/server/src/kin/lead/layer.test.ts @@ -44,7 +44,8 @@ vi.mock("../memory/Memory.ts", async (importOriginal) => { anchor: () => Effect.die("unused"), record: () => Effect.die("unused"), recover: () => Effect.die("unused"), - reclone: () => Effect.die("unused"), + withHistoryRewrite: () => Effect.die("unused"), + purgeInFlight: () => Effect.die("unused"), ensureCheckouts: () => Effect.succeed({ writer: "/private/tmp", read: "/private/tmp" }), prepareWakeup: () => Effect.sync(() => { diff --git a/apps/server/src/kin/memory/Memory.ts b/apps/server/src/kin/memory/Memory.ts index f0dc60b358bad..9a0cf53f48239 100644 --- a/apps/server/src/kin/memory/Memory.ts +++ b/apps/server/src/kin/memory/Memory.ts @@ -142,6 +142,13 @@ export class MemorySyncHooks extends Context.Reference<{ defaultValue: () => ({ syncItems: () => Effect.void }), }) {} +export interface MemoryHistoryRewrite { + /** Writes the durable marker, then deletes the writer and read checkouts that hold the old history. */ + readonly quarantine: (marker: string) => Effect.Effect; + /** Clones fresh, records the remote heads without preserving old anchors, rebuilds, then clears the marker. */ + readonly reclone: () => Effect.Effect; +} + export class Memory extends Context.Service< Memory, { @@ -162,8 +169,13 @@ export class Memory extends Context.Service< head: string, ) => Effect.Effect; readonly recover: (orgId: KinOrgId) => Effect.Effect; - /** Server-only, for history rewrites: the caller holds the writer lock. Never preserves old anchors. */ - readonly reclone: (orgId: KinOrgId) => Effect.Effect; + /** Server-only, for `KinPurge`: runs `body` under the writer lock with the checkout operations a history rewrite needs. */ + readonly withHistoryRewrite: ( + orgId: KinOrgId, + body: (rewrite: MemoryHistoryRewrite) => Effect.Effect, + ) => Effect.Effect; + /** The durable marker of an unfinished history rewrite; while present, nothing pushes old anchors. */ + readonly purgeInFlight: (orgId: KinOrgId) => Effect.Effect; readonly ensureCheckouts: (orgId: KinOrgId) => Effect.Effect; readonly prepareWakeup: ( orgId: KinOrgId, @@ -440,6 +452,24 @@ const make = Effect.gen(function* () { return { repo, root, writer, read }; }); + const markerPath = (root: string) => path.join(root, "purge-in-flight"); + const purgeInFlight = Effect.fn("Memory.purgeInFlight")(function* (orgId: KinOrgId) { + const { root } = yield* pathsFor(orgId); + if (!(yield* io(orgId, "purgeMarker", fs.exists(markerPath(root))))) return undefined; + return yield* io(orgId, "purgeMarker", fs.readFileString(markerPath(root))); + }); + const refuseDuringPurge = Effect.fn("Memory.refuseDuringPurge")(function* ( + orgId: KinOrgId, + operation: string, + ) { + if ((yield* purgeInFlight(orgId)) !== undefined) + return yield* new MemorySyncConflict({ + orgId, + operation, + reason: "A memory purge is in flight; old history must not be pushed", + }); + }); + const walkAncestors = (...args: Parameters) => ancestors(...args).pipe( Effect.provideService(FileSystem.FileSystem, fs), @@ -847,6 +877,7 @@ const make = Effect.gen(function* () { head: string, wakeupId?: string, ) { + yield* refuseDuringPurge(orgId, "preserve"); const now = yield* Clock.currentTimeMillis; const id = wakeupId ?? `recovery-${now}-${head.slice(0, 12)}`; if (!/^[a-zA-Z0-9_-]+$/.test(id) || !/^[a-zA-Z0-9_-]+$/.test(orgId)) @@ -1244,13 +1275,15 @@ const make = Effect.gen(function* () { } } if (next !== head) { - if (next !== remote) + if (next !== remote) { + yield* refuseDuringPurge(orgId, "merge"); yield* git(orgId, paths.writer, [ "push", "--", repo.repoUrl, `${next}:refs/heads/${branch}`, ]); + } yield* git(orgId, paths.writer, ["update-ref", `refs/heads/${branch}`, next, head]); yield* record(orgId, branch, head, next); head = next; @@ -1291,20 +1324,55 @@ const make = Effect.gen(function* () { return { head, indexAsOf: DateTime.toUtc(indexAsOf) }; }); - const reclone = Effect.fn("Memory.reclone")(function* (orgId: KinOrgId) { + const dropCheckouts = Effect.fn("Memory.dropCheckouts")(function* (orgId: KinOrgId) { const paths = yield* pathsFor(orgId); for (const directory of [paths.writer, paths.read]) - yield* io(orgId, "reclone", fs.remove(directory, { recursive: true, force: true })); + yield* io(orgId, "quarantine", fs.remove(directory, { recursive: true, force: true })); initializedCheckouts.delete(orgId); - // With no writer checkout, recover clones fresh and records the rewritten heads. - yield* recover(orgId); - yield* prepare(orgId); - yield* rebuildIndex(orgId); + pinnedHeads.delete(orgId); + snapshots.delete(orgId); + blobCaches.delete(orgId); + return paths; }); + const historyRewrite = (orgId: KinOrgId): MemoryHistoryRewrite => ({ + quarantine: Effect.fn("Memory.quarantine")((marker: string) => + locked( + orgId, + Effect.gen(function* () { + const { root } = yield* pathsFor(orgId); + yield* io(orgId, "mkdir", fs.makeDirectory(root, { recursive: true })); + yield* io(orgId, "purgeMarker", fs.writeFileString(`${markerPath(root)}.tmp`, marker)); + yield* io(orgId, "purgeMarker", fs.rename(`${markerPath(root)}.tmp`, markerPath(root))); + yield* dropCheckouts(orgId); + }), + ), + ), + reclone: Effect.fn("Memory.reclone")(() => + locked( + orgId, + Effect.gen(function* () { + const { root } = yield* dropCheckouts(orgId); + // With no writer checkout, recover clones fresh and records the rewritten heads. + yield* recover(orgId); + yield* prepare(orgId); + yield* rebuildIndex(orgId); + yield* io(orgId, "purgeMarker", fs.remove(markerPath(root), { force: true })); + }), + ), + ), + }); + + const withHistoryRewrite = ( + orgId: KinOrgId, + body: (rewrite: MemoryHistoryRewrite) => Effect.Effect, + ) => withWriterLock(orgId, body(historyRewrite(orgId))); return Memory.of({ withWriterLock, - reclone: Effect.fn("Memory.recloneLocked")((orgId) => locked(orgId, reclone(orgId))), + withHistoryRewrite, + purgeInFlight: Effect.fn("Memory.purgeInFlightLocked")((orgId) => + locked(orgId, purgeInFlight(orgId)), + ), recover: Effect.fn("Memory.recoverLocked")((orgId) => withWriterLock(orgId, locked(orgId, recover(orgId))), ), diff --git a/apps/server/src/kin/memory/purge.test.ts b/apps/server/src/kin/memory/purge.test.ts index 464f0778889cd..079e5b613123b 100644 --- a/apps/server/src/kin/memory/purge.test.ts +++ b/apps/server/src/kin/memory/purge.test.ts @@ -2,10 +2,14 @@ import * as NodeChildProcess from "node:child_process"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { describe, expect, it } from "@effect/vitest"; +import { vi } from "vite-plus/test"; import { KinMemoryItemId, KinOrgId } from "@t3tools/contracts"; +import * as Cause from "effect/Cause"; import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; +import { ChildProcessSpawner } from "effect/process"; import * as Sqlite from "../../persistence/Sqlite.ts"; import * as Persistence from "../persistence/KinPersistence.ts"; import * as Vcs from "../../vcs/VcsProcess.ts"; @@ -14,6 +18,7 @@ import { parseFrontMatter } from "./frontMatter.ts"; import * as Memory from "./Memory.ts"; import { MemoryConfig } from "./MemoryConfig.ts"; import * as Purge from "./purge.ts"; +import * as PurgePush from "./purgePush.ts"; import * as Scan from "./secretScan.ts"; const orgId = KinOrgId.make("fixture-org"); @@ -21,180 +26,210 @@ const token = ["gh", "p_"].join("") + "A1b2C3d4E5".repeat(3) + "abcdef"; // TRIG-20 installs git-filter-repo on the Kin host; machines without it skip the real rewrite. const filterRepoMissing = NodeChildProcess.spawnSync("git-filter-repo", ["--version"]).status !== 0; -const authorityFor = (plan: Purge.PurgePlan): Purge.PurgeAuthority => ({ - action: { - id: "act-1", - orgId, - kind: "memory_purge", - revision: 1, - target: { _tag: "content", path: "purge/act-1.json", sha256: Purge.purgePlanDigest(plan) }, - }, - grant: { - id: "grant-1", - orgId, - actionId: "act-1", - revision: 1, - jti: "jti-1", - approverHumanId: "fixture-owner", - keyThumbprint: "thumb-1", - }, - auditId: "audit-1", +const grant: Purge.PurgeGrant = { + id: "grant-1", + orgId, + actionId: "act-1", + revision: 1, + jti: "jti-1", + approverHumanId: "fixture-owner", + keyThumbprint: "thumb-1", +}; +const actionFor = (plan: Purge.PurgePlan): Purge.PurgeAction => ({ + id: "act-1", + orgId, + kind: "memory_purge", + revision: 1, + target: Purge.purgePlanJson(plan).target, }); +const gateOpen = Effect.provideService(Purge.PurgeExecutorGate, true); -describe.skipIf(filterRepoMissing)("purge with git-filter-repo", () => { - it.live.each([ - "drop_file", - "redact_text", - "wrong pattern", - "push fails", - "missing tool", - ] as const)("purge on a fixture repo: %s", (scenario) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const vcs = yield* Vcs.VcsProcess; - const root = yield* fs.realPath(yield* fs.makeTempDirectoryScoped()); - const source = `${root}/seed`; - const remote = `${root}/remote.git`; - // A file:// URL transfers reachable objects only, like a real remote; a bare path copies them all. - const remoteUrl = `file://${remote}`; - const git = (cwd: string, args: ReadonlyArray) => - vcs - .run({ - operation: "test.purge", - command: "git", - cwd, - args: cwd === remote ? [`--git-dir=${remote}`, ...args] : args, - maxOutputBytes: 64 * 1024 * 1024, - }) - .pipe(Effect.map((result) => result.stdout.trim())); - const commit = (message: string) => - git(source, [ - "-c", - "user.name=Fixture Bot", - "-c", - "user.email=fixture@example.invalid", - "commit", - "-q", - "-m", - message, - ]); - yield* fs.makeDirectory(`${source}/claims`, { recursive: true }); - yield* git(source, ["init", "-q", "-b", "main"]); - yield* fs.writeFileString(`${source}/rules.md`, "Invented rules\n"); - yield* fs.writeFileString(`${source}/INDEX.md`, "As of 2026-11-08T00:00:00.000Z\n"); - yield* fs.writeFileString( - `${source}/claims/leak.md`, - fixtureItem({ id: "claim-leak", kind: "claim", title: "Invented vendor note" }) + - `vendor key: ${token}\n`, - ); - yield* fs.writeFileString( - `${source}/claims/keep.md`, - fixtureItem({ id: "claim-keep", kind: "claim", title: "Invented keeper" }), - ); - yield* git(source, ["add", "--", "."]); - yield* commit(scenario === "redact_text" ? `fixture ${token}` : "fixture"); - yield* git(source, ["switch", "-q", "-c", "side"]); - yield* fs.writeFileString(`${source}/claims/leak.md`, `side copy ${token}\n`); - yield* git(source, ["add", "--", "."]); - yield* commit("side"); - yield* git(source, ["switch", "-q", "main"]); - yield* fs.writeFileString(`${source}/rules.md`, "Invented rules, revised\n"); - yield* git(source, ["add", "--", "."]); - yield* commit("revise rules"); - yield* git(root, ["clone", "-q", "--bare", "--", source, remote]); - - const events: string[] = []; - const config = Layer.succeed(MemoryConfig, { - baseDirectory: root, - repositories: [{ orgId, repoUrl: remoteUrl, path: "state" }], - }); - const memoryLayer = Memory.layer.pipe( - Layer.provide(config), - Layer.provide(Layer.succeed(Memory.MemoryRecoveryHooks, { notify: () => Effect.void })), - ); - const push = - scenario === "push fails" - ? Layer.succeed(Purge.KinPurgePush, { - push: () => Effect.fail(new Purge.KinPurgePushError({ reason: "fixture refusal" })), - }) - : Purge.gitPushLayer({}).pipe(Layer.provide(config)); - const layer = Purge.layer.pipe( - Layer.provideMerge(memoryLayer), - Layer.provide(config), - Layer.provide(Scan.layer), - Layer.provide(push), - Layer.provide( - Layer.succeed( - Purge.GitFilterRepoCommand, - scenario === "missing tool" ? `${root}/missing/git-filter-repo` : "git-filter-repo", - ), - ), - Layer.provide( - Layer.succeed(Purge.KinPurgeQueue, { - pause: () => Effect.sync(() => events.push("pause")), - resume: () => Effect.sync(() => events.push("resume")), +const services = Layer.mergeAll( + Vcs.layer.pipe(Layer.provide(NodeServices.layer)), + NodeServices.layer, + Persistence.layer.pipe(Layer.provideMerge(Sqlite.layerMemory)), +); + +interface PurgeOptions { + readonly push?: ( + real: PurgePush.KinPurgePush["Service"], + ) => Partial | undefined; + readonly scan?: Layer.Layer; + readonly filterRepo?: string; + readonly queue?: "default"; +} + +const fixture = Effect.fn(function* (tokenInMessage = false) { + const fs = yield* FileSystem.FileSystem; + const vcs = yield* Vcs.VcsProcess; + const root = yield* fs.realPath(yield* fs.makeTempDirectoryScoped()); + const source = `${root}/seed`; + const remote = `${root}/remote.git`; + // A file:// URL transfers reachable objects only, like a real remote; a bare path copies them all. + const remoteUrl = `file://${remote}`; + const git = (cwd: string, args: ReadonlyArray) => + vcs + .run({ + operation: "test.purge", + command: "git", + cwd, + args: cwd === remote ? [`--git-dir=${remote}`, ...args] : args, + maxOutputBytes: 64 * 1024 * 1024, + }) + .pipe(Effect.map((result) => result.stdout.trim())); + const commit = (message: string) => + git(source, [ + "-c", + "user.name=Fixture Bot", + "-c", + "user.email=fixture@example.invalid", + "commit", + "-q", + "-m", + message, + ]); + yield* fs.makeDirectory(`${source}/claims`, { recursive: true }); + yield* git(source, ["init", "-q", "-b", "main"]); + yield* fs.writeFileString(`${source}/rules.md`, "Invented rules\n"); + yield* fs.writeFileString(`${source}/INDEX.md`, "As of 2026-11-08T00:00:00.000Z\n"); + yield* fs.writeFileString( + `${source}/claims/leak.md`, + fixtureItem({ id: "claim-leak", kind: "claim", title: "Invented vendor note" }) + + `vendor key: ${token}\n`, + ); + yield* fs.writeFileString( + `${source}/claims/keep.md`, + fixtureItem({ id: "claim-keep", kind: "claim", title: "Invented keeper" }), + ); + yield* git(source, ["add", "--", "."]); + yield* commit(tokenInMessage ? `fixture ${token}` : "fixture"); + yield* git(source, ["switch", "-q", "-c", "side"]); + yield* fs.writeFileString(`${source}/claims/leak.md`, `side copy ${token}\n`); + yield* git(source, ["add", "--", "."]); + yield* commit("side"); + yield* git(source, ["switch", "-q", "main"]); + yield* fs.writeFileString(`${source}/rules.md`, "Invented rules, revised\n"); + yield* git(source, ["add", "--", "."]); + yield* commit("revise rules"); + yield* git(root, ["clone", "-q", "--bare", "--", source, remote]); + + const events: string[] = []; + const config = Layer.succeed(MemoryConfig, { + baseDirectory: root, + repositories: [{ orgId, repoUrl: remoteUrl, path: "state" }], + }); + const purgeLayer = (options: PurgeOptions = {}) => { + const realPush = PurgePush.gitPushLayer({ credentialHelper: "fixture-helper" }); + const push = options.push + ? Layer.effect( + PurgePush.KinPurgePush, + Effect.gen(function* () { + const real = yield* PurgePush.KinPurgePush; + return PurgePush.KinPurgePush.of({ ...real, ...options.push?.(real) }); }), + ).pipe(Layer.provide(realPush)) + : realPush; + return Purge.layer.pipe( + Layer.provideMerge( + Memory.layer.pipe( + Layer.provide(config), + Layer.provide(Layer.succeed(Memory.MemoryRecoveryHooks, { notify: () => Effect.void })), ), - ); + ), + Layer.provide(config), + Layer.provide(options.scan ?? Scan.layer), + Layer.provide(push), + Layer.provide( + Layer.succeed(Purge.GitFilterRepoCommand, options.filterRepo ?? "git-filter-repo"), + ), + Layer.provide( + options.queue === "default" + ? Layer.empty + : Layer.succeed(Purge.KinPurgeQueue, { + pause: () => Effect.sync(() => events.push("pause")), + resume: () => Effect.sync(() => events.push("resume")), + }), + ), + ); + }; + const writerHistory = (writer: string) => + Effect.gen(function* () { + if (!(yield* fs.exists(`${writer}/.git`))) return ""; + return [ + yield* git(writer, ["log", "--all", "-p"]), + yield* git(writer, ["cat-file", "--batch-all-objects", "--batch"]), + ].join("\n"); + }); + /** Seeds anchors and checkouts the way a lead's first wake-up does. */ + const prepare = Effect.gen(function* () { + const memory = yield* Memory.Memory; + const paths = yield* memory.ensureCheckouts(orgId); + yield* memory.prepareWakeup(orgId); + return paths; + }); + return { + fs, + git, + root, + source, + remote, + remoteUrl, + events, + commit, + purgeLayer, + writerHistory, + prepare, + }; +}); + +const dropPlan = (remoteUrl: string, paths = ["claims/leak.md"]): Purge.PurgePlan => ({ + orgId, + repo: remoteUrl, + mode: "drop_file", + paths, +}); +const runPurge = (plan: Purge.PurgePlan) => + Effect.gen(function* () { + const target = yield* (yield* Purge.KinPurge).storePlan(plan); + expect(target).toEqual(actionFor(plan).target); + return yield* Purge.memoryPurgeHandler.run(actionFor(plan), grant).pipe(gateOpen, Effect.exit); + }); +const reconcile = (plan: Purge.PurgePlan) => + Purge.memoryPurgeHandler.reconcile(actionFor(plan)).pipe(gateOpen); +const failureOf = (exit: Exit.Exit) => + Exit.isFailure(exit) ? Cause.findErrorOption(exit.cause) : undefined; +describe.skipIf(filterRepoMissing)("purge with git-filter-repo", () => { + it.live.each(["drop_file", "redact_text"] as const)("purges a fixture repo: %s", (mode) => + Effect.gen(function* () { + const { fs, git, root, remote, remoteUrl, events, purgeLayer, writerHistory, prepare } = + yield* fixture(mode === "redact_text"); yield* Effect.gen(function* () { const memory = yield* Memory.Memory; - const paths = yield* memory.ensureCheckouts(orgId); - yield* memory.prepareWakeup(orgId); + const paths = yield* prepare; const oldMain = yield* git(remote, ["rev-parse", "refs/heads/main"]); const oldSide = yield* git(remote, ["rev-parse", "refs/heads/side"]); const plan: Purge.PurgePlan = - scenario === "redact_text" - ? { - orgId, - repo: remoteUrl, - mode: "redact_text", - paths: ["claims/leak.md"], - patterns: [token], - } - : scenario === "wrong pattern" - ? { - orgId, - repo: remoteUrl, - mode: "redact_text", - paths: ["claims/leak.md"], - patterns: ["Invented vendor note"], - } - : { orgId, repo: remoteUrl, mode: "drop_file", paths: ["claims/leak.md"] }; - const result = yield* Purge.memoryPurgeHandler - .run(plan, authorityFor(plan)) - .pipe(Effect.provideService(Purge.PurgeExecutorGate, true), Effect.result); - - if (scenario !== "drop_file" && scenario !== "redact_text") { - expect(result._tag).toBe("Failure"); - if (result._tag === "Failure") { - expect(result.failure).toMatchObject({ - _tag: "KinPurgeFailed", - ...{ - "wrong pattern": { stage: "verify", queueHeld: false }, - "push fails": { stage: "push", queueHeld: true }, - "missing tool": { stage: "rewrite", queueHeld: false }, - }[scenario], - }); - expect(JSON.stringify(result.failure)).not.toContain(token); - } - expect(events).toEqual(scenario === "push fails" ? ["pause"] : ["pause", "resume"]); - expect(yield* git(remote, ["rev-parse", "refs/heads/main"])).toBe(oldMain); - expect(yield* memory.anchor(orgId, "main")).toBe(oldMain); - return; - } + mode === "redact_text" + ? { orgId, repo: remoteUrl, mode, patterns: [token] } + : dropPlan(remoteUrl); + yield* fs.makeDirectory(`${root}/state/purge-scratch/mirror-stale`, { recursive: true }); + const exit = yield* runPurge(plan); - expect(result._tag).toBe("Success"); - if (result._tag !== "Success") return; + expect(Exit.isSuccess(exit)).toBe(true); + expect(yield* fs.readDirectory(`${root}/state/purge-scratch`)).toEqual([]); + if (!Exit.isSuccess(exit)) return; + const result = exit.value; const newMain = yield* git(remote, ["rev-parse", "refs/heads/main"]); const newSide = yield* git(remote, ["rev-parse", "refs/heads/side"]); expect(events).toEqual(["pause", "resume"]); - expect(result.success).toMatchObject({ oldHead: oldMain, newHead: newMain }); - expect(result.success.rewrittenRefs).toEqual([ + expect(yield* memory.purgeInFlight(orgId)).toBeUndefined(); + expect(result).toMatchObject({ oldHead: oldMain, newHead: newMain }); + expect(result.rewrittenRefs).toEqual([ { ref: "refs/heads/main", old: oldMain, new: newMain }, { ref: "refs/heads/side", old: oldSide, new: newSide }, ]); - expect(result.success.audit).toMatchObject({ + expect(result.audit).toMatchObject({ orgId, event: "execution_finished", actionId: "act-1", @@ -203,53 +238,227 @@ describe.skipIf(filterRepoMissing)("purge with git-filter-repo", () => { keyThumbprint: "thumb-1", headSha: newMain, }); - expect(JSON.parse(result.success.audit.result)).toMatchObject({ + expect(JSON.parse(result.audit.result)).toMatchObject({ oldHead: oldMain, newHead: newMain, }); + expect(result.audit.result).not.toContain(token); - for (const history of [ - yield* git(remote, ["log", "--all", "-p"]), - yield* git(paths.writer, ["log", "--all", "-p"]), - yield* git(paths.writer, ["cat-file", "--batch-all-objects", "--batch"]), - ]) - expect(history).not.toContain(token); + expect(yield* git(remote, ["log", "--all", "-p"])).not.toContain(token); + expect(yield* writerHistory(paths.writer)).not.toContain(token); expect(yield* memory.anchor(orgId, "main")).toBe(newMain); expect(yield* git(remote, ["show", `${newMain}:claims/keep.md`])).toContain("claim-keep"); const leak = yield* git(remote, ["show", `${newMain}:claims/leak.md`]); - if (scenario === "redact_text") { - expect(result.success.tombstones).toEqual([]); + if (mode === "redact_text") { + expect(result.tombstones).toEqual([]); expect(leak).toContain("vendor key: ***REMOVED***"); return; } - expect(result.success.tombstones).toEqual(["claims/leak.md"]); + expect(result.tombstones).toEqual(["claims/leak.md"]); const tombstone = yield* parseFrontMatter(orgId, "claims/leak.md", leak + "\n"); expect(tombstone.frontMatter).toMatchObject({ id: "claim-leak", kind: "claim", status: "cancelled", said_by: "fixture-owner", + source: "kin-action:act-1", }); expect(tombstone.frontMatter.invalid_at).not.toBeNull(); expect(tombstone.body).toMatch( - /^Purged on \d{4}-\d{2}-\d{2} by the owner \(audit audit-1\)/, + /^Purged on \d{4}-\d{2}-\d{2} by the owner \(action act-1\)/, ); expect( (yield* memory.readItemQuietly(orgId, KinMemoryItemId.make("claim-leak"))).head.status, ).toBe("cancelled"); expect(yield* fs.readFileString(`${paths.read}/INDEX.md`)).toContain("Purged item"); - }).pipe(Effect.provide(layer)); - }).pipe( - Effect.provide( - Layer.mergeAll( - Vcs.layer.pipe(Layer.provide(NodeServices.layer)), - NodeServices.layer, - Persistence.layer.pipe(Layer.provideMerge(Sqlite.layerMemory)), - ), - ), - Effect.scoped, - ), + }).pipe(Effect.provide(purgeLayer())); + }).pipe(Effect.provide(services), Effect.scoped), + ); + + it.live.each([ + ["wrong pattern", { _tag: "KinPurgeFailed", stage: "verify", queueHeld: false }], + ["missing tool", { _tag: "KinPurgeFailed", stage: "rewrite", queueHeld: false }], + ["path typo", { _tag: "KinPurgeRejected" }], + ["interrupted", undefined], + ["no pause", { _tag: "KinPurgePauseUnavailable" }], + ] as const)( + "a purge that stops before the push resumes the queue and changes nothing: %s", + ([scenario, expected]) => + Effect.gen(function* () { + const { fs, git, root, remote, remoteUrl, events, purgeLayer, writerHistory, prepare } = + yield* fixture(); + const layer = purgeLayer({ + ...(scenario === "missing tool" ? { filterRepo: `${root}/missing/git-filter-repo` } : {}), + ...(scenario === "interrupted" + ? { scan: Layer.succeed(Scan.KinSecretScan, { scan: () => Effect.interrupt }) } + : {}), + ...(scenario === "no pause" ? { queue: "default" as const } : {}), + }); + yield* Effect.gen(function* () { + const memory = yield* Memory.Memory; + const paths = yield* prepare; + const oldMain = yield* git(remote, ["rev-parse", "refs/heads/main"]); + const plan: Purge.PurgePlan = + scenario === "wrong pattern" + ? { orgId, repo: remoteUrl, mode: "redact_text", patterns: ["Invented vendor note"] } + : dropPlan(remoteUrl, scenario === "path typo" ? ["claims/leek.md"] : undefined); + const exit = yield* runPurge(plan); + + expect(Exit.isFailure(exit)).toBe(true); + if (!Exit.isFailure(exit)) return; + if (expected === undefined) expect(Cause.hasInterruptsOnly(exit.cause)).toBe(true); + else expect(failureOf(exit)).toMatchObject({ value: expected }); + if (scenario === "wrong pattern") + expect(failureOf(exit)).toMatchObject({ + value: { reason: expect.stringMatching(/Secret scan/) }, + }); + if (scenario === "path typo") + expect(failureOf(exit)).toMatchObject({ + value: { reason: expect.stringMatching(/not in history/) }, + }); + expect(JSON.stringify(exit)).not.toContain(token); + expect(events).toEqual(scenario === "no pause" ? [] : ["pause", "resume"]); + expect(yield* git(remote, ["rev-parse", "refs/heads/main"])).toBe(oldMain); + expect(yield* memory.anchor(orgId, "main")).toBe(oldMain); + expect(yield* memory.purgeInFlight(orgId)).toBeUndefined(); + expect(yield* writerHistory(paths.writer)).toContain(token); + expect( + yield* fs + .readDirectory(`${root}/state/purge-scratch`) + .pipe(Effect.orElseSucceed(() => [])), + ).toEqual([]); + }).pipe(Effect.provide(layer)); + }).pipe(Effect.provide(services), Effect.scoped), + ); + + it.live.each(["error", "crash", "interrupt"] as const)( + "a push that lands and then fails never republishes the purged history: %s", + (scenario) => + Effect.gen(function* () { + const { fs, git, remote, remoteUrl, events, purgeLayer, writerHistory, prepare } = + yield* fixture(); + const plan = dropPlan(remoteUrl); + const landing = purgeLayer({ + push: (real) => ({ + push: (input) => + real + .push(input) + .pipe( + Effect.andThen( + scenario === "error" + ? Effect.fail(new PurgePush.KinPurgePushError({ reason: "timeout" })) + : scenario === "crash" + ? Effect.die("fixture crash after the push") + : Effect.interrupt, + ), + ), + }), + }); + const paths = yield* Effect.gen(function* () { + const paths = yield* prepare; + const exit = yield* runPurge(plan); + expect(Exit.isFailure(exit)).toBe(true); + if (scenario === "interrupt") + expect(Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause)).toBe(true); + else + expect(failureOf(exit)).toMatchObject({ + value: { _tag: "KinPurgeFailed", stage: "push", queueHeld: true }, + }); + expect(events).toEqual(["pause"]); + expect(yield* (yield* Memory.Memory).purgeInFlight(orgId)).toBeDefined(); + expect(yield* fs.exists(paths.writer)).toBe(false); + expect(yield* fs.exists(paths.read)).toBe(false); + return paths; + }).pipe(Effect.provide(landing)); + + // A crash restarts the server: fresh Memory and purge services over the same database and files. + const after = scenario === "crash" ? purgeLayer() : landing; + yield* Effect.gen(function* () { + const memory = yield* Memory.Memory; + yield* memory.prepareWakeup(orgId).pipe(Effect.result); + yield* memory.recover(orgId).pipe(Effect.result); + expect(yield* git(remote, ["log", "--all", "-p"])).not.toContain(token); + expect(yield* git(remote, ["for-each-ref", "--format=%(refname)"])).not.toContain( + "kin/conflict", + ); + + expect(yield* reconcile(plan)).toBe("done"); + expect(events).toEqual(["pause", "resume"]); + expect(yield* memory.purgeInFlight(orgId)).toBeUndefined(); + const newMain = yield* git(remote, ["rev-parse", "refs/heads/main"]); + expect((yield* memory.prepareWakeup(orgId)).head).toBe(newMain); + expect(yield* writerHistory(paths.writer)).not.toContain(token); + expect(yield* git(remote, ["log", "--all", "-p"])).not.toContain(token); + }).pipe(Effect.provide(after)); + }).pipe(Effect.provide(services), Effect.scoped), + ); + + it.live.each([ + ["refused", "remote_rejected", "not_done"], + ["lease rejected", "lease_rejected", "unknown"], + ] as const)( + "a push that does not land holds the queue until reconcile: %s", + ([scenario, reason, verdict]) => + Effect.gen(function* () { + const { + fs, + git, + source, + remote, + remoteUrl, + events, + commit, + purgeLayer, + writerHistory, + prepare, + } = yield* fixture(); + const plan = dropPlan(remoteUrl); + const layer = purgeLayer({ + push: (real) => ({ + push: (input) => + scenario === "refused" + ? Effect.fail(new PurgePush.KinPurgePushError({ reason: "remote_rejected" })) + : Effect.gen(function* () { + // The remote moves after the purge's mirror clone. + yield* fs.writeFileString(`${source}/claims/other.md`, "Invented later note\n"); + yield* git(source, ["add", "--", "."]); + yield* commit("later"); + yield* git(source, ["push", "-q", remote, "main"]); + }).pipe(Effect.orDie, Effect.andThen(real.push(input))), + }), + }); + yield* Effect.gen(function* () { + const memory = yield* Memory.Memory; + const paths = yield* prepare; + const oldMain = yield* git(remote, ["rev-parse", "refs/heads/main"]); + const exit = yield* runPurge(plan); + expect(failureOf(exit)).toMatchObject({ + value: { + _tag: "KinPurgeFailed", + stage: "push", + queueHeld: true, + reason: `Force-push failed: ${reason}`, + }, + }); + expect(events).toEqual(["pause"]); + const remoteMain = yield* git(remote, ["rev-parse", "refs/heads/main"]); + if (scenario === "refused") expect(remoteMain).toBe(oldMain); + else expect(remoteMain).not.toBe(oldMain); + expect(yield* git(remote, ["log", "--all", "-p"])).toContain(token); + + expect(yield* reconcile(plan)).toBe(verdict); + if (verdict === "unknown") { + expect(events).toEqual(["pause"]); + expect(yield* memory.purgeInFlight(orgId)).toBeDefined(); + return; + } + expect(events).toEqual(["pause", "resume"]); + expect(yield* memory.purgeInFlight(orgId)).toBeUndefined(); + expect(yield* memory.anchor(orgId, "main")).toBe(oldMain); + expect(yield* writerHistory(paths.writer)).toContain(token); + }).pipe(Effect.provide(layer)); + }).pipe(Effect.provide(services), Effect.scoped), ); }); @@ -260,13 +469,186 @@ const drop: Purge.PurgePlan = { paths: ["claims/leak.md"], }; -it.effect("the executor gate refuses a purge until APPR-9 and APPR-10 land", () => +const unitLayer = (baseDirectory: string) => + Purge.layer.pipe( + Layer.provide(Layer.mock(Memory.Memory)({ purgeInFlight: () => Effect.undefined })), + Layer.provide( + Layer.succeed(MemoryConfig, { + baseDirectory, + repositories: [{ orgId, repoUrl: "fixture-remote", path: "state" }], + }), + ), + Layer.provide(Layer.mock(Scan.KinSecretScan)({})), + Layer.provide(Layer.mock(PurgePush.KinPurgePush)({})), + Layer.provide(Layer.mock(Vcs.VcsProcess)({})), + Layer.provide(NodeServices.layer), + ); + +it.effect("the executor gate refuses a purge until APPR-9, APPR-10 and LEAD-11 land", () => + Effect.gen(function* () { + const action = actionFor(drop); + for (const attempt of [ + Purge.memoryPurgeHandler.precheck(action), + Purge.memoryPurgeHandler.run(action, grant), + Purge.memoryPurgeHandler.reconcile(action), + Effect.gen(function* () { + return yield* (yield* Purge.KinPurge).purge(action, grant); + }), + ]) { + const result = yield* Effect.result(attempt); + expect(result).toMatchObject({ failure: { _tag: "KinPurgeExecutorNotReady" } }); + } + }).pipe(Effect.provide(unitLayer("/nonexistent"))), +); + +it.effect("a stored plan binds its file to the action digest until it is discarded", () => Effect.gen(function* () { - const result = yield* Purge.memoryPurgeHandler - .run(drop, authorityFor(drop)) - .pipe(Effect.provide(Layer.mock(Purge.KinPurge)({})), Effect.result); - expect(result._tag).toBe("Failure"); - if (result._tag === "Failure") expect(result.failure._tag).toBe("KinPurgeExecutorNotReady"); + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped(); + yield* Effect.gen(function* () { + const purge = yield* Purge.KinPurge; + const action = actionFor(drop); + const target = yield* purge.storePlan(drop); + expect(target).toEqual(action.target); + expect(yield* fs.readFileString(`${root}/${target.path}`)).toBe( + Purge.purgePlanJson(drop).json, + ); + yield* purge.precheck(action).pipe(gateOpen); + + yield* fs.writeFileString( + `${root}/${target.path}`, + Purge.purgePlanJson({ ...drop, paths: ["claims/keep.md"] }).json, + ); + expect(yield* purge.precheck(action).pipe(gateOpen, Effect.result)).toMatchObject({ + failure: { _tag: "KinPurgeRejected", reason: "Plan does not match the approved digest" }, + }); + expect( + yield* purge + .precheck({ ...action, target: { ...action.target, path: "../escape.json" } }) + .pipe(gateOpen, Effect.result), + ).toMatchObject({ failure: { _tag: "KinPurgeRejected", reason: "Invalid plan path" } }); + + yield* purge.discardPlan(target); + expect(yield* fs.exists(`${root}/${target.path}`)).toBe(false); + expect(yield* purge.precheck(action).pipe(gateOpen, Effect.result)).toMatchObject({ + failure: { _tag: "KinPurgeRejected", reason: "Plan file is missing" }, + }); + expect( + yield* purge.storePlan({ ...drop, repo: "other-remote" }).pipe(Effect.result), + ).toMatchObject({ failure: { _tag: "KinPurgeRejected" } }); + }).pipe(Effect.provide(unitLayer(root))); + }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), +); + +const pushInput = { + orgId, + repoUrl: "https://example.invalid/memory.git", + gitDir: "/nonexistent", + updates: [{ ref: "refs/heads/main", old: "a".repeat(40), new: "b".repeat(40) }], +}; +const pushWith = ( + owner: Parameters[0], + output: { readonly code: number; readonly stdout?: string; readonly stderr?: string }, +) => { + const calls: Array = []; + return Effect.gen(function* () { + const result = yield* (yield* PurgePush.KinPurgePush).push(pushInput).pipe(Effect.result); + return { result, calls }; + }).pipe( + Effect.provide( + PurgePush.gitPushLayer(owner).pipe( + Layer.provide( + Layer.mock(Vcs.VcsProcess)({ + run: (input) => + Effect.sync(() => { + calls.push(input); + return { + exitCode: ChildProcessSpawner.ExitCode(output.code), + stdout: output.stdout ?? "", + stderr: output.stderr ?? "", + stdoutTruncated: false, + stderrTruncated: false, + }; + }), + }), + ), + ), + ), + ); +}; + +it.effect.each([ + ["no credential", {}], + ["inline helper script", { credentialHelper: "!f() { echo password=hunter2; }; f" }], + ["shell in ssh command", { sshCommand: "ssh -i key; cat /etc/passwd" }], +] as const)("the owner push fails closed without a credential reference: %s", ([, owner]) => + Effect.gen(function* () { + const { result, calls } = yield* pushWith(owner, { code: 0 }); + expect(result).toMatchObject({ failure: { reason: "credential_unavailable" } }); + expect(calls).toEqual([]); + }), +); + +it.effect("the owner push passes only credential references and never prompts", () => + Effect.gen(function* () { + const { result, calls } = yield* Effect.acquireUseRelease( + Effect.sync(() => vi.stubEnv("GIT_CONFIG_PARAMETERS", "'core.hookspath'='/invented'")), + () => + pushWith( + { credentialHelper: "osxkeychain", sshCommand: "ssh -i /keys/owner" }, + { code: 0 }, + ), + () => Effect.sync(() => vi.unstubAllEnvs()), + ); + expect(result._tag).toBe("Success"); + const [call] = calls; + expect(call?.args).toContain("credential.helper=osxkeychain"); + expect(call?.env).toMatchObject({ + GIT_TERMINAL_PROMPT: "0", + GIT_ASKPASS: "false", + SSH_ASKPASS: "false", + SSH_ASKPASS_REQUIRE: "never", + GIT_SSH_COMMAND: + "ssh -i /keys/owner -o BatchMode=yes -o IdentitiesOnly=yes -o IdentityAgent=none", + }); + expect(call?.env).toHaveProperty("GIT_CONFIG_PARAMETERS", undefined); + expect(call?.env).toHaveProperty("SSH_AUTH_SOCK", undefined); + + const helperOnly = yield* pushWith({ credentialHelper: "osxkeychain" }, { code: 0 }); + expect(helperOnly.calls[0]?.env?.GIT_SSH_COMMAND).toBe("false"); + }), +); + +it.effect.each([ + [ + "stale lease", + { stdout: "To x\n!\trefs/heads/main:refs/heads/main\t[rejected] (stale info)\nDone\n" }, + "lease_rejected", + ], + [ + "ruleset", + { + stdout: + "To x\n!\trefs/heads/main:refs/heads/main\t[remote rejected] (protected branch hook declined)\n", + }, + "remote_rejected", + ], + [ + "auth", + { stderr: `fatal: Authentication failed for 'https://x:${token}@example.invalid/'\n` }, + "auth", + ], + [ + "network", + { stderr: "fatal: unable to access 'https://example.invalid/': Could not resolve host\n" }, + "network", + ], + ["other", { stderr: "fatal: something else\n" }, "unknown"], +] as const)("classifies a failed owner push: %s", ([, output, reason]) => + Effect.gen(function* () { + const { result } = yield* pushWith({ credentialHelper: "osxkeychain" }, { code: 1, ...output }); + expect(result).toMatchObject({ failure: { _tag: "KinPurgePushError", reason } }); + expect(JSON.stringify(result)).not.toContain(token); }), ); @@ -282,39 +664,57 @@ it.each([ ["dot segment", { paths: ["claims/./leak.md"] }], ["backslash", { paths: ["claims\\leak.md"] }], ["newline", { paths: ["claims/leak.md\nrules.md"] }], + ["decomposed", { paths: ["claims/café.md"] }], ["not markdown", { paths: ["claims"] }], ["rules", { paths: ["rules.md"] }], ["no paths", { paths: [] }], ["duplicate", { paths: ["claims/leak.md", "claims/leak.md"] }], - ["drop with patterns", { patterns: ["Invented text"] }], - ["no patterns", { mode: "redact_text" }], - ["short pattern", { mode: "redact_text", patterns: ["short"] }], - ["rule injection", { mode: "redact_text", patterns: ["Invented==>regex:.*"] }], - ["extra rule", { mode: "redact_text", patterns: ["Invented text\nregex:.*"] }], -] as const)("rejects an unsafe plan: %s", (_name, overrides) => { +] as const)("rejects an unsafe drop plan: %s", (_name, overrides) => { const plan = { ...drop, ...overrides }; - expect(Purge.purgeRejection(plan, authorityFor(plan))).toBeDefined(); + expect(Purge.purgeRejection(plan, actionFor(plan), grant)).toBeDefined(); +}); + +it.each([ + ["no patterns", []], + ["short pattern", ["short"]], + ["duplicate", ["Invented text", "Invented text"]], + ["rule injection", ["Invented==>regex:.*"]], + ["extra rule", ["Invented text\nregex:.*"]], +] as const)("rejects an unsafe redact plan: %s", (_name, patterns) => { + const plan: Purge.PurgePlan = { orgId, repo: "fixture-remote", mode: "redact_text", patterns }; + expect(Purge.purgeRejection(plan, actionFor(plan), grant)).toBeDefined(); }); it.each([ [ "digest", - (a: Purge.PurgeAuthority) => ({ - ...a, - action: { ...a.action, target: { ...a.action.target, sha256: "0".repeat(64) } }, - }), + (a: Purge.PurgeAction, g: Purge.PurgeGrant) => + [{ ...a, target: { ...a.target, sha256: "0".repeat(64) } }, g] as const, + ], + [ + "plan path", + (a: Purge.PurgeAction, g: Purge.PurgeGrant) => + [{ ...a, target: { ...a.target, path: "purge-plans/other.json" } }, g] as const, ], - ["action", (a: Purge.PurgeAuthority) => ({ ...a, grant: { ...a.grant, actionId: "act-2" } })], - ["revision", (a: Purge.PurgeAuthority) => ({ ...a, grant: { ...a.grant, revision: 2 } })], + [ + "action", + (a: Purge.PurgeAction, g: Purge.PurgeGrant) => [a, { ...g, actionId: "act-2" }] as const, + ], + ["revision", (a: Purge.PurgeAction, g: Purge.PurgeGrant) => [a, { ...g, revision: 2 }] as const], [ "org", - (a: Purge.PurgeAuthority) => ({ - ...a, - grant: { ...a.grant, orgId: KinOrgId.make("other-org") }, - }), + (a: Purge.PurgeAction, g: Purge.PurgeGrant) => + [a, { ...g, orgId: KinOrgId.make("other-org") }] as const, + ], + [ + "action id", + (a: Purge.PurgeAction, g: Purge.PurgeGrant) => + [ + { ...a, id: "act 1\nforged" }, + { ...g, actionId: "act 1\nforged" }, + ] as const, ], - ["audit id", (a: Purge.PurgeAuthority) => ({ ...a, auditId: "audit 1\nforged" })], ] as const)("rejects authority that does not bind the plan: %s", (_name, tamper) => { - expect(Purge.purgeRejection(drop, authorityFor(drop))).toBeUndefined(); - expect(Purge.purgeRejection(drop, tamper(authorityFor(drop)))).toBeDefined(); + expect(Purge.purgeRejection(drop, actionFor(drop), grant)).toBeUndefined(); + expect(Purge.purgeRejection(drop, ...tamper(actionFor(drop), grant))).toBeDefined(); }); diff --git a/apps/server/src/kin/memory/purge.ts b/apps/server/src/kin/memory/purge.ts index 1189d0286bb0f..de4a01747bf90 100644 --- a/apps/server/src/kin/memory/purge.ts +++ b/apps/server/src/kin/memory/purge.ts @@ -1,6 +1,7 @@ // @effect-diagnostics-next-line nodeBuiltinImport:off -- The plan digest must match the grant's sha256 content target. import * as NodeCrypto from "node:crypto"; -import type { KinOrgId, MemoryItemKind } from "@t3tools/contracts"; +import { KinOrgId, type MemoryItemKind } from "@t3tools/contracts"; +import * as Cause from "effect/Cause"; import * as Clock from "effect/Clock"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; @@ -8,24 +9,44 @@ import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; import * as Schema from "effect/Schema"; import { stringify } from "yaml"; import { VcsProcess } from "../../vcs/VcsProcess.ts"; +import { KinPurgePush, hardenedGitEnv, scrubbedGitEnv, type PurgeRefUpdate } from "./purgePush.ts"; import { parseFrontMatter } from "./frontMatter.ts"; import { memoryGit } from "./git.ts"; -import { Memory } from "./Memory.ts"; +import { Memory, type MemoryHistoryRewrite } from "./Memory.ts"; import { MemoryConfig } from "./MemoryConfig.ts"; import { readObjects } from "./objects.ts"; import { KinSecretScan, type SecretFinding } from "./secretScan.ts"; -export interface PurgePlan { - readonly orgId: KinOrgId; - readonly repo: string; - readonly paths: ReadonlyArray; - readonly mode: "drop_file" | "redact_text"; - readonly patterns?: ReadonlyArray; -} +/** `redact_text` rewrites every blob and commit message in history, so it takes no paths. */ +const PurgePlanSchema = Schema.Union([ + Schema.Struct({ + orgId: KinOrgId, + repo: Schema.String, + mode: Schema.Literal("drop_file"), + paths: Schema.Array(Schema.String), + }), + Schema.Struct({ + orgId: KinOrgId, + repo: Schema.String, + mode: Schema.Literal("redact_text"), + patterns: Schema.Array(Schema.String), + }), +]); +export type PurgePlan = typeof PurgePlanSchema.Type; +const decodePlan = Schema.decodeUnknownEffect(Schema.fromJsonString(PurgePlanSchema)); + +const PurgeMarker = Schema.Struct({ + sha256: Schema.String, + actionId: Schema.String, + oldHead: Schema.String, +}); +const decodeMarker = Schema.decodeUnknownOption(Schema.fromJsonString(PurgeMarker)); /** Lane B's `KinGrant` (APPR-4) fields the purge binds to. The executor verifies the signature. */ export interface PurgeGrant { @@ -36,9 +57,10 @@ export interface PurgeGrant { readonly jti: string; readonly approverHumanId: string; readonly keyThumbprint: string; + readonly deviceLabel?: string; } -/** Lane B's `KinPendingAction` (APPR-2) for kind `memory_purge`; the target is the plan's digest. */ +/** Lane B's `KinPendingAction` (APPR-2) for kind `memory_purge`; the target is `purgePlanJson(plan).target`. */ export interface PurgeAction { readonly id: string; readonly orgId: KinOrgId; @@ -47,14 +69,6 @@ export interface PurgeAction { readonly target: { readonly _tag: "content"; readonly path: string; readonly sha256: string }; } -export interface PurgeAuthority { - readonly action: PurgeAction; - readonly grant: PurgeGrant; - /** Id of the executor's `execution_started` audit entry, written before `run`. */ - readonly auditId: string; - readonly deviceLabel?: string; -} - /** Lane B's `KinAuditAppendInput` (APPR-10) for the `execution_finished` entry. */ export interface PurgeAuditInput { readonly orgId: KinOrgId; @@ -68,12 +82,6 @@ export interface PurgeAuditInput { readonly result: string; } -export interface PurgeRefUpdate { - readonly ref: string; - readonly old: string; - readonly new: string | null; -} - export interface PurgeResult { readonly mode: PurgePlan["mode"]; readonly paths: ReadonlyArray; @@ -91,25 +99,36 @@ export class KinPurgeExecutorNotReady extends Schema.TaggedError()( - "KinPurgePushError", - { reason: Schema.String }, +export class KinPurgePauseUnavailable extends Schema.TaggedError()( + "KinPurgePauseUnavailable", + { orgId: Schema.String }, ) {} export class KinPurgeFailed extends Schema.TaggedError()("KinPurgeFailed", { orgId: Schema.String, - stage: Schema.Literals(["clone", "rewrite", "verify", "tombstone", "push", "rebuild"]), + stage: Schema.Literals([ + "clone", + "rewrite", + "verify", + "tombstone", + "quarantine", + "push", + "rebuild", + ]), reason: Schema.String, - /** Once the push starts, local checkouts may hold the old history, so the queue stays paused. */ + /** Once the checkouts are quarantined for the push, the queue stays paused until `reconcile`. */ queueHeld: Schema.Boolean, cause: Schema.optional(Schema.Defect()), }) {} -/** LEAD-11 supplies the real org pause; the default is a no-op that the purge still calls. */ +/** LEAD-11 supplies the real org pause; until then the purge can't run. */ export class KinPurgeQueue extends Context.Reference<{ - readonly pause: (orgId: KinOrgId) => Effect.Effect; + readonly pause: (orgId: KinOrgId) => Effect.Effect; readonly resume: (orgId: KinOrgId) => Effect.Effect; }>("t3/kin/memory/purge/KinPurgeQueue", { - defaultValue: () => ({ pause: () => Effect.void, resume: () => Effect.void }), + defaultValue: () => ({ + pause: (orgId) => Effect.fail(new KinPurgePauseUnavailable({ orgId })), + resume: () => Effect.void, + }), }) {} export const GitFilterRepoCommand = Context.Reference( @@ -117,56 +136,7 @@ export const GitFilterRepoCommand = Context.Reference( { defaultValue: () => "git-filter-repo" }, ); -export class KinPurgePush extends Context.Service< - KinPurgePush, - { - readonly push: (input: { - readonly orgId: KinOrgId; - readonly repoUrl: string; - readonly gitDir: string; - readonly updates: ReadonlyArray; - }) => Effect.Effect; - } ->()("t3/kin/memory/purge/KinPurgePush") {} - -/** Pushes with the owner's git credential helper or SSH command; T3 never reads the secret. */ -export const gitPushLayer = (owner: { - readonly credentialHelper?: string; - readonly sshCommand?: string; -}) => - Layer.effect( - KinPurgePush, - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const vcs = yield* VcsProcess; - const config = yield* MemoryConfig; - const ownerConfig = MemoryConfig.of({ - baseDirectory: config.baseDirectory, - repositories: config.repositories, - ...(owner.credentialHelper ? { credentialHelper: owner.credentialHelper } : {}), - ...(owner.sshCommand ? { sshCommand: owner.sshCommand } : {}), - }); - return KinPurgePush.of({ - push: Effect.fn("KinPurgePush.git")(function* ({ repoUrl, gitDir, updates }) { - yield* memoryGit(gitDir, [ - "push", - "--atomic", - "--porcelain", - ...updates.map((update) => `--force-with-lease=${update.ref}:${update.old}`), - "--", - repoUrl, - ...updates.map((update) => `${update.new ?? ""}:${update.ref}`), - ]).pipe( - Effect.provideService(FileSystem.FileSystem, fs), - Effect.provideService(VcsProcess, vcs), - Effect.provideService(MemoryConfig, ownerConfig), - // The cause can echo the remote URL or helper output; keep both out of errors and logs. - Effect.mapError(() => new KinPurgePushError({ reason: "Force-push failed" })), - ); - }), - }); - }), - ); +const literalPathspecs = { GIT_LITERAL_PATHSPECS: "1" }; const itemKinds = new Map([ ["objectives", "objective"], @@ -202,45 +172,53 @@ const isPattern = (pattern: string) => !/[\u0000-\u001f\u007f]/.test(pattern) && !pattern.includes("==>"); -export const purgePlanDigest = (plan: PurgePlan) => - NodeCrypto.createHash("sha256") - .update( - JSON.stringify({ - orgId: plan.orgId, - repo: plan.repo, - mode: plan.mode, - paths: plan.paths, - patterns: plan.patterns ?? [], - }), - ) - .digest("hex"); +const sha256 = (text: string) => NodeCrypto.createHash("sha256").update(text).digest("hex"); +const planFilePath = /^purge-plans\/[a-f0-9]{64}\.json$/; + +/** The stored plan file and the action target that binds it, from one serialization. */ +export const purgePlanJson = (plan: PurgePlan) => { + const json = JSON.stringify( + plan.mode === "drop_file" + ? { orgId: plan.orgId, repo: plan.repo, mode: plan.mode, paths: plan.paths } + : { orgId: plan.orgId, repo: plan.repo, mode: plan.mode, patterns: plan.patterns }, + ); + const digest = sha256(json); + return { + json, + target: { _tag: "content" as const, path: `purge-plans/${digest}.json`, sha256: digest }, + }; +}; + +const planRejection = (plan: PurgePlan) => { + if (plan.mode === "drop_file") { + if (plan.paths.length === 0 || plan.paths.length > 64) return "Expected 1-64 paths"; + if (new Set(plan.paths).size !== plan.paths.length) return "Duplicate paths"; + if (!plan.paths.every(isPurgePath)) return "Invalid path"; + return undefined; + } + if (plan.patterns.length === 0 || plan.patterns.length > 64) return "Expected 1-64 patterns"; + if (new Set(plan.patterns).size !== plan.patterns.length) return "Duplicate patterns"; + if (!plan.patterns.every(isPattern)) return "Invalid pattern"; + return undefined; +}; -/** Returns why the plan or its authority is unacceptable, or `undefined`. */ -export const purgeRejection = (plan: PurgePlan, authority: PurgeAuthority) => { - const { action, grant } = authority; +/** Returns why the plan, its action or its grant is unacceptable, or `undefined`. */ +export const purgeRejection = (plan: PurgePlan, action: PurgeAction, grant?: PurgeGrant) => { if (action.kind !== "memory_purge" || action.target._tag !== "content") return "Action is not a memory purge"; - if (action.orgId !== plan.orgId || grant.orgId !== plan.orgId) + if (action.orgId !== plan.orgId || (grant !== undefined && grant.orgId !== plan.orgId)) return "Grant, action and plan organizations differ"; - if (grant.actionId !== action.id || grant.revision !== action.revision) + if (grant !== undefined && (grant.actionId !== action.id || grant.revision !== action.revision)) return "Grant does not bind this action revision"; - if (action.target.sha256 !== purgePlanDigest(plan)) + const { target } = purgePlanJson(plan); + if (action.target.sha256 !== target.sha256 || action.target.path !== target.path) return "Plan does not match the approved digest"; - if (!/^[A-Za-z0-9_-]{1,128}$/.test(authority.auditId)) return "Invalid audit id"; - if (plan.paths.length === 0 || plan.paths.length > 64) return "Expected 1-64 paths"; - if (new Set(plan.paths).size !== plan.paths.length) return "Duplicate paths"; - if (!plan.paths.every(isPurgePath)) return "Invalid path"; - const patterns = plan.patterns ?? []; - if (plan.mode === "drop_file" && patterns.length > 0) return "drop_file takes no patterns"; - if (plan.mode === "redact_text") { - if (patterns.length === 0 || patterns.length > 64) return "Expected 1-64 patterns"; - if (new Set(patterns).size !== patterns.length) return "Duplicate patterns"; - if (!patterns.every(isPattern)) return "Invalid pattern"; - } - return undefined; + if (!/^[A-Za-z0-9_-]{1,128}$/.test(action.id)) return "Invalid action id"; + return planRejection(plan); }; const isPurgeFailed = Schema.is(KinPurgeFailed); +const markerJson = (marker: typeof PurgeMarker.Type) => JSON.stringify(marker); const auditResult = (result: { readonly mode: PurgePlan["mode"]; readonly paths: ReadonlyArray; @@ -255,14 +233,40 @@ const scanBatchBytes = 4 * 1024 * 1024; export class KinPurge extends Context.Service< KinPurge, { + /** Writes the plan file a `memory_purge` action targets; the proposer calls it before `request_action`. */ + readonly storePlan: ( + plan: PurgePlan, + ) => Effect.Effect; + /** Deletes the plan file once the action is terminal; it can hold the redacted secret. */ + readonly discardPlan: (target: PurgeAction["target"]) => Effect.Effect; + readonly precheck: ( + action: PurgeAction, + ) => Effect.Effect; /** Server-only: reached through `memoryPurgeHandler`, never a model, MCP tool or RPC. */ readonly purge: ( - plan: PurgePlan, - authority: PurgeAuthority, - ) => Effect.Effect; + action: PurgeAction, + grant: PurgeGrant, + ) => Effect.Effect< + PurgeResult, + KinPurgeExecutorNotReady | KinPurgeRejected | KinPurgePauseUnavailable | KinPurgeFailed + >; + readonly reconcile: ( + action: PurgeAction, + ) => Effect.Effect< + "done" | "not_done" | "unknown", + KinPurgeExecutorNotReady | KinPurgeRejected | KinPurgeFailed + >; } >()("t3/kin/memory/purge/KinPurge") {} +// Gated until APPR-9 (executor), APPR-10 (audit) and LEAD-11 (queue pause) land. +const purgeExecutorReady = false; +/** Test seam for the release gate; production always reads `purgeExecutorReady`. */ +export const PurgeExecutorGate = Context.Reference( + "t3/kin/memory/purge/PurgeExecutorGate", + { defaultValue: () => purgeExecutorReady }, +); + const make = Effect.gen(function* () { const memory = yield* Memory; const config = yield* MemoryConfig; @@ -271,6 +275,7 @@ const make = Effect.gen(function* () { const queue = yield* KinPurgeQueue; const filterRepo = yield* GitFilterRepoCommand; const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; const vcs = yield* VcsProcess; const provide = (effect: Effect.Effect) => effect.pipe( @@ -283,7 +288,7 @@ const make = Effect.gen(function* () { stdin?: string, env: NodeJS.ProcessEnv = {}, ) => - provide(memoryGit(work, args, stdin, env)).pipe( + provide(memoryGit(work, args, stdin, { ...scrubbedGitEnv(), ...hardenedGitEnv, ...env })).pipe( Effect.provideService(MemoryConfig, config), Effect.map((stdout) => stdout.trim()), ); @@ -299,6 +304,55 @@ const make = Effect.gen(function* () { ); const fail = (orgId: KinOrgId, stage: KinPurgeFailed["stage"], reason: string) => Effect.fail(new KinPurgeFailed({ orgId, stage, reason, queueHeld: false })); + const reject = (reason: string) => Effect.fail(new KinPurgeRejected({ reason })); + + const gate = Effect.fn("KinPurge.gate")(function* (action: PurgeAction) { + if (!(yield* PurgeExecutorGate)) + return yield* new KinPurgeExecutorNotReady({ actionId: action.id }); + }); + + const repoFor = (plan: PurgePlan) => { + const repo = config.repositories.find((entry) => entry.orgId === plan.orgId); + return repo && repo.repoUrl === plan.repo ? repo : undefined; + }; + + const planFile = (target: PurgeAction["target"]) => + planFilePath.test(target.path) && target.path === `purge-plans/${target.sha256}.json` + ? Effect.succeed(path.resolve(config.baseDirectory, target.path)) + : reject("Invalid plan path"); + + const loadPlan = Effect.fn("KinPurge.loadPlan")(function* (action: PurgeAction) { + if (action.kind !== "memory_purge" || action.target._tag !== "content") + return yield* reject("Action is not a memory purge"); + const file = yield* planFile(action.target); + const json = yield* fs + .readFileString(file) + .pipe(Effect.mapError(() => new KinPurgeRejected({ reason: "Plan file is missing" }))); + const plan = yield* decodePlan(json).pipe( + Effect.mapError(() => new KinPurgeRejected({ reason: "Plan file is malformed" })), + ); + if (purgePlanJson(plan).json !== json) + return yield* reject("Plan does not match the approved digest"); + return plan; + }); + + /** Validates the action against its stored plan and the org's config, and returns any purge marker. */ + const checked = Effect.fn("KinPurge.checked")(function* ( + action: PurgeAction, + grant?: PurgeGrant, + ) { + yield* gate(action); + const plan = yield* loadPlan(action); + const rejection = purgeRejection(plan, action, grant); + if (rejection !== undefined) return yield* reject(rejection); + const repo = repoFor(plan); + if (!repo) return yield* reject("Plan repo is not this organization's memory"); + const inFlight = yield* memory + .purgeInFlight(plan.orgId) + .pipe(failed(plan.orgId, "clone", "Could not read the purge marker")); + const marker = inFlight === undefined ? Option.none() : decodeMarker(inFlight); + return { plan, repo, inFlight, marker: Option.getOrUndefined(marker) }; + }); const branchRefs = Effect.fn("KinPurge.branchRefs")(function* (work: string) { const lines = yield* git(work, [ @@ -315,20 +369,28 @@ const make = Effect.gen(function* () { ); }); + /** Every blob ever introduced, merge sides included, plus every commit object unless `paths` narrows it. */ const historyFiles = Effect.fn("KinPurge.historyFiles")(function* ( orgId: KinOrgId, work: string, + paths?: ReadonlyArray, ) { - const raw = yield* git(work, [ - "log", - "--all", - "--raw", - "--no-abbrev", - "--no-renames", - "--diff-merges=separate", - "--format=", - "-z", - ]); + const raw = yield* git( + work, + [ + "log", + "--all", + "--raw", + "--no-abbrev", + "--no-renames", + "--diff-merges=separate", + "--format=", + "-z", + ...(paths ? ["--", ...paths] : []), + ], + undefined, + literalPathspecs, + ); const blobs = new Map(); const tokens = raw.split("\0"); for (let index = 0; index < tokens.length; index++) { @@ -339,7 +401,9 @@ const make = Effect.gen(function* () { if (!sha || /^0+$/.test(sha) || mode === "160000") continue; blobs.set(`${sha} ${path}`, { sha, path }); } - const commits = (yield* git(work, ["rev-list", "--all"])).split("\n").filter(Boolean); + const commits = paths + ? [] + : (yield* git(work, ["rev-list", "--all"])).split("\n").filter(Boolean); const requests = [ ...[...blobs.values()].map(({ sha, path }) => ({ sha, type: "blob" as const, path })), ...commits.map((sha) => ({ sha, type: "commit" as const, path: `commit ${sha}` })), @@ -356,6 +420,46 @@ const make = Effect.gen(function* () { return files; }); + const containsPattern = ( + files: ReadonlyArray<{ readonly content: Uint8Array }>, + patterns: ReadonlyArray, + ) => { + const needles = patterns.map((pattern) => Buffer.from(pattern, "utf8")); + return files.some((file) => + needles.some((needle) => Buffer.from(file.content).includes(needle)), + ); + }; + + const pathInHistory = (work: string, file: string) => + git(work, ["rev-list", "--all", "-n", "1", "--", file], undefined, literalPathspecs).pipe( + Effect.map((commits) => commits.length > 0), + ); + + /** Whether any reachable commit still holds what the plan removes. */ + const planInHistory = Effect.fn("KinPurge.planInHistory")(function* ( + plan: PurgePlan, + work: string, + ) { + if (plan.mode === "drop_file") { + // The purge's own tombstones live at the purged paths; anything else there is old content. + for (const file of yield* historyFiles(plan.orgId, work, plan.paths)) { + const parsed = yield* parseFrontMatter( + plan.orgId, + file.path, + Buffer.from(file.content).toString("utf8"), + ).pipe(Effect.option); + if ( + Option.isNone(parsed) || + parsed.value.frontMatter.title !== "Purged item" || + !parsed.value.frontMatter.source.startsWith("kin-action:") + ) + return true; + } + return false; + } + return containsPattern(yield* historyFiles(plan.orgId, work), plan.patterns); + }); + const scanAll = Effect.fn("KinPurge.scanAll")(function* ( orgId: KinOrgId, files: ReadonlyArray<{ path: string; content: Uint8Array }>, @@ -383,7 +487,8 @@ const make = Effect.gen(function* () { work: string, oldHead: string, path: string, - authority: PurgeAuthority, + action: PurgeAction, + grant: PurgeGrant, now: number, ) { const folder = path.split("/")[0] ?? ""; @@ -417,30 +522,28 @@ const make = Effect.gen(function* () { invalid_at: iso, superseded_by: null, review_by: null, - source: `kin-audit:${authority.auditId}`, - said_by: authority.grant.approverHumanId, + source: `kin-action:${action.id}`, + said_by: grant.approverHumanId, trust: "owner", links: [], }, - `Purged on ${iso.slice(0, 10)} by the owner (audit ${authority.auditId}).\n`, + `Purged on ${iso.slice(0, 10)} by the owner (action ${action.id}).\n`, ); yield* parseFrontMatter(orgId, path, content); return content; }); const commitTombstones = Effect.fn("KinPurge.commitTombstones")(function* ( - orgId: KinOrgId, work: string, branch: string, head: string, tombstones: ReadonlyMap, - auditId: string, + actionId: string, now: number, ) { - const scratch = yield* fs.makeTempDirectoryScoped({ prefix: "kin-purge-index-" }); const date = `${Math.floor(now / 1000)} +0000`; const env = { - GIT_INDEX_FILE: `${scratch}/index`, + GIT_INDEX_FILE: `${work}/tombstone-index`, GIT_AUTHOR_NAME: config.botName ?? "Kin Memory Bot", GIT_AUTHOR_EMAIL: config.botEmail ?? "kin-memory@localhost", GIT_COMMITTER_NAME: config.botName ?? "Kin Memory Bot", @@ -458,7 +561,7 @@ const make = Effect.gen(function* () { const tree = yield* git(work, ["write-tree"], undefined, env); const commit = yield* git( work, - ["commit-tree", tree, "-p", head, "-m", `Purge tombstones (audit ${auditId})`], + ["commit-tree", tree, "-p", head, "-m", `Purge tombstones (action ${actionId})`], undefined, env, ); @@ -466,50 +569,96 @@ const make = Effect.gen(function* () { return commit; }); - const rewrite = Effect.fn("KinPurge.rewrite")(function* ( - plan: PurgePlan, - authority: PurgeAuthority, - repoUrl: string, - onPush: () => void, + /** A fresh mirror under the org's purge directory; stale ones from a crashed purge are swept first. */ + const mirror = Effect.fn("KinPurge.mirror")(function* ( + orgId: KinOrgId, + repo: { readonly repoUrl: string; readonly path: string }, ) { - const { orgId } = plan; - const branch = config.defaultBranch ?? "main"; - const now = yield* Clock.currentTimeMillis; - const work = yield* fs - .makeTempDirectoryScoped({ prefix: "kin-purge-" }) - .pipe(failed(orgId, "clone", "Could not create a scratch directory")); + const root = path.resolve(config.baseDirectory, repo.path, "purge-scratch"); + const work = yield* Effect.gen(function* () { + yield* fs.remove(root, { recursive: true, force: true }); + yield* fs.makeDirectory(root, { recursive: true }); + return yield* fs.makeTempDirectoryScoped({ directory: root, prefix: "mirror-" }); + }).pipe(failed(orgId, "clone", "Could not create a scratch directory")); yield* git(work, [ "clone", "--mirror", "--no-local", "--template=", "--", - repoUrl, + repo.repoUrl, `${work}/.git`, ]).pipe(failed(orgId, "clone", "Mirror clone failed")); const before = yield* branchRefs(work).pipe(failed(orgId, "clone", "Could not list refs")); - const oldHead = before.get(`refs/heads/${branch}`); - if (!oldHead) return yield* fail(orgId, "clone", "Remote default branch is missing"); + const head = before.get(`refs/heads/${config.defaultBranch ?? "main"}`); + if (!head) return yield* fail(orgId, "clone", "Remote default branch is missing"); + return { work, before, head }; + }); + + const rewrite = Effect.fn("KinPurge.rewrite")(function* (input: { + readonly plan: PurgePlan; + readonly action: PurgeAction; + readonly grant: PurgeGrant; + readonly repo: { readonly repoUrl: string; readonly path: string }; + readonly inFlight: string | undefined; + readonly marker: typeof PurgeMarker.Type | undefined; + readonly history: MemoryHistoryRewrite; + readonly commit: () => void; + }) { + const { plan, action, grant, repo, history } = input; + const { orgId } = plan; + const branch = config.defaultBranch ?? "main"; + const now = yield* Clock.currentTimeMillis; + if ( + (yield* memory + .purgeInFlight(orgId) + .pipe(failed(orgId, "clone", "Could not read the purge marker"))) !== input.inFlight + ) { + input.commit(); + return yield* fail(orgId, "clone", "The purge marker changed while waiting for the lock"); + } + const { work, before, head: oldHead } = yield* mirror(orgId, repo); + + // A re-run after a push that landed: nothing to rewrite, but the checkouts still need a reclone. + if (input.inFlight !== undefined) { + const remaining = yield* planInHistory(plan, work).pipe( + failed(orgId, "verify", "Could not check history"), + ); + if (!remaining) { + yield* history + .reclone() + .pipe(failed(orgId, "rebuild", "Could not rebuild local memory checkouts")); + return { + oldHead: input.marker?.oldHead ?? oldHead, + newHead: oldHead, + updates: [], + tombstones: [], + }; + } + } const tombstones = new Map(); if (plan.mode === "drop_file") - for (const path of plan.paths) { - const content = yield* tombstone(orgId, work, oldHead, path, authority, now).pipe( + for (const file of plan.paths) { + const present = yield* pathInHistory(work, file).pipe( + failed(orgId, "verify", "Could not check history"), + ); + if (!present) + return yield* reject( + "A purge path is not in history; check its spelling and Unicode normalization", + ); + const content = yield* tombstone(orgId, work, oldHead, file, action, grant, now).pipe( failed(orgId, "tombstone", "Could not build a tombstone"), ); - if (content !== undefined) tombstones.set(path, content); + if (content !== undefined) tombstones.set(file, content); } - const scratch = yield* fs - .makeTempDirectoryScoped({ prefix: "kin-purge-rules-" }) - .pipe(failed(orgId, "rewrite", "Could not create a scratch directory")); - const rules = `${scratch}/rules.txt`; + const rules = `${work}/rules.txt`; if (plan.mode === "redact_text") yield* fs .writeFileString( rules, - (plan.patterns ?? []).map((pattern) => `literal:${pattern}==>***REMOVED***`).join("\n") + - "\n", + plan.patterns.map((pattern) => `literal:${pattern}==>***REMOVED***`).join("\n") + "\n", ) .pipe(failed(orgId, "rewrite", "Could not write replacement rules")); yield* vcs @@ -520,16 +669,14 @@ const make = Effect.gen(function* () { args: [ "--quiet", ...(plan.mode === "drop_file" - ? ["--invert-paths", ...plan.paths.flatMap((path) => ["--path", path])] + ? ["--invert-paths", ...plan.paths.flatMap((file) => ["--path", file])] : ["--replace-text", rules, "--replace-message", rules]), ], env: { + ...scrubbedGitEnv(), + ...hardenedGitEnv, // filter-repo accepts only a fresh clone opened as GIT_DIR=. from its own directory. GIT_DIR: ".", - GIT_CONFIG_NOSYSTEM: "1", - GIT_CONFIG_GLOBAL: "/dev/null", - GIT_TERMINAL_PROMPT: "0", - GIT_NO_REPLACE_OBJECTS: "1", GIT_CONFIG_COUNT: "3", GIT_CONFIG_KEY_0: "core.hooksPath", GIT_CONFIG_VALUE_0: "/dev/null", @@ -549,31 +696,22 @@ const make = Effect.gen(function* () { return yield* fail(orgId, "rewrite", "Nothing in history matched the plan"); if (plan.mode === "drop_file") { - const remaining = yield* git(work, ["rev-list", "--all", "--", ...plan.paths]).pipe( + const remaining = yield* planInHistory(plan, work).pipe( failed(orgId, "verify", "Could not check rewritten history"), ); if (remaining) return yield* fail(orgId, "verify", "Purged paths remain in history"); } const newHead = tombstones.size > 0 - ? yield* commitTombstones( - orgId, - work, - branch, - rewrittenHead, - tombstones, - authority.auditId, - now, - ).pipe(failed(orgId, "tombstone", "Could not commit tombstones")) + ? yield* commitTombstones(work, branch, rewrittenHead, tombstones, action.id, now).pipe( + failed(orgId, "tombstone", "Could not commit tombstones"), + ) : rewrittenHead; const files = yield* historyFiles(orgId, work).pipe( failed(orgId, "verify", "Could not read rewritten history"), ); - const leftovers = (plan.patterns ?? []).map((pattern) => Buffer.from(pattern, "utf8")); - if ( - files.some((file) => leftovers.some((pattern) => Buffer.from(file.content).includes(pattern))) - ) + if (plan.mode === "redact_text" && containsPattern(files, plan.patterns)) return yield* fail(orgId, "verify", "Redacted text remains in history"); const findings = yield* scanAll(orgId, files).pipe( failed(orgId, "verify", "Secret scan of rewritten history failed"), @@ -591,102 +729,188 @@ const make = Effect.gen(function* () { const updates: PurgeRefUpdate[] = [...before] .filter(([ref, sha]) => after.get(ref) !== sha) .map(([ref, old]) => ({ ref, old, new: after.get(ref) ?? null })); - onPush(); - yield* pusher - .push({ orgId, repoUrl, gitDir: work, updates }) - .pipe(failed(orgId, "push", "Force-push failed")); - yield* memory - .reclone(orgId) + + // From here a push may land at any moment, so local checkouts holding the old history must go + // first; the marker keeps preserve and merges from pushing it back until a reclone or reconcile. + input.commit(); + yield* history + .quarantine( + input.inFlight ?? + markerJson({ sha256: action.target.sha256, actionId: action.id, oldHead }), + ) + .pipe(failed(orgId, "quarantine", "Could not quarantine local memory checkouts")); + yield* pusher.push({ orgId, repoUrl: repo.repoUrl, gitDir: work, updates }).pipe( + Effect.mapError( + (error) => + new KinPurgeFailed({ + orgId, + stage: "push", + reason: `Force-push failed: ${error.reason}`, + queueHeld: true, + }), + ), + ); + yield* history + .reclone() .pipe(failed(orgId, "rebuild", "Could not rebuild local memory checkouts")); - return { oldHead, newHead, updates, tombstones: [...tombstones.keys()] }; + return { + oldHead: input.marker?.oldHead ?? oldHead, + newHead, + updates, + tombstones: [...tombstones.keys()], + }; }); return KinPurge.of({ - purge: Effect.fn("KinPurge.purge")(function* (plan, authority) { - const rejection = purgeRejection(plan, authority); - if (rejection !== undefined) return yield* new KinPurgeRejected({ reason: rejection }); - const repo = config.repositories.find((entry) => entry.orgId === plan.orgId); - if (!repo || repo.repoUrl !== plan.repo) - return yield* new KinPurgeRejected({ - reason: "Plan repo is not this organization's memory", - }); - let pushStarted = false; - yield* queue.pause(plan.orgId); - const exit = yield* memory - .withWriterLock( - plan.orgId, + storePlan: Effect.fn("KinPurge.storePlan")(function* (plan) { + const rejection = planRejection(plan) ?? (repoFor(plan) ? undefined : "Unknown memory repo"); + if (rejection !== undefined) return yield* reject(rejection); + const { json, target } = purgePlanJson(plan); + const file = yield* planFile(target); + yield* Effect.gen(function* () { + yield* fs.makeDirectory(path.dirname(file), { recursive: true }); + yield* fs.writeFileString(`${file}.tmp`, json); + yield* fs.rename(`${file}.tmp`, file); + }).pipe(failed(plan.orgId, "clone", "Could not store the plan")); + return target; + }), + + discardPlan: Effect.fn("KinPurge.discardPlan")(function* (target) { + const file = yield* planFile(target); + yield* fs + .remove(file, { force: true }) + .pipe(Effect.mapError(() => new KinPurgeRejected({ reason: "Could not delete the plan" }))); + }), + + precheck: Effect.fn("KinPurge.precheck")(function* (action) { + const { inFlight, marker } = yield* checked(action); + if (inFlight !== undefined && marker?.sha256 !== action.target.sha256) + return yield* reject("Another purge is in flight; reconcile it first"); + }), + + purge: Effect.fn("KinPurge.purge")(function* (action, grant) { + const { plan, repo, inFlight, marker } = yield* checked(action, grant); + if (inFlight !== undefined && marker?.sha256 !== action.target.sha256) + return yield* reject("Another purge is in flight; reconcile it first"); + const { orgId } = plan; + // An unresolved marker means the queue is already held for this plan; never resume it early. + let committed = inFlight !== undefined; + yield* queue.pause(orgId); + const { oldHead, newHead, updates, tombstones } = yield* memory + .withHistoryRewrite(orgId, (history) => Effect.scoped( - rewrite(plan, authority, repo.repoUrl, () => { - pushStarted = true; + rewrite({ + plan, + action, + grant, + repo, + inFlight, + marker, + history, + commit: () => { + committed = true; + }, }), ), ) - .pipe(Effect.exit); - if (Exit.isFailure(exit)) { - if (!pushStarted) { - yield* queue.resume(plan.orgId); - return yield* Effect.failCause(exit.cause); - } - return yield* Effect.failCause(exit.cause).pipe( - Effect.mapError( - (error) => + .pipe( + Effect.onExit((exit) => + Exit.isSuccess(exit) || !committed ? queue.resume(orgId) : Effect.void, + ), + Effect.catchCause((cause) => { + if (!committed || Cause.hasInterruptsOnly(cause)) return Effect.failCause(cause); + const error = Option.getOrUndefined(Cause.findErrorOption(cause)); + return Effect.fail( new KinPurgeFailed({ - orgId: error.orgId, - stage: error.stage, - reason: error.reason, + orgId, + stage: isPurgeFailed(error) ? error.stage : "push", + reason: isPurgeFailed(error) + ? error.reason + : "Purge stopped after quarantining local checkouts", queueHeld: true, - ...(error.cause === undefined ? {} : { cause: error.cause }), + ...(isPurgeFailed(error) && error.cause !== undefined + ? { cause: error.cause } + : {}), }), - ), + ); + }), ); - } - yield* queue.resume(plan.orgId); - const { oldHead, newHead, updates, tombstones } = exit.value; + const paths = plan.mode === "drop_file" ? plan.paths : []; return { mode: plan.mode, - paths: plan.paths, + paths, oldHead, newHead, rewrittenRefs: updates, tombstones, audit: { - orgId: plan.orgId, + orgId, event: "execution_finished", - actionId: authority.action.id, + actionId: action.id, kind: "memory_purge", - grantJti: authority.grant.jti, - keyThumbprint: authority.grant.keyThumbprint, - ...(authority.deviceLabel === undefined ? {} : { deviceLabel: authority.deviceLabel }), + grantJti: grant.jti, + keyThumbprint: grant.keyThumbprint, + ...(grant.deviceLabel === undefined ? {} : { deviceLabel: grant.deviceLabel }), headSha: newHead, - result: auditResult({ - mode: plan.mode, - paths: plan.paths, - oldHead, - newHead, - refs: updates, - }), + result: auditResult({ mode: plan.mode, paths, oldHead, newHead, refs: updates }), }, } satisfies PurgeResult; }), + + reconcile: Effect.fn("KinPurge.reconcile")(function* (action) { + const { plan, repo, inFlight, marker } = yield* checked(action); + if (inFlight !== undefined && marker?.sha256 !== action.target.sha256) return "unknown"; + const { orgId } = plan; + return yield* memory.withHistoryRewrite(orgId, (history) => + Effect.scoped( + Effect.gen(function* () { + const observed = yield* mirror(orgId, repo).pipe( + Effect.flatMap(({ work, head }) => + planInHistory(plan, work).pipe(Effect.map((remaining) => ({ head, remaining }))), + ), + Effect.option, + ); + if (Option.isNone(observed)) return "unknown" as const; + const { head, remaining } = observed.value; + const verdict = !remaining + ? ("done" as const) + : marker === undefined || head === marker.oldHead + ? ("not_done" as const) + : ("unknown" as const); + if (verdict === "unknown") return verdict; + if (inFlight !== undefined) + yield* history.reclone().pipe( + Effect.mapError( + () => + new KinPurgeFailed({ + orgId, + stage: "rebuild", + reason: "Could not rebuild local memory checkouts", + queueHeld: true, + }), + ), + ); + yield* queue.resume(orgId); + return verdict; + }), + ), + ); + }), }); }); export const layer = Layer.effect(KinPurge, make); -// Gated until APPR-9 (executor) and APPR-10 (audit) land. -const purgeExecutorReady = false; -/** Test seam for the release gate; production always reads `purgeExecutorReady`. */ -export const PurgeExecutorGate = Context.Reference( - "t3/kin/memory/purge/PurgeExecutorGate", - { defaultValue: () => purgeExecutorReady }, -); - -/** The only way to purge: APPR-9's executor calls `run` with a verified grant. */ +/** APPR-9's `KinActionHandler` for `memory_purge`; the executor calls it with a verified grant. */ export const memoryPurgeHandler = { kind: "memory_purge", - run: Effect.fn("KinPurge.handler")(function* (plan: PurgePlan, authority: PurgeAuthority) { - if (!(yield* PurgeExecutorGate)) - return yield* new KinPurgeExecutorNotReady({ actionId: authority.action.id }); - return yield* (yield* KinPurge).purge(plan, authority); + precheck: Effect.fn("KinPurge.handlerPrecheck")(function* (action: PurgeAction) { + return yield* (yield* KinPurge).precheck(action); + }), + run: Effect.fn("KinPurge.handlerRun")(function* (action: PurgeAction, grant: PurgeGrant) { + return yield* (yield* KinPurge).purge(action, grant); + }), + reconcile: Effect.fn("KinPurge.handlerReconcile")(function* (action: PurgeAction) { + return yield* (yield* KinPurge).reconcile(action); }), } as const; diff --git a/apps/server/src/kin/memory/purgePush.ts b/apps/server/src/kin/memory/purgePush.ts new file mode 100644 index 0000000000000..4bfee31d49914 --- /dev/null +++ b/apps/server/src/kin/memory/purgePush.ts @@ -0,0 +1,162 @@ +import type { KinOrgId } from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; + +import { VcsProcess } from "../../vcs/VcsProcess.ts"; + +export interface PurgeRefUpdate { + readonly ref: string; + readonly old: string; + readonly new: string | null; +} + +export class KinPurgePushError extends Schema.TaggedError()( + "KinPurgePushError", + { + reason: Schema.Literals([ + "credential_unavailable", + "lease_rejected", + "remote_rejected", + "rejected", + "auth", + "network", + "timeout", + "unknown", + ]), + }, +) {} +export class KinPurgePush extends Context.Service< + KinPurgePush, + { + readonly push: (input: { + readonly orgId: KinOrgId; + readonly repoUrl: string; + readonly gitDir: string; + readonly updates: ReadonlyArray; + }) => Effect.Effect; + } +>()("t3/kin/memory/purgePush/KinPurgePush") {} + +/** Inherited `GIT_*` variables can point git at another repository, config or object store. */ +export const scrubbedGitEnv = (): NodeJS.ProcessEnv => + Object.fromEntries( + Object.keys(process.env) + .filter((name) => /^GIT_/i.test(name)) + .map((name) => [name, undefined]), + ); +export const hardenedGitEnv = { + GIT_CONFIG_NOSYSTEM: "1", + GIT_CONFIG_GLOBAL: "/dev/null", + GIT_TERMINAL_PROMPT: "0", + GIT_ATTR_NOSYSTEM: "1", + GIT_NO_REPLACE_OBJECTS: "1", + GIT_ASKPASS: "", +}; +const pushFailure = (stdout: string, stderr: string): KinPurgePushError["reason"] => { + const rejected = stdout.split("\n").filter((line) => line.startsWith("!")); + if (rejected.some((line) => line.includes("(stale info)"))) return "lease_rejected"; + if (rejected.some((line) => line.includes("[remote rejected]"))) return "remote_rejected"; + if (rejected.length > 0) return "rejected"; + if ( + /authentication failed|permission denied|could not read (username|password)|terminal prompts disabled|returned error: 40[13]/i.test( + stderr, + ) + ) + return "auth"; + if ( + /could not resolve host|connection (refused|reset|timed out)|operation timed out|network is unreachable|unable to access|early eof/i.test( + stderr, + ) + ) + return "network"; + return "unknown"; +}; + +// A helper name or absolute path, never an inline `!` script that could carry the secret itself. +const isCredentialHelperReference = (helper: string) => + /^(?:[A-Za-z0-9][A-Za-z0-9._-]*|\/[A-Za-z0-9._/-]+)$/.test(helper); +const isSshCommandReference = (command: string) => /^[A-Za-z0-9 ._/=,:@+-]+$/.test(command); + +/** + * Pushes with a reference to the owner's credential: a git credential helper or an SSH command naming + * its key file. The secret never enters argv or env, and with neither configured the push fails closed. + */ +export const gitPushLayer = (owner: { + readonly credentialHelper?: string; + readonly sshCommand?: string; +}) => + Layer.effect( + KinPurgePush, + Effect.gen(function* () { + const vcs = yield* VcsProcess; + return KinPurgePush.of({ + push: Effect.fn("KinPurgePush.git")(function* ({ repoUrl, gitDir, updates }) { + const { credentialHelper, sshCommand } = owner; + if ( + (credentialHelper === undefined && sshCommand === undefined) || + (credentialHelper !== undefined && !isCredentialHelperReference(credentialHelper)) || + (sshCommand !== undefined && !isSshCommandReference(sshCommand)) + ) + return yield* new KinPurgePushError({ reason: "credential_unavailable" }); + const result = yield* vcs + .run({ + operation: "KinPurgePush.git", + command: "git", + cwd: gitDir, + args: [ + "--no-replace-objects", + `--git-dir=${gitDir}/.git`, + "-c", + "core.hooksPath=/dev/null", + "-c", + "core.fsmonitor=false", + "-c", + "core.attributesFile=/dev/null", + "-c", + "protocol.file.allow=user", + "-c", + "credential.helper=", + ...(credentialHelper ? ["-c", `credential.helper=${credentialHelper}`] : []), + "push", + "--atomic", + "--porcelain", + ...updates.map((update) => `--force-with-lease=${update.ref}:${update.old}`), + "--", + repoUrl, + ...updates.map((update) => `${update.new ?? ""}:${update.ref}`), + ], + env: { + ...scrubbedGitEnv(), + ...hardenedGitEnv, + GIT_ASKPASS: "false", + SSH_ASKPASS: "false", + SSH_ASKPASS_REQUIRE: "never", + SSH_AUTH_SOCK: undefined, + GCM_INTERACTIVE: "never", + // Without an owner SSH command, refuse SSH instead of using the host's keys or agent. + GIT_SSH_COMMAND: sshCommand + ? `${sshCommand} -o BatchMode=yes -o IdentitiesOnly=yes -o IdentityAgent=none` + : "false", + }, + allowNonZeroExit: true, + timeoutMs: 600_000, + }) + .pipe( + // The cause can echo the remote URL or helper output; keep both out of errors and logs. + Effect.mapError( + (error) => + new KinPurgePushError({ + reason: error._tag === "VcsProcessTimeoutError" ? "timeout" : "unknown", + }), + ), + ); + if (Number(result.exitCode) !== 0) + return yield* new KinPurgePushError({ + reason: pushFailure(result.stdout, result.stderr), + }); + }), + }); + }), + ); diff --git a/docs/kin/specs/triggers.md b/docs/kin/specs/triggers.md index 6ad54f2aef1c9..379084f146420 100644 --- a/docs/kin/specs/triggers.md +++ b/docs/kin/specs/triggers.md @@ -426,20 +426,25 @@ Known gaps, accepted for phase 1: JWE (five-part) tokens and JWT headers longer ### Owner-signed purge (gap 5) -Pending action `memory_purge`. Its `content` target holds the sha256 of the plan `{ orgId, repo, paths, mode: "drop_file" | "redact_text", patterns? }` (`purgePlanDigest`), so the grant binds the exact plan. The owner opens it from an item's menu, or Kin proposes it when the weekly `dream_full` scans full history and finds a secret. Approval uses a native card and a signed grant (APPR). `memoryPurgeHandler` in `apps/server/src/kin/memory/purge.ts` is the only entry point; no model, MCP tool or RPC reaches `KinPurge`. It refuses with `KinPurgeExecutorNotReady` until APPR-9 and APPR-10 land. +Pending action `memory_purge`. The plan is `{ orgId, repo, mode: "drop_file", paths }` or `{ orgId, repo, mode: "redact_text", patterns }`; `redact_text` rewrites every blob and commit message in history, so it takes no paths. `KinPurge.storePlan` writes `purgePlanJson(plan)` to `/purge-plans/.json` and returns the action's `content` target, whose path and sha256 come from that same serialization, so the grant binds the exact plan; `discardPlan` deletes it once the action is terminal, since it can hold the secret. The owner opens the action from an item's menu, or Kin proposes it when the weekly `dream_full` scans full history and finds a secret. Approval uses a native card and a signed grant (APPR). `memoryPurgeHandler` in `apps/server/src/kin/memory/purge.ts` is APPR-9's `KinActionHandler` (`precheck`, `run`, `reconcile`) and the only entry point; no model, MCP tool or RPC reaches `KinPurge`. Every `KinPurge` method refuses with `KinPurgeExecutorNotReady` until APPR-9, APPR-10 and LEAD-11 land. -Before anything runs, the grant, action and plan must share the org, the grant must bind the action id and revision, and `repo` must equal the org's configured memory URL. Paths: 1–64, unique, relative `.md` files other than `rules.md`, with no `..`, `.`, empty, `-`-leading or `.git` segment, backslash or control character. `redact_text` takes 1–64 literal patterns of 8–1024 characters without control characters or `==>`; each becomes one `literal:` rule, so a pattern can't add a rule or switch to regex. `drop_file` takes none. Every argument goes to git and `git-filter-repo` as an argv entry, never through a shell. +Before anything runs, the stored plan must re-serialize to the same bytes and match the target; the grant, action and plan must share the org; the grant must bind the action id and revision; and `repo` must equal the org's configured memory URL. Paths: 1–64, unique, NFC, relative `.md` files other than `rules.md`, with no `..`, `.`, empty, `-`-leading or `.git` segment, backslash or control character, and each must exist in history (literal pathspecs), so a typo or an NFD name fails instead of purging nothing. `redact_text` takes 1–64 literal patterns of 8–1024 characters without control characters or `==>`; each becomes one `literal:` rule, so a pattern can't add a rule or switch to regex. Every argument goes to git and `git-filter-repo` as an argv entry, never through a shell, with inherited `GIT_*` variables removed. -The executor then: +`run` then: -1. Pauses the org's lead queue (`KinPurgeQueue`, a no-op until LEAD-11 builds the pause) and takes the memory writer lock. -2. Mirror-clones the remote into a scratch directory and runs `git filter-repo --invert-paths --path

` (`drop_file`) or `--replace-text --replace-message ` (`redact_text`). A plan that changes no ref fails. -3. For `drop_file`, checks that no reachable commit touches the paths, then commits a tombstone on the default branch for each item path present at the old head: the original `id` and the folder's kind, `title: Purged item`, `status: cancelled`, `invalid_at: now`, `said_by` the approving owner, body "Purged on by the owner (audit )", where `` is the executor's `execution_started` audit entry. Memory still shows that something was there. Journal and other non-item paths get no tombstone. +1. Pauses the org's lead queue through `KinPurgeQueue`. The default fails `KinPurgePauseUnavailable` until LEAD-11 builds the pause, because a reclone deletes the read checkout a running lead uses as its workspace. It takes the memory writer lock through `Memory.withHistoryRewrite`, which is the only way to reach the quarantine and reclone operations. +2. Mirror-clones the remote into `//purge-scratch` (stale mirrors from a crashed purge are swept first) and runs `git filter-repo --invert-paths --path

` (`drop_file`) or `--replace-text --replace-message ` (`redact_text`). A plan that changes no ref fails. +3. For `drop_file`, checks that no reachable commit touches the paths, then commits a tombstone on the default branch for each item path present at the old head: the original `id` and the folder's kind, `title: Purged item`, `status: cancelled`, `invalid_at: now`, `source: kin-action:`, `said_by` the approving owner, body "Purged on by the owner (action )". Memory still shows that something was there. Journal and other non-item paths get no tombstone. 4. Reads every blob ever introduced at every path, merge sides included, and every commit object. It fails if a redacted pattern remains, then runs `KinSecretScan` over all of it and fails closed on **any** finding: findings carry no fingerprint, so the purge can't prove one unrelated, and a copy under an older path would otherwise survive. Allowlisted fingerprints don't count. Annotated tag messages are not scanned. -5. Force-pushes the changed `refs/heads/*` and `refs/tags/*` in one atomic push, each leased at its cloned value, through `KinPurgePush`. `gitPushLayer` runs git with the owner's credential helper or SSH command, so T3 never reads the credential, and push errors drop their cause so neither the URL nor helper output is logged. This needs the owner as a bypass actor on the memory repo's ruleset. Closing open dream PRs that touch the paths waits on the GitHub client. -6. `Memory.reclone` deletes the writer and read checkouts, clones fresh, records the rewritten anchors without preserving the old ones (a conflict branch would push the purged history back), and rebuilds the read checkout, cache and `INDEX.md`. The queue resumes. A failure before the push resumes the queue with nothing changed; from the push on, `KinPurgeFailed` carries `queueHeld: true` and the queue stays paused for owner recovery. -7. Returns the `execution_finished` audit input (grant `jti`, key thumbprint, device, new head) with the paths, old head, new head and rewritten refs; APPR-10 writes it as a `KinAuditEntry`. -8. Tells the owner what a purge can't reach: GitHub keeps `refs/pull/*` and cached PR diffs (removing those needs a GitHub Support request), forks, local backups (Time Machine), and the chat database. A leaked credential must be rotated whatever happens. +5. Writes the durable marker `//purge-in-flight` (plan digest, action id, old head) and deletes the writer and read checkouts, before the push. A push can land and still report failure (a timeout, a crash, a restart), and those checkouts hold the old history: `Memory.preserve` would push it back as a `kin/conflict/*` branch. With them gone every later path clones fresh, and while the marker exists `preserve` and the wake-up merge refuse to push. +6. Force-pushes the changed `refs/heads/*` and `refs/tags/*` in one atomic push, each leased at its cloned value, through `KinPurgePush`. Failures are classified from `--porcelain` output (`lease_rejected`, `remote_rejected`, `auth`, `network`, `timeout`) without the cause, so neither the URL nor helper output is logged. This needs the owner as a bypass actor on the memory repo's ruleset. Closing open dream PRs that touch the paths waits on the GitHub client. +7. Reclones: clones fresh, records the rewritten anchors without preserving the old ones, rebuilds the read checkout, cache and `INDEX.md`, clears the marker, and resumes the queue. A failure or interruption before step 5 resumes the queue with nothing changed; from step 5 on, any failure, defect or interruption leaves `KinPurgeFailed` with `queueHeld: true` and the queue paused. +8. Returns the `execution_finished` audit input (grant `jti`, key thumbprint, device, new head) with the paths, old head, new head and rewritten refs; APPR-10 writes it as a `KinAuditEntry`. +9. Tells the owner what a purge can't reach: GitHub keeps `refs/pull/*` and cached PR diffs (removing those needs a GitHub Support request), forks, local backups (Time Machine), and the chat database. A leaked credential must be rotated whatever happens. + +`reconcile` mirror-clones the remote and checks whether the plan's paths (other than purge tombstones) or patterns are still in history. Gone is `done`; still there with the default branch at the marker's old head is `not_done`; anything else, or an unreachable remote, is `unknown` and changes nothing. `done` and `not_done` reclone, clear the marker and resume the queue. A re-run of the same plan while its marker exists reclones instead of resuming against old checkouts, and reports success when the push had already landed. + +**Owner credential.** `gitPushLayer` takes a reference to the owner's credential, never the secret: a git credential helper name or absolute path (no inline `!` script), or an SSH command naming its key file (no shell metacharacters; T3 appends `BatchMode`, `IdentitiesOnly` and `IdentityAgent=none`). This is the one exception to APPR's rule that the owner credential never reaches a child process: the helper or key is read by git or ssh, not by T3. With neither configured the push fails `credential_unavailable`, SSH without an owner command is refused rather than using the host's keys or agent, and `GIT_TERMINAL_PROMPT=0`, `GIT_ASKPASS`, `SSH_ASKPASS` and `SSH_ASKPASS_REQUIRE=never` keep anything from prompting. **Dependency:** `git-filter-repo` (git's recommended history rewriter; MIT, also usable under git's GPL), checked at v2.47.0. TRIG-20 installs it on the Kin host; `GitFilterRepoCommand` overrides the binary path. The real-rewrite tests skip on machines without it. @@ -562,7 +567,7 @@ Kin runs on one always-on machine, the **Kin host**. The laptop and phone are cl - **Deferral:** unchanged recheck, then heads-up, then silence, then `started` at due. A linked issue closed before due leads to `asked`, and nothing starts. With the stop switch on, it asks. [Monday] re-arms to next Monday 09:00. A `due` fire consumed two hours late asks; a queued fire for a disarmed or moved deferral is dropped. - **Dream:** a dream touching `rules.md` fails, and `main` is unchanged. A dream with one journal fold and one `decisions/` change opens two PRs, and only the auto one merges. - **Secret scan:** a journal line holding `ghp_` plus 36 characters blocks the entire commit, and the owner gets one typed notification without the value. -- **Purge:** on a fixture repo, the file is gone from `git log --all -p` and the recloned checkout, a tombstone exists and parses, and the returned audit input carries the old and new heads. A surviving secret fails closed before the push. +- **Purge:** on a fixture repo, the file is gone from `git log --all -p` and the recloned checkout, a tombstone exists and parses, and the returned audit input carries the old and new heads. A surviving secret fails closed before the push. A push that lands and then errors, crashes or is interrupted never republishes the old history through the next wake-up, and `reconcile` reports `done`. - **Contract tests:** `vp test run kin-contract` passes on today's upstream. Renaming `scheduledTaskId` in a scratch branch makes it fail. ## Tasks @@ -587,7 +592,7 @@ Kin runs on one always-on machine, the **Kin host**. The laptop and phone are cl | TRIG-14 | Dream split, auto-merge under delegation, review card, rejections; full weekly variant | `classifyDream.ts`, `DreamRunner.ts` | TRIG-13, APPR-4; auto-merge waits for APPR-12 (phase 3), and until then the owner approves both PRs | the two-PR test passes; a rejected item is not proposed again | backend | | TRIG-15 | Memory repo guard rails | `kin-memory`: `CODEOWNERS`, `memory-check.yml`, ruleset | APPR-11 | a PR touching `decisions/` can't merge without owner review; a planted token fails the check | backend | | TRIG-16 | Secret scanner, writer hook, allowlist | `secretScan.ts` | LEAD-3 | rule fixtures (one hit and one near-miss per rule) pass; commit refusal and typed owner notification tested | backend | -| TRIG-17 | Owner-signed purge | `purge.ts` | TRIG-16, APPR-9, APPR-10 | purge test passes on a fixture repo; `git-filter-repo` provenance noted in the PR | backend | +| TRIG-17 | Owner-signed purge | `purge.ts` | TRIG-16, APPR-9, APPR-10, LEAD-11 | purge test passes on a fixture repo; `git-filter-repo` provenance noted in the PR | backend | | TRIG-18 | Server contract tests: triggers, relay, `delegate_task`, managed policy | `apps/server/src/kin/contract/*` | TRIG-4 | `cd apps/server && vp test run kin-contract` passes | backend | | TRIG-19 | `McpAppWebView` contract test | `apps/mobile/src/features/chats/contract/*` | — | `cd apps/mobile && vp test run kin-contract` passes | UI | | TRIG-20 | Kin host setup: systemd unit, `kin` user, `setup.sh`, `backup.sh`, verify source-build `up --connect` and hold | `ops/kin/host/*` | — | after a reboot, the server is reachable from the phone over T3 Connect; a GitHub redelivery while it's stopped arrives on start; a restore drill from backup works | backend | From 826f40e9a8b4b0ae0a3753245d51732fea6c979c Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Sun, 8 Nov 2026 23:53:14 +0530 Subject: [PATCH 3/7] fix(kin): purge blocks clones and pushes until it lands (TRIG-17) A dream or wake-up could clone the old history from the remote while the purge's push was in flight, and a later dream branch pushed it back. While the purge marker exists, Memory no longer clones a writer outside the purge's own reclone, and MemoryWriter refuses commitWakeup, commitDream and deleteDreamBranch. - The purge clone is bare (heads and tags only), so GitHub's read-only refs/pull/* no longer keep reconcile from answering done. - The marker carries the plan, and discardPlan refuses while that plan's marker exists, so a stuck marker always has a way out; the manual way out is in docs/operations/kin-memory-purge.md. - The owner push runs with an empty HOME and `ssh -F none`, and refuses a remote whose scheme doesn't match the configured credential. - Tombstones match only this action's source; the reclone also drops /dreams/. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/kin/memory/Memory.ts | 10 +- apps/server/src/kin/memory/MemoryWriter.ts | 14 ++ apps/server/src/kin/memory/purge.test.ts | 201 +++++++++++++++++---- apps/server/src/kin/memory/purge.ts | 72 ++++++-- apps/server/src/kin/memory/purgePush.ts | 30 ++- docs/kin/specs/triggers.md | 14 +- docs/operations/kin-memory-purge.md | 9 + 7 files changed, 287 insertions(+), 63 deletions(-) create mode 100644 docs/operations/kin-memory-purge.md diff --git a/apps/server/src/kin/memory/Memory.ts b/apps/server/src/kin/memory/Memory.ts index 9a0cf53f48239..8c910d19c3fce 100644 --- a/apps/server/src/kin/memory/Memory.ts +++ b/apps/server/src/kin/memory/Memory.ts @@ -536,6 +536,8 @@ const make = Effect.gen(function* () { if (yield* io(orgId, "exists", fs.exists(paths.writer))) { yield* validateWriter(orgId, paths, false); } else { + // The remote can still hold the old history until the purge's push lands. + yield* refuseDuringPurge(orgId, "clone"); for (const entry of yield* io(orgId, "clones", fs.readDirectory(paths.root))) { if (/^memory\.clone-[A-Za-z0-9]+$/.test(entry)) yield* io( @@ -916,10 +918,11 @@ const make = Effect.gen(function* () { return branch; }); - const recover = Effect.fn("Memory.recover")(function* (orgId: KinOrgId) { + const recover = Effect.fn("Memory.recover")(function* (orgId: KinOrgId, reclone = false) { const paths = yield* pathsFor(orgId); yield* io(orgId, "mkdir", fs.makeDirectory(paths.root, { recursive: true })); const exists = yield* io(orgId, "exists", fs.exists(paths.writer)); + if (!exists && !reclone) yield* refuseDuringPurge(orgId, "clone"); if (!exists) yield* git(orgId, paths.root, [ "clone", @@ -1326,7 +1329,8 @@ const make = Effect.gen(function* () { const dropCheckouts = Effect.fn("Memory.dropCheckouts")(function* (orgId: KinOrgId) { const paths = yield* pathsFor(orgId); - for (const directory of [paths.writer, paths.read]) + // `dreams/` holds plain copies of memory files for a running dream. + for (const directory of [paths.writer, paths.read, path.join(paths.root, "dreams")]) yield* io(orgId, "quarantine", fs.remove(directory, { recursive: true, force: true })); initializedCheckouts.delete(orgId); pinnedHeads.delete(orgId); @@ -1353,7 +1357,7 @@ const make = Effect.gen(function* () { Effect.gen(function* () { const { root } = yield* dropCheckouts(orgId); // With no writer checkout, recover clones fresh and records the rewritten heads. - yield* recover(orgId); + yield* recover(orgId, true); yield* prepare(orgId); yield* rebuildIndex(orgId); yield* io(orgId, "purgeMarker", fs.remove(markerPath(root), { force: true })); diff --git a/apps/server/src/kin/memory/MemoryWriter.ts b/apps/server/src/kin/memory/MemoryWriter.ts index 86a40b2ead561..42d95d7e726d1 100644 --- a/apps/server/src/kin/memory/MemoryWriter.ts +++ b/apps/server/src/kin/memory/MemoryWriter.ts @@ -377,6 +377,17 @@ const make = Effect.gen(function* () { const paths = yield* memory.ensureCheckouts(orgId); return paths; }); + // Every caller holds the writer lock, and the marker only changes under it. + const refuseDuringPurge = Effect.fn("MemoryWriter.refuseDuringPurge")(function* ( + orgId: KinOrgId, + ) { + if ((yield* memory.purgeInFlight(orgId)) !== undefined) + return yield* new MemorySyncConflict({ + orgId, + operation: "push", + reason: "A memory purge is in flight; old history must not be pushed", + }); + }); const report = (orgId: KinOrgId, wakeupId: string, effect: Effect.Effect) => effect.pipe( Effect.catchTags({ @@ -770,6 +781,7 @@ const make = Effect.gen(function* () { Effect.gen(function* () { const { ctx, state } = yield* session(orgId, wakeupId); const { writer } = yield* validate(orgId); + yield* refuseDuringPurge(orgId); const repo = config.repositories.find((entry) => entry.orgId === orgId); if (!repo) return yield* new MemoryConfigError({ @@ -899,6 +911,7 @@ const make = Effect.gen(function* () { if (!dreamBranch.test(input.branch)) return yield* new MemoryWriteRejected({ reason: "Invalid dream branch" }); const { writer } = yield* validate(orgId); + yield* refuseDuringPurge(orgId); const repo = yield* repoFor(orgId); const baseFiles = yield* filesAt(orgId, writer, input.base); const regular = yield* regularPaths(orgId, writer, input.base); @@ -964,6 +977,7 @@ const make = Effect.gen(function* () { if (!dreamBranch.test(branch)) return yield* new MemoryWriteRejected({ reason: "Invalid dream branch" }); const { writer } = yield* validate(orgId); + yield* refuseDuringPurge(orgId); const repo = yield* repoFor(orgId); const advertised = yield* git(orgId, writer, [ "ls-remote", diff --git a/apps/server/src/kin/memory/purge.test.ts b/apps/server/src/kin/memory/purge.test.ts index 079e5b613123b..cd125cd3ef10e 100644 --- a/apps/server/src/kin/memory/purge.test.ts +++ b/apps/server/src/kin/memory/purge.test.ts @@ -3,7 +3,7 @@ import * as NodeChildProcess from "node:child_process"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { describe, expect, it } from "@effect/vitest"; import { vi } from "vite-plus/test"; -import { KinMemoryItemId, KinOrgId } from "@t3tools/contracts"; +import { KinAgentId, KinMemoryItemId, KinOrgId } from "@t3tools/contracts"; import * as Cause from "effect/Cause"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; @@ -17,6 +17,7 @@ import { fixtureItem } from "./__fixtures__/items.ts"; import { parseFrontMatter } from "./frontMatter.ts"; import * as Memory from "./Memory.ts"; import { MemoryConfig } from "./MemoryConfig.ts"; +import * as Writer from "./MemoryWriter.ts"; import * as Purge from "./purge.ts"; import * as PurgePush from "./purgePush.ts"; import * as Scan from "./secretScan.ts"; @@ -112,6 +113,10 @@ const fixture = Effect.fn(function* (tokenInMessage = false) { yield* git(source, ["add", "--", "."]); yield* commit("revise rules"); yield* git(root, ["clone", "-q", "--bare", "--", source, remote]); + // GitHub's read-only pull refs: a purge can't rewrite them and must not wait on them. + yield* git(remote, ["update-ref", "refs/pull/1/head", "refs/heads/side"]); + const pullHead = yield* git(remote, ["rev-parse", "refs/pull/1/head"]); + const remoteHistory = git(remote, ["log", "--exclude=refs/pull/*", "--all", "-p"]); const events: string[] = []; const config = Layer.succeed(MemoryConfig, { @@ -152,6 +157,30 @@ const fixture = Effect.fn(function* (tokenInMessage = false) { ), ); }; + /** The purge plus the dream and wake-up writer, which push to the same remote. */ + const writerLayer = (options: PurgeOptions = {}) => + Writer.layer.pipe( + Layer.provideMerge(purgeLayer(options)), + Layer.provide(config), + Layer.provide(Scan.layer), + Layer.provide(Layer.succeed(Memory.MemoryRecoveryHooks, { notify: () => Effect.void })), + Layer.provide(Layer.succeed(Writer.MemoryWriterHooks, { notify: () => Effect.void })), + Layer.provide( + Layer.succeed(Writer.MemoryWriterAuthority, { + resolve: (_org, wakeupId) => + Effect.succeed({ + orgId, + wakeupId, + agentId: KinAgentId.make("fixture-lead"), + level: "L1", + timeZone: "UTC", + startedAt: "2026-11-08T00:00:00.000Z", + eventIds: [], + sources: new Map(), + }), + }), + ), + ); const writerHistory = (writer: string) => Effect.gen(function* () { if (!(yield* fs.exists(`${writer}/.git`))) return ""; @@ -177,8 +206,11 @@ const fixture = Effect.fn(function* (tokenInMessage = false) { events, commit, purgeLayer, + writerLayer, writerHistory, prepare, + pullHead, + remoteHistory, }; }); @@ -202,8 +234,19 @@ const failureOf = (exit: Exit.Exit) => describe.skipIf(filterRepoMissing)("purge with git-filter-repo", () => { it.live.each(["drop_file", "redact_text"] as const)("purges a fixture repo: %s", (mode) => Effect.gen(function* () { - const { fs, git, root, remote, remoteUrl, events, purgeLayer, writerHistory, prepare } = - yield* fixture(mode === "redact_text"); + const { + fs, + git, + root, + remote, + remoteUrl, + events, + purgeLayer, + writerHistory, + prepare, + pullHead, + remoteHistory, + } = yield* fixture(mode === "redact_text"); yield* Effect.gen(function* () { const memory = yield* Memory.Memory; const paths = yield* prepare; @@ -224,6 +267,8 @@ describe.skipIf(filterRepoMissing)("purge with git-filter-repo", () => { const newSide = yield* git(remote, ["rev-parse", "refs/heads/side"]); expect(events).toEqual(["pause", "resume"]); expect(yield* memory.purgeInFlight(orgId)).toBeUndefined(); + expect(yield* reconcile(plan)).toBe("done"); + expect(yield* git(remote, ["rev-parse", "refs/pull/1/head"])).toBe(pullHead); expect(result).toMatchObject({ oldHead: oldMain, newHead: newMain }); expect(result.rewrittenRefs).toEqual([ { ref: "refs/heads/main", old: oldMain, new: newMain }, @@ -244,7 +289,7 @@ describe.skipIf(filterRepoMissing)("purge with git-filter-repo", () => { }); expect(result.audit.result).not.toContain(token); - expect(yield* git(remote, ["log", "--all", "-p"])).not.toContain(token); + expect(yield* remoteHistory).not.toContain(token); expect(yield* writerHistory(paths.writer)).not.toContain(token); expect(yield* memory.anchor(orgId, "main")).toBe(newMain); expect(yield* git(remote, ["show", `${newMain}:claims/keep.md`])).toContain("claim-keep"); @@ -332,32 +377,48 @@ describe.skipIf(filterRepoMissing)("purge with git-filter-repo", () => { }).pipe(Effect.provide(services), Effect.scoped), ); - it.live.each(["error", "crash", "interrupt"] as const)( + it.live.each(["error", "checkout during push", "crash", "interrupt"] as const)( "a push that lands and then fails never republishes the purged history: %s", (scenario) => Effect.gen(function* () { - const { fs, git, remote, remoteUrl, events, purgeLayer, writerHistory, prepare } = + const { fs, git, root, remote, remoteUrl, events, writerLayer, writerHistory, prepare } = yield* fixture(); const plan = dropPlan(remoteUrl); - const landing = purgeLayer({ + // A dream or wake-up asking for checkouts while the remote still holds the old history. + let beforePush: Effect.Effect = Effect.void; + const landing = writerLayer({ push: (real) => ({ push: (input) => - real - .push(input) - .pipe( - Effect.andThen( - scenario === "error" - ? Effect.fail(new PurgePush.KinPurgePushError({ reason: "timeout" })) - : scenario === "crash" - ? Effect.die("fixture crash after the push") - : Effect.interrupt, - ), + Effect.suspend(() => beforePush).pipe( + Effect.andThen(real.push(input)), + Effect.andThen( + scenario === "crash" + ? Effect.die("fixture crash after the push") + : scenario === "interrupt" + ? Effect.interrupt + : Effect.fail(new PurgePush.KinPurgePushError({ reason: "timeout" })), ), + ), }), }); + const target = Purge.purgePlanJson(plan).target; const paths = yield* Effect.gen(function* () { + const memory = yield* Memory.Memory; const paths = yield* prepare; + const oldMain = yield* git(remote, ["rev-parse", "refs/heads/main"]); + let checkout: Exit.Exit | undefined; + if (scenario === "checkout during push") + beforePush = memory.ensureCheckouts(orgId).pipe( + Effect.exit, + Effect.map((exit) => { + checkout = exit; + }), + ); const exit = yield* runPurge(plan); + if (scenario === "checkout during push") + expect(checkout && failureOf(checkout)).toMatchObject({ + value: { _tag: "MemorySyncConflict" }, + }); expect(Exit.isFailure(exit)).toBe(true); if (scenario === "interrupt") expect(Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause)).toBe(true); @@ -366,30 +427,61 @@ describe.skipIf(filterRepoMissing)("purge with git-filter-repo", () => { value: { _tag: "KinPurgeFailed", stage: "push", queueHeld: true }, }); expect(events).toEqual(["pause"]); - expect(yield* (yield* Memory.Memory).purgeInFlight(orgId)).toBeDefined(); + expect(yield* memory.purgeInFlight(orgId)).toBeDefined(); expect(yield* fs.exists(paths.writer)).toBe(false); expect(yield* fs.exists(paths.read)).toBe(false); + + const writer = yield* Writer.MemoryWriter; + const dream = yield* writer + .commitDream(orgId, { + branch: "dream/2026-11-08", + base: oldMain, + files: new Map([ + [ + "claims/dream.md", + fixtureItem({ id: "claim-dream", kind: "claim", title: "Invented dream" }), + ], + ]), + message: "dream", + }) + .pipe(Effect.result); + expect(dream._tag).toBe("Failure"); + expect( + (yield* writer.commitWakeup(orgId, "wake1", "completed").pipe(Effect.result))._tag, + ).toBe("Failure"); return paths; }).pipe(Effect.provide(landing)); // A crash restarts the server: fresh Memory and purge services over the same database and files. - const after = scenario === "crash" ? purgeLayer() : landing; + const after = scenario === "crash" ? writerLayer() : landing; yield* Effect.gen(function* () { const memory = yield* Memory.Memory; + const purge = yield* Purge.KinPurge; yield* memory.prepareWakeup(orgId).pipe(Effect.result); yield* memory.recover(orgId).pipe(Effect.result); - expect(yield* git(remote, ["log", "--all", "-p"])).not.toContain(token); - expect(yield* git(remote, ["for-each-ref", "--format=%(refname)"])).not.toContain( - "kin/conflict", + expect(yield* fs.exists(paths.writer)).toBe(false); + expect(yield* git(remote, ["log", "--exclude=refs/pull/*", "--all", "-p"])).not.toContain( + token, ); + expect( + yield* git(remote, ["for-each-ref", "--format=%(refname)", "refs/heads/"]), + ).not.toMatch(/kin\/conflict|dream\//); + // The marker keeps the plan, so a lost plan file can't strand the held queue. + expect(yield* purge.discardPlan(target).pipe(Effect.result)).toMatchObject({ + failure: { _tag: "KinPurgeRejected" }, + }); + yield* fs.remove(`${root}/${target.path}`); expect(yield* reconcile(plan)).toBe("done"); expect(events).toEqual(["pause", "resume"]); expect(yield* memory.purgeInFlight(orgId)).toBeUndefined(); const newMain = yield* git(remote, ["rev-parse", "refs/heads/main"]); expect((yield* memory.prepareWakeup(orgId)).head).toBe(newMain); expect(yield* writerHistory(paths.writer)).not.toContain(token); - expect(yield* git(remote, ["log", "--all", "-p"])).not.toContain(token); + expect(yield* git(remote, ["log", "--exclude=refs/pull/*", "--all", "-p"])).not.toContain( + token, + ); + yield* purge.discardPlan(target); }).pipe(Effect.provide(after)); }).pipe(Effect.provide(services), Effect.scoped), ); @@ -549,14 +641,18 @@ const pushInput = { const pushWith = ( owner: Parameters[0], output: { readonly code: number; readonly stdout?: string; readonly stderr?: string }, + repoUrl = pushInput.repoUrl, ) => { const calls: Array = []; return Effect.gen(function* () { - const result = yield* (yield* PurgePush.KinPurgePush).push(pushInput).pipe(Effect.result); + const result = yield* (yield* PurgePush.KinPurgePush) + .push({ ...pushInput, repoUrl }) + .pipe(Effect.result); return { result, calls }; }).pipe( Effect.provide( PurgePush.gitPushLayer(owner).pipe( + Layer.provide(NodeServices.layer), Layer.provide( Layer.mock(Vcs.VcsProcess)({ run: (input) => @@ -578,15 +674,51 @@ const pushWith = ( }; it.effect.each([ - ["no credential", {}], - ["inline helper script", { credentialHelper: "!f() { echo password=hunter2; }; f" }], - ["shell in ssh command", { sshCommand: "ssh -i key; cat /etc/passwd" }], -] as const)("the owner push fails closed without a credential reference: %s", ([, owner]) => - Effect.gen(function* () { - const { result, calls } = yield* pushWith(owner, { code: 0 }); - expect(result).toMatchObject({ failure: { reason: "credential_unavailable" } }); - expect(calls).toEqual([]); - }), + ["no credential", {}, pushInput.repoUrl, "credential_unavailable"], + [ + "inline helper script", + { credentialHelper: "!f() { echo password=hunter2; }; f" }, + pushInput.repoUrl, + "credential_unavailable", + ], + [ + "shell in ssh command", + { sshCommand: "ssh -i key; cat /etc/passwd" }, + pushInput.repoUrl, + "credential_unavailable", + ], + [ + "https without a helper", + { sshCommand: "ssh -i /keys/owner" }, + pushInput.repoUrl, + "credential_unavailable", + ], + [ + "ssh without a command", + { credentialHelper: "osxkeychain" }, + "git@example.invalid:memory.git", + "credential_unavailable", + ], + [ + "plain http", + { credentialHelper: "osxkeychain" }, + "http://example.invalid/memory.git", + "unsupported_remote", + ], + [ + "remote helper", + { credentialHelper: "osxkeychain", sshCommand: "ssh -i /keys/owner" }, + "ext::ssh -i /keys/owner example.invalid %S memory.git", + "unsupported_remote", + ], +] as const)( + "the owner push fails closed without a matching credential reference: %s", + ([, owner, repoUrl, reason]) => + Effect.gen(function* () { + const { result, calls } = yield* pushWith(owner, { code: 0 }, repoUrl); + expect(result).toMatchObject({ failure: { reason } }); + expect(calls).toEqual([]); + }), ); it.effect("the owner push passes only credential references and never prompts", () => @@ -609,8 +741,9 @@ it.effect("the owner push passes only credential references and never prompts", SSH_ASKPASS: "false", SSH_ASKPASS_REQUIRE: "never", GIT_SSH_COMMAND: - "ssh -i /keys/owner -o BatchMode=yes -o IdentitiesOnly=yes -o IdentityAgent=none", + "ssh -i /keys/owner -F none -o BatchMode=yes -o IdentitiesOnly=yes -o IdentityAgent=none", }); + expect(call?.env?.HOME).toMatch(/kin-push-home-/); expect(call?.env).toHaveProperty("GIT_CONFIG_PARAMETERS", undefined); expect(call?.env).toHaveProperty("SSH_AUTH_SOCK", undefined); diff --git a/apps/server/src/kin/memory/purge.ts b/apps/server/src/kin/memory/purge.ts index de4a01747bf90..0f16e5b1c18f2 100644 --- a/apps/server/src/kin/memory/purge.ts +++ b/apps/server/src/kin/memory/purge.ts @@ -41,10 +41,12 @@ const PurgePlanSchema = Schema.Union([ export type PurgePlan = typeof PurgePlanSchema.Type; const decodePlan = Schema.decodeUnknownEffect(Schema.fromJsonString(PurgePlanSchema)); +/** Carries the plan so `reconcile` and re-runs still work if the plan file is lost. */ const PurgeMarker = Schema.Struct({ sha256: Schema.String, actionId: Schema.String, oldHead: Schema.String, + plan: Schema.String, }); const decodeMarker = Schema.decodeUnknownOption(Schema.fromJsonString(PurgeMarker)); @@ -321,13 +323,30 @@ const make = Effect.gen(function* () { ? Effect.succeed(path.resolve(config.baseDirectory, target.path)) : reject("Invalid plan path"); + const markerFor = (orgId: KinOrgId) => + memory.purgeInFlight(orgId).pipe( + Effect.map((inFlight) => ({ + inFlight, + marker: inFlight === undefined ? undefined : Option.getOrUndefined(decodeMarker(inFlight)), + })), + ); + const loadPlan = Effect.fn("KinPurge.loadPlan")(function* (action: PurgeAction) { if (action.kind !== "memory_purge" || action.target._tag !== "content") return yield* reject("Action is not a memory purge"); const file = yield* planFile(action.target); - const json = yield* fs - .readFileString(file) - .pipe(Effect.mapError(() => new KinPurgeRejected({ reason: "Plan file is missing" }))); + const stored = yield* fs.readFileString(file).pipe(Effect.option); + const json = Option.isSome(stored) + ? stored.value + : config.repositories.some((entry) => entry.orgId === action.orgId) + ? yield* markerFor(action.orgId).pipe( + Effect.map(({ marker }) => + marker?.sha256 === action.target.sha256 ? marker.plan : undefined, + ), + Effect.orElseSucceed(() => undefined), + ) + : undefined; + if (json === undefined) return yield* reject("Plan file is missing"); const plan = yield* decodePlan(json).pipe( Effect.mapError(() => new KinPurgeRejected({ reason: "Plan file is malformed" })), ); @@ -347,11 +366,10 @@ const make = Effect.gen(function* () { if (rejection !== undefined) return yield* reject(rejection); const repo = repoFor(plan); if (!repo) return yield* reject("Plan repo is not this organization's memory"); - const inFlight = yield* memory - .purgeInFlight(plan.orgId) - .pipe(failed(plan.orgId, "clone", "Could not read the purge marker")); - const marker = inFlight === undefined ? Option.none() : decodeMarker(inFlight); - return { plan, repo, inFlight, marker: Option.getOrUndefined(marker) }; + const { inFlight, marker } = yield* markerFor(plan.orgId).pipe( + failed(plan.orgId, "clone", "Could not read the purge marker"), + ); + return { plan, repo, inFlight, marker }; }); const branchRefs = Effect.fn("KinPurge.branchRefs")(function* (work: string) { @@ -438,10 +456,11 @@ const make = Effect.gen(function* () { /** Whether any reachable commit still holds what the plan removes. */ const planInHistory = Effect.fn("KinPurge.planInHistory")(function* ( plan: PurgePlan, + actionId: string, work: string, ) { if (plan.mode === "drop_file") { - // The purge's own tombstones live at the purged paths; anything else there is old content. + // This action's own tombstones live at the purged paths; anything else there is old content. for (const file of yield* historyFiles(plan.orgId, work, plan.paths)) { const parsed = yield* parseFrontMatter( plan.orgId, @@ -451,7 +470,7 @@ const make = Effect.gen(function* () { if ( Option.isNone(parsed) || parsed.value.frontMatter.title !== "Purged item" || - !parsed.value.frontMatter.source.startsWith("kin-action:") + parsed.value.frontMatter.source !== `kin-action:${actionId}` ) return true; } @@ -569,7 +588,10 @@ const make = Effect.gen(function* () { return commit; }); - /** A fresh mirror under the org's purge directory; stale ones from a crashed purge are swept first. */ + /** + * A fresh bare clone of `refs/heads/*` and `refs/tags/*` under the org's purge directory; stale ones + * from a crashed purge are swept first. GitHub's read-only `refs/pull/*` stay out of reach. + */ const mirror = Effect.fn("KinPurge.mirror")(function* ( orgId: KinOrgId, repo: { readonly repoUrl: string; readonly path: string }, @@ -582,13 +604,13 @@ const make = Effect.gen(function* () { }).pipe(failed(orgId, "clone", "Could not create a scratch directory")); yield* git(work, [ "clone", - "--mirror", + "--bare", "--no-local", "--template=", "--", repo.repoUrl, `${work}/.git`, - ]).pipe(failed(orgId, "clone", "Mirror clone failed")); + ]).pipe(failed(orgId, "clone", "Bare clone failed")); const before = yield* branchRefs(work).pipe(failed(orgId, "clone", "Could not list refs")); const head = before.get(`refs/heads/${config.defaultBranch ?? "main"}`); if (!head) return yield* fail(orgId, "clone", "Remote default branch is missing"); @@ -621,7 +643,7 @@ const make = Effect.gen(function* () { // A re-run after a push that landed: nothing to rewrite, but the checkouts still need a reclone. if (input.inFlight !== undefined) { - const remaining = yield* planInHistory(plan, work).pipe( + const remaining = yield* planInHistory(plan, action.id, work).pipe( failed(orgId, "verify", "Could not check history"), ); if (!remaining) { @@ -696,7 +718,7 @@ const make = Effect.gen(function* () { return yield* fail(orgId, "rewrite", "Nothing in history matched the plan"); if (plan.mode === "drop_file") { - const remaining = yield* planInHistory(plan, work).pipe( + const remaining = yield* planInHistory(plan, action.id, work).pipe( failed(orgId, "verify", "Could not check rewritten history"), ); if (remaining) return yield* fail(orgId, "verify", "Purged paths remain in history"); @@ -736,7 +758,12 @@ const make = Effect.gen(function* () { yield* history .quarantine( input.inFlight ?? - markerJson({ sha256: action.target.sha256, actionId: action.id, oldHead }), + markerJson({ + sha256: action.target.sha256, + actionId: action.id, + oldHead, + plan: purgePlanJson(plan).json, + }), ) .pipe(failed(orgId, "quarantine", "Could not quarantine local memory checkouts")); yield* pusher.push({ orgId, repoUrl: repo.repoUrl, gitDir: work, updates }).pipe( @@ -777,6 +804,15 @@ const make = Effect.gen(function* () { discardPlan: Effect.fn("KinPurge.discardPlan")(function* (target) { const file = yield* planFile(target); + for (const { orgId } of config.repositories) { + const { inFlight, marker } = yield* markerFor(orgId).pipe( + Effect.mapError( + () => new KinPurgeRejected({ reason: "Could not read the purge marker" }), + ), + ); + if (inFlight !== undefined && (marker === undefined || marker.sha256 === target.sha256)) + return yield* reject("This plan's purge is in flight; reconcile it first"); + } yield* fs .remove(file, { force: true }) .pipe(Effect.mapError(() => new KinPurgeRejected({ reason: "Could not delete the plan" }))); @@ -866,7 +902,9 @@ const make = Effect.gen(function* () { Effect.gen(function* () { const observed = yield* mirror(orgId, repo).pipe( Effect.flatMap(({ work, head }) => - planInHistory(plan, work).pipe(Effect.map((remaining) => ({ head, remaining }))), + planInHistory(plan, action.id, work).pipe( + Effect.map((remaining) => ({ head, remaining })), + ), ), Effect.option, ); diff --git a/apps/server/src/kin/memory/purgePush.ts b/apps/server/src/kin/memory/purgePush.ts index 4bfee31d49914..6399ec8883299 100644 --- a/apps/server/src/kin/memory/purgePush.ts +++ b/apps/server/src/kin/memory/purgePush.ts @@ -1,6 +1,7 @@ import type { KinOrgId } from "@t3tools/contracts"; import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Schema from "effect/Schema"; @@ -17,6 +18,7 @@ export class KinPurgePushError extends Schema.TaggedError()( { reason: Schema.Literals([ "credential_unavailable", + "unsupported_remote", "lease_rejected", "remote_rejected", "rejected", @@ -78,6 +80,15 @@ const pushFailure = (stdout: string, stderr: string): KinPurgePushError["reason" const isCredentialHelperReference = (helper: string) => /^(?:[A-Za-z0-9][A-Za-z0-9._-]*|\/[A-Za-z0-9._/-]+)$/.test(helper); const isSshCommandReference = (command: string) => /^[A-Za-z0-9 ._/=,:@+-]+$/.test(command); +/** `file://` carries no credential; https needs the helper and SSH the owner's command. */ +const remoteKind = (url: string) => + /^https:\/\//i.test(url) + ? "https" + : /^ssh:\/\//i.test(url) || /^[A-Za-z0-9._-]+@[A-Za-z0-9.-]+:(?!\/\/)/.test(url) + ? "ssh" + : /^file:\/\//i.test(url) + ? "file" + : undefined; /** * Pushes with a reference to the owner's credential: a git credential helper or an SSH command naming @@ -91,6 +102,7 @@ export const gitPushLayer = (owner: { KinPurgePush, Effect.gen(function* () { const vcs = yield* VcsProcess; + const fs = yield* FileSystem.FileSystem; return KinPurgePush.of({ push: Effect.fn("KinPurgePush.git")(function* ({ repoUrl, gitDir, updates }) { const { credentialHelper, sshCommand } = owner; @@ -100,6 +112,18 @@ export const gitPushLayer = (owner: { (sshCommand !== undefined && !isSshCommandReference(sshCommand)) ) return yield* new KinPurgePushError({ reason: "credential_unavailable" }); + const kind = remoteKind(repoUrl); + if (kind === undefined) + return yield* new KinPurgePushError({ reason: "unsupported_remote" }); + if ( + (kind === "https" && credentialHelper === undefined) || + (kind === "ssh" && sshCommand === undefined) + ) + return yield* new KinPurgePushError({ reason: "credential_unavailable" }); + // An empty HOME keeps curl's `.netrc` out; `-F none` keeps `~/.ssh/config` out. + const home = yield* fs + .makeTempDirectoryScoped({ prefix: "kin-push-home-" }) + .pipe(Effect.mapError(() => new KinPurgePushError({ reason: "unknown" }))); const result = yield* vcs .run({ operation: "KinPurgePush.git", @@ -130,6 +154,8 @@ export const gitPushLayer = (owner: { env: { ...scrubbedGitEnv(), ...hardenedGitEnv, + HOME: home, + XDG_CONFIG_HOME: home, GIT_ASKPASS: "false", SSH_ASKPASS: "false", SSH_ASKPASS_REQUIRE: "never", @@ -137,7 +163,7 @@ export const gitPushLayer = (owner: { GCM_INTERACTIVE: "never", // Without an owner SSH command, refuse SSH instead of using the host's keys or agent. GIT_SSH_COMMAND: sshCommand - ? `${sshCommand} -o BatchMode=yes -o IdentitiesOnly=yes -o IdentityAgent=none` + ? `${sshCommand} -F none -o BatchMode=yes -o IdentitiesOnly=yes -o IdentityAgent=none` : "false", }, allowNonZeroExit: true, @@ -156,7 +182,7 @@ export const gitPushLayer = (owner: { return yield* new KinPurgePushError({ reason: pushFailure(result.stdout, result.stderr), }); - }), + }, Effect.scoped), }); }), ); diff --git a/docs/kin/specs/triggers.md b/docs/kin/specs/triggers.md index 379084f146420..75e6845888f05 100644 --- a/docs/kin/specs/triggers.md +++ b/docs/kin/specs/triggers.md @@ -426,25 +426,25 @@ Known gaps, accepted for phase 1: JWE (five-part) tokens and JWT headers longer ### Owner-signed purge (gap 5) -Pending action `memory_purge`. The plan is `{ orgId, repo, mode: "drop_file", paths }` or `{ orgId, repo, mode: "redact_text", patterns }`; `redact_text` rewrites every blob and commit message in history, so it takes no paths. `KinPurge.storePlan` writes `purgePlanJson(plan)` to `/purge-plans/.json` and returns the action's `content` target, whose path and sha256 come from that same serialization, so the grant binds the exact plan; `discardPlan` deletes it once the action is terminal, since it can hold the secret. The owner opens the action from an item's menu, or Kin proposes it when the weekly `dream_full` scans full history and finds a secret. Approval uses a native card and a signed grant (APPR). `memoryPurgeHandler` in `apps/server/src/kin/memory/purge.ts` is APPR-9's `KinActionHandler` (`precheck`, `run`, `reconcile`) and the only entry point; no model, MCP tool or RPC reaches `KinPurge`. Every `KinPurge` method refuses with `KinPurgeExecutorNotReady` until APPR-9, APPR-10 and LEAD-11 land. +Pending action `memory_purge`. The plan is `{ orgId, repo, mode: "drop_file", paths }` or `{ orgId, repo, mode: "redact_text", patterns }`; `redact_text` rewrites every blob and commit message in history, so it takes no paths. `KinPurge.storePlan` writes `purgePlanJson(plan)` to `/purge-plans/.json` and returns the action's `content` target, whose path and sha256 come from that same serialization, so the grant binds the exact plan; `discardPlan` deletes it once the action is terminal, since it can hold the secret, and refuses while that plan's purge marker exists. The owner opens the action from an item's menu, or Kin proposes it when the weekly `dream_full` scans full history and finds a secret. Approval uses a native card and a signed grant (APPR). `memoryPurgeHandler` in `apps/server/src/kin/memory/purge.ts` is APPR-9's `KinActionHandler` (`precheck`, `run`, `reconcile`) and the only entry point; no model, MCP tool or RPC reaches `KinPurge`. Every `KinPurge` method refuses with `KinPurgeExecutorNotReady` until APPR-9, APPR-10 and LEAD-11 land. Before anything runs, the stored plan must re-serialize to the same bytes and match the target; the grant, action and plan must share the org; the grant must bind the action id and revision; and `repo` must equal the org's configured memory URL. Paths: 1–64, unique, NFC, relative `.md` files other than `rules.md`, with no `..`, `.`, empty, `-`-leading or `.git` segment, backslash or control character, and each must exist in history (literal pathspecs), so a typo or an NFD name fails instead of purging nothing. `redact_text` takes 1–64 literal patterns of 8–1024 characters without control characters or `==>`; each becomes one `literal:` rule, so a pattern can't add a rule or switch to regex. Every argument goes to git and `git-filter-repo` as an argv entry, never through a shell, with inherited `GIT_*` variables removed. `run` then: 1. Pauses the org's lead queue through `KinPurgeQueue`. The default fails `KinPurgePauseUnavailable` until LEAD-11 builds the pause, because a reclone deletes the read checkout a running lead uses as its workspace. It takes the memory writer lock through `Memory.withHistoryRewrite`, which is the only way to reach the quarantine and reclone operations. -2. Mirror-clones the remote into `//purge-scratch` (stale mirrors from a crashed purge are swept first) and runs `git filter-repo --invert-paths --path

` (`drop_file`) or `--replace-text --replace-message ` (`redact_text`). A plan that changes no ref fails. +2. Bare-clones the remote's `refs/heads/*` and `refs/tags/*` into `//purge-scratch` (stale clones from a crashed purge are swept first) and runs `git filter-repo --invert-paths --path

` (`drop_file`) or `--replace-text --replace-message ` (`redact_text`). A plan that changes no ref fails. 3. For `drop_file`, checks that no reachable commit touches the paths, then commits a tombstone on the default branch for each item path present at the old head: the original `id` and the folder's kind, `title: Purged item`, `status: cancelled`, `invalid_at: now`, `source: kin-action:`, `said_by` the approving owner, body "Purged on by the owner (action )". Memory still shows that something was there. Journal and other non-item paths get no tombstone. 4. Reads every blob ever introduced at every path, merge sides included, and every commit object. It fails if a redacted pattern remains, then runs `KinSecretScan` over all of it and fails closed on **any** finding: findings carry no fingerprint, so the purge can't prove one unrelated, and a copy under an older path would otherwise survive. Allowlisted fingerprints don't count. Annotated tag messages are not scanned. -5. Writes the durable marker `//purge-in-flight` (plan digest, action id, old head) and deletes the writer and read checkouts, before the push. A push can land and still report failure (a timeout, a crash, a restart), and those checkouts hold the old history: `Memory.preserve` would push it back as a `kin/conflict/*` branch. With them gone every later path clones fresh, and while the marker exists `preserve` and the wake-up merge refuse to push. +5. Writes the durable marker `//purge-in-flight` (plan digest, action id, old head and the plan itself) and deletes the writer and read checkouts and the dream scratch `/dreams/`, before the push. A push can land and still report failure (a timeout, a crash, a restart), and those checkouts hold the old history: `Memory.preserve` would push it back as a `kin/conflict/*` branch, and a dream as `dream/`. While the marker exists, `ensureCheckouts` and `recover` won't clone (until the push lands the remote still has the old history), and `preserve`, the wake-up merge, `commitWakeup`, `commitDream` and `deleteDreamBranch` refuse to push. Only the purge's own reclone clones. 6. Force-pushes the changed `refs/heads/*` and `refs/tags/*` in one atomic push, each leased at its cloned value, through `KinPurgePush`. Failures are classified from `--porcelain` output (`lease_rejected`, `remote_rejected`, `auth`, `network`, `timeout`) without the cause, so neither the URL nor helper output is logged. This needs the owner as a bypass actor on the memory repo's ruleset. Closing open dream PRs that touch the paths waits on the GitHub client. 7. Reclones: clones fresh, records the rewritten anchors without preserving the old ones, rebuilds the read checkout, cache and `INDEX.md`, clears the marker, and resumes the queue. A failure or interruption before step 5 resumes the queue with nothing changed; from step 5 on, any failure, defect or interruption leaves `KinPurgeFailed` with `queueHeld: true` and the queue paused. 8. Returns the `execution_finished` audit input (grant `jti`, key thumbprint, device, new head) with the paths, old head, new head and rewritten refs; APPR-10 writes it as a `KinAuditEntry`. -9. Tells the owner what a purge can't reach: GitHub keeps `refs/pull/*` and cached PR diffs (removing those needs a GitHub Support request), forks, local backups (Time Machine), and the chat database. A leaked credential must be rotated whatever happens. +9. Tells the owner what a purge can't reach: GitHub keeps `refs/pull/*` (read-only, so the clone leaves them out and `reconcile` doesn't wait on them) and cached PR diffs (removing those needs a GitHub Support request), forks, local backups (Time Machine), and the chat database. A leaked credential must be rotated whatever happens. -`reconcile` mirror-clones the remote and checks whether the plan's paths (other than purge tombstones) or patterns are still in history. Gone is `done`; still there with the default branch at the marker's old head is `not_done`; anything else, or an unreachable remote, is `unknown` and changes nothing. `done` and `not_done` reclone, clear the marker and resume the queue. A re-run of the same plan while its marker exists reclones instead of resuming against old checkouts, and reports success when the push had already landed. +`reconcile` bare-clones the remote the same way and checks whether the plan's paths (other than this action's tombstones) or patterns are still in history; it reads the plan from the marker if the plan file is gone. Gone is `done`; still there with the default branch at the marker's old head is `not_done`; anything else, or an unreachable remote, is `unknown` and changes nothing. `done` and `not_done` reclone, clear the marker and resume the queue. A re-run of the same plan while its marker exists reclones instead of resuming against old checkouts, and reports success when the push had already landed. [Kin memory purge](../../operations/kin-memory-purge.md) covers clearing a stuck marker by hand. -**Owner credential.** `gitPushLayer` takes a reference to the owner's credential, never the secret: a git credential helper name or absolute path (no inline `!` script), or an SSH command naming its key file (no shell metacharacters; T3 appends `BatchMode`, `IdentitiesOnly` and `IdentityAgent=none`). This is the one exception to APPR's rule that the owner credential never reaches a child process: the helper or key is read by git or ssh, not by T3. With neither configured the push fails `credential_unavailable`, SSH without an owner command is refused rather than using the host's keys or agent, and `GIT_TERMINAL_PROMPT=0`, `GIT_ASKPASS`, `SSH_ASKPASS` and `SSH_ASKPASS_REQUIRE=never` keep anything from prompting. +**Owner credential.** `gitPushLayer` takes a reference to the owner's credential, never the secret: a git credential helper name or absolute path (no inline `!` script), or an SSH command naming its key file (no shell metacharacters; T3 appends `-F none`, `BatchMode`, `IdentitiesOnly` and `IdentityAgent=none`). An `https://` remote needs the helper and an SSH remote the command; `file://` takes neither, and any other transport fails `unsupported_remote`. git runs with an empty scratch `HOME`, so `~/.netrc` and `~/.ssh/config` can't supply another credential. This is the one exception to APPR's rule that the owner credential never reaches a child process: the helper or key is read by git or ssh, not by T3. With neither configured the push fails `credential_unavailable`, SSH without an owner command is refused rather than using the host's keys or agent, and `GIT_TERMINAL_PROMPT=0`, `GIT_ASKPASS`, `SSH_ASKPASS` and `SSH_ASKPASS_REQUIRE=never` keep anything from prompting. **Dependency:** `git-filter-repo` (git's recommended history rewriter; MIT, also usable under git's GPL), checked at v2.47.0. TRIG-20 installs it on the Kin host; `GitFilterRepoCommand` overrides the binary path. The real-rewrite tests skip on machines without it. @@ -567,7 +567,7 @@ Kin runs on one always-on machine, the **Kin host**. The laptop and phone are cl - **Deferral:** unchanged recheck, then heads-up, then silence, then `started` at due. A linked issue closed before due leads to `asked`, and nothing starts. With the stop switch on, it asks. [Monday] re-arms to next Monday 09:00. A `due` fire consumed two hours late asks; a queued fire for a disarmed or moved deferral is dropped. - **Dream:** a dream touching `rules.md` fails, and `main` is unchanged. A dream with one journal fold and one `decisions/` change opens two PRs, and only the auto one merges. - **Secret scan:** a journal line holding `ghp_` plus 36 characters blocks the entire commit, and the owner gets one typed notification without the value. -- **Purge:** on a fixture repo, the file is gone from `git log --all -p` and the recloned checkout, a tombstone exists and parses, and the returned audit input carries the old and new heads. A surviving secret fails closed before the push. A push that lands and then errors, crashes or is interrupted never republishes the old history through the next wake-up, and `reconcile` reports `done`. +- **Purge:** on a fixture repo, the file is gone from `git log --all -p` and the recloned checkout, a tombstone exists and parses, and the returned audit input carries the old and new heads. A surviving secret fails closed before the push. A push that lands and then errors, crashes or is interrupted never republishes the old history through the next wake-up, a dream branch or a checkout cloned during the push, and `reconcile` reports `done` with a `refs/pull/*` ref still holding it. - **Contract tests:** `vp test run kin-contract` passes on today's upstream. Renaming `scheduledTaskId` in a scratch branch makes it fail. ## Tasks diff --git a/docs/operations/kin-memory-purge.md b/docs/operations/kin-memory-purge.md new file mode 100644 index 0000000000000..a14bc4e8cb642 --- /dev/null +++ b/docs/operations/kin-memory-purge.md @@ -0,0 +1,9 @@ +# Kin memory purge + +> For maintainers operating a Kin host. The design is in [the triggers spec](../kin/specs/triggers.md#owner-signed-purge-gap-5). + +## A stuck purge marker + +A purge writes `/kin//purge-in-flight` before it force-pushes, and leaves it there when the push fails, crashes or is interrupted. While it exists, the org's lead queue stays paused, memory won't clone a checkout, and nothing pushes memory history. `reconcile` on the action clears it: it reads the plan from the marker, so it works even after the plan file is gone. Retry `reconcile` first; it answers `unknown` while the remote is unreachable or has moved. + +Delete the marker by hand only when `reconcile` keeps answering `unknown` and you have checked the remote yourself. Deleting it tells Kin that no local copy of the old history will be pushed again, so first delete `/memory`, `/memory-read` and `/dreams` too, and confirm with `git log --branches --tags -p` on a fresh clone whether the purged content is still on the remote. If it is, run the purge again from a new owner-approved action; if it's gone, the next wake-up clones the rewritten history. Then resume the org's queue. From 2d5819bb45c0fcbc8b65fe39ec9191835fa2e9f0 Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Mon, 9 Nov 2026 00:52:07 +0530 Subject: [PATCH 4/7] fix(kin): purge refuses foreign refs and stale wake-ups (TRIG-17) - Lists remote refs with the owner credential and fails closed at clone when anything outside heads, tags and pulls is advertised (notes, custom refs); reconcile answers unknown. - Memory bumps a per-org history generation on quarantine and reclone; a MemoryWriter session staged under an older generation fails MemorySyncConflict and is dropped instead of re-committing purged content. - Stores the exact tombstone blob ids in the marker and beside the plan, so planInHistory no longer trusts look-alike front matter. - dropCheckouts also sweeps half-finished memory.clone-* directories. - Uses KinPendingAction and KinGrantPayload from contracts. - Ops doc: refs a purge can't rewrite, stop the server before manual recovery, every path to delete, and no resume path before LEAD-11. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/kin/lead/LeadPrompt.test.ts | 1 + apps/server/src/kin/lead/layer.test.ts | 1 + apps/server/src/kin/memory/Memory.ts | 31 ++- apps/server/src/kin/memory/MemoryWriter.ts | 12 + apps/server/src/kin/memory/purge.test.ts | 230 +++++++++++++++--- apps/server/src/kin/memory/purge.ts | 245 ++++++++++++-------- apps/server/src/kin/memory/purgePush.ts | 184 ++++++++------- docs/kin/specs/triggers.md | 16 +- docs/operations/kin-memory-purge.md | 17 +- 9 files changed, 504 insertions(+), 233 deletions(-) diff --git a/apps/server/src/kin/lead/LeadPrompt.test.ts b/apps/server/src/kin/lead/LeadPrompt.test.ts index 92b5771034382..e064bf6219cc4 100644 --- a/apps/server/src/kin/lead/LeadPrompt.test.ts +++ b/apps/server/src/kin/lead/LeadPrompt.test.ts @@ -65,6 +65,7 @@ const memory = Layer.succeed( recover: () => Effect.void, withHistoryRewrite: () => Effect.die("Unused"), purgeInFlight: () => Effect.die("Unused"), + historyGeneration: () => Effect.die("Unused"), readPromptSnapshot: (org, head) => org !== orgId || !["head-one", "head-two"].includes(head) ? Effect.fail(new Memory.MemoryError({ orgId: org, operation: "promptSnapshot" })) diff --git a/apps/server/src/kin/lead/layer.test.ts b/apps/server/src/kin/lead/layer.test.ts index f1d1e8a6722c8..dd1c1f2318a5f 100644 --- a/apps/server/src/kin/lead/layer.test.ts +++ b/apps/server/src/kin/lead/layer.test.ts @@ -46,6 +46,7 @@ vi.mock("../memory/Memory.ts", async (importOriginal) => { recover: () => Effect.die("unused"), withHistoryRewrite: () => Effect.die("unused"), purgeInFlight: () => Effect.die("unused"), + historyGeneration: () => Effect.die("unused"), ensureCheckouts: () => Effect.succeed({ writer: "/private/tmp", read: "/private/tmp" }), prepareWakeup: () => Effect.sync(() => { diff --git a/apps/server/src/kin/memory/Memory.ts b/apps/server/src/kin/memory/Memory.ts index 8c910d19c3fce..a4d4da7437d81 100644 --- a/apps/server/src/kin/memory/Memory.ts +++ b/apps/server/src/kin/memory/Memory.ts @@ -176,6 +176,8 @@ export class Memory extends Context.Service< ) => Effect.Effect; /** The durable marker of an unfinished history rewrite; while present, nothing pushes old anchors. */ readonly purgeInFlight: (orgId: KinOrgId) => Effect.Effect; + /** Bumped whenever a history rewrite drops the checkouts; work staged under an older value is stale. */ + readonly historyGeneration: (orgId: KinOrgId) => Effect.Effect; readonly ensureCheckouts: (orgId: KinOrgId) => Effect.Effect; readonly prepareWakeup: ( orgId: KinOrgId, @@ -328,6 +330,7 @@ const make = Effect.gen(function* () { const cache = yield* MemoryCache.MemoryCache; const locks = new Map(); const initializedCheckouts = new Set(); + const historyGenerations = new Map(); const duplicateWarnings = new Map>(); const snapshots = new Map(); const blobCaches = new Map(); @@ -453,6 +456,19 @@ const make = Effect.gen(function* () { }); const markerPath = (root: string) => path.join(root, "purge-in-flight"); + /** Half-finished clones from a crash; after a purge they can hold the old history. */ + const removeStaleClones = Effect.fn("Memory.removeStaleClones")(function* ( + orgId: KinOrgId, + root: string, + ) { + for (const entry of yield* io(orgId, "clones", fs.readDirectory(root))) + if (/^memory\.clone-[A-Za-z0-9]+$/.test(entry)) + yield* io( + orgId, + "recoverClone", + fs.remove(path.join(root, entry), { recursive: true, force: true }), + ); + }); const purgeInFlight = Effect.fn("Memory.purgeInFlight")(function* (orgId: KinOrgId) { const { root } = yield* pathsFor(orgId); if (!(yield* io(orgId, "purgeMarker", fs.exists(markerPath(root))))) return undefined; @@ -538,14 +554,7 @@ const make = Effect.gen(function* () { } else { // The remote can still hold the old history until the purge's push lands. yield* refuseDuringPurge(orgId, "clone"); - for (const entry of yield* io(orgId, "clones", fs.readDirectory(paths.root))) { - if (/^memory\.clone-[A-Za-z0-9]+$/.test(entry)) - yield* io( - orgId, - "recoverClone", - fs.remove(path.join(paths.root, entry), { recursive: true, force: true }), - ); - } + yield* removeStaleClones(orgId, paths.root); yield* io( orgId, "clone", @@ -1329,9 +1338,12 @@ const make = Effect.gen(function* () { const dropCheckouts = Effect.fn("Memory.dropCheckouts")(function* (orgId: KinOrgId) { const paths = yield* pathsFor(orgId); + historyGenerations.set(orgId, (historyGenerations.get(orgId) ?? 0) + 1); // `dreams/` holds plain copies of memory files for a running dream. for (const directory of [paths.writer, paths.read, path.join(paths.root, "dreams")]) yield* io(orgId, "quarantine", fs.remove(directory, { recursive: true, force: true })); + if (yield* io(orgId, "exists", fs.exists(paths.root))) + yield* removeStaleClones(orgId, paths.root); initializedCheckouts.delete(orgId); pinnedHeads.delete(orgId); snapshots.delete(orgId); @@ -1377,6 +1389,9 @@ const make = Effect.gen(function* () { purgeInFlight: Effect.fn("Memory.purgeInFlightLocked")((orgId) => locked(orgId, purgeInFlight(orgId)), ), + historyGeneration: Effect.fn("Memory.historyGeneration")(function* (orgId) { + return historyGenerations.get(orgId) ?? 0; + }), recover: Effect.fn("Memory.recoverLocked")((orgId) => withWriterLock(orgId, locked(orgId, recover(orgId))), ), diff --git a/apps/server/src/kin/memory/MemoryWriter.ts b/apps/server/src/kin/memory/MemoryWriter.ts index 42d95d7e726d1..3aa32c0c88cbb 100644 --- a/apps/server/src/kin/memory/MemoryWriter.ts +++ b/apps/server/src/kin/memory/MemoryWriter.ts @@ -196,6 +196,7 @@ const make = Effect.gen(function* () { KinOrgId, { wakeupId: string; + generation: number; files: Map; lines: string[]; supersessions: Map; @@ -307,8 +308,19 @@ const make = Effect.gen(function* () { const previous = sessions.get(orgId); if (previous && previous.wakeupId !== wakeupId) return yield* new MemoryWriteRejected({ reason: "Another wake-up owns the writer" }); + const generation = yield* memory.historyGeneration(orgId); + // A purge rewrote history after this wake-up staged; committing it could restore purged content. + if (previous && previous.generation !== generation) { + sessions.delete(orgId); + return yield* new MemorySyncConflict({ + orgId, + operation: "push", + reason: "Memory history was rewritten during this wake-up; its staged changes were dropped", + }); + } const state = previous ?? { wakeupId, + generation, files: new Map(), lines: [], supersessions: new Map(), diff --git a/apps/server/src/kin/memory/purge.test.ts b/apps/server/src/kin/memory/purge.test.ts index cd125cd3ef10e..b0d124776e9db 100644 --- a/apps/server/src/kin/memory/purge.test.ts +++ b/apps/server/src/kin/memory/purge.test.ts @@ -3,8 +3,18 @@ import * as NodeChildProcess from "node:child_process"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { describe, expect, it } from "@effect/vitest"; import { vi } from "vite-plus/test"; -import { KinAgentId, KinMemoryItemId, KinOrgId } from "@t3tools/contracts"; +import { + KinActionId, + KinAgentId, + KinMemoryItemId, + KinOrgId, + KinPersonId, + ThreadId, + type KinGrantPayload, + type KinPendingAction, +} from "@t3tools/contracts"; import * as Cause from "effect/Cause"; +import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import * as FileSystem from "effect/FileSystem"; @@ -27,21 +37,42 @@ const token = ["gh", "p_"].join("") + "A1b2C3d4E5".repeat(3) + "abcdef"; // TRIG-20 installs git-filter-repo on the Kin host; machines without it skip the real rewrite. const filterRepoMissing = NodeChildProcess.spawnSync("git-filter-repo", ["--version"]).status !== 0; -const grant: Purge.PurgeGrant = { - id: "grant-1", +const actionId = KinActionId.make("act-1"); +const thumbprint = "T".repeat(43); +const targetDigest = "d".repeat(64); +const actionFor = (plan: Purge.PurgePlan): KinPendingAction => ({ + id: actionId, orgId, - actionId: "act-1", + idempotencyKey: "e".repeat(64), + kind: "memory_purge", + target: Purge.purgePlanJson(plan).target, + resolved: { _tag: "none" }, + targetDigest, + nonce: "fixture-nonce", revision: 1, - jti: "jti-1", - approverHumanId: "fixture-owner", - keyThumbprint: "thumb-1", -}; -const actionFor = (plan: Purge.PurgePlan): Purge.PurgeAction => ({ - id: "act-1", + reason: "Invented purge", + requestedBy: { agentId: KinAgentId.make("fixture-lead"), threadId: ThreadId.make("thread-1") }, + provenance: { startedBy: "schedule", startedByRole: "none", chain: [] }, + requiredApprovers: [{ role: "owner" }], + state: "executing", + createdAt: DateTime.makeUnsafe("2026-11-08T00:00:00.000Z"), + updatedAt: DateTime.makeUnsafe("2026-11-08T00:00:00.000Z"), + expiresAt: DateTime.makeUnsafe("2026-11-09T00:00:00.000Z"), +}); +const grantFor = (plan: Purge.PurgePlan): KinGrantPayload => ({ + v: 1, orgId, - kind: "memory_purge", + actionId, revision: 1, + nonce: "fixture-nonce", + jti: "jti-1", + kind: "memory_purge", + targetDigest, target: Purge.purgePlanJson(plan).target, + approver: { humanId: KinPersonId.make("fixture-owner"), kid: thumbprint }, + decision: "approve", + iat: 0, + exp: 900, }); const gateOpen = Effect.provideService(Purge.PurgeExecutorGate, true); @@ -224,7 +255,9 @@ const runPurge = (plan: Purge.PurgePlan) => Effect.gen(function* () { const target = yield* (yield* Purge.KinPurge).storePlan(plan); expect(target).toEqual(actionFor(plan).target); - return yield* Purge.memoryPurgeHandler.run(actionFor(plan), grant).pipe(gateOpen, Effect.exit); + return yield* Purge.memoryPurgeHandler + .run(actionFor(plan), grantFor(plan)) + .pipe(gateOpen, Effect.exit); }); const reconcile = (plan: Purge.PurgePlan) => Purge.memoryPurgeHandler.reconcile(actionFor(plan)).pipe(gateOpen); @@ -241,7 +274,7 @@ describe.skipIf(filterRepoMissing)("purge with git-filter-repo", () => { remote, remoteUrl, events, - purgeLayer, + writerLayer, writerHistory, prepare, pullHead, @@ -249,7 +282,18 @@ describe.skipIf(filterRepoMissing)("purge with git-filter-repo", () => { } = yield* fixture(mode === "redact_text"); yield* Effect.gen(function* () { const memory = yield* Memory.Memory; + const writer = yield* Writer.MemoryWriter; const paths = yield* prepare; + // A wake-up staged against the old history before the queue paused. + yield* writer.stage(orgId, "wake1", { + path: "claims/staged.md", + content: fixtureItem({ + id: "claim-staged", + kind: "claim", + title: "Invented staged", + links: [], + }), + }); const oldMain = yield* git(remote, ["rev-parse", "refs/heads/main"]); const oldSide = yield* git(remote, ["rev-parse", "refs/heads/side"]); const plan: Purge.PurgePlan = @@ -267,6 +311,10 @@ describe.skipIf(filterRepoMissing)("purge with git-filter-repo", () => { const newSide = yield* git(remote, ["rev-parse", "refs/heads/side"]); expect(events).toEqual(["pause", "resume"]); expect(yield* memory.purgeInFlight(orgId)).toBeUndefined(); + expect( + yield* writer.commitWakeup(orgId, "wake1", "completed").pipe(Effect.result), + ).toMatchObject({ failure: { _tag: "MemorySyncConflict" } }); + expect(yield* git(remote, ["rev-parse", "refs/heads/main"])).toBe(newMain); expect(yield* reconcile(plan)).toBe("done"); expect(yield* git(remote, ["rev-parse", "refs/pull/1/head"])).toBe(pullHead); expect(result).toMatchObject({ oldHead: oldMain, newHead: newMain }); @@ -280,7 +328,7 @@ describe.skipIf(filterRepoMissing)("purge with git-filter-repo", () => { actionId: "act-1", kind: "memory_purge", grantJti: "jti-1", - keyThumbprint: "thumb-1", + keyThumbprint: thumbprint, headSha: newMain, }); expect(JSON.parse(result.audit.result)).toMatchObject({ @@ -301,6 +349,21 @@ describe.skipIf(filterRepoMissing)("purge with git-filter-repo", () => { return; } expect(result.tombstones).toEqual(["claims/leak.md"]); + // Only this purge's exact tombstone counts as purged; a look-alike at the path is old content. + const later = `${root}/later`; + yield* git(root, ["clone", "-q", "--", remote, later]); + yield* fs.writeFileString(`${later}/claims/leak.md`, `${leak}\nInvented restored note\n`); + yield* git(later, [ + "-c", + "user.name=Fixture Bot", + "-c", + "user.email=fixture@example.invalid", + "commit", + "-qam", + "look-alike tombstone", + ]); + yield* git(later, ["push", "-q", "origin", "main"]); + expect(yield* reconcile(plan)).toBe("not_done"); const tombstone = yield* parseFrontMatter(orgId, "claims/leak.md", leak + "\n"); expect(tombstone.frontMatter).toMatchObject({ id: "claim-leak", @@ -317,7 +380,7 @@ describe.skipIf(filterRepoMissing)("purge with git-filter-repo", () => { (yield* memory.readItemQuietly(orgId, KinMemoryItemId.make("claim-leak"))).head.status, ).toBe("cancelled"); expect(yield* fs.readFileString(`${paths.read}/INDEX.md`)).toContain("Purged item"); - }).pipe(Effect.provide(purgeLayer())); + }).pipe(Effect.provide(writerLayer())); }).pipe(Effect.provide(services), Effect.scoped), ); @@ -327,6 +390,8 @@ describe.skipIf(filterRepoMissing)("purge with git-filter-repo", () => { ["path typo", { _tag: "KinPurgeRejected" }], ["interrupted", undefined], ["no pause", { _tag: "KinPurgePauseUnavailable" }], + // `clone --bare` leaves notes and custom refs behind, and they would keep the old history. + ["notes ref", { _tag: "KinPurgeFailed", stage: "clone", queueHeld: false }], ] as const)( "a purge that stops before the push resumes the queue and changes nothing: %s", ([scenario, expected]) => @@ -348,6 +413,8 @@ describe.skipIf(filterRepoMissing)("purge with git-filter-repo", () => { scenario === "wrong pattern" ? { orgId, repo: remoteUrl, mode: "redact_text", patterns: ["Invented vendor note"] } : dropPlan(remoteUrl, scenario === "path typo" ? ["claims/leek.md"] : undefined); + if (scenario === "notes ref") + yield* git(remote, ["update-ref", "refs/notes/commits", "refs/heads/side"]); const exit = yield* runPurge(plan); expect(Exit.isFailure(exit)).toBe(true); @@ -362,6 +429,12 @@ describe.skipIf(filterRepoMissing)("purge with git-filter-repo", () => { expect(failureOf(exit)).toMatchObject({ value: { reason: expect.stringMatching(/not in history/) }, }); + if (scenario === "notes ref") { + expect(failureOf(exit)).toMatchObject({ + value: { reason: expect.stringContaining("refs/notes/commits") }, + }); + expect(yield* reconcile(plan)).toBe("unknown"); + } expect(JSON.stringify(exit)).not.toContain(token); expect(events).toEqual(scenario === "no pause" ? [] : ["pause", "resume"]); expect(yield* git(remote, ["rev-parse", "refs/heads/main"])).toBe(oldMain); @@ -445,10 +518,10 @@ describe.skipIf(filterRepoMissing)("purge with git-filter-repo", () => { message: "dream", }) .pipe(Effect.result); - expect(dream._tag).toBe("Failure"); + expect(dream).toMatchObject({ failure: { _tag: "MemorySyncConflict" } }); expect( - (yield* writer.commitWakeup(orgId, "wake1", "completed").pipe(Effect.result))._tag, - ).toBe("Failure"); + yield* writer.commitWakeup(orgId, "wake1", "completed").pipe(Effect.result), + ).toMatchObject({ failure: { _tag: "MemorySyncConflict" } }); return paths; }).pipe(Effect.provide(landing)); @@ -554,6 +627,73 @@ describe.skipIf(filterRepoMissing)("purge with git-filter-repo", () => { ); }); +it.live("a marker left before the checkouts dropped blocks every writer push until reconcile", () => + Effect.gen(function* () { + const { fs, git, root, remote, remoteUrl, events, writerLayer, prepare } = yield* fixture(); + const plan = dropPlan(remoteUrl); + yield* Effect.gen(function* () { + const memory = yield* Memory.Memory; + const writer = yield* Writer.MemoryWriter; + const paths = yield* prepare; + const oldMain = yield* git(remote, ["rev-parse", "refs/heads/main"]); + const { json, target } = Purge.purgePlanJson(plan); + yield* (yield* Purge.KinPurge).storePlan(plan); + // A crash after the marker rename and before `dropCheckouts`. + yield* fs.writeFileString( + `${root}/state/purge-in-flight`, + JSON.stringify({ + sha256: target.sha256, + actionId, + oldHead: oldMain, + plan: json, + tombstones: [], + }), + ); + expect(yield* fs.exists(paths.writer)).toBe(true); + const guarded = { + failure: { + _tag: "MemorySyncConflict", + operation: "push", + reason: expect.stringMatching(/purge is in flight/), + }, + }; + yield* writer.stage(orgId, "wake1", { + path: "claims/staged.md", + content: fixtureItem({ + id: "claim-staged", + kind: "claim", + title: "Invented staged", + links: [], + }), + }); + expect( + yield* writer.commitWakeup(orgId, "wake1", "completed").pipe(Effect.result), + ).toMatchObject(guarded); + expect( + yield* writer + .commitDream(orgId, { + branch: "dream/2026-11-08", + base: oldMain, + files: new Map(), + message: "dream", + }) + .pipe(Effect.result), + ).toMatchObject(guarded); + expect( + yield* writer.deleteDreamBranch(orgId, "dream/2026-11-08").pipe(Effect.result), + ).toMatchObject(guarded); + expect(yield* git(remote, ["rev-parse", "refs/heads/main"])).toBe(oldMain); + expect(yield* git(remote, ["for-each-ref", "--format=%(refname)", "refs/heads/dream/"])).toBe( + "", + ); + + expect(yield* reconcile(plan)).toBe("not_done"); + expect(events).toEqual(["resume"]); + expect(yield* memory.purgeInFlight(orgId)).toBeUndefined(); + }).pipe(Effect.provide(writerLayer())); + }).pipe(Effect.provide(services), Effect.scoped), +); + const drop: Purge.PurgePlan = { orgId, repo: "fixture-remote", @@ -581,10 +721,10 @@ it.effect("the executor gate refuses a purge until APPR-9, APPR-10 and LEAD-11 l const action = actionFor(drop); for (const attempt of [ Purge.memoryPurgeHandler.precheck(action), - Purge.memoryPurgeHandler.run(action, grant), + Purge.memoryPurgeHandler.run(action, grantFor(drop)), Purge.memoryPurgeHandler.reconcile(action), Effect.gen(function* () { - return yield* (yield* Purge.KinPurge).purge(action, grant); + return yield* (yield* Purge.KinPurge).purge(action, grantFor(drop)); }), ]) { const result = yield* Effect.result(attempt); @@ -616,7 +756,7 @@ it.effect("a stored plan binds its file to the action digest until it is discard }); expect( yield* purge - .precheck({ ...action, target: { ...action.target, path: "../escape.json" } }) + .precheck({ ...action, target: { ...target, path: "../escape.json" } }) .pipe(gateOpen, Effect.result), ).toMatchObject({ failure: { _tag: "KinPurgeRejected", reason: "Invalid plan path" } }); @@ -804,7 +944,7 @@ it.each([ ["duplicate", { paths: ["claims/leak.md", "claims/leak.md"] }], ] as const)("rejects an unsafe drop plan: %s", (_name, overrides) => { const plan = { ...drop, ...overrides }; - expect(Purge.purgeRejection(plan, actionFor(plan), grant)).toBeDefined(); + expect(Purge.purgeRejection(plan, actionFor(plan), grantFor(plan))).toBeDefined(); }); it.each([ @@ -815,39 +955,59 @@ it.each([ ["extra rule", ["Invented text\nregex:.*"]], ] as const)("rejects an unsafe redact plan: %s", (_name, patterns) => { const plan: Purge.PurgePlan = { orgId, repo: "fixture-remote", mode: "redact_text", patterns }; - expect(Purge.purgeRejection(plan, actionFor(plan), grant)).toBeDefined(); + expect(Purge.purgeRejection(plan, actionFor(plan), grantFor(plan))).toBeDefined(); }); it.each([ [ "digest", - (a: Purge.PurgeAction, g: Purge.PurgeGrant) => - [{ ...a, target: { ...a.target, sha256: "0".repeat(64) } }, g] as const, + (a: KinPendingAction, g: KinGrantPayload) => + [ + { ...a, target: { ...Purge.purgePlanJson(drop).target, sha256: "0".repeat(64) } }, + g, + ] as const, ], [ "plan path", - (a: Purge.PurgeAction, g: Purge.PurgeGrant) => - [{ ...a, target: { ...a.target, path: "purge-plans/other.json" } }, g] as const, + (a: KinPendingAction, g: KinGrantPayload) => + [ + { ...a, target: { ...Purge.purgePlanJson(drop).target, path: "purge-plans/other.json" } }, + g, + ] as const, ], [ "action", - (a: Purge.PurgeAction, g: Purge.PurgeGrant) => [a, { ...g, actionId: "act-2" }] as const, + (a: KinPendingAction, g: KinGrantPayload) => + [a, { ...g, actionId: KinActionId.make("act-2") }] as const, + ], + ["revision", (a: KinPendingAction, g: KinGrantPayload) => [a, { ...g, revision: 2 }] as const], + [ + "grant target", + (a: KinPendingAction, g: KinGrantPayload) => + [ + a, + { ...g, target: Purge.purgePlanJson({ ...drop, paths: ["claims/keep.md"] }).target }, + ] as const, + ], + [ + "grant digest", + (a: KinPendingAction, g: KinGrantPayload) => + [a, { ...g, targetDigest: "0".repeat(64) }] as const, ], - ["revision", (a: Purge.PurgeAction, g: Purge.PurgeGrant) => [a, { ...g, revision: 2 }] as const], [ "org", - (a: Purge.PurgeAction, g: Purge.PurgeGrant) => + (a: KinPendingAction, g: KinGrantPayload) => [a, { ...g, orgId: KinOrgId.make("other-org") }] as const, ], [ "action id", - (a: Purge.PurgeAction, g: Purge.PurgeGrant) => + (a: KinPendingAction, g: KinGrantPayload) => [ - { ...a, id: "act 1\nforged" }, - { ...g, actionId: "act 1\nforged" }, + { ...a, id: KinActionId.make("act 1\nforged") }, + { ...g, actionId: KinActionId.make("act 1\nforged") }, ] as const, ], ] as const)("rejects authority that does not bind the plan: %s", (_name, tamper) => { - expect(Purge.purgeRejection(drop, actionFor(drop), grant)).toBeUndefined(); - expect(Purge.purgeRejection(drop, ...tamper(actionFor(drop), grant))).toBeDefined(); + expect(Purge.purgeRejection(drop, actionFor(drop), grantFor(drop))).toBeUndefined(); + expect(Purge.purgeRejection(drop, ...tamper(actionFor(drop), grantFor(drop)))).toBeDefined(); }); diff --git a/apps/server/src/kin/memory/purge.ts b/apps/server/src/kin/memory/purge.ts index 0f16e5b1c18f2..7b03fe9d0f189 100644 --- a/apps/server/src/kin/memory/purge.ts +++ b/apps/server/src/kin/memory/purge.ts @@ -1,6 +1,12 @@ // @effect-diagnostics-next-line nodeBuiltinImport:off -- The plan digest must match the grant's sha256 content target. import * as NodeCrypto from "node:crypto"; -import { KinOrgId, type MemoryItemKind } from "@t3tools/contracts"; +import { + KinOrgId, + type KinActionTarget, + type KinGrantPayload, + type KinPendingAction, + type MemoryItemKind, +} from "@t3tools/contracts"; import * as Cause from "effect/Cause"; import * as Clock from "effect/Clock"; import * as Context from "effect/Context"; @@ -41,35 +47,24 @@ const PurgePlanSchema = Schema.Union([ export type PurgePlan = typeof PurgePlanSchema.Type; const decodePlan = Schema.decodeUnknownEffect(Schema.fromJsonString(PurgePlanSchema)); +const Tombstones = Schema.Array(Schema.String); /** Carries the plan so `reconcile` and re-runs still work if the plan file is lost. */ const PurgeMarker = Schema.Struct({ sha256: Schema.String, actionId: Schema.String, oldHead: Schema.String, plan: Schema.String, + /** Blob ids of this purge's own tombstones; any other blob at a purged path is old content. */ + tombstones: Tombstones, }); const decodeMarker = Schema.decodeUnknownOption(Schema.fromJsonString(PurgeMarker)); +const decodeTombstones = Schema.decodeUnknownOption(Schema.fromJsonString(Tombstones)); +const encodeTombstones = Schema.encodeEffect(Schema.fromJsonString(Tombstones)); -/** Lane B's `KinGrant` (APPR-4) fields the purge binds to. The executor verifies the signature. */ -export interface PurgeGrant { - readonly id: string; - readonly orgId: KinOrgId; - readonly actionId: string; - readonly revision: number; - readonly jti: string; - readonly approverHumanId: string; - readonly keyThumbprint: string; - readonly deviceLabel?: string; -} - -/** Lane B's `KinPendingAction` (APPR-2) for kind `memory_purge`; the target is `purgePlanJson(plan).target`. */ -export interface PurgeAction { - readonly id: string; - readonly orgId: KinOrgId; - readonly kind: "memory_purge"; - readonly revision: number; - readonly target: { readonly _tag: "content"; readonly path: string; readonly sha256: string }; -} +/** A `memory_purge` action's target: the stored plan file and its sha256. */ +export type PurgeTarget = Extract; +const purgeTarget = (action: KinPendingAction) => + action.kind === "memory_purge" && action.target._tag === "content" ? action.target : undefined; /** Lane B's `KinAuditAppendInput` (APPR-10) for the `execution_finished` entry. */ export interface PurgeAuditInput { @@ -79,7 +74,6 @@ export interface PurgeAuditInput { readonly kind: "memory_purge"; readonly grantJti: string; readonly keyThumbprint: string; - readonly deviceLabel?: string; readonly headSha: string; readonly result: string; } @@ -122,7 +116,10 @@ export class KinPurgeFailed extends Schema.TaggedError()("KinPur cause: Schema.optional(Schema.Defect()), }) {} -/** LEAD-11 supplies the real org pause; until then the purge can't run. */ +/** + * LEAD-11 supplies the real org pause; until then the purge can't run. `pause` must not return while a + * wake-up or dream is still running for the org: their staged files and dream copies predate the rewrite. + */ export class KinPurgeQueue extends Context.Reference<{ readonly pause: (orgId: KinOrgId) => Effect.Effect; readonly resume: (orgId: KinOrgId) => Effect.Effect; @@ -187,7 +184,11 @@ export const purgePlanJson = (plan: PurgePlan) => { const digest = sha256(json); return { json, - target: { _tag: "content" as const, path: `purge-plans/${digest}.json`, sha256: digest }, + target: { + _tag: "content", + path: `purge-plans/${digest}.json`, + sha256: digest, + } satisfies PurgeTarget, }; }; @@ -205,15 +206,27 @@ const planRejection = (plan: PurgePlan) => { }; /** Returns why the plan, its action or its grant is unacceptable, or `undefined`. */ -export const purgeRejection = (plan: PurgePlan, action: PurgeAction, grant?: PurgeGrant) => { - if (action.kind !== "memory_purge" || action.target._tag !== "content") - return "Action is not a memory purge"; +export const purgeRejection = ( + plan: PurgePlan, + action: KinPendingAction, + grant?: KinGrantPayload, +) => { + const actionTarget = purgeTarget(action); + if (!actionTarget) return "Action is not a memory purge"; if (action.orgId !== plan.orgId || (grant !== undefined && grant.orgId !== plan.orgId)) return "Grant, action and plan organizations differ"; - if (grant !== undefined && (grant.actionId !== action.id || grant.revision !== action.revision)) + if ( + grant !== undefined && + (grant.actionId !== action.id || + grant.revision !== action.revision || + grant.kind !== action.kind || + grant.targetDigest !== action.targetDigest || + grant.target._tag !== "content" || + grant.target.sha256 !== actionTarget.sha256) + ) return "Grant does not bind this action revision"; const { target } = purgePlanJson(plan); - if (action.target.sha256 !== target.sha256 || action.target.path !== target.path) + if (actionTarget.sha256 !== target.sha256 || actionTarget.path !== target.path) return "Plan does not match the approved digest"; if (!/^[A-Za-z0-9_-]{1,128}$/.test(action.id)) return "Invalid action id"; return planRejection(plan); @@ -238,22 +251,22 @@ export class KinPurge extends Context.Service< /** Writes the plan file a `memory_purge` action targets; the proposer calls it before `request_action`. */ readonly storePlan: ( plan: PurgePlan, - ) => Effect.Effect; + ) => Effect.Effect; /** Deletes the plan file once the action is terminal; it can hold the redacted secret. */ - readonly discardPlan: (target: PurgeAction["target"]) => Effect.Effect; + readonly discardPlan: (target: PurgeTarget) => Effect.Effect; readonly precheck: ( - action: PurgeAction, + action: KinPendingAction, ) => Effect.Effect; /** Server-only: reached through `memoryPurgeHandler`, never a model, MCP tool or RPC. */ readonly purge: ( - action: PurgeAction, - grant: PurgeGrant, + action: KinPendingAction, + grant: KinGrantPayload, ) => Effect.Effect< PurgeResult, KinPurgeExecutorNotReady | KinPurgeRejected | KinPurgePauseUnavailable | KinPurgeFailed >; readonly reconcile: ( - action: PurgeAction, + action: KinPendingAction, ) => Effect.Effect< "done" | "not_done" | "unknown", KinPurgeExecutorNotReady | KinPurgeRejected | KinPurgeFailed @@ -308,7 +321,7 @@ const make = Effect.gen(function* () { Effect.fail(new KinPurgeFailed({ orgId, stage, reason, queueHeld: false })); const reject = (reason: string) => Effect.fail(new KinPurgeRejected({ reason })); - const gate = Effect.fn("KinPurge.gate")(function* (action: PurgeAction) { + const gate = Effect.fn("KinPurge.gate")(function* (action: KinPendingAction) { if (!(yield* PurgeExecutorGate)) return yield* new KinPurgeExecutorNotReady({ actionId: action.id }); }); @@ -318,10 +331,12 @@ const make = Effect.gen(function* () { return repo && repo.repoUrl === plan.repo ? repo : undefined; }; - const planFile = (target: PurgeAction["target"]) => + const planFile = (target: PurgeTarget) => planFilePath.test(target.path) && target.path === `purge-plans/${target.sha256}.json` ? Effect.succeed(path.resolve(config.baseDirectory, target.path)) : reject("Invalid plan path"); + /** The tombstone ids outlive the marker, so a `reconcile` after a finished purge still knows them. */ + const tombstonesFile = (planPath: string) => planPath.replace(/\.json$/, ".tombstones.json"); const markerFor = (orgId: KinOrgId) => memory.purgeInFlight(orgId).pipe( @@ -331,17 +346,18 @@ const make = Effect.gen(function* () { })), ); - const loadPlan = Effect.fn("KinPurge.loadPlan")(function* (action: PurgeAction) { - if (action.kind !== "memory_purge" || action.target._tag !== "content") - return yield* reject("Action is not a memory purge"); - const file = yield* planFile(action.target); + const loadPlan = Effect.fn("KinPurge.loadPlan")(function* ( + action: KinPendingAction, + target: PurgeTarget, + ) { + const file = yield* planFile(target); const stored = yield* fs.readFileString(file).pipe(Effect.option); const json = Option.isSome(stored) ? stored.value : config.repositories.some((entry) => entry.orgId === action.orgId) ? yield* markerFor(action.orgId).pipe( Effect.map(({ marker }) => - marker?.sha256 === action.target.sha256 ? marker.plan : undefined, + marker?.sha256 === target.sha256 ? marker.plan : undefined, ), Effect.orElseSucceed(() => undefined), ) @@ -352,16 +368,18 @@ const make = Effect.gen(function* () { ); if (purgePlanJson(plan).json !== json) return yield* reject("Plan does not match the approved digest"); - return plan; + return { plan, file }; }); /** Validates the action against its stored plan and the org's config, and returns any purge marker. */ const checked = Effect.fn("KinPurge.checked")(function* ( - action: PurgeAction, - grant?: PurgeGrant, + action: KinPendingAction, + grant?: KinGrantPayload, ) { yield* gate(action); - const plan = yield* loadPlan(action); + const target = purgeTarget(action); + if (!target) return yield* reject("Action is not a memory purge"); + const { plan, file } = yield* loadPlan(action, target); const rejection = purgeRejection(plan, action, grant); if (rejection !== undefined) return yield* reject(rejection); const repo = repoFor(plan); @@ -369,7 +387,7 @@ const make = Effect.gen(function* () { const { inFlight, marker } = yield* markerFor(plan.orgId).pipe( failed(plan.orgId, "clone", "Could not read the purge marker"), ); - return { plan, repo, inFlight, marker }; + return { plan, file, target, repo, inFlight, marker }; }); const branchRefs = Effect.fn("KinPurge.branchRefs")(function* (work: string) { @@ -426,13 +444,13 @@ const make = Effect.gen(function* () { ...[...blobs.values()].map(({ sha, path }) => ({ sha, type: "blob" as const, path })), ...commits.map((sha) => ({ sha, type: "commit" as const, path: `commit ${sha}` })), ]; - const files: Array<{ path: string; content: Uint8Array }> = []; + const files: Array<{ sha: string; path: string; content: Uint8Array }> = []; for (let index = 0; index < requests.length; index += 32) { const batch = requests.slice(index, index + 32); const objects = yield* provide(readObjects(orgId, work, batch)); for (const [position, request] of batch.entries()) { const content = objects[position]; - if (content) files.push({ path: request.path, content }); + if (content) files.push({ sha: request.sha, path: request.path, content }); } } return files; @@ -453,29 +471,16 @@ const make = Effect.gen(function* () { Effect.map((commits) => commits.length > 0), ); - /** Whether any reachable commit still holds what the plan removes. */ + /** Whether any reachable commit still holds what the plan removes, besides this purge's `tombstones`. */ const planInHistory = Effect.fn("KinPurge.planInHistory")(function* ( plan: PurgePlan, - actionId: string, + tombstones: ReadonlyArray, work: string, ) { - if (plan.mode === "drop_file") { - // This action's own tombstones live at the purged paths; anything else there is old content. - for (const file of yield* historyFiles(plan.orgId, work, plan.paths)) { - const parsed = yield* parseFrontMatter( - plan.orgId, - file.path, - Buffer.from(file.content).toString("utf8"), - ).pipe(Effect.option); - if ( - Option.isNone(parsed) || - parsed.value.frontMatter.title !== "Purged item" || - parsed.value.frontMatter.source !== `kin-action:${actionId}` - ) - return true; - } - return false; - } + if (plan.mode === "drop_file") + return (yield* historyFiles(plan.orgId, work, plan.paths)).some( + (file) => !tombstones.includes(file.sha), + ); return containsPattern(yield* historyFiles(plan.orgId, work), plan.patterns); }); @@ -506,8 +511,8 @@ const make = Effect.gen(function* () { work: string, oldHead: string, path: string, - action: PurgeAction, - grant: PurgeGrant, + action: KinPendingAction, + grant: KinGrantPayload, now: number, ) { const folder = path.split("/")[0] ?? ""; @@ -542,7 +547,7 @@ const make = Effect.gen(function* () { superseded_by: null, review_by: null, source: `kin-action:${action.id}`, - said_by: grant.approverHumanId, + said_by: grant.approver.humanId, trust: "owner", links: [], }, @@ -571,9 +576,11 @@ const make = Effect.gen(function* () { GIT_COMMITTER_DATE: date, }; yield* git(work, ["read-tree", head], undefined, env); + const blobs: string[] = []; const entries: string[] = []; for (const [path, content] of tombstones) { const blob = yield* git(work, ["hash-object", "-w", "--stdin"], content); + blobs.push(blob); entries.push(`100644 blob ${blob}\t${path}`); } yield* git(work, ["update-index", "--index-info"], entries.join("\n") + "\n", env); @@ -585,17 +592,30 @@ const make = Effect.gen(function* () { env, ); yield* git(work, ["update-ref", `refs/heads/${branch}`, commit, head]); - return commit; + return { commit, blobs }; }); /** * A fresh bare clone of `refs/heads/*` and `refs/tags/*` under the org's purge directory; stale ones - * from a crashed purge are swept first. GitHub's read-only `refs/pull/*` stay out of reach. + * from a crashed purge are swept first. GitHub's read-only `refs/pull/*` stay out of reach. Any other + * advertised ref (`refs/notes/*`, custom namespaces) would keep the old history, so it fails closed. */ const mirror = Effect.fn("KinPurge.mirror")(function* ( orgId: KinOrgId, repo: { readonly repoUrl: string; readonly path: string }, ) { + const advertised = yield* pusher + .listRefs({ orgId, repoUrl: repo.repoUrl }) + .pipe(failed(orgId, "clone", "Could not list remote refs")); + const foreign = advertised.filter( + (ref) => ref !== "HEAD" && !/^refs\/(heads|tags|pull)\//.test(ref), + ); + if (foreign.length > 0) + return yield* fail( + orgId, + "clone", + `Remote has refs a purge can't rewrite; remove them first: ${foreign.slice(0, 10).join(", ")}`, + ); const root = path.resolve(config.baseDirectory, repo.path, "purge-scratch"); const work = yield* Effect.gen(function* () { yield* fs.remove(root, { recursive: true, force: true }); @@ -619,15 +639,17 @@ const make = Effect.gen(function* () { const rewrite = Effect.fn("KinPurge.rewrite")(function* (input: { readonly plan: PurgePlan; - readonly action: PurgeAction; - readonly grant: PurgeGrant; + readonly planFile: string; + readonly target: PurgeTarget; + readonly action: KinPendingAction; + readonly grant: KinGrantPayload; readonly repo: { readonly repoUrl: string; readonly path: string }; readonly inFlight: string | undefined; readonly marker: typeof PurgeMarker.Type | undefined; readonly history: MemoryHistoryRewrite; readonly commit: () => void; }) { - const { plan, action, grant, repo, history } = input; + const { plan, target, action, grant, repo, history } = input; const { orgId } = plan; const branch = config.defaultBranch ?? "main"; const now = yield* Clock.currentTimeMillis; @@ -643,7 +665,7 @@ const make = Effect.gen(function* () { // A re-run after a push that landed: nothing to rewrite, but the checkouts still need a reclone. if (input.inFlight !== undefined) { - const remaining = yield* planInHistory(plan, action.id, work).pipe( + const remaining = yield* planInHistory(plan, input.marker?.tombstones ?? [], work).pipe( failed(orgId, "verify", "Could not check history"), ); if (!remaining) { @@ -718,17 +740,17 @@ const make = Effect.gen(function* () { return yield* fail(orgId, "rewrite", "Nothing in history matched the plan"); if (plan.mode === "drop_file") { - const remaining = yield* planInHistory(plan, action.id, work).pipe( + const remaining = yield* planInHistory(plan, [], work).pipe( failed(orgId, "verify", "Could not check rewritten history"), ); if (remaining) return yield* fail(orgId, "verify", "Purged paths remain in history"); } - const newHead = + const { commit: newHead, blobs } = tombstones.size > 0 ? yield* commitTombstones(work, branch, rewrittenHead, tombstones, action.id, now).pipe( failed(orgId, "tombstone", "Could not commit tombstones"), ) - : rewrittenHead; + : { commit: rewrittenHead, blobs: [] }; const files = yield* historyFiles(orgId, work).pipe( failed(orgId, "verify", "Could not read rewritten history"), @@ -752,18 +774,23 @@ const make = Effect.gen(function* () { .filter(([ref, sha]) => after.get(ref) !== sha) .map(([ref, old]) => ({ ref, old, new: after.get(ref) ?? null })); + yield* Effect.gen(function* () { + const file = tombstonesFile(input.planFile); + yield* fs.writeFileString(`${file}.tmp`, yield* encodeTombstones(blobs)); + yield* fs.rename(`${file}.tmp`, file); + }).pipe(failed(orgId, "tombstone", "Could not record the tombstones")); // From here a push may land at any moment, so local checkouts holding the old history must go // first; the marker keeps preserve and merges from pushing it back until a reclone or reconcile. input.commit(); yield* history .quarantine( - input.inFlight ?? - markerJson({ - sha256: action.target.sha256, - actionId: action.id, - oldHead, - plan: purgePlanJson(plan).json, - }), + markerJson({ + sha256: target.sha256, + actionId: action.id, + oldHead: input.marker?.oldHead ?? oldHead, + plan: purgePlanJson(plan).json, + tombstones: blobs, + }), ) .pipe(failed(orgId, "quarantine", "Could not quarantine local memory checkouts")); yield* pusher.push({ orgId, repoUrl: repo.repoUrl, gitDir: work, updates }).pipe( @@ -813,20 +840,21 @@ const make = Effect.gen(function* () { if (inFlight !== undefined && (marker === undefined || marker.sha256 === target.sha256)) return yield* reject("This plan's purge is in flight; reconcile it first"); } - yield* fs - .remove(file, { force: true }) - .pipe(Effect.mapError(() => new KinPurgeRejected({ reason: "Could not delete the plan" }))); + yield* Effect.all([ + fs.remove(file, { force: true }), + fs.remove(tombstonesFile(file), { force: true }), + ]).pipe(Effect.mapError(() => new KinPurgeRejected({ reason: "Could not delete the plan" }))); }), precheck: Effect.fn("KinPurge.precheck")(function* (action) { - const { inFlight, marker } = yield* checked(action); - if (inFlight !== undefined && marker?.sha256 !== action.target.sha256) + const { target, inFlight, marker } = yield* checked(action); + if (inFlight !== undefined && marker?.sha256 !== target.sha256) return yield* reject("Another purge is in flight; reconcile it first"); }), purge: Effect.fn("KinPurge.purge")(function* (action, grant) { - const { plan, repo, inFlight, marker } = yield* checked(action, grant); - if (inFlight !== undefined && marker?.sha256 !== action.target.sha256) + const { plan, file, target, repo, inFlight, marker } = yield* checked(action, grant); + if (inFlight !== undefined && marker?.sha256 !== target.sha256) return yield* reject("Another purge is in flight; reconcile it first"); const { orgId } = plan; // An unresolved marker means the queue is already held for this plan; never resume it early. @@ -837,6 +865,8 @@ const make = Effect.gen(function* () { Effect.scoped( rewrite({ plan, + planFile: file, + target, action, grant, repo, @@ -885,8 +915,7 @@ const make = Effect.gen(function* () { actionId: action.id, kind: "memory_purge", grantJti: grant.jti, - keyThumbprint: grant.keyThumbprint, - ...(grant.deviceLabel === undefined ? {} : { deviceLabel: grant.deviceLabel }), + keyThumbprint: grant.approver.kid, headSha: newHead, result: auditResult({ mode: plan.mode, paths, oldHead, newHead, refs: updates }), }, @@ -894,15 +923,24 @@ const make = Effect.gen(function* () { }), reconcile: Effect.fn("KinPurge.reconcile")(function* (action) { - const { plan, repo, inFlight, marker } = yield* checked(action); - if (inFlight !== undefined && marker?.sha256 !== action.target.sha256) return "unknown"; + const { plan, file, target, repo, inFlight, marker } = yield* checked(action); + if (inFlight !== undefined && marker?.sha256 !== target.sha256) return "unknown"; const { orgId } = plan; + const tombstones = + marker?.tombstones ?? + Option.getOrElse( + Option.flatMap( + yield* fs.readFileString(tombstonesFile(file)).pipe(Effect.option), + decodeTombstones, + ), + () => [], + ); return yield* memory.withHistoryRewrite(orgId, (history) => Effect.scoped( Effect.gen(function* () { const observed = yield* mirror(orgId, repo).pipe( Effect.flatMap(({ work, head }) => - planInHistory(plan, action.id, work).pipe( + planInHistory(plan, tombstones, work).pipe( Effect.map((remaining) => ({ head, remaining })), ), ), @@ -942,13 +980,16 @@ export const layer = Layer.effect(KinPurge, make); /** APPR-9's `KinActionHandler` for `memory_purge`; the executor calls it with a verified grant. */ export const memoryPurgeHandler = { kind: "memory_purge", - precheck: Effect.fn("KinPurge.handlerPrecheck")(function* (action: PurgeAction) { + precheck: Effect.fn("KinPurge.handlerPrecheck")(function* (action: KinPendingAction) { return yield* (yield* KinPurge).precheck(action); }), - run: Effect.fn("KinPurge.handlerRun")(function* (action: PurgeAction, grant: PurgeGrant) { + run: Effect.fn("KinPurge.handlerRun")(function* ( + action: KinPendingAction, + grant: KinGrantPayload, + ) { return yield* (yield* KinPurge).purge(action, grant); }), - reconcile: Effect.fn("KinPurge.handlerReconcile")(function* (action: PurgeAction) { + reconcile: Effect.fn("KinPurge.handlerReconcile")(function* (action: KinPendingAction) { return yield* (yield* KinPurge).reconcile(action); }), } as const; diff --git a/apps/server/src/kin/memory/purgePush.ts b/apps/server/src/kin/memory/purgePush.ts index 6399ec8883299..2c80689fad72e 100644 --- a/apps/server/src/kin/memory/purgePush.ts +++ b/apps/server/src/kin/memory/purgePush.ts @@ -38,6 +38,11 @@ export class KinPurgePush extends Context.Service< readonly gitDir: string; readonly updates: ReadonlyArray; }) => Effect.Effect; + /** Every ref the remote advertises, read with the same owner credential as `push`. */ + readonly listRefs: (input: { + readonly orgId: KinOrgId; + readonly repoUrl: string; + }) => Effect.Effect, KinPurgePushError>; } >()("t3/kin/memory/purgePush/KinPurgePush") {} @@ -91,7 +96,7 @@ const remoteKind = (url: string) => : undefined; /** - * Pushes with a reference to the owner's credential: a git credential helper or an SSH command naming + * Pushes and lists refs with a reference to the owner's credential: a git credential helper or an SSH command naming * its key file. The secret never enters argv or env, and with neither configured the push fails closed. */ export const gitPushLayer = (owner: { @@ -103,85 +108,108 @@ export const gitPushLayer = (owner: { Effect.gen(function* () { const vcs = yield* VcsProcess; const fs = yield* FileSystem.FileSystem; + /** Runs git against the remote with only the owner's credential reference, in a scratch HOME. */ + const run = Effect.fn("KinPurgePush.run")(function* ( + repoUrl: string, + args: ReadonlyArray, + cwd?: string, + ) { + const { credentialHelper, sshCommand } = owner; + if ( + (credentialHelper === undefined && sshCommand === undefined) || + (credentialHelper !== undefined && !isCredentialHelperReference(credentialHelper)) || + (sshCommand !== undefined && !isSshCommandReference(sshCommand)) + ) + return yield* new KinPurgePushError({ reason: "credential_unavailable" }); + const kind = remoteKind(repoUrl); + if (kind === undefined) + return yield* new KinPurgePushError({ reason: "unsupported_remote" }); + if ( + (kind === "https" && credentialHelper === undefined) || + (kind === "ssh" && sshCommand === undefined) + ) + return yield* new KinPurgePushError({ reason: "credential_unavailable" }); + // An empty HOME keeps curl's `.netrc` out; `-F none` keeps `~/.ssh/config` out. + const home = yield* fs + .makeTempDirectoryScoped({ prefix: "kin-push-home-" }) + .pipe(Effect.mapError(() => new KinPurgePushError({ reason: "unknown" }))); + const result = yield* vcs + .run({ + operation: "KinPurgePush.git", + command: "git", + cwd: cwd ?? home, + args: [ + "--no-replace-objects", + ...(cwd ? [`--git-dir=${cwd}/.git`] : []), + "-c", + "core.hooksPath=/dev/null", + "-c", + "core.fsmonitor=false", + "-c", + "core.attributesFile=/dev/null", + "-c", + "protocol.file.allow=user", + "-c", + "credential.helper=", + ...(credentialHelper ? ["-c", `credential.helper=${credentialHelper}`] : []), + ...args, + ], + env: { + ...scrubbedGitEnv(), + ...hardenedGitEnv, + HOME: home, + XDG_CONFIG_HOME: home, + GIT_ASKPASS: "false", + SSH_ASKPASS: "false", + SSH_ASKPASS_REQUIRE: "never", + SSH_AUTH_SOCK: undefined, + GCM_INTERACTIVE: "never", + // Without an owner SSH command, refuse SSH instead of using the host's keys or agent. + GIT_SSH_COMMAND: sshCommand + ? `${sshCommand} -F none -o BatchMode=yes -o IdentitiesOnly=yes -o IdentityAgent=none` + : "false", + }, + allowNonZeroExit: true, + maxOutputBytes: 16 * 1024 * 1024, + timeoutMs: 600_000, + }) + .pipe( + // The cause can echo the remote URL or helper output; keep both out of errors and logs. + Effect.mapError( + (error) => + new KinPurgePushError({ + reason: error._tag === "VcsProcessTimeoutError" ? "timeout" : "unknown", + }), + ), + ); + if (Number(result.exitCode) !== 0) + return yield* new KinPurgePushError({ + reason: pushFailure(result.stdout, result.stderr), + }); + return result.stdout; + }); return KinPurgePush.of({ push: Effect.fn("KinPurgePush.git")(function* ({ repoUrl, gitDir, updates }) { - const { credentialHelper, sshCommand } = owner; - if ( - (credentialHelper === undefined && sshCommand === undefined) || - (credentialHelper !== undefined && !isCredentialHelperReference(credentialHelper)) || - (sshCommand !== undefined && !isSshCommandReference(sshCommand)) - ) - return yield* new KinPurgePushError({ reason: "credential_unavailable" }); - const kind = remoteKind(repoUrl); - if (kind === undefined) - return yield* new KinPurgePushError({ reason: "unsupported_remote" }); - if ( - (kind === "https" && credentialHelper === undefined) || - (kind === "ssh" && sshCommand === undefined) - ) - return yield* new KinPurgePushError({ reason: "credential_unavailable" }); - // An empty HOME keeps curl's `.netrc` out; `-F none` keeps `~/.ssh/config` out. - const home = yield* fs - .makeTempDirectoryScoped({ prefix: "kin-push-home-" }) - .pipe(Effect.mapError(() => new KinPurgePushError({ reason: "unknown" }))); - const result = yield* vcs - .run({ - operation: "KinPurgePush.git", - command: "git", - cwd: gitDir, - args: [ - "--no-replace-objects", - `--git-dir=${gitDir}/.git`, - "-c", - "core.hooksPath=/dev/null", - "-c", - "core.fsmonitor=false", - "-c", - "core.attributesFile=/dev/null", - "-c", - "protocol.file.allow=user", - "-c", - "credential.helper=", - ...(credentialHelper ? ["-c", `credential.helper=${credentialHelper}`] : []), - "push", - "--atomic", - "--porcelain", - ...updates.map((update) => `--force-with-lease=${update.ref}:${update.old}`), - "--", - repoUrl, - ...updates.map((update) => `${update.new ?? ""}:${update.ref}`), - ], - env: { - ...scrubbedGitEnv(), - ...hardenedGitEnv, - HOME: home, - XDG_CONFIG_HOME: home, - GIT_ASKPASS: "false", - SSH_ASKPASS: "false", - SSH_ASKPASS_REQUIRE: "never", - SSH_AUTH_SOCK: undefined, - GCM_INTERACTIVE: "never", - // Without an owner SSH command, refuse SSH instead of using the host's keys or agent. - GIT_SSH_COMMAND: sshCommand - ? `${sshCommand} -F none -o BatchMode=yes -o IdentitiesOnly=yes -o IdentityAgent=none` - : "false", - }, - allowNonZeroExit: true, - timeoutMs: 600_000, - }) - .pipe( - // The cause can echo the remote URL or helper output; keep both out of errors and logs. - Effect.mapError( - (error) => - new KinPurgePushError({ - reason: error._tag === "VcsProcessTimeoutError" ? "timeout" : "unknown", - }), - ), - ); - if (Number(result.exitCode) !== 0) - return yield* new KinPurgePushError({ - reason: pushFailure(result.stdout, result.stderr), - }); + yield* run( + repoUrl, + [ + "push", + "--atomic", + "--porcelain", + ...updates.map((update) => `--force-with-lease=${update.ref}:${update.old}`), + "--", + repoUrl, + ...updates.map((update) => `${update.new ?? ""}:${update.ref}`), + ], + gitDir, + ); + }, Effect.scoped), + listRefs: Effect.fn("KinPurgePush.listRefs")(function* ({ repoUrl }) { + const stdout = yield* run(repoUrl, ["ls-remote", "--", repoUrl]); + return stdout.split("\n").flatMap((line) => { + const ref = line.split("\t")[1]; + return ref ? [ref] : []; + }); }, Effect.scoped), }); }), diff --git a/docs/kin/specs/triggers.md b/docs/kin/specs/triggers.md index 75e6845888f05..3ab6bb8801bbf 100644 --- a/docs/kin/specs/triggers.md +++ b/docs/kin/specs/triggers.md @@ -428,21 +428,21 @@ Known gaps, accepted for phase 1: JWE (five-part) tokens and JWT headers longer Pending action `memory_purge`. The plan is `{ orgId, repo, mode: "drop_file", paths }` or `{ orgId, repo, mode: "redact_text", patterns }`; `redact_text` rewrites every blob and commit message in history, so it takes no paths. `KinPurge.storePlan` writes `purgePlanJson(plan)` to `/purge-plans/.json` and returns the action's `content` target, whose path and sha256 come from that same serialization, so the grant binds the exact plan; `discardPlan` deletes it once the action is terminal, since it can hold the secret, and refuses while that plan's purge marker exists. The owner opens the action from an item's menu, or Kin proposes it when the weekly `dream_full` scans full history and finds a secret. Approval uses a native card and a signed grant (APPR). `memoryPurgeHandler` in `apps/server/src/kin/memory/purge.ts` is APPR-9's `KinActionHandler` (`precheck`, `run`, `reconcile`) and the only entry point; no model, MCP tool or RPC reaches `KinPurge`. Every `KinPurge` method refuses with `KinPurgeExecutorNotReady` until APPR-9, APPR-10 and LEAD-11 land. -Before anything runs, the stored plan must re-serialize to the same bytes and match the target; the grant, action and plan must share the org; the grant must bind the action id and revision; and `repo` must equal the org's configured memory URL. Paths: 1–64, unique, NFC, relative `.md` files other than `rules.md`, with no `..`, `.`, empty, `-`-leading or `.git` segment, backslash or control character, and each must exist in history (literal pathspecs), so a typo or an NFD name fails instead of purging nothing. `redact_text` takes 1–64 literal patterns of 8–1024 characters without control characters or `==>`; each becomes one `literal:` rule, so a pattern can't add a rule or switch to regex. Every argument goes to git and `git-filter-repo` as an argv entry, never through a shell, with inherited `GIT_*` variables removed. +Before anything runs, the stored plan must re-serialize to the same bytes and match the target; the grant, action and plan must share the org; the grant must bind the action id, revision, kind, `targetDigest` and plan digest; and `repo` must equal the org's configured memory URL. Paths: 1–64, unique, NFC, relative `.md` files other than `rules.md`, with no `..`, `.`, empty, `-`-leading or `.git` segment, backslash or control character, and each must exist in history (literal pathspecs), so a typo or an NFD name fails instead of purging nothing. `redact_text` takes 1–64 literal patterns of 8–1024 characters without control characters or `==>`; each becomes one `literal:` rule, so a pattern can't add a rule or switch to regex. Every argument goes to git and `git-filter-repo` as an argv entry, never through a shell, with inherited `GIT_*` variables removed. `run` then: -1. Pauses the org's lead queue through `KinPurgeQueue`. The default fails `KinPurgePauseUnavailable` until LEAD-11 builds the pause, because a reclone deletes the read checkout a running lead uses as its workspace. It takes the memory writer lock through `Memory.withHistoryRewrite`, which is the only way to reach the quarantine and reclone operations. -2. Bare-clones the remote's `refs/heads/*` and `refs/tags/*` into `//purge-scratch` (stale clones from a crashed purge are swept first) and runs `git filter-repo --invert-paths --path

` (`drop_file`) or `--replace-text --replace-message ` (`redact_text`). A plan that changes no ref fails. +1. Pauses the org's lead queue through `KinPurgeQueue`. The default fails `KinPurgePauseUnavailable` until LEAD-11 builds the pause, because a reclone deletes the read checkout a running lead uses as its workspace. `pause` must not return while a wake-up or dream is running: their staged files and dream copies predate the rewrite. As a backstop, every quarantine and reclone bumps `Memory.historyGeneration`, and a `MemoryWriter` session started under an older generation fails `MemorySyncConflict` and drops its staged changes instead of committing them onto the rewritten head. It takes the memory writer lock through `Memory.withHistoryRewrite`, which is the only way to reach the quarantine and reclone operations. +2. Lists the remote's refs with `git ls-remote` under the owner credential `KinPurgePush` uses, and fails closed at the clone stage if it advertises anything other than `HEAD`, `refs/heads/*`, `refs/tags/*` and `refs/pull/*`: `refs/notes/*` or a custom namespace such as `refs/custom/*` would keep the old history out of reach. It then bare-clones `refs/heads/*` and `refs/tags/*` into `//purge-scratch` (stale clones from a crashed purge are swept first) and runs `git filter-repo --invert-paths --path

` (`drop_file`) or `--replace-text --replace-message ` (`redact_text`). A plan that changes no ref fails. 3. For `drop_file`, checks that no reachable commit touches the paths, then commits a tombstone on the default branch for each item path present at the old head: the original `id` and the folder's kind, `title: Purged item`, `status: cancelled`, `invalid_at: now`, `source: kin-action:`, `said_by` the approving owner, body "Purged on by the owner (action )". Memory still shows that something was there. Journal and other non-item paths get no tombstone. 4. Reads every blob ever introduced at every path, merge sides included, and every commit object. It fails if a redacted pattern remains, then runs `KinSecretScan` over all of it and fails closed on **any** finding: findings carry no fingerprint, so the purge can't prove one unrelated, and a copy under an older path would otherwise survive. Allowlisted fingerprints don't count. Annotated tag messages are not scanned. -5. Writes the durable marker `//purge-in-flight` (plan digest, action id, old head and the plan itself) and deletes the writer and read checkouts and the dream scratch `/dreams/`, before the push. A push can land and still report failure (a timeout, a crash, a restart), and those checkouts hold the old history: `Memory.preserve` would push it back as a `kin/conflict/*` branch, and a dream as `dream/`. While the marker exists, `ensureCheckouts` and `recover` won't clone (until the push lands the remote still has the old history), and `preserve`, the wake-up merge, `commitWakeup`, `commitDream` and `deleteDreamBranch` refuse to push. Only the purge's own reclone clones. +5. Records the tombstones' blob ids in `purge-plans/.tombstones.json`, writes the durable marker `//purge-in-flight` (plan digest, action id, old head, the plan itself and the tombstone ids), and deletes the writer and read checkouts, any half-finished `memory.clone-*` and the dream scratch `/dreams/`, before the push. A push can land and still report failure (a timeout, a crash, a restart), and those checkouts hold the old history: `Memory.preserve` would push it back as a `kin/conflict/*` branch, and a dream as `dream/`. While the marker exists, `ensureCheckouts` and `recover` won't clone (until the push lands the remote still has the old history), and `preserve`, the wake-up merge, `commitWakeup`, `commitDream` and `deleteDreamBranch` refuse to push. Only the purge's own reclone clones. 6. Force-pushes the changed `refs/heads/*` and `refs/tags/*` in one atomic push, each leased at its cloned value, through `KinPurgePush`. Failures are classified from `--porcelain` output (`lease_rejected`, `remote_rejected`, `auth`, `network`, `timeout`) without the cause, so neither the URL nor helper output is logged. This needs the owner as a bypass actor on the memory repo's ruleset. Closing open dream PRs that touch the paths waits on the GitHub client. 7. Reclones: clones fresh, records the rewritten anchors without preserving the old ones, rebuilds the read checkout, cache and `INDEX.md`, clears the marker, and resumes the queue. A failure or interruption before step 5 resumes the queue with nothing changed; from step 5 on, any failure, defect or interruption leaves `KinPurgeFailed` with `queueHeld: true` and the queue paused. -8. Returns the `execution_finished` audit input (grant `jti`, key thumbprint, device, new head) with the paths, old head, new head and rewritten refs; APPR-10 writes it as a `KinAuditEntry`. -9. Tells the owner what a purge can't reach: GitHub keeps `refs/pull/*` (read-only, so the clone leaves them out and `reconcile` doesn't wait on them) and cached PR diffs (removing those needs a GitHub Support request), forks, local backups (Time Machine), and the chat database. A leaked credential must be rotated whatever happens. +8. Returns the `execution_finished` audit input (grant `jti`, key thumbprint, new head; APPR-10 adds the device label from the key) with the paths, old head, new head and rewritten refs; APPR-10 writes it as a `KinAuditEntry`. +9. Tells the owner what a purge can't reach: GitHub keeps `refs/pull/*` (read-only, so the clone leaves them out and `reconcile` doesn't wait on them) and cached PR diffs (removing those needs a GitHub Support request); forks, local backups (Time Machine) and the chat database keep their copies. It won't rewrite `refs/notes/*` or custom refs such as `refs/custom/*` either: their presence fails the purge in step 2 and makes `reconcile` answer `unknown`, so the owner deletes them on the remote first. A leaked credential must be rotated whatever happens. -`reconcile` bare-clones the remote the same way and checks whether the plan's paths (other than this action's tombstones) or patterns are still in history; it reads the plan from the marker if the plan file is gone. Gone is `done`; still there with the default branch at the marker's old head is `not_done`; anything else, or an unreachable remote, is `unknown` and changes nothing. `done` and `not_done` reclone, clear the marker and resume the queue. A re-run of the same plan while its marker exists reclones instead of resuming against old checkouts, and reports success when the push had already landed. [Kin memory purge](../../operations/kin-memory-purge.md) covers clearing a stuck marker by hand. +`reconcile` bare-clones the remote the same way and checks whether the plan's paths or patterns are still in history, counting only blobs whose ids match this purge's recorded tombstones as purged, so a look-alike tombstone counts as old content; it reads the plan from the marker if the plan file is gone. Gone is `done`; still there with the default branch at the marker's old head is `not_done`; anything else, an unreachable remote, or a remote advertising refs outside heads, tags and pulls is `unknown` and changes nothing. `done` and `not_done` reclone, clear the marker and resume the queue. A re-run of the same plan while its marker exists reclones instead of resuming against old checkouts, and reports success when the push had already landed. [Kin memory purge](../../operations/kin-memory-purge.md) covers clearing a stuck marker by hand. **Owner credential.** `gitPushLayer` takes a reference to the owner's credential, never the secret: a git credential helper name or absolute path (no inline `!` script), or an SSH command naming its key file (no shell metacharacters; T3 appends `-F none`, `BatchMode`, `IdentitiesOnly` and `IdentityAgent=none`). An `https://` remote needs the helper and an SSH remote the command; `file://` takes neither, and any other transport fails `unsupported_remote`. git runs with an empty scratch `HOME`, so `~/.netrc` and `~/.ssh/config` can't supply another credential. This is the one exception to APPR's rule that the owner credential never reaches a child process: the helper or key is read by git or ssh, not by T3. With neither configured the push fails `credential_unavailable`, SSH without an owner command is refused rather than using the host's keys or agent, and `GIT_TERMINAL_PROMPT=0`, `GIT_ASKPASS`, `SSH_ASKPASS` and `SSH_ASKPASS_REQUIRE=never` keep anything from prompting. @@ -567,7 +567,7 @@ Kin runs on one always-on machine, the **Kin host**. The laptop and phone are cl - **Deferral:** unchanged recheck, then heads-up, then silence, then `started` at due. A linked issue closed before due leads to `asked`, and nothing starts. With the stop switch on, it asks. [Monday] re-arms to next Monday 09:00. A `due` fire consumed two hours late asks; a queued fire for a disarmed or moved deferral is dropped. - **Dream:** a dream touching `rules.md` fails, and `main` is unchanged. A dream with one journal fold and one `decisions/` change opens two PRs, and only the auto one merges. - **Secret scan:** a journal line holding `ghp_` plus 36 characters blocks the entire commit, and the owner gets one typed notification without the value. -- **Purge:** on a fixture repo, the file is gone from `git log --all -p` and the recloned checkout, a tombstone exists and parses, and the returned audit input carries the old and new heads. A surviving secret fails closed before the push. A push that lands and then errors, crashes or is interrupted never republishes the old history through the next wake-up, a dream branch or a checkout cloned during the push, and `reconcile` reports `done` with a `refs/pull/*` ref still holding it. +- **Purge:** on a fixture repo, the file is gone from `git log --all -p` and the recloned checkout, a tombstone exists and parses, and the returned audit input carries the old and new heads. A surviving secret fails closed before the push. A push that lands and then errors, crashes or is interrupted never republishes the old history through the next wake-up, a dream branch or a checkout cloned during the push, and `reconcile` reports `done` with a `refs/pull/*` ref still holding it. A `refs/notes/*` ref fails the purge at the clone and makes `reconcile` answer `unknown`; a wake-up that staged before the purge can't commit after it; a look-alike tombstone makes `reconcile` answer `not_done`. - **Contract tests:** `vp test run kin-contract` passes on today's upstream. Renaming `scheduledTaskId` in a scratch branch makes it fail. ## Tasks diff --git a/docs/operations/kin-memory-purge.md b/docs/operations/kin-memory-purge.md index a14bc4e8cb642..e9351b3298a51 100644 --- a/docs/operations/kin-memory-purge.md +++ b/docs/operations/kin-memory-purge.md @@ -2,8 +2,21 @@ > For maintainers operating a Kin host. The design is in [the triggers spec](../kin/specs/triggers.md#owner-signed-purge-gap-5). +## Refs a purge can't rewrite + +A purge rewrites only `refs/heads/*` and `refs/tags/*`. GitHub's read-only `refs/pull/*` are left alone. If the remote advertises any other ref, such as `refs/notes/commits` or a custom `refs/custom/*`, the purge fails at the clone stage, changes nothing and resumes the queue, and `reconcile` answers `unknown`. Those refs can still hold the purged content, so list them with `git ls-remote `, delete them on the remote (`git push --delete `), and run the purge again. + ## A stuck purge marker -A purge writes `/kin//purge-in-flight` before it force-pushes, and leaves it there when the push fails, crashes or is interrupted. While it exists, the org's lead queue stays paused, memory won't clone a checkout, and nothing pushes memory history. `reconcile` on the action clears it: it reads the plan from the marker, so it works even after the plan file is gone. Retry `reconcile` first; it answers `unknown` while the remote is unreachable or has moved. +A purge writes `/kin//purge-in-flight` before it force-pushes, and leaves it there when the push fails, crashes or is interrupted. While it exists, the org's lead queue stays paused, memory won't clone a checkout, and nothing pushes memory history. `reconcile` on the action clears it: it reads the plan from the marker, so it works even after the plan file is gone. Retry `reconcile` first; it answers `unknown` while the remote is unreachable, has moved, or advertises refs a purge can't rewrite. + +Delete the marker by hand only when `reconcile` keeps answering `unknown` and you have checked the remote yourself. First stop the T3 server, or confirm that no purge or `reconcile` for the org is running: both hold the memory writer lock and write these paths. Deleting the marker tells Kin that no local copy of the old history will be pushed again, so first delete everything under `` that can hold it: + +- `memory` and `memory-read` (the writer and read checkouts); +- `dreams` (dream scratch copies); +- any `memory.clone-*` (a clone a crash left half-finished); +- `purge-scratch` and any `purge-scratch/mirror-*` inside it (the purge's rewritten bare clone, which also holds old objects). + +Then confirm with `git log --branches --tags -p` on a fresh clone, and `git ls-remote` for refs outside heads, tags and pulls, whether the purged content is still on the remote. If it is, run the purge again from a new owner-approved action; if it's gone, the next wake-up clones the rewritten history. -Delete the marker by hand only when `reconcile` keeps answering `unknown` and you have checked the remote yourself. Deleting it tells Kin that no local copy of the old history will be pushed again, so first delete `/memory`, `/memory-read` and `/dreams` too, and confirm with `git log --branches --tags -p` on a fresh clone whether the purged content is still on the remote. If it is, run the purge again from a new owner-approved action; if it's gone, the next wake-up clones the rewritten history. Then resume the org's queue. +Last, resume the org's queue. Until LEAD-11 builds the org pause there is no operator path to resume it: the purge can't run then either, because `KinPurgeQueue.pause` fails by default, so a stuck marker can only come from a host running LEAD-11's pause. From 9b8505dec1073aadfbd6153aa9f4a6051dea08fe Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Mon, 9 Nov 2026 01:15:12 +0530 Subject: [PATCH 5/7] fix(kin): purge re-checks the remote after its push (TRIG-17) The push leases only refs the purge saw at clone time, so a branch, tag or notes ref created since could keep the purged content while the purge reported success. After the push it now mirrors the remote again and fails at the push stage with the queue held unless history is clean. The mirror also fails closed on tags that don't peel to a commit, which filter-repo and the history checks both skip. A dream whose base a purge rewrote away now fails with MemorySyncConflict instead of a checkout-integrity error. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/kin/memory/MemoryWriter.ts | 12 ++++ apps/server/src/kin/memory/purge.test.ts | 65 ++++++++++++++++++---- apps/server/src/kin/memory/purge.ts | 30 ++++++++++ docs/operations/kin-memory-purge.md | 4 +- 4 files changed, 98 insertions(+), 13 deletions(-) diff --git a/apps/server/src/kin/memory/MemoryWriter.ts b/apps/server/src/kin/memory/MemoryWriter.ts index 3aa32c0c88cbb..1b69e0424e4ec 100644 --- a/apps/server/src/kin/memory/MemoryWriter.ts +++ b/apps/server/src/kin/memory/MemoryWriter.ts @@ -925,6 +925,18 @@ const make = Effect.gen(function* () { const { writer } = yield* validate(orgId); yield* refuseDuringPurge(orgId); const repo = yield* repoFor(orgId); + // A purge's reclone drops the old history, so a dream started before it has no base here. + if ( + !/^[a-f0-9]{40}([a-f0-9]{24})?$/.test(input.base) || + (yield* git(orgId, writer, ["cat-file", "-e", `${input.base}^{commit}`]).pipe( + Effect.result, + ))._tag === "Failure" + ) + return yield* new MemorySyncConflict({ + orgId, + operation: "push", + reason: "The dream's base is no longer in memory history", + }); const baseFiles = yield* filesAt(orgId, writer, input.base); const regular = yield* regularPaths(orgId, writer, input.base); const combined = new Map( diff --git a/apps/server/src/kin/memory/purge.test.ts b/apps/server/src/kin/memory/purge.test.ts index b0d124776e9db..9291cc291e38e 100644 --- a/apps/server/src/kin/memory/purge.test.ts +++ b/apps/server/src/kin/memory/purge.test.ts @@ -314,6 +314,17 @@ describe.skipIf(filterRepoMissing)("purge with git-filter-repo", () => { expect( yield* writer.commitWakeup(orgId, "wake1", "completed").pipe(Effect.result), ).toMatchObject({ failure: { _tag: "MemorySyncConflict" } }); + // A dream that read its base before the purge. + expect( + yield* writer + .commitDream(orgId, { + branch: "dream/2026-11-08", + base: oldMain, + files: new Map(), + message: "dream", + }) + .pipe(Effect.result), + ).toMatchObject({ failure: { _tag: "MemorySyncConflict" } }); expect(yield* git(remote, ["rev-parse", "refs/heads/main"])).toBe(newMain); expect(yield* reconcile(plan)).toBe("done"); expect(yield* git(remote, ["rev-parse", "refs/pull/1/head"])).toBe(pullHead); @@ -392,6 +403,8 @@ describe.skipIf(filterRepoMissing)("purge with git-filter-repo", () => { ["no pause", { _tag: "KinPurgePauseUnavailable" }], // `clone --bare` leaves notes and custom refs behind, and they would keep the old history. ["notes ref", { _tag: "KinPurgeFailed", stage: "clone", queueHeld: false }], + // filter-repo leaves a tag on a blob or tree alone, and history checks walk commits only. + ["blob tag", { _tag: "KinPurgeFailed", stage: "clone", queueHeld: false }], ] as const)( "a purge that stops before the push resumes the queue and changes nothing: %s", ([scenario, expected]) => @@ -415,6 +428,12 @@ describe.skipIf(filterRepoMissing)("purge with git-filter-repo", () => { : dropPlan(remoteUrl, scenario === "path typo" ? ["claims/leek.md"] : undefined); if (scenario === "notes ref") yield* git(remote, ["update-ref", "refs/notes/commits", "refs/heads/side"]); + if (scenario === "blob tag") + yield* git(remote, [ + "update-ref", + "refs/tags/blobtag", + "refs/heads/side:claims/leak.md", + ]); const exit = yield* runPurge(plan); expect(Exit.isFailure(exit)).toBe(true); @@ -429,9 +448,13 @@ describe.skipIf(filterRepoMissing)("purge with git-filter-repo", () => { expect(failureOf(exit)).toMatchObject({ value: { reason: expect.stringMatching(/not in history/) }, }); - if (scenario === "notes ref") { + if (scenario === "notes ref" || scenario === "blob tag") { expect(failureOf(exit)).toMatchObject({ - value: { reason: expect.stringContaining("refs/notes/commits") }, + value: { + reason: expect.stringContaining( + scenario === "notes ref" ? "refs/notes/commits" : "refs/tags/blobtag", + ), + }, }); expect(yield* reconcile(plan)).toBe("unknown"); } @@ -559,9 +582,14 @@ describe.skipIf(filterRepoMissing)("purge with git-filter-repo", () => { }).pipe(Effect.provide(services), Effect.scoped), ); + const lateRef = "The remote may still hold purged history after the push; reconcile it"; it.live.each([ - ["refused", "remote_rejected", "not_done"], - ["lease rejected", "lease_rejected", "unknown"], + ["refused", "Force-push failed: remote_rejected", "not_done"], + ["lease rejected", "Force-push failed: lease_rejected", "unknown"], + // Created after the mirror clone, so the push's lease never covers them. + ["late branch", lateRef, "unknown"], + ["late tag", lateRef, "unknown"], + ["late notes ref", lateRef, "unknown"], ] as const)( "a push that does not land holds the queue until reconcile: %s", ([scenario, reason, verdict]) => @@ -584,13 +612,26 @@ describe.skipIf(filterRepoMissing)("purge with git-filter-repo", () => { push: (input) => scenario === "refused" ? Effect.fail(new PurgePush.KinPurgePushError({ reason: "remote_rejected" })) - : Effect.gen(function* () { - // The remote moves after the purge's mirror clone. - yield* fs.writeFileString(`${source}/claims/other.md`, "Invented later note\n"); - yield* git(source, ["add", "--", "."]); - yield* commit("later"); - yield* git(source, ["push", "-q", remote, "main"]); - }).pipe(Effect.orDie, Effect.andThen(real.push(input))), + : scenario !== "lease rejected" + ? git(remote, [ + "update-ref", + { + "late branch": "refs/heads/sneaky", + "late tag": "refs/tags/sneaky", + "late notes ref": "refs/notes/commits", + }[scenario], + "refs/heads/side", + ]).pipe(Effect.orDie, Effect.andThen(real.push(input))) + : Effect.gen(function* () { + // The remote moves after the purge's mirror clone. + yield* fs.writeFileString( + `${source}/claims/other.md`, + "Invented later note\n", + ); + yield* git(source, ["add", "--", "."]); + yield* commit("later"); + yield* git(source, ["push", "-q", remote, "main"]); + }).pipe(Effect.orDie, Effect.andThen(real.push(input))), }), }); yield* Effect.gen(function* () { @@ -603,7 +644,7 @@ describe.skipIf(filterRepoMissing)("purge with git-filter-repo", () => { _tag: "KinPurgeFailed", stage: "push", queueHeld: true, - reason: `Force-push failed: ${reason}`, + reason, }, }); expect(events).toEqual(["pause"]); diff --git a/apps/server/src/kin/memory/purge.ts b/apps/server/src/kin/memory/purge.ts index 7b03fe9d0f189..88bcdce0fe3ae 100644 --- a/apps/server/src/kin/memory/purge.ts +++ b/apps/server/src/kin/memory/purge.ts @@ -631,6 +631,23 @@ const make = Effect.gen(function* () { repo.repoUrl, `${work}/.git`, ]).pipe(failed(orgId, "clone", "Bare clone failed")); + // filter-repo rewrites commits only, and `historyFiles` walks commits only. + const nonCommitTags = (yield* git(work, [ + "for-each-ref", + "--format=%(refname) %(objecttype) %(*objecttype)", + "refs/tags/", + ]).pipe(failed(orgId, "clone", "Could not list tags"))) + .split("\n") + .flatMap((line) => { + const [ref, type, peeled] = line.split(" "); + return ref && type !== "commit" && !(type === "tag" && peeled === "commit") ? [ref] : []; + }); + if (nonCommitTags.length > 0) + return yield* fail( + orgId, + "clone", + `Remote has tags that don't point at a commit; remove them first: ${nonCommitTags.slice(0, 10).join(", ")}`, + ); const before = yield* branchRefs(work).pipe(failed(orgId, "clone", "Could not list refs")); const head = before.get(`refs/heads/${config.defaultBranch ?? "main"}`); if (!head) return yield* fail(orgId, "clone", "Remote default branch is missing"); @@ -804,6 +821,19 @@ const make = Effect.gen(function* () { }), ), ); + // The lease covers only refs this purge saw, so one created since can still hold the old history. + const clean = yield* mirror(orgId, repo).pipe( + Effect.flatMap((fresh) => planInHistory(plan, blobs, fresh.work)), + Effect.map((remaining) => !remaining), + Effect.orElseSucceed(() => false), + ); + if (!clean) + return yield* new KinPurgeFailed({ + orgId, + stage: "push", + reason: "The remote may still hold purged history after the push; reconcile it", + queueHeld: true, + }); yield* history .reclone() .pipe(failed(orgId, "rebuild", "Could not rebuild local memory checkouts")); diff --git a/docs/operations/kin-memory-purge.md b/docs/operations/kin-memory-purge.md index e9351b3298a51..45284abc7d6ba 100644 --- a/docs/operations/kin-memory-purge.md +++ b/docs/operations/kin-memory-purge.md @@ -4,12 +4,14 @@ ## Refs a purge can't rewrite -A purge rewrites only `refs/heads/*` and `refs/tags/*`. GitHub's read-only `refs/pull/*` are left alone. If the remote advertises any other ref, such as `refs/notes/commits` or a custom `refs/custom/*`, the purge fails at the clone stage, changes nothing and resumes the queue, and `reconcile` answers `unknown`. Those refs can still hold the purged content, so list them with `git ls-remote `, delete them on the remote (`git push --delete `), and run the purge again. +A purge rewrites only `refs/heads/*` and `refs/tags/*`. GitHub's read-only `refs/pull/*` are left alone. If the remote advertises any other ref, such as `refs/notes/commits` or a custom `refs/custom/*`, the purge fails at the clone stage, changes nothing and resumes the queue, and `reconcile` answers `unknown`. Those refs can still hold the purged content, so list them with `git ls-remote `, delete them on the remote (`git push --delete `), and run the purge again. A tag that points at a blob or tree fails the same way. A ref created on the remote while a purge runs fails it at the push stage with the queue held, and `reconcile` answers `unknown` until you delete that ref. ## A stuck purge marker A purge writes `/kin//purge-in-flight` before it force-pushes, and leaves it there when the push fails, crashes or is interrupted. While it exists, the org's lead queue stays paused, memory won't clone a checkout, and nothing pushes memory history. `reconcile` on the action clears it: it reads the plan from the marker, so it works even after the plan file is gone. Retry `reconcile` first; it answers `unknown` while the remote is unreachable, has moved, or advertises refs a purge can't rewrite. +The marker and the `purge-plans/*.tombstones.json` files live beside the lead's read workspace, and `reconcile` trusts them, so a lead with full filesystem access could tamper with them. + Delete the marker by hand only when `reconcile` keeps answering `unknown` and you have checked the remote yourself. First stop the T3 server, or confirm that no purge or `reconcile` for the org is running: both hold the memory writer lock and write these paths. Deleting the marker tells Kin that no local copy of the old history will be pushed again, so first delete everything under `` that can hold it: - `memory` and `memory-read` (the writer and read checkouts); From 3952150108e48fc10ad2527f9265c0bfa206bd5c Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Mon, 9 Nov 2026 01:20:56 +0530 Subject: [PATCH 6/7] style(kin): read the history generation without an empty generator Co-Authored-By: Claude Opus 5.5 --- apps/server/src/kin/memory/Memory.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/server/src/kin/memory/Memory.ts b/apps/server/src/kin/memory/Memory.ts index a4d4da7437d81..037fc25fc2784 100644 --- a/apps/server/src/kin/memory/Memory.ts +++ b/apps/server/src/kin/memory/Memory.ts @@ -1389,9 +1389,9 @@ const make = Effect.gen(function* () { purgeInFlight: Effect.fn("Memory.purgeInFlightLocked")((orgId) => locked(orgId, purgeInFlight(orgId)), ), - historyGeneration: Effect.fn("Memory.historyGeneration")(function* (orgId) { - return historyGenerations.get(orgId) ?? 0; - }), + historyGeneration: Effect.fn("Memory.historyGeneration")((orgId) => + Effect.sync(() => historyGenerations.get(orgId) ?? 0), + ), recover: Effect.fn("Memory.recoverLocked")((orgId) => withWriterLock(orgId, locked(orgId, recover(orgId))), ), From 63c629d5389687f0da5d6674c874fd73e8ee5692 Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Mon, 9 Nov 2026 01:36:39 +0530 Subject: [PATCH 7/7] fix(kin): check tag objects after a purge and state the late-ref window Tagger idents survive filter-repo's message rewrite, so a redacted pattern there now fails verification instead of passing. The ops doc and spec say the post-push check covers refs created before it, not pushes from an old clone after it. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/kin/memory/objects.ts | 4 ++-- apps/server/src/kin/memory/purge.test.ts | 24 +++++++++++++++++++++++- apps/server/src/kin/memory/purge.ts | 16 +++++++++++++++- docs/kin/specs/triggers.md | 6 +++--- docs/operations/kin-memory-purge.md | 2 +- 5 files changed, 44 insertions(+), 8 deletions(-) diff --git a/apps/server/src/kin/memory/objects.ts b/apps/server/src/kin/memory/objects.ts index e7c4d33155168..1d285684a9fdf 100644 --- a/apps/server/src/kin/memory/objects.ts +++ b/apps/server/src/kin/memory/objects.ts @@ -9,7 +9,7 @@ import { MemoryCheckoutInvalid } from "./Memory.ts"; export const readObjects = Effect.fnUntraced(function* ( orgId: KinOrgId, cwd: string, - requests: ReadonlyArray<{ sha: string; type: "blob" | "tree" | "commit" }>, + requests: ReadonlyArray<{ sha: string; type: "blob" | "tree" | "commit" | "tag" }>, ) { const invalid = () => new MemoryCheckoutInvalid({ @@ -34,7 +34,7 @@ export const readObjects = Effect.fnUntraced(function* ( for (const { sha, type } of requests) { const end = buffer.indexOf(10, offset); const header = buffer.subarray(offset, end).toString("ascii"); - const match = /^(\w+) (blob|tree|commit) (\d+)$/.exec(header); + const match = /^(\w+) (blob|tree|commit|tag) (\d+)$/.exec(header); if (end < 0 || !match || match[1] !== sha || match[2] !== type) return yield* invalid(); const length = Number(match[3]); offset = end + 1; diff --git a/apps/server/src/kin/memory/purge.test.ts b/apps/server/src/kin/memory/purge.test.ts index 9291cc291e38e..38ab8a9c6f42a 100644 --- a/apps/server/src/kin/memory/purge.test.ts +++ b/apps/server/src/kin/memory/purge.test.ts @@ -405,6 +405,8 @@ describe.skipIf(filterRepoMissing)("purge with git-filter-repo", () => { ["notes ref", { _tag: "KinPurgeFailed", stage: "clone", queueHeld: false }], // filter-repo leaves a tag on a blob or tree alone, and history checks walk commits only. ["blob tag", { _tag: "KinPurgeFailed", stage: "clone", queueHeld: false }], + // filter-repo rewrites tag messages but not tagger idents. + ["tagger ident", { _tag: "KinPurgeFailed", stage: "verify", queueHeld: false }], ] as const)( "a purge that stops before the push resumes the queue and changes nothing: %s", ([scenario, expected]) => @@ -425,9 +427,29 @@ describe.skipIf(filterRepoMissing)("purge with git-filter-repo", () => { const plan: Purge.PurgePlan = scenario === "wrong pattern" ? { orgId, repo: remoteUrl, mode: "redact_text", patterns: ["Invented vendor note"] } - : dropPlan(remoteUrl, scenario === "path typo" ? ["claims/leek.md"] : undefined); + : scenario === "tagger ident" + ? { + orgId, + repo: remoteUrl, + mode: "redact_text", + patterns: [token], + } + : dropPlan(remoteUrl, scenario === "path typo" ? ["claims/leek.md"] : undefined); if (scenario === "notes ref") yield* git(remote, ["update-ref", "refs/notes/commits", "refs/heads/side"]); + if (scenario === "tagger ident") + yield* git(remote, [ + "-c", + "user.name=Tagger", + "-c", + `user.email=${token}`, + "tag", + "-a", + "v1", + "-m", + `release ${token}`, + "refs/heads/side", + ]); if (scenario === "blob tag") yield* git(remote, [ "update-ref", diff --git a/apps/server/src/kin/memory/purge.ts b/apps/server/src/kin/memory/purge.ts index 88bcdce0fe3ae..b84450f257f02 100644 --- a/apps/server/src/kin/memory/purge.ts +++ b/apps/server/src/kin/memory/purge.ts @@ -405,7 +405,7 @@ const make = Effect.gen(function* () { ); }); - /** Every blob ever introduced, merge sides included, plus every commit object unless `paths` narrows it. */ + /** Every blob ever introduced, merge sides included, plus every commit and tag object unless `paths` narrows it. */ const historyFiles = Effect.fn("KinPurge.historyFiles")(function* ( orgId: KinOrgId, work: string, @@ -440,9 +440,23 @@ const make = Effect.gen(function* () { const commits = paths ? [] : (yield* git(work, ["rev-list", "--all"])).split("\n").filter(Boolean); + // Tagger idents survive filter-repo's text and message rewrites; nested tags included. + const tags = paths + ? [] + : (yield* git(work, [ + "cat-file", + "--batch-all-objects", + "--batch-check=%(objectname) %(objecttype)", + ])) + .split("\n") + .flatMap((line) => { + const [sha, type] = line.split(" "); + return sha && type === "tag" ? [sha] : []; + }); const requests = [ ...[...blobs.values()].map(({ sha, path }) => ({ sha, type: "blob" as const, path })), ...commits.map((sha) => ({ sha, type: "commit" as const, path: `commit ${sha}` })), + ...tags.map((sha) => ({ sha, type: "tag" as const, path: `tag ${sha}` })), ]; const files: Array<{ sha: string; path: string; content: Uint8Array }> = []; for (let index = 0; index < requests.length; index += 32) { diff --git a/docs/kin/specs/triggers.md b/docs/kin/specs/triggers.md index 3ab6bb8801bbf..9c9f0a30d6b10 100644 --- a/docs/kin/specs/triggers.md +++ b/docs/kin/specs/triggers.md @@ -433,12 +433,12 @@ Before anything runs, the stored plan must re-serialize to the same bytes and ma `run` then: 1. Pauses the org's lead queue through `KinPurgeQueue`. The default fails `KinPurgePauseUnavailable` until LEAD-11 builds the pause, because a reclone deletes the read checkout a running lead uses as its workspace. `pause` must not return while a wake-up or dream is running: their staged files and dream copies predate the rewrite. As a backstop, every quarantine and reclone bumps `Memory.historyGeneration`, and a `MemoryWriter` session started under an older generation fails `MemorySyncConflict` and drops its staged changes instead of committing them onto the rewritten head. It takes the memory writer lock through `Memory.withHistoryRewrite`, which is the only way to reach the quarantine and reclone operations. -2. Lists the remote's refs with `git ls-remote` under the owner credential `KinPurgePush` uses, and fails closed at the clone stage if it advertises anything other than `HEAD`, `refs/heads/*`, `refs/tags/*` and `refs/pull/*`: `refs/notes/*` or a custom namespace such as `refs/custom/*` would keep the old history out of reach. It then bare-clones `refs/heads/*` and `refs/tags/*` into `//purge-scratch` (stale clones from a crashed purge are swept first) and runs `git filter-repo --invert-paths --path

` (`drop_file`) or `--replace-text --replace-message ` (`redact_text`). A plan that changes no ref fails. +2. Lists the remote's refs with `git ls-remote` under the owner credential `KinPurgePush` uses, and fails closed at the clone stage if it advertises anything other than `HEAD`, `refs/heads/*`, `refs/tags/*` and `refs/pull/*`: `refs/notes/*` or a custom namespace such as `refs/custom/*` would keep the old history out of reach. It then bare-clones `refs/heads/*` and `refs/tags/*` into `//purge-scratch` (stale clones from a crashed purge are swept first) and runs `git filter-repo --invert-paths --path

` (`drop_file`) or `--replace-text --replace-message ` (`redact_text`). It also fails closed at the clone stage on any tag that doesn't peel to a commit, because filter-repo leaves tags on blobs and trees alone. A plan that changes no ref fails. 3. For `drop_file`, checks that no reachable commit touches the paths, then commits a tombstone on the default branch for each item path present at the old head: the original `id` and the folder's kind, `title: Purged item`, `status: cancelled`, `invalid_at: now`, `source: kin-action:`, `said_by` the approving owner, body "Purged on by the owner (action )". Memory still shows that something was there. Journal and other non-item paths get no tombstone. -4. Reads every blob ever introduced at every path, merge sides included, and every commit object. It fails if a redacted pattern remains, then runs `KinSecretScan` over all of it and fails closed on **any** finding: findings carry no fingerprint, so the purge can't prove one unrelated, and a copy under an older path would otherwise survive. Allowlisted fingerprints don't count. Annotated tag messages are not scanned. +4. Reads every blob ever introduced at every path, merge sides included, every commit object and every tag object (tagger idents survive `--replace-message`). It fails if a redacted pattern remains, then runs `KinSecretScan` over all of it and fails closed on **any** finding: findings carry no fingerprint, so the purge can't prove one unrelated, and a copy under an older path would otherwise survive. Allowlisted fingerprints don't count. 5. Records the tombstones' blob ids in `purge-plans/.tombstones.json`, writes the durable marker `//purge-in-flight` (plan digest, action id, old head, the plan itself and the tombstone ids), and deletes the writer and read checkouts, any half-finished `memory.clone-*` and the dream scratch `/dreams/`, before the push. A push can land and still report failure (a timeout, a crash, a restart), and those checkouts hold the old history: `Memory.preserve` would push it back as a `kin/conflict/*` branch, and a dream as `dream/`. While the marker exists, `ensureCheckouts` and `recover` won't clone (until the push lands the remote still has the old history), and `preserve`, the wake-up merge, `commitWakeup`, `commitDream` and `deleteDreamBranch` refuse to push. Only the purge's own reclone clones. 6. Force-pushes the changed `refs/heads/*` and `refs/tags/*` in one atomic push, each leased at its cloned value, through `KinPurgePush`. Failures are classified from `--porcelain` output (`lease_rejected`, `remote_rejected`, `auth`, `network`, `timeout`) without the cause, so neither the URL nor helper output is logged. This needs the owner as a bypass actor on the memory repo's ruleset. Closing open dream PRs that touch the paths waits on the GitHub client. -7. Reclones: clones fresh, records the rewritten anchors without preserving the old ones, rebuilds the read checkout, cache and `INDEX.md`, clears the marker, and resumes the queue. A failure or interruption before step 5 resumes the queue with nothing changed; from step 5 on, any failure, defect or interruption leaves `KinPurgeFailed` with `queueHeld: true` and the queue paused. +7. Re-lists and re-mirrors the remote and runs the same history check as `reconcile`; a ref created during the purge that still holds the content fails at the push stage with `queueHeld: true`. Then reclones: clones fresh, records the rewritten anchors without preserving the old ones, rebuilds the read checkout, cache and `INDEX.md`, clears the marker, and resumes the queue. A failure or interruption before step 5 resumes the queue with nothing changed; from step 5 on, any failure, defect or interruption leaves `KinPurgeFailed` with `queueHeld: true` and the queue paused. 8. Returns the `execution_finished` audit input (grant `jti`, key thumbprint, new head; APPR-10 adds the device label from the key) with the paths, old head, new head and rewritten refs; APPR-10 writes it as a `KinAuditEntry`. 9. Tells the owner what a purge can't reach: GitHub keeps `refs/pull/*` (read-only, so the clone leaves them out and `reconcile` doesn't wait on them) and cached PR diffs (removing those needs a GitHub Support request); forks, local backups (Time Machine) and the chat database keep their copies. It won't rewrite `refs/notes/*` or custom refs such as `refs/custom/*` either: their presence fails the purge in step 2 and makes `reconcile` answer `unknown`, so the owner deletes them on the remote first. A leaked credential must be rotated whatever happens. diff --git a/docs/operations/kin-memory-purge.md b/docs/operations/kin-memory-purge.md index 45284abc7d6ba..d1f160b5ea11e 100644 --- a/docs/operations/kin-memory-purge.md +++ b/docs/operations/kin-memory-purge.md @@ -4,7 +4,7 @@ ## Refs a purge can't rewrite -A purge rewrites only `refs/heads/*` and `refs/tags/*`. GitHub's read-only `refs/pull/*` are left alone. If the remote advertises any other ref, such as `refs/notes/commits` or a custom `refs/custom/*`, the purge fails at the clone stage, changes nothing and resumes the queue, and `reconcile` answers `unknown`. Those refs can still hold the purged content, so list them with `git ls-remote `, delete them on the remote (`git push --delete `), and run the purge again. A tag that points at a blob or tree fails the same way. A ref created on the remote while a purge runs fails it at the push stage with the queue held, and `reconcile` answers `unknown` until you delete that ref. +A purge rewrites only `refs/heads/*` and `refs/tags/*`. GitHub's read-only `refs/pull/*` are left alone. If the remote advertises any other ref, such as `refs/notes/commits` or a custom `refs/custom/*`, the purge fails at the clone stage, changes nothing and resumes the queue, and `reconcile` answers `unknown`. Those refs can still hold the purged content, so list them with `git ls-remote `, delete them on the remote (`git push --delete `), and run the purge again. A tag that points at a blob or tree fails the same way. A ref created on the remote before the purge's post-push check fails it at the push stage with the queue held, and `reconcile` answers `unknown` until you delete that ref. Anyone holding an old clone can still push the content back after that check. ## A stuck purge marker