diff --git a/apps/server/src/kin/lead/LeadPrompt.test.ts b/apps/server/src/kin/lead/LeadPrompt.test.ts index 488f2874c9c27..fbc72428b0691 100644 --- a/apps/server/src/kin/lead/LeadPrompt.test.ts +++ b/apps/server/src/kin/lead/LeadPrompt.test.ts @@ -59,6 +59,10 @@ const repository = Chat.layer.pipe(Layer.provide(database)); const memory = Layer.succeed( Memory.Memory, Memory.Memory.of({ + withWriterLock: (_org, effect) => effect, + anchor: () => Effect.succeed("fixture"), + record: () => Effect.void, + recover: () => Effect.void, readPromptSnapshot: (org, head) => org !== orgId || !["head-one", "head-two"].includes(head) ? Effect.fail(new Memory.MemoryError({ orgId: org, operation: "promptSnapshot" })) diff --git a/apps/server/src/kin/lead/WakeupRunner.ts b/apps/server/src/kin/lead/WakeupRunner.ts index 1a551a23f4176..cac394b78d9c1 100644 --- a/apps/server/src/kin/lead/WakeupRunner.ts +++ b/apps/server/src/kin/lead/WakeupRunner.ts @@ -351,7 +351,7 @@ const make = Effect.gen(function* () { } if (events.length === 0) return; const budgetContext = yield* budget.promptContext(orgId); - const pinned = yield* memory.prepareWakeup(orgId).pipe( + const pinned = yield* memory.prepareWakeup(orgId, wakeupId).pipe( Effect.tapError((error) => Effect.logError("Kin memory preparation failed", { orgId, diff --git a/apps/server/src/kin/lead/layer.test.ts b/apps/server/src/kin/lead/layer.test.ts index 62b54d3d8a6d1..c2d16b667511a 100644 --- a/apps/server/src/kin/lead/layer.test.ts +++ b/apps/server/src/kin/lead/layer.test.ts @@ -30,28 +30,36 @@ vi.mock("../memory/Memory.ts", async (importOriginal) => { const DateTime = await import("effect/DateTime"); return { ...original, - layer: Layer.sync(original.Memory, () => { - let prepared = false; - return original.Memory.of({ - ensureCheckouts: () => Effect.succeed({ writer: "/private/tmp", read: "/private/tmp" }), - prepareWakeup: () => - Effect.sync(() => { - prepared = true; - return { head: "fixture-head", indexAsOf: DateTime.makeUnsafe(0) }; - }), - readPromptSnapshot: (_orgId, head) => - prepared && head === "fixture-head" - ? Effect.succeed({ rules: "Fixture standing rules", index: "Fixture memory index" }) - : Effect.die("Prompt and runner must share pinned memory"), - readIndex: () => Effect.die("unused"), - readRules: () => Effect.die("unused"), - query: () => Effect.die("unused"), - readItem: () => Effect.die("unused"), - readItemQuietly: () => Effect.die("unused"), - rebuildIndex: () => Effect.die("unused"), - rebuildCache: () => Effect.die("unused"), - }); - }), + layer: Layer.effect( + original.Memory, + Effect.gen(function* () { + yield* original.MemoryRecoveryHooks; + let prepared = false; + return original.Memory.of({ + withWriterLock: (_orgId, effect) => effect, + anchor: () => Effect.die("unused"), + record: () => Effect.die("unused"), + recover: () => Effect.die("unused"), + ensureCheckouts: () => Effect.succeed({ writer: "/private/tmp", read: "/private/tmp" }), + prepareWakeup: () => + Effect.sync(() => { + prepared = true; + return { head: "fixture-head", indexAsOf: DateTime.makeUnsafe(0) }; + }), + readPromptSnapshot: (_orgId, head) => + prepared && head === "fixture-head" + ? Effect.succeed({ rules: "Fixture standing rules", index: "Fixture memory index" }) + : Effect.die("Prompt and runner must share pinned memory"), + readIndex: () => Effect.die("unused"), + readRules: () => Effect.die("unused"), + query: () => Effect.die("unused"), + readItem: () => Effect.die("unused"), + readItemQuietly: () => Effect.die("unused"), + rebuildIndex: () => Effect.die("unused"), + rebuildCache: () => Effect.die("unused"), + }); + }), + ), }; }); diff --git a/apps/server/src/kin/lead/layer.ts b/apps/server/src/kin/lead/layer.ts index ad30be0ad29b5..56cdbaeb98469 100644 --- a/apps/server/src/kin/lead/layer.ts +++ b/apps/server/src/kin/lead/layer.ts @@ -44,7 +44,20 @@ export const layer = Layer.effectDiscard( Layer.provide(WakeupRunner.layer), Layer.provide( Layer.mergeAll(KinLeadRegistry.layer, KinLeadPrompt.layer).pipe( - Layer.provideMerge(Layer.mergeAll(Memory.layer, KinChatRepository.layer)), + Layer.provideMerge( + Layer.mergeAll( + Memory.layer.pipe( + Layer.provide( + Layer.succeed(Memory.MemoryRecoveryHooks, { + notify: Effect.fn("KinMemoryRecovery.notify")((notice) => + Effect.logWarning("Kin memory recovery requires operator attention", notice), + ), + }), + ), + ), + KinChatRepository.layer, + ), + ), ), ), Layer.provide(KinPersistence.layer), diff --git a/apps/server/src/kin/memory/Memory.test.ts b/apps/server/src/kin/memory/Memory.test.ts index 02f6a25184f36..402e17e08e711 100644 --- a/apps/server/src/kin/memory/Memory.test.ts +++ b/apps/server/src/kin/memory/Memory.test.ts @@ -1,3 +1,6 @@ +import * as NodeZlib from "node:zlib"; +// @effect-diagnostics-next-line nodeBuiltinImport:off -- Git object identity uses the native byte-oriented hash API. +import * as NodeCrypto from "node:crypto"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { expect, it } from "@effect/vitest"; import { KinAgentId, KinWakeupId, KinMemoryItemId, KinOrgId } from "@t3tools/contracts"; @@ -21,6 +24,13 @@ import * as MemoryConfig from "./MemoryConfig.ts"; import * as ServerConfig from "../../config.ts"; import { fixtureItem } from "./__fixtures__/items.ts"; +const memoryTestLayer = Memory.layer.pipe( + Layer.provide( + Layer.succeed(Memory.MemoryRecoveryHooks, { + notify: (notice) => Effect.logWarning("Memory recovery", notice), + }), + ), +); const orgA = KinOrgId.make("demo-a"); const orgB = KinOrgId.make("demo-b"); const itemId = KinMemoryItemId.make("def-001"); @@ -34,17 +44,43 @@ const databaseLayer = Layer.effectDiscard( }), ).pipe(Layer.provideMerge(KinPersistence.layer.pipe(Layer.provideMerge(Sqlite.layerMemory)))); -const fakeRepository = () => { +const fakeRepository = (fs: FileSystem.FileSystem) => { const files = new Map(); - const hashes = new Map(); - const hash = (text: string) => { - let sha = hashes.get(text); - if (!sha) { - sha = (hashes.size + 1).toString(16).padStart(40, "0"); - hashes.set(text, sha); - } + const objects = new Map(); + const object = (type: string, bytes: Buffer) => { + const sha = NodeCrypto.createHash("sha1") + .update(`${type} ${bytes.length}\0`) + .update(bytes) + .digest("hex"); + objects.set(sha, { type, bytes }); return sha; }; + const head = () => { + const tree = (prefix: string): string => { + const names = [ + ...new Set( + [...files.keys()] + .filter((name) => name.startsWith(prefix)) + .map((name) => name.slice(prefix.length).split("/")[0] ?? ""), + ), + ].sort(); + return object( + "tree", + Buffer.concat( + names.map((name) => { + const text = files.get(prefix + name); + const sha = + text === undefined ? tree(prefix + name + "/") : object("blob", Buffer.from(text)); + return Buffer.concat([ + Buffer.from(`${text === undefined ? "40000" : "100644"} ${name}\0`), + Buffer.from(sha, "hex"), + ]); + }), + ), + ); + }; + return object("commit", Buffer.from(`tree ${tree("")}\n\nfixture ${revision}\n`)); + }; let validations = 0; let treeReads = 0; let blobReads = 0; @@ -53,72 +89,84 @@ const fakeRepository = () => { const layer = Layer.succeed( VcsProcess.VcsProcess, VcsProcess.VcsProcess.of({ - run: Effect.fn("Test.git")(({ args, stdin, cwd, onStdoutChunk }) => { + run: Effect.fn("Test.git")(({ args: rawArgs, stdin, cwd, onStdoutChunk }) => { + const args: string[] = []; + for (let i = 0; i < rawArgs.length; i++) { + const arg = rawArgs[i]; + if (arg === "-c") { + i++; + continue; + } + if ( + arg && + !arg.startsWith("--git-dir=") && + !arg.startsWith("--work-tree=") && + arg !== "--no-replace-objects" + ) + args.push(arg); + } let stdout = ""; switch (args[0]) { + case "fetch": case "rev-parse": + if (args.includes("refs/heads/main")) validations++; stdout = args.includes("--show-toplevel") ? cwd : args.includes("--git-common-dir") ? cwd.replace(/memory-read$/, "memory/.git") - : `fixture-head-${revision}`; - break; - case "remote": - validations++; - stdout = "unused"; + : head(); break; - case "ls-tree": { - treeReads++; - stdout = [...files] - .sort(([a], [b]) => a.localeCompare(b)) - .map( - ([file, text]) => `100644 blob ${hash(text)} ${Buffer.byteLength(text)}\t${file}\0`, - ) - .join(""); + case "ls-remote": + stdout = `${head()}\trefs/heads/main\n`; break; - } case "cat-file": { - blobReads++; - if (args[1] === "blob") { - stdout = - [...files.values()].find( - (value) => hash(value) === args[2] && value !== "unreadable fixture", - ) ?? ""; - break; - } - stdout = (stdin?.trim().split("\n") ?? []) - .map((sha) => { - const text = [...files.values()].find( - (value) => hash(value) === sha && value !== "unreadable fixture", - ); - return text === undefined - ? `${sha} missing\n` - : `${sha} blob ${Buffer.byteLength(text)}\n${text}\n`; - }) - .join(""); + const chunks = (stdin?.trim().split("\n") ?? []).map((sha) => { + const entry = objects.get(sha); + if (!entry) return Buffer.from(`${sha} missing\n`); + if (args.includes("--batch-check")) + return Buffer.from(`${sha} ${entry.type} ${entry.bytes.length}\n`); + if (entry.type === "blob") blobReads++; + if (entry.type === "tree") treeReads++; + return Buffer.concat([ + Buffer.from(`${sha} ${entry.type} ${entry.bytes.length}\n`), + entry.bytes, + Buffer.from("\n"), + ]); + }); if (corruptNextBatch) { - stdout = "invalid header\n" + stdout; + chunks.unshift(Buffer.from("invalid header\n")); corruptNextBatch = false; } + if (args.includes("--batch-check")) stdout = Buffer.concat(chunks).toString("utf8"); + else onStdoutChunk?.(Buffer.concat(chunks)); break; } } onStdoutChunk?.(Buffer.from(stdout)); - return Effect.succeed({ - exitCode: ChildProcessSpawner.ExitCode(0), - stdout, - stderr: "", - stdoutTruncated: false, - stderrTruncated: false, + return Effect.gen(function* () { + if (args[0] === "clone" || args[0] === "clone-snapshot") { + yield* fs + .makeDirectory(`${args.at(-1)}/.git/objects`, { recursive: true }) + .pipe(Effect.orDie); + } + return { + exitCode: ChildProcessSpawner.ExitCode(0), + stdout, + stderr: "", + stdoutTruncated: false, + stderrTruncated: false, + }; }); }), }), ); return { files, + head, layer, commit: () => { revision++; + return head(); }, counts: () => ({ treeReads, blobReads, validations }), corruptBatch: () => { @@ -135,7 +183,7 @@ const configLayer = (baseDirectory: string, repoUrl: string) => ], }); const memoryLayer = (base: string, repo: string) => - Memory.layer.pipe( + memoryTestLayer.pipe( Layer.provide(configLayer(base, repo)), Layer.provide(gitLayer), Layer.provide(NodeServices.layer), @@ -365,7 +413,7 @@ it.effect( const fs = yield* FileSystem.FileSystem; const temporary = yield* fs.makeTempDirectoryScoped(); const root = yield* fs.realPath(temporary); - const repository = fakeRepository(); + const repository = fakeRepository(fs); const messages: string[] = []; const loggerLayer = Logger.layer( [Logger.make(({ message }) => messages.push(String(message)))], @@ -373,7 +421,8 @@ it.effect( ); const checkout = `${root}/a/memory-read`; yield* fs.makeDirectory(`${checkout}/deferrals`, { recursive: true }); - yield* fs.makeDirectory(`${root}/a/memory/.git`, { recursive: true }); + yield* fs.makeDirectory(`${root}/a/memory/.git/objects`, { recursive: true }); + yield* fs.makeDirectory(`${root}/a/memory/.git/refs`, { recursive: true }); repository.files.set( "deferrals/def-001.md", fixtureItem({ recorded_at: "2026-11-14T00:00:00.000Z" }), @@ -396,6 +445,7 @@ it.effect( VALUES (${orgA}, 'Asia/Kolkata', '08:30', '18:30', '02:00', 0, 5, '12:00', '{"from":"22:00","to":"07:00"}') `; const memory = yield* Memory.Memory; + yield* memory.anchor(orgA, "main", repository.head()); yield* memory.rebuildIndex(orgA); expect(yield* memory.readIndex(orgA)).not.toContain("## Due in 7 days"); expect(yield* memory.readIndex(orgA)).toContain("As of 2026-11-14T00:00:00.000Z"); @@ -419,14 +469,14 @@ it.effect( expect(moved).toBeDefined(); repository.files.delete("deferrals/def-001.md"); repository.files.set("deferrals/moved.md", moved ?? ""); - repository.commit(); + yield* memory.record(orgA, "main", yield* memory.anchor(orgA, "main"), repository.commit()); yield* memory.prepareWakeup(orgA); yield* TestClock.adjust("1 week"); yield* memory.rebuildIndex(orgA); expect(yield* memory.readIndex(orgA)).toContain("Invented service maintenance"); expect(yield* memory.readIndex(orgA)).not.toContain("Never opened fixture"); }); - const layer = Memory.layer.pipe( + const layer = memoryTestLayer.pipe( Layer.provide(configLayer(root, "unused")), Layer.provide(repository.layer), Layer.provide(NodeServices.layer), @@ -443,8 +493,8 @@ it.effect( expect(yield* memory.query(orgA, {})).toHaveLength(2); const beforeRebuild = repository.counts(); yield* memory.rebuildCache(orgA); - expect(repository.counts().treeReads).toBe(beforeRebuild.treeReads + 1); - expect(repository.counts().blobReads).toBe(beforeRebuild.blobReads + 1); + expect(repository.counts().treeReads).toBe(beforeRebuild.treeReads + 2); + expect(repository.counts().blobReads).toBe(beforeRebuild.blobReads); yield* memory.readItem(orgA, itemId); yield* memory.rebuildIndex(orgA); expect(yield* memory.readIndex(orgA)).toContain("Invented service maintenance"); @@ -458,7 +508,7 @@ it.effect( }), ); } - repository.commit(); + yield* memory.record(orgA, "main", yield* memory.anchor(orgA, "main"), repository.commit()); yield* memory.prepareWakeup(orgA); yield* memory.rebuildIndex(orgA); expect((yield* memory.readIndex(orgA)).trimEnd().split("\n")).toHaveLength(200); @@ -476,7 +526,12 @@ it.effect( "# index\nAs of 2027-02-28T00:00:00Z\n", ]) { repository.files.set("INDEX.md", index); - repository.commit(); + yield* memory.record( + orgA, + "main", + yield* memory.anchor(orgA, "main"), + repository.commit(), + ); const result = yield* memory.prepareWakeup(orgA); expect(DateTime.toEpochMillis(result.indexAsOf)).toBe(yield* Clock.currentTimeMillis); } @@ -484,7 +539,7 @@ it.effect( "deferrals/duplicate.md", fixtureItem({ recorded_at: "2027-02-27T00:00:00Z" }), ); - repository.commit(); + yield* memory.record(orgA, "main", yield* memory.anchor(orgA, "main"), repository.commit()); yield* memory.prepareWakeup(orgA); const duplicates = yield* memory.query(orgA, {}); expect( @@ -501,8 +556,9 @@ it.effect( for (const id of ["fallback-1", "fallback-2"]) { repository.files.set(`deferrals/${id}.md`, fixtureItem({ id })); } - repository.commit(); + yield* memory.record(orgA, "main", yield* memory.anchor(orgA, "main"), repository.commit()); repository.corruptBatch(); + expect((yield* memory.prepareWakeup(orgA).pipe(Effect.result))._tag).toBe("Failure"); yield* memory.prepareWakeup(orgA); const afterFallback = yield* memory.query(orgA, {}); expect(afterFallback.filter((item) => !item.id.startsWith("fallback-"))).toEqual( @@ -513,10 +569,10 @@ it.effect( expect(messages.filter((message) => message.includes("Duplicate memory id"))).toHaveLength( 1, ); - expect(messages.some((message) => message.includes("batch framing failed"))).toBe(true); + const sql = yield* SqlClient.SqlClient; yield* sql`UPDATE kin_trigger_settings SET time_zone = 'UTC' WHERE org_id = ${orgA}`; - repository.commit(); + yield* memory.record(orgA, "main", yield* memory.anchor(orgA, "main"), repository.commit()); yield* memory.prepareWakeup(orgA); const utcIndex = yield* memory.readIndex(orgA); yield* Effect.gen(function* () { @@ -534,7 +590,12 @@ it.effect( yield* sql`UPDATE kin_trigger_settings SET time_zone = ${row.zone} WHERE org_id = ${orgA}`; expect(yield* cache.readOwnerTimeZone(orgA)).toEqual(row.expected); expect(yield* cache.readOwnerTimeZone(orgB)).toEqual({ _tag: "missing" }); - repository.commit(); + yield* memory.record( + orgA, + "main", + yield* memory.anchor(orgA, "main"), + repository.commit(), + ); yield* memory.prepareWakeup(orgA); if (row.expected._tag !== "zone") expect(yield* memory.readIndex(orgA)).toBe(utcIndex); } @@ -545,7 +606,7 @@ it.effect( expect(yield* cache.readOwnerTimeZone(orgA)).toEqual({ _tag: "unavailable" }); }).pipe(Effect.provide(MemoryCache.layer)); yield* sql`DROP TABLE kin_memory_reads`; - repository.commit(); + yield* memory.record(orgA, "main", yield* memory.anchor(orgA, "main"), repository.commit()); yield* memory.prepareWakeup(orgA); expect(yield* memory.readIndex(orgA)).toContain("As of "); expect( @@ -559,7 +620,7 @@ it.effect( ); expect( messages.filter((message) => message.includes("Skipping malformed memory item")), - ).toHaveLength(2); + ).toHaveLength(4); }).pipe(Effect.provide(Layer.merge(layer, loggerLayer))); }).pipe(Effect.provide(Layer.merge(NodeServices.layer, databaseLayer)), Effect.scoped), ); @@ -583,7 +644,7 @@ it.effect.each([ expect(yield* fs.readFileString(`${outside}/keep`)).toBe("untouched"); }).pipe( Effect.provide( - Memory.layer.pipe( + memoryTestLayer.pipe( Layer.provide( Layer.succeed(MemoryConfig.MemoryConfig, { baseDirectory: root, @@ -594,7 +655,7 @@ it.effect.each([ })), }), ), - Layer.provide(fakeRepository().layer), + Layer.provide(fakeRepository(fs).layer), ), ), ); @@ -722,7 +783,7 @@ it.live.each([ expect(yield* fs.readDirectory(base)).toEqual([first]); }).pipe( Effect.provide( - Memory.layer.pipe( + memoryTestLayer.pipe( Layer.provide( Layer.succeed(MemoryConfig.MemoryConfig, { baseDirectory: base, @@ -740,7 +801,7 @@ it.live.each([ ), ); -it.live("refuses reads from a writer with the wrong origin after service restart", () => +it.live("ignores lead-controlled origin after service restart", () => Effect.gen(function* () { const { root, source, git } = yield* fixtureRepository(); const base = `${root}/state`; @@ -752,8 +813,7 @@ it.live("refuses reads from a writer with the wrong origin after service restart yield* Effect.gen(function* () { const memory = yield* Memory.Memory; const result = yield* memory.query(orgA, {}).pipe(Effect.result); - expect(result._tag).toBe("Failure"); - if (result._tag === "Failure") expect(result.failure._tag).toBe("MemoryCheckoutInvalid"); + expect(result._tag).toBe("Success"); }).pipe(Effect.provide(memoryLayer(base, source))); }).pipe( Effect.provide(Layer.mergeAll(gitLayer, NodeServices.layer, databaseLayer)), @@ -775,6 +835,10 @@ it.live("uses filesystem identity for a case-mismatched base without recloning", }).pipe(Effect.provide(memoryLayer(base, source))); yield* fs.writeFileString(`${writer}/keep`, "unpushed work"); const head = yield* commit(writer); + yield* Effect.gen(function* () { + const memory = yield* Memory.Memory; + yield* memory.record(orgA, "main", yield* memory.anchor(orgA, "main"), head); + }).pipe(Effect.provide(memoryLayer(base, source))); const before = yield* fs.stat(writer); yield* Effect.gen(function* () { const memory = yield* Memory.Memory; @@ -792,80 +856,271 @@ it.live("uses filesystem identity for a case-mismatched base without recloning", ), ); -it.live.each(["diverged", "conflict", "network down", "dirty writer", "rebase in progress"])( - "syncs the writer with $0", - (scenario) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const { root, source, git, commit } = yield* fixtureRepository(); - const base = `${root}/state`; - const messages: string[] = []; - const logger = Logger.layer([Logger.make(({ message }) => messages.push(String(message)))], { - mergeWithExisting: false, - }); - yield* Effect.gen(function* () { - const memory = yield* Memory.Memory; - const initial = yield* memory.prepareWakeup(orgA); - const { writer } = yield* memory.ensureCheckouts(orgA); - yield* git(writer, ["config", "user.name", "Fixture"]); - yield* git(writer, ["config", "user.email", "fixture@example.invalid"]); - yield* fs.writeFileString(`${writer}/rules.md`, "Local invented rules\n"); - const local = yield* commit(writer); +it.live.each([ + "diverged", + "conflict", + "network down", + "dirty writer", + "rebase in progress", + "journal conflict", + "H2-merge-swap", + "unrelated", + "criss-cross", + "push retry", +])("syncs the writer with $0", (scenario) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const { root, source, git, commit } = yield* fixtureRepository(); + const base = `${root}/state`; + const remote = `${root}/sync-remote.git`; + const h2Base = + "said_by: fixture-lead\ntrust: agent\n" + + "unchanged fixture context\n".repeat(12) + + "base tail\n"; + const h2Local = h2Base.replace("fixture-lead", "local-lead"); + if (scenario === "H2-merge-swap") { + yield* fs.writeFileString(`${source}/rules.md`, h2Base); + yield* commit(source); + } + yield* git(root, ["clone", "--bare", "--", source, remote]); + const vcs = yield* VcsProcess.VcsProcess; + let attack: { sha: string; path: string } | undefined; + let swapped = false; + const intercepted = Layer.succeed( + VcsProcess.VcsProcess, + VcsProcess.VcsProcess.of({ + run: Effect.fn("MemoryTest.mergeSwap")(function* (input) { + if ( + attack && + (input.args.includes("merge-tree") || + (input.args.includes("--batch") && input.stdin?.includes(attack.sha))) + ) { + const forged = Buffer.from( + h2Local.replace("local-lead", "real-owner").replace("trust: agent", "trust: owner"), + ); + yield* fs.remove(attack.path).pipe(Effect.orDie); + yield* fs + .writeFile( + attack.path, + NodeZlib.deflateSync( + Buffer.concat([Buffer.from(`blob ${forged.length}\0`), forged]), + ), + ) + .pipe(Effect.orDie); + attack = undefined; + swapped = true; + } + return yield* vcs.run(input); + }), + }), + ); + const messages: string[] = []; + const logger = Logger.layer([Logger.make(({ message }) => messages.push(String(message)))], { + mergeWithExisting: false, + }); + yield* Effect.gen(function* () { + const memory = yield* Memory.Memory; + const initial = yield* memory.prepareWakeup(orgA); + const { writer } = yield* memory.ensureCheckouts(orgA); + yield* git(writer, ["config", "user.name", "Fixture"]); + yield* git(writer, ["config", "user.email", "fixture@example.invalid"]); + if (scenario === "journal conflict" || scenario === "push retry") { + yield* fs.makeDirectory(`${writer}/journal`, { recursive: true }); + yield* fs.makeDirectory(`${source}/journal`, { recursive: true }); + yield* fs.writeFileString(`${writer}/journal/day.md`, "local entry\n"); + yield* fs.writeFileString(`${writer}/INDEX.md`, "local generated index\n"); + } else yield* fs.writeFileString( - `${source}/${scenario === "diverged" || scenario === "network down" || scenario === "dirty writer" ? "remote-only" : "rules.md"}`, - "Remote invented edit\n", + `${writer}/rules.md`, + scenario === "H2-merge-swap" ? h2Local : "Local invented rules\n", ); - const remote = yield* commit(source); - if (scenario === "network down") yield* fs.rename(source, `${root}/offline`); - if (scenario === "dirty writer") - yield* fs.writeFileString(`${writer}/rules.md`, "Uncommitted fixture edit\n"); - if (scenario === "rebase in progress") { - yield* git(writer, ["fetch", "origin"]); - yield* git(writer, ["rebase", "@{u}"]).pipe(Effect.ignore); - yield* fs.writeFileString(`${writer}/rules.md`, "Half-resolved fixture edit\n"); + let local = yield* commit(writer); + if (scenario === "push retry") { + yield* git(writer, [ + "commit", + "--allow-empty", + "-m", + "Earlier journal outcome\n\nEvents: fixture-earlier", + ]); + yield* git(writer, [ + "commit", + "--allow-empty", + "-m", + "Later journal outcome\n\nEvents: fixture-later", + ]); + local = yield* git(writer, ["rev-parse", "HEAD"]); + } + yield* memory.record(orgA, "main", initial.head, local); + yield* fs.writeFileString( + `${source}/${scenario === "push retry" || scenario === "diverged" || scenario === "network down" || scenario === "dirty writer" || scenario === "H2-merge-swap" ? "remote-only" : "rules.md"}`, + "Remote invented edit\n", + ); + if (scenario === "journal conflict" || scenario === "push retry") { + yield* fs.writeFileString(`${source}/journal/day.md`, "remote entry\n"); + yield* fs.writeFileString(`${source}/INDEX.md`, "remote generated index\n"); + } + if (scenario === "H2-merge-swap") + yield* fs.writeFileString(`${source}/rules.md`, h2Base.replace("base tail", "remote tail")); + yield* commit(source); + yield* git(source, ["push", "--", remote, "main"]); + if (scenario === "network down") yield* fs.rename(remote, `${root}/offline`); + if (scenario === "dirty writer") + yield* fs.writeFileString(`${writer}/rules.md`, "Uncommitted fixture edit\n"); + if (scenario === "rebase in progress") { + yield* git(writer, ["fetch", "origin"]); + yield* git(writer, ["rebase", "@{u}"]).pipe(Effect.ignore); + yield* fs.writeFileString(`${writer}/rules.md`, "Half-resolved fixture edit\n"); + } + if (scenario === "H2-merge-swap") { + const sha = yield* git(writer, ["rev-parse", `${local}:rules.md`]); + attack = { sha, path: `${writer}/.git/objects/${sha.slice(0, 2)}/${sha.slice(2)}` }; + } + if (scenario === "unrelated" || scenario === "criss-cross") { + yield* git(writer, ["fetch", "--", remote, "main"]); + const remoteHead = yield* git(source, ["rev-parse", "HEAD"]); + const tree = yield* git(writer, ["rev-parse", `${local}^{tree}`]); + const parents = scenario === "criss-cross" ? ["-p", local, "-p", remoteHead] : []; + const replacement = yield* git(writer, [ + "commit-tree", + tree, + ...parents, + "-m", + "invented local merge", + ]); + if (scenario === "criss-cross") { + const other = yield* git(writer, [ + "commit-tree", + tree, + "-p", + remoteHead, + "-p", + local, + "-m", + "invented remote merge", + ]); + yield* git(writer, ["push", "--", remote, `${other}:refs/heads/main`]); } - const result = yield* memory.prepareWakeup(orgA).pipe(Effect.result); - if (scenario === "rebase in progress") { - expect(result._tag).toBe("Failure"); - if (result._tag === "Failure") expect(result.failure._tag).toBe("MemoryCheckoutInvalid"); - expect(yield* fs.exists(`${writer}/.git/rebase-merge`)).toBe(true); - expect(yield* fs.readFileString(`${writer}/rules.md`)).toBe( - "Half-resolved fixture edit\n", + yield* git(writer, ["update-ref", "refs/heads/main", replacement, local]); + yield* memory.record(orgA, "main", local, replacement); + local = replacement; + } + const result = yield* memory.prepareWakeup(orgA, "fixture-wakeup").pipe(Effect.result); + if (scenario === "unrelated" || scenario === "criss-cross") { + expect(result._tag).toBe("Failure"); + if (result._tag === "Failure") expect(result.failure._tag).toBe("MemorySyncConflict"); + expect(yield* git(writer, ["rev-parse", "refs/heads/main"])).toBe(local); + expect( + yield* git(root, [ + `--git-dir=${remote}`, + "for-each-ref", + "--format=%(objectname)", + `refs/heads/kin/conflict/${orgA}/fixture-wakeup-*`, + ]), + ).toBe(local); + expect( + messages.some( + (message) => + message.includes("No unique verified merge base") && + message.includes("fixture-wakeup"), + ), + ).toBe(true); + const again = yield* memory.prepareWakeup(orgA, "fixture-wakeup-2").pipe(Effect.result); + expect(again._tag).toBe("Failure"); + expect( + yield* git(root, [ + `--git-dir=${remote}`, + "for-each-ref", + "--format=%(objectname)", + `refs/heads/kin/conflict/${orgA}/`, + ]), + ).toBe(local); + yield* memory.recover(orgA); + expect( + (yield* memory.prepareWakeup(orgA, "fixture-wakeup-3").pipe(Effect.result))._tag, + ).toBe("Success"); + } else if (scenario === "push retry") { + expect(result._tag).toBe("Success"); + if (result._tag === "Success") { + const merged = result.success.head; + expect((yield* git(writer, ["show", "-s", "--format=%P", merged])).split(" ")).toContain( + local, ); - } else if (scenario === "dirty writer") { - expect(result._tag).toBe("Failure"); - if (result._tag === "Failure") expect(result.failure._tag).toBe("MemoryError"); - expect(yield* git(writer, ["rev-parse", "HEAD"])).toBe(local); - expect(yield* fs.readFileString(`${writer}/rules.md`)).toBe("Uncommitted fixture edit\n"); - } else if (scenario === "conflict") { - expect(result._tag).toBe("Failure"); - if (result._tag === "Failure") expect(result.failure._tag).toBe("MemorySyncConflict"); - expect(yield* git(writer, ["rev-parse", "HEAD"])).toBe(local); - expect(yield* fs.readFileString(`${writer}/rules.md`)).toBe("Local invented rules\n"); - expect(yield* git(writer, ["status", "--porcelain"])).toBe(""); - expect(yield* fs.exists(`${writer}/.git/rebase-merge`)).toBe(false); - expect(yield* memory.readRules(orgA)).toBe("Invented standing rules.\n"); - } else { - expect(result._tag).toBe("Success"); - if (result._tag !== "Success") return; - expect(result.success.head).not.toBe(initial.head); - expect(yield* memory.readRules(orgA)).toBe("Local invented rules\n"); - if (scenario === "diverged") { + const messages = yield* git(writer, ["log", "--format=%B", merged]); + expect(messages).toContain("Events: fixture-earlier"); + expect(messages).toContain("Events: fixture-later"); + const journalBefore = yield* git(writer, ["show", `${merged}:journal/day.md`]); + yield* git(writer, ["update-ref", "refs/heads/main", local, merged]); + yield* memory.record(orgA, "main", merged, local); + expect((yield* memory.prepareWakeup(orgA)).head).toBe(merged); + expect(yield* git(writer, ["show", `${merged}:journal/day.md`])).toBe(journalBefore); + } + } else if (scenario === "H2-merge-swap") { + expect(swapped).toBe(true); + expect(result._tag).toBe("Failure"); + expect(yield* git(root, [`--git-dir=${remote}`, "show", "main:rules.md"])).not.toContain( + "FORGED", + ); + } else if ( + scenario === "network down" || + scenario === "diverged" || + scenario === "dirty writer" + ) { + expect(result._tag).toBe("Success"); + if (result._tag === "Success") { + if (scenario === "network down") expect(result.success.head).toBe(local); + else { expect(result.success.head).not.toBe(local); - expect(yield* git(writer, ["merge-base", "HEAD", remote])).toBe(remote); - expect(yield* fs.readFileString(`${writer}/remote-only`)).toBe( - "Remote invented edit\n", - ); - } else { - expect(result.success.head).toBe(local); - expect(messages.some((message) => message.includes("using local writer HEAD"))).toBe( - true, + expect(yield* memory.readRules(orgA)).toBe("Local invented rules\n"); + expect(yield* git(writer, ["show", `${result.success.head}:remote-only`])).toBe( + "Remote invented edit", ); } } - }).pipe(Effect.provide(Layer.merge(memoryLayer(base, source), logger))); + } else if (scenario === "journal conflict" || scenario === "push retry") { + expect(result._tag).toBe("Success"); + if (result._tag === "Success") + expect(yield* git(writer, ["show", `${result.success.head}:journal/day.md`])).toBe( + "remote entry\nlocal entry", + ); + } else if (scenario === "conflict") { + expect(result._tag).toBe("Success"); + const backup = (yield* git(root, [ + `--git-dir=${remote}`, + "for-each-ref", + "--format=%(objectname)", + `refs/heads/kin/conflict/${orgA}/`, + ])).trim(); + expect(backup).toBe(local); + expect((yield* memory.prepareWakeup(orgA)).head).toBe( + yield* git(source, ["rev-parse", "HEAD"]), + ); + } else { + expect(result._tag).toBe("Failure"); + if (result._tag === "Failure") expect(result.failure._tag).toBe("MemoryCheckoutInvalid"); + } + if (scenario === "rebase in progress" || scenario === "network down") + expect(yield* git(writer, ["rev-parse", "refs/heads/main"])).toBe(local); }).pipe( - Effect.provide(Layer.mergeAll(gitLayer, NodeServices.layer, databaseLayer)), - Effect.scoped, - ), + Effect.provide( + Layer.merge( + Memory.layer.pipe( + Layer.provide(configLayer(base, remote)), + Layer.provide(intercepted), + Layer.provide( + Layer.succeed(Memory.MemoryRecoveryHooks, { + notify: (notice) => + Effect.sync(() => { + messages.push(JSON.stringify(notice)); + }), + }), + ), + ), + logger, + ), + ), + ); + }).pipe( + Effect.provide(Layer.mergeAll(gitLayer, NodeServices.layer, databaseLayer)), + Effect.scoped, + ), ); diff --git a/apps/server/src/kin/memory/Memory.ts b/apps/server/src/kin/memory/Memory.ts index 5a1531675c9ee..4ecc9ded5fd74 100644 --- a/apps/server/src/kin/memory/Memory.ts +++ b/apps/server/src/kin/memory/Memory.ts @@ -1,8 +1,16 @@ +import { + readTree, + readObjects, + ancestors, + readCommit, + readFiles, + objectHash, + verifyTree, +} from "./objects.ts"; import { type KinMemoryItemId, type KinOrgId, MemoryItemHead, - VcsProcessExitError, type MemoryItemKind, type MemoryItemFrontMatter, } from "@t3tools/contracts"; @@ -12,14 +20,16 @@ import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; -import * as Option from "effect/Option"; import * as Path from "effect/Path"; import * as Schema from "effect/Schema"; import * as Semaphore from "effect/Semaphore"; -import { isNonRepositoryGitStderr } from "../../vcs/gitErrors.ts"; -import * as VcsProcess from "../../vcs/VcsProcess.ts"; +import * as SqlClient from "effect/sql/SqlClient"; +import { toPersistenceSqlError } from "../../persistence/Errors.ts"; +import { memoryGit as runMemoryGit } from "./git.ts"; +import { VcsProcess } from "../../vcs/VcsProcess.ts"; import * as MemoryCache from "./MemoryCache.ts"; +import { mergeMemoryFiles, uniqueMergeBase } from "./merge.ts"; import { buildIndex } from "./buildIndex.ts"; import { MemoryConfig } from "./MemoryConfig.ts"; import { @@ -106,12 +116,42 @@ export interface MemoryQuery { readonly limit?: number; } +export class MemoryRecoveryHooks extends Context.Service< + MemoryRecoveryHooks, + { + readonly notify: (notice: { + orgId: KinOrgId; + branch: string; + reason: string; + conflictBranch?: string; + }) => Effect.Effect; + } +>()("t3/kin/memory/Memory/MemoryRecoveryHooks") {} + export class Memory extends Context.Service< Memory, { + readonly withWriterLock: ( + orgId: KinOrgId, + effect: Effect.Effect, + ) => Effect.Effect; + readonly anchor: ( + orgId: KinOrgId, + branch: string, + initial?: string, + ) => Effect.Effect; + /** Server-only; never register as an agent tool. */ + readonly record: ( + orgId: KinOrgId, + branch: string, + previous: string, + head: string, + ) => Effect.Effect; + readonly recover: (orgId: KinOrgId) => Effect.Effect; readonly ensureCheckouts: (orgId: KinOrgId) => Effect.Effect; readonly prepareWakeup: ( orgId: KinOrgId, + wakeupId?: string, ) => Effect.Effect<{ head: string; indexAsOf: DateTime.Utc }, MemoryFailure>; readonly readPromptSnapshot: ( orgId: KinOrgId, @@ -140,16 +180,6 @@ export class Memory extends Context.Service< >()("t3/kin/memory/Memory") {} const decodeHead = Schema.decodeEffect(MemoryItemHead); -const isVcsProcessExitError = Schema.is(VcsProcessExitError); -// Mirrors STATUS_UPSTREAM_REFRESH_ENV in vcs/GitVcsDriverCore.ts. -const memoryGitEnv = Object.freeze({ - GCM_INTERACTIVE: "never", - GIT_ASKPASS: "", - GIT_TERMINAL_PROMPT: "0", - SSH_ASKPASS: "", - SSH_ASKPASS_REQUIRE: "never", -}); - const folders = ["objectives", "commitments", "deferrals", "decisions", "people", "claims"]; const maxBlobBytes = 256 * 1024; @@ -175,8 +205,94 @@ interface BlobCache { const make = Effect.gen(function* () { const config = yield* MemoryConfig; const fs = yield* FileSystem.FileSystem; + const { notify } = yield* MemoryRecoveryHooks; + const vcs = yield* VcsProcess; + const memoryGit = (...args: Parameters) => + runMemoryGit(...args).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(VcsProcess, vcs), + Effect.provideService(MemoryConfig, config), + ); const path = yield* Path.Path; - const vcs = yield* VcsProcess.VcsProcess; + const sql = yield* SqlClient.SqlClient; + const writerLocks = new Map(); + const withWriterLock = (orgId: KinOrgId, effect: Effect.Effect) => { + let lock = writerLocks.get(orgId); + if (!lock) { + lock = Semaphore.makeUnsafe(1); + writerLocks.set(orgId, lock); + } + return lock.withPermits(1)(effect); + }; + const persistence = (orgId: KinOrgId, effect: Effect.Effect) => + effect.pipe( + Effect.mapError(toPersistenceSqlError("Memory.anchor")), + Effect.mapError((cause) => new MemoryError({ orgId, operation: "anchor", cause })), + ); + const anchor = Effect.fn("Memory.anchor")(function* ( + orgId: KinOrgId, + branch: string, + initial?: string, + ) { + return yield* persistence( + orgId, + sql.withTransaction( + Effect.gen(function* () { + if (initial !== undefined) + yield* sql`INSERT INTO kin_memory_heads (org_id, branch, head) + VALUES (${orgId}, ${branch}, ${initial}) ON CONFLICT DO NOTHING RETURNING head`; + const rows = yield* sql<{ + head: string; + }>`SELECT head FROM kin_memory_heads WHERE org_id = ${orgId} AND branch = ${branch} ORDER BY branch`; + return rows[0]?.head; + }), + ), + ).pipe( + Effect.filterOrFail( + (head): head is string => head !== undefined, + () => + new MemoryCheckoutInvalid({ + orgId, + operation: "anchor", + reason: "Missing SQLite anchor", + }), + ), + ); + }); + const record = Effect.fn("Memory.record")(function* ( + orgId: KinOrgId, + branch: string, + previous: string, + head: string, + ) { + yield* sql + .withTransaction( + Effect.gen(function* () { + const rows = yield* persistence( + orgId, + sql`UPDATE kin_memory_heads SET head = ${head} + WHERE org_id = ${orgId} AND branch = ${branch} AND head = ${previous} RETURNING head`, + ); + if (rows.length !== 1) + return yield* new MemoryCheckoutInvalid({ + orgId, + operation: "record", + reason: "Anchor changed", + }); + }), + ) + .pipe( + Effect.mapError((cause) => + cause._tag === "MemoryCheckoutInvalid" || cause._tag === "MemoryError" + ? cause + : new MemoryError({ + orgId, + operation: "record", + cause: toPersistenceSqlError("Memory.record")(cause), + }), + ), + ); + }); const cache = yield* MemoryCache.MemoryCache; const locks = new Map(); const pinnedHeads = new Map(); @@ -204,33 +320,47 @@ const make = Effect.gen(function* () { stdin?: string, onStdoutChunk?: (chunk: Uint8Array) => void, ) { - const result = yield* vcs - .run({ - operation: "Memory.git", - command: "git", - cwd, - args, - ...(stdin === undefined ? {} : { stdin }), - env: memoryGitEnv, - ...(onStdoutChunk === undefined ? {} : { onStdoutChunk }), - classifyFailure: (stderr) => (isNonRepositoryGitStderr(stderr) ? "not-found" : null), - timeoutMs: 120_000, - maxOutputBytes: 64 * 1024 * 1024, - outputMode: "error", - }) - .pipe( - Effect.tapError((cause) => - Effect.logWarning("Memory git command failed", { - orgId, - command: args[0], - errorTag: cause._tag, - }), - ), - Effect.mapError((cause) => new MemoryError({ orgId, operation: "git", cause })), - ); - return result.stdout; + return yield* memoryGit(cwd, args, stdin, {}, onStdoutChunk).pipe( + Effect.mapError((cause) => + cause._tag === "MemoryGitMissingRepository" + ? new MemoryCheckoutInvalid({ + orgId, + operation: "git", + reason: "Writer .git is missing or not a directory", + path: cwd, + cause, + }) + : new MemoryError({ orgId, operation: "git", cause }), + ), + ); }); - const io = (orgId: KinOrgId, operation: string, effect: Effect.Effect) => + const advertisedHead = Effect.fn("Memory.advertisedHead")(function* ( + orgId: KinOrgId, + writer: string, + url: string, + branch: string, + ) { + const line = (yield* git(orgId, writer, [ + "ls-remote", + "--", + url, + `refs/heads/${branch}`, + ])).trim(); + const [head, name] = line.split(/\s+/); + if (!head || !/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(head) || name !== `refs/heads/${branch}`) + return yield* new MemoryCheckoutInvalid({ + orgId, + operation: "remote", + reason: "Missing advertised head", + }); + yield* git(orgId, writer, ["fetch", "--", url, head]); + yield* readFiles(orgId, writer, head).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(VcsProcess, vcs), + ); + return head; + }); + const io = (orgId: KinOrgId, operation: string, effect: Effect.Effect) => effect.pipe(Effect.mapError((cause) => new MemoryError({ orgId, operation, cause }))); const pathsFor = Effect.fn("Memory.pathsFor")(function* (orgId: KinOrgId) { @@ -291,21 +421,16 @@ const make = Effect.gen(function* () { return { repo, root, writer, read }; }); - const sameDirectory = Effect.fn("Memory.sameDirectory")(function* ( - orgId: KinOrgId, - left: string, - right: string, - ) { - const a = yield* io(orgId, "stat", fs.stat(left)); - const b = yield* io(orgId, "stat", fs.stat(right)); - return ( - a.dev === b.dev && Option.isSome(a.ino) && Option.isSome(b.ino) && a.ino.value === b.ino.value + const walkAncestors = (...args: Parameters) => + ancestors(...args).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(VcsProcess, vcs), ); - }); const validateWriter = Effect.fn("Memory.validateWriter")(function* ( orgId: KinOrgId, paths: { writer: string; repo: { repoUrl: string } }, + checkTip = true, ) { yield* Effect.gen(function* () { const entry = yield* fs.lstat(path.join(paths.writer, ".git")); @@ -315,21 +440,26 @@ const make = Effect.gen(function* () { operation: "source", reason: "Writer .git must be a directory", }); - const root = (yield* git(orgId, paths.writer, ["rev-parse", "--show-toplevel"])).trim(); - if (!(yield* sameDirectory(orgId, root, paths.writer))) - return yield* new MemoryError({ - orgId, - operation: "source", - reason: "Writer is not the repository root", - }); - yield* git(orgId, paths.writer, ["rev-parse", "--verify", "HEAD^{commit}"]); - const origin = (yield* git(orgId, paths.writer, ["remote", "get-url", "origin"])).trim(); - if (origin !== paths.repo.repoUrl) - return yield* new MemoryError({ + const branch = config.defaultBranch ?? "main"; + const expected = yield* anchor(orgId, branch); + for (const name of ["objects", "refs"]) { + if ((yield* fs.lstat(`${paths.writer}/.git/${name}`)).type !== "Directory") + return yield* new MemoryCheckoutInvalid({ + orgId, + operation: "validate", + reason: "Invalid git directory", + }); + } + const head = (yield* git(orgId, paths.writer, ["rev-parse", `refs/heads/${branch}`])).trim(); + if (!checkTip) return; + if (head !== expected) { + yield* git(orgId, paths.writer, ["update-ref", `refs/heads/${branch}`, expected, head]); + yield* notify({ orgId, - operation: "origin", - reason: "Repository origin does not match configuration", + branch, + reason: "Reset tampered branch to anchor; unpushed work at the replaced tip is lost", }); + } }).pipe( Effect.mapError( (cause) => @@ -349,12 +479,13 @@ const make = Effect.gen(function* () { reusePin = false, ) { const paths = yield* pathsFor(orgId); - if (reusePin && initializedCheckouts.has(orgId) && pinnedHeads.has(orgId)) + if (reusePin && initializedCheckouts.has(orgId) && pinnedHeads.has(orgId)) { + yield* validateWriter(orgId, paths, false); return { writer: paths.writer, read: paths.read }; + } yield* io(orgId, "mkdir", fs.makeDirectory(paths.root, { recursive: true })); - const gitDirectory = path.join(paths.writer, ".git"); if (yield* io(orgId, "exists", fs.exists(paths.writer))) { - yield* validateWriter(orgId, paths); + yield* validateWriter(orgId, paths, false); } else { for (const entry of yield* io(orgId, "clones", fs.readDirectory(paths.root))) { if (/^memory\.clone-[A-Za-z0-9]+$/.test(entry)) @@ -372,49 +503,36 @@ const make = Effect.gen(function* () { directory: paths.root, prefix: "memory.clone-", }); - yield* git(orgId, paths.root, ["clone", "--", paths.repo.repoUrl, temporary]); - yield* validateWriter(orgId, { ...paths, writer: temporary }); + yield* git(orgId, paths.root, [ + "clone", + "--branch", + config.defaultBranch ?? "main", + "--", + paths.repo.repoUrl, + temporary, + ]); + const branch = config.defaultBranch ?? "main"; + const head = yield* advertisedHead(orgId, temporary, paths.repo.repoUrl, branch); + const expected = yield* anchor(orgId, branch, head); + if (expected !== head) + return yield* new MemoryCheckoutInvalid({ + orgId, + operation: "bootstrap", + reason: "Existing anchor differs", + }); yield* fs.rename(temporary, paths.writer); }).pipe(Effect.scoped), ); } - const readExists = yield* io(orgId, "exists", fs.exists(paths.read)); - const validRead = - readExists && - (yield* git(orgId, paths.read, [ - "rev-parse", - "--path-format=absolute", - "--git-common-dir", - ]).pipe( - Effect.flatMap((common) => sameDirectory(orgId, common.trim(), gitDirectory)), - Effect.flatMap((same) => - same - ? git(orgId, paths.read, ["rev-parse", "--show-toplevel"]).pipe( - Effect.flatMap((root) => sameDirectory(orgId, root.trim(), paths.read)), - ) - : Effect.succeed(false), - ), - Effect.catchTags({ - MemoryError: (error) => - isVcsProcessExitError(error.cause) && error.cause.failureKind === "not-found" - ? Effect.succeed(false) - : Effect.fail(error), - }), - )); - if (!validRead) { - yield* io(orgId, "recoverWorktree", fs.remove(paths.read, { recursive: true, force: true })); - yield* git(orgId, paths.writer, ["worktree", "prune"]); - yield* git(orgId, paths.writer, ["worktree", "add", "--detach", paths.read, "HEAD"]); - } initializedCheckouts.add(orgId); return { writer: paths.writer, read: paths.read }; }); const load = Effect.fn("Memory.load")(function* (orgId: KinOrgId, force = false) { const paths = yield* pathsFor(orgId); - const head = - pinnedHeads.get(orgId) ?? (yield* git(orgId, paths.writer, ["rev-parse", "HEAD"])).trim(); - if (pinnedHeads.get(orgId) !== head) yield* validateWriter(orgId, paths); + const pinned = pinnedHeads.get(orgId); + if (pinned === undefined) yield* validateWriter(orgId, paths, false); + const head = pinned ?? (yield* anchor(orgId, config.defaultBranch ?? "main")); pinnedHeads.set(orgId, head); const previous = snapshots.get(orgId); if (!force && previous?.head === head) return previous; @@ -425,25 +543,35 @@ const make = Effect.gen(function* () { }; blobCaches.set(orgId, blobCache); const { blobs, parsedBlobs, mismatchWarnings } = blobCache; - const tree = yield* git(orgId, paths.writer, [ - "ls-tree", - "-r", - "-l", - "-z", - head, - "--", - ...folders, - "rules.md", - "INDEX.md", - ]); - const entries = tree.split("\0").flatMap((entry) => { - const match = /^(\d+) (\S+) ([0-9a-f]+) +([0-9]+)\t([\s\S]+)$/.exec(entry); - if (!match) return []; - const [, mode, type, sha, sizeText, relative] = match; - if (!sha || !relative || mode === "120000" || type !== "blob" || !relative.endsWith(".md")) - return []; - return [{ sha, relative, size: Number(sizeText) }]; - }); + const verifiedEntries = yield* readTree(orgId, paths.writer, head).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(VcsProcess, vcs), + ); + const sizes = new Map(); + const blobsToSize = verifiedEntries.filter((entry) => + ["100644", "100755"].includes(entry.mode), + ); + if (blobsToSize.length) { + const headers = yield* git( + orgId, + paths.writer, + ["cat-file", "--batch-check"], + blobsToSize.map((entry) => entry.sha).join("\n") + "\n", + ); + for (const line of headers.trim().split("\n")) { + const [sha, type, size] = line.split(" "); + if (sha && type === "blob" && size) sizes.set(sha, Number(size)); + } + } + const entries = verifiedEntries + .filter( + ({ relative, mode }) => + ["100644", "100755"].includes(mode) && + relative.endsWith(".md") && + (folders.some((folder) => relative.startsWith(`${folder}/`)) || + ["rules.md", "INDEX.md"].includes(relative)), + ) + .map((entry) => ({ ...entry, size: sizes.get(entry.sha) ?? Infinity })); const pending = [ ...new Map( entries @@ -479,67 +607,18 @@ const make = Effect.gen(function* () { }); for (let index = 0; index < pending.length; index += 32) { const batch = pending.slice(index, index + 32); - const chunks: Uint8Array[] = []; - const result = yield* git( + const objects = yield* readObjects( orgId, paths.writer, - ["cat-file", "--batch"], - batch.map((entry) => entry.sha).join("\n") + "\n", - (chunk) => chunks.push(chunk), - ).pipe(Effect.result); - const buffer = Buffer.concat(chunks); - let offset = 0; - let fallback = result._tag === "Failure"; - for (const entry of batch) { - if (!fallback) { - const end = buffer.indexOf(10, offset); - const header = end < 0 ? "" : buffer.subarray(offset, end).toString("ascii"); - offset = end + 1; - if (header === `${entry.sha} missing`) continue; - if ( - end < 0 || - header !== `${entry.sha} blob ${entry.size}` || - buffer.length < offset + entry.size + 1 || - buffer[offset + entry.size] !== 10 - ) { - fallback = true; - const error = new MemoryError({ - orgId, - operation: "batch", - reason: "Invalid git blob batch framing", - }); - yield* Effect.logWarning( - "Memory blob batch framing failed; reading remaining blobs individually", - { - orgId, - errorTag: error._tag, - path: entry.relative, - }, - ); - } else { - yield* decodeBlob(entry, buffer.subarray(offset, offset + entry.size)); - offset += entry.size + 1; - continue; - } - } - const singleChunks: Uint8Array[] = []; - const single = yield* git( - orgId, - paths.writer, - ["cat-file", "blob", entry.sha], - undefined, - (chunk) => singleChunks.push(chunk), - ).pipe(Effect.result); - if (single._tag === "Success") { - const bytes = Buffer.concat(singleChunks); - if (bytes.length === entry.size) yield* decodeBlob(entry, bytes); - else - yield* Effect.logWarning("Skipping invalid memory blob length", { - orgId, - path: entry.relative, - errorTag: "MemoryError", - }); - } + batch.map(({ sha }) => ({ sha, type: "blob" })), + ).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(VcsProcess, vcs), + ); + for (const [index, entry] of batch.entries()) { + const bytes = objects[index]; + if (bytes && bytes.length <= maxBlobBytes) yield* decodeBlob(entry, bytes); + else blobs.set(entry.sha, undefined); } } const items: Array = []; @@ -732,62 +811,410 @@ const make = Effect.gen(function* () { return text; }); - const prepare = Effect.fn("Memory.prepare")(function* (orgId: KinOrgId) { + let conflictSequence = 0; + const preserve = Effect.fn("Memory.preserve")(function* ( + orgId: KinOrgId, + writer: string, + url: string, + head: string, + wakeupId?: string, + ) { + const now = yield* Clock.currentTimeMillis; + const id = wakeupId ?? `recovery-${now}-${head.slice(0, 12)}`; + if (!/^[a-zA-Z0-9_-]+$/.test(id) || !/^[a-zA-Z0-9_-]+$/.test(orgId)) + return yield* new MemoryCheckoutInvalid({ + orgId, + operation: "preserve", + reason: "Invalid wake-up id", + }); + const existing = (yield* git(orgId, writer, [ + "ls-remote", + "--", + url, + `refs/heads/kin/conflict/${orgId}/*`, + ])) + .split("\n") + .flatMap((line) => { + const [sha, ref] = line.split("\t"); + return sha === head && ref?.startsWith("refs/heads/") + ? [ref.slice("refs/heads/".length)] + : []; + }) + .toSorted()[0]; + if (existing) return existing; + const branch = `kin/conflict/${orgId}/${id}-${now.toString(36)}-${++conflictSequence}`; + if ((yield* git(orgId, writer, ["ls-remote", "--", url, `refs/heads/${branch}`])).trim()) + return yield* new MemorySyncConflict({ + orgId, + operation: "preserve", + reason: "Conflict branch already exists", + }); + yield* readFiles(orgId, writer, head).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(VcsProcess, vcs), + ); + yield* git(orgId, writer, ["push", "--", url, `${head}:refs/heads/${branch}`]); + return branch; + }); + + const recover = Effect.fn("Memory.recover")(function* (orgId: KinOrgId) { + const paths = yield* pathsFor(orgId); + yield* io(orgId, "mkdir", fs.makeDirectory(paths.root, { recursive: true })); + const exists = yield* io(orgId, "exists", fs.exists(paths.writer)); + if (!exists) + yield* git(orgId, paths.root, [ + "clone", + "--no-checkout", + "--", + paths.repo.repoUrl, + paths.writer, + ]); + for (const name of [".git", ".git/objects", ".git/refs"]) { + const stat = yield* io(orgId, "recoverStructure", fs.lstat(`${paths.writer}/${name}`)); + if (stat.type !== "Directory") + return yield* new MemoryCheckoutInvalid({ + orgId, + operation: "recover", + reason: "Invalid writer directory", + }); + } + const refs = (yield* git(orgId, paths.writer, [ + "ls-remote", + "--", + paths.repo.repoUrl, + "refs/heads/*", + ])) + .trim() + .split("\n") + .filter(Boolean); + const targets = new Map(); + for (const ref of refs) { + const [head, name] = ref.split(/\s+/); + const branch = name?.replace(/^refs\/heads\//, ""); + if ( + branch && + head && + /^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(head) && + (branch === (config.defaultBranch ?? "main") || branch.startsWith("kin/shadow/")) + ) { + yield* git(orgId, paths.writer, ["fetch", "--", paths.repo.repoUrl, head]); + targets.set(branch, head); + } + } + // Local enumeration is deletion-only; it can never introduce a recovery target. + const localShadows = (yield* git(orgId, paths.writer, [ + "for-each-ref", + "--format=%(refname) %(objectname)", + "refs/heads/kin/shadow/", + ])) + .trim() + .split("\n") + .filter(Boolean); + for (const ref of localShadows) { + const [name, sha] = ref.split(" "); + if (name && sha && !targets.has(name.slice("refs/heads/".length))) { + yield* git(orgId, paths.writer, ["update-ref", "-d", name, sha]); + yield* notify({ orgId, branch: name, reason: "Deleted untrusted local shadow branch" }); + } + } + const main = targets.get(config.defaultBranch ?? "main"); + if (!main) + return yield* new MemoryCheckoutInvalid({ + orgId, + operation: "recover", + reason: "Missing remote main branch", + }); + const rows = yield* persistence( + orgId, + sql<{ + branch: string; + head: string; + }>`SELECT branch, head FROM kin_memory_heads WHERE org_id = ${orgId} ORDER BY branch`, + ); + for (const row of rows) + if (row.branch.startsWith("kin/shadow/") && !targets.has(row.branch)) + targets.set(row.branch, main); + const preserved = new Set(); + for (const [branch, head] of targets) { + yield* readFiles(orgId, paths.writer, head).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(VcsProcess, vcs), + ); + const old = yield* anchor(orgId, branch, head); + if (old !== head && exists) { + const local = yield* readCommit(orgId, paths.writer, old).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(VcsProcess, vcs), + Effect.result, + ); + const present = yield* walkAncestors(orgId, paths.writer, head); + if (local._tag === "Success" && !present.has(old) && !preserved.has(old)) { + const conflictBranch = yield* preserve(orgId, paths.writer, paths.repo.repoUrl, old); + preserved.add(old); + yield* notify({ + orgId, + branch, + reason: "Preserved old anchor before recovery", + conflictBranch, + }); + } + } + const local = yield* git(orgId, paths.writer, [ + "rev-parse", + "--verify", + `refs/heads/${branch}`, + ]).pipe(Effect.result); + yield* git(orgId, paths.writer, [ + "update-ref", + `refs/heads/${branch}`, + head, + local._tag === "Success" ? local.success.trim() : "0".repeat(head.length), + ]); + if (old !== head) yield* record(orgId, branch, old, head); + yield* notify({ orgId, branch, reason: "Recovered from remote head" }); + } + pinnedHeads.delete(orgId); + snapshots.delete(orgId); + blobCaches.delete(orgId); + }); + + const prepare = Effect.fn("Memory.prepare")(function* (orgId: KinOrgId, wakeupId?: string) { const paths = yield* ensureCheckouts(orgId); + yield* validateWriter(orgId, yield* pathsFor(orgId)); + yield* git(orgId, paths.writer, [ + "fsck", + "--full", + "--strict", + "--no-reflogs", + yield* anchor(orgId, config.defaultBranch ?? "main"), + ]); for (const marker of ["rebase-merge", "rebase-apply", "MERGE_HEAD", "CHERRY_PICK_HEAD"]) { - if (yield* io(orgId, "sync", fs.exists(path.join(paths.writer, ".git", marker)))) + if (yield* io(orgId, "sync", fs.exists(`${paths.writer}/.git/${marker}`))) return yield* new MemoryCheckoutInvalid({ orgId, operation: "sync", - path: paths.writer, - reason: `Writer has an unfinished git operation (${marker}); manual recovery required`, + reason: "Unfinished Git operation", }); } - const fetched = yield* git(orgId, paths.writer, ["fetch", "origin"]).pipe(Effect.result); - if (fetched._tag === "Failure") { - yield* Effect.logWarning("Memory sync unavailable; using local writer HEAD", { orgId }); - } else { - yield* git(orgId, paths.writer, ["-c", "rebase.autoStash=false", "pull", "--rebase"]).pipe( - Effect.catch((cause) => - Effect.gen(function* () { - const rebasing = yield* io( - orgId, - "sync", - fs.exists(path.join(paths.writer, ".git", "rebase-merge")), - ); - const applying = yield* io( - orgId, - "sync", - fs.exists(path.join(paths.writer, ".git", "rebase-apply")), - ); - if (!rebasing && !applying) { - const reachable = yield* git(orgId, paths.writer, ["fetch", "origin"]).pipe( - Effect.result, + const { repo } = yield* pathsFor(orgId); + const branch = config.defaultBranch ?? "main"; + let head = yield* anchor(orgId, branch); + const fetched = yield* advertisedHead(orgId, paths.writer, repo.repoUrl, branch).pipe( + Effect.result, + ); + if (fetched._tag === "Failure" && fetched.failure._tag === "MemoryCheckoutInvalid") + return yield* fetched.failure; + if (fetched._tag === "Success") { + const remote = fetched.success.trim(); + if (remote !== head) { + const remoteAncestors = yield* walkAncestors(orgId, paths.writer, remote, head); + let next = remote; + if (!remoteAncestors.has(head)) { + const localAncestors = yield* walkAncestors(orgId, paths.writer, head); + if (localAncestors.has(remote)) next = head; + else { + const mergeBase = uniqueMergeBase(localAncestors, remoteAncestors); + if (!mergeBase) { + const conflictBranch = yield* preserve( + orgId, + paths.writer, + repo.repoUrl, + head, + wakeupId, ); - if (reachable._tag === "Success") return yield* cause; - yield* Effect.logWarning("Memory pull unavailable; using local writer HEAD", { + yield* notify({ orgId, + branch, + reason: "No unique verified merge base", + conflictBranch, + }); + return yield* new MemorySyncConflict({ + orgId, + operation: "merge", + reason: "No unique verified merge base", }); - return; } - yield* git(orgId, paths.writer, ["rebase", "--abort"]); - return yield* new MemorySyncConflict({ - orgId, - operation: "sync", - path: paths.writer, - reason: "Rebase conflicted; writer restored", - cause, - }); - }), - ), - ); + const commit = head; + const parent = mergeBase; + let tree: string | undefined; + { + const read = (sha: string) => + readFiles(orgId, paths.writer, sha).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(VcsProcess, vcs), + ); + const base = yield* read(parent); + const local = yield* read(commit); + const remoteFiles = yield* read(next); + const metadata = new Map>(); + for (const sha of [parent, commit, next]) { + const entries = yield* readTree(orgId, paths.writer, sha).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(VcsProcess, vcs), + ); + metadata.set(sha, new Map(entries.map((entry) => [entry.relative, entry]))); + } + const merged = mergeMemoryFiles( + base, + local, + remoteFiles, + metadata.get(parent)!, + metadata.get(commit)!, + metadata.get(next)!, + ); + if (!merged) { + const conflictBranch = yield* preserve( + orgId, + paths.writer, + repo.repoUrl, + head, + wakeupId, + ); + next = remote; + yield* notify({ + orgId, + branch, + reason: "Content conflict preserved", + conflictBranch, + }); + } else { + const { files: mergedFiles, modes } = merged; + remoteFiles.clear(); + for (const [file, bytes] of mergedFiles) remoteFiles.set(file, bytes); + const heads = []; + for (const [file, bytes] of remoteFiles) { + if (!["100644", "100755"].includes(modes.get(file) ?? "")) continue; + if (!folders.some((folder) => file.startsWith(`${folder}/`))) continue; + const parsed = yield* parseFrontMatter(orgId, file, bytes.toString("utf8")).pipe( + Effect.result, + ); + if (parsed._tag === "Success") + heads.push({ ...parsed.success.frontMatter, orgId, path: file }); + } + const reads = yield* io(orgId, "mergeIndex", cache.lastReads(orgId)); + const zone = yield* io(orgId, "mergeIndex", cache.ownerTimeZone(orgId)); + modes.set("INDEX.md", "100644"); + remoteFiles.set( + "INDEX.md", + Buffer.from(buildIndex(heads, reads, yield* Clock.currentTimeMillis, zone)), + ); + tree = yield* Effect.scoped( + Effect.gen(function* () { + const temp = yield* io(orgId, "journalMerge", fs.makeTempDirectoryScoped()); + const env = { GIT_INDEX_FILE: `${temp}/index` }; + yield* memoryGit(paths.writer, ["read-tree", "--empty"], undefined, env).pipe( + Effect.mapError( + (cause) => new MemoryError({ orgId, operation: "merge", cause }), + ), + ); + const expected = new Map(); + for (const [file, bytes] of remoteFiles) { + yield* io(orgId, "mergeBlob", fs.writeFile(`${temp}/blob`, bytes)); + const mode = modes.get(file) ?? "100644"; + const blob = + mode === "160000" + ? bytes.toString("ascii") + : (yield* git(orgId, paths.writer, [ + "hash-object", + "-w", + "--no-filters", + `${temp}/blob`, + ])).trim(); + expected.set(file, { + mode, + sha: + mode === "160000" + ? bytes.toString("ascii") + : objectHash(bytes, head.length), + }); + yield* memoryGit( + paths.writer, + ["update-index", "--add", "--cacheinfo", `${mode},${blob},${file}`], + undefined, + env, + ).pipe( + Effect.mapError( + (cause) => new MemoryError({ orgId, operation: "merge", cause }), + ), + ); + } + const written = (yield* memoryGit( + paths.writer, + ["write-tree"], + undefined, + env, + ).pipe( + Effect.mapError( + (cause) => new MemoryError({ orgId, operation: "merge", cause }), + ), + )).trim(); + yield* verifyTree(orgId, paths.writer, written, expected).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(VcsProcess, vcs), + ); + return written; + }), + ); + } + } + if (next !== remote || tree) { + if (!tree || !/^[0-9a-f]{40,64}$/.test(tree)) + return yield* new MemorySyncConflict({ + orgId, + operation: "sync", + reason: "Invalid merge tree", + }); + const { message } = yield* readCommit(orgId, paths.writer, commit).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(VcsProcess, vcs), + ); + next = (yield* memoryGit( + paths.writer, + ["commit-tree", tree, "-p", remote, "-p", head], + message, + { + GIT_AUTHOR_NAME: config.botName ?? "Kin Memory Bot", + GIT_AUTHOR_EMAIL: config.botEmail ?? "kin-memory@localhost", + GIT_COMMITTER_NAME: config.botName ?? "Kin Memory Bot", + GIT_COMMITTER_EMAIL: config.botEmail ?? "kin-memory@localhost", + }, + ).pipe( + Effect.mapError((cause) => new MemoryError({ orgId, operation: "rebase", cause })), + )).trim(); + } + } + } + if (next !== head) { + if (next !== remote) + yield* git(orgId, paths.writer, [ + "push", + "--", + repo.repoUrl, + `${next}:refs/heads/${branch}`, + ]); + yield* git(orgId, paths.writer, ["update-ref", `refs/heads/${branch}`, next, head]); + yield* record(orgId, branch, head, next); + head = next; + } + } } - const head = (yield* git(orgId, paths.writer, ["rev-parse", "HEAD"])).trim(); - yield* git(orgId, paths.read, ["checkout", "--force", "--detach", head]); - yield* git(orgId, paths.read, ["reset", "--hard", head]); - yield* git(orgId, paths.read, ["clean", "-fdx"]); + yield* io( + orgId, + "readSnapshot", + Effect.scoped( + Effect.gen(function* () { + const temporary = yield* fs.makeTempDirectoryScoped(); + yield* git(orgId, paths.writer, ["clone-snapshot", `${temporary}/read`]); + yield* git(orgId, `${temporary}/read`, ["checkout", "--detach", head]); + yield* fs.remove(paths.read, { recursive: true, force: true }); + yield* fs.rename(`${temporary}/read`, paths.read); + }), + ), + ); pinnedHeads.set(orgId, head); - const index = yield* readFile(orgId, "INDEX.md").pipe(Effect.orElseSucceed(() => "")); + const index = yield* readFile(orgId, "INDEX.md").pipe( + Effect.catchTags({ MemoryError: () => Effect.succeed("") }), + ); const asOf = /^As of ([^\r\n]+)(?:\r?\n|$)/.exec(index)?.[1]; const now = yield* Clock.currentTimeMillis; if ( @@ -806,10 +1233,18 @@ const make = Effect.gen(function* () { }); return Memory.of({ + withWriterLock, + recover: Effect.fn("Memory.recoverLocked")((orgId) => + withWriterLock(orgId, locked(orgId, recover(orgId))), + ), + anchor, + record, ensureCheckouts: Effect.fn("Memory.ensureCheckoutsLocked")((orgId) => locked(orgId, ensureCheckouts(orgId, true)), ), - prepareWakeup: Effect.fn("Memory.prepareWakeup")((orgId) => locked(orgId, prepare(orgId))), + prepareWakeup: Effect.fn("Memory.prepareWakeup")((orgId, wakeupId) => + withWriterLock(orgId, locked(orgId, prepare(orgId, wakeupId))), + ), readPromptSnapshot: Effect.fn("Memory.readPromptSnapshot")((orgId, expectedHead) => locked( orgId, diff --git a/apps/server/src/kin/memory/MemoryConfig.ts b/apps/server/src/kin/memory/MemoryConfig.ts index a8eea522ea190..045cee1547640 100644 --- a/apps/server/src/kin/memory/MemoryConfig.ts +++ b/apps/server/src/kin/memory/MemoryConfig.ts @@ -10,6 +10,11 @@ export class MemoryConfig extends Context.Service< MemoryConfig, { readonly baseDirectory: string; + readonly defaultBranch?: string; + readonly botName?: string; + readonly credentialHelper?: string; + readonly sshCommand?: string; + readonly botEmail?: string; readonly repositories: ReadonlyArray<{ readonly orgId: KinOrgId; readonly repoUrl: string; diff --git a/apps/server/src/kin/memory/MemoryWriter.test.ts b/apps/server/src/kin/memory/MemoryWriter.test.ts new file mode 100644 index 0000000000000..5dc139d5b196c --- /dev/null +++ b/apps/server/src/kin/memory/MemoryWriter.test.ts @@ -0,0 +1,701 @@ +import { memoryGit } from "./git.ts"; +import * as NodeZlib from "node:zlib"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { expect, it } from "@effect/vitest"; +import { KinAgentId, KinOrgId } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Sqlite from "../../persistence/Sqlite.ts"; +import * as Persistence from "../persistence/KinPersistence.ts"; +import * as Vcs from "../../vcs/VcsProcess.ts"; +import * as Memory from "./Memory.ts"; +import { MemoryConfig } from "./MemoryConfig.ts"; +import * as Writer from "./MemoryWriter.ts"; +import * as Scan from "./secretScan.ts"; +import { journalEntry } from "./journal.ts"; +import { fixtureItem } from "./__fixtures__/items.ts"; + +const orgId = KinOrgId.make("fixture-org"); +const agentId = KinAgentId.make("fixture-lead"); +const gitLayer = Vcs.layer.pipe(Layer.provide(NodeServices.layer)); +it.live.each([ + "completed", + "failed", + "blocked", + "forged", + "conflict", + "B1-shadow", + "B2-anchor", + "B3-execution", + "B4-pushurl", + "B5-journal", + "B6-owner-open", + "B6-owner-closed", + "B6-agent-own", + "B7-identity", + "P3-object", + "P4-journal", + "P6-malformed", + "P7-crash", + "P8-lost-writer", + "P9-shadow", + "P10-lost-git-dir", + "sha256", + "review-default", + "review-null", + "credentials", + "read-during-record", + "H3-packed-ref", + "M1-unanchored", + "M2-nondescendant", + "M4-modes", + "tree-substitution", +] as const)("writer real git: %s", (scenario) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const vcs = yield* Vcs.VcsProcess; + const tmp = yield* fs.makeTempDirectoryScoped(); + const root = yield* fs.realPath(tmp); + const source = `${root}/seed`; + const remote = `${root}/remote.git`; + yield* fs.makeDirectory(source); + const git = Effect.fn("WriterTest.git")((cwd: string, args: ReadonlyArray) => + vcs + .run({ operation: "test.writer", command: "git", cwd, args }) + .pipe(Effect.map((r) => r.stdout.trim())), + ); + const commit = (cwd: string) => + git(cwd, [ + "-c", + "user.name=Fixture Bot", + "-c", + "user.email=fixture@example.invalid", + "commit", + "-m", + "fixture", + ]); + yield* git(source, [ + "init", + "-b", + "main", + ...(scenario === "sha256" ? ["--object-format=sha256"] : []), + ]); + yield* fs.writeFileString(`${source}/rules.md`, "Invented rules\n"); + yield* fs.writeFileString(`${source}/INDEX.md`, "As of 2026-11-08T00:00:00.000Z\n"); + if (scenario.startsWith("B6") || scenario === "P3-object" || scenario === "P6-malformed") { + yield* fs.makeDirectory(`${source}/claims`); + yield* fs.writeFileString( + `${source}/claims/existing.md`, + scenario === "P6-malformed" + ? "---\nid: [\n---\n" + : fixtureItem({ + kind: "claim", + status: scenario === "B6-owner-open" ? "open" : "done", + trust: scenario === "B6-agent-own" ? "agent" : "owner", + said_by: scenario === "B6-agent-own" ? agentId : "fixture-owner", + }), + ); + } + if (scenario === "B5-journal" || scenario === "P4-journal") { + yield* fs.makeDirectory(`${source}/journal/2026`, { recursive: true }); + yield* fs.writeFile( + `${source}/journal/2026/11-08.md`, + Buffer.concat([ + scenario === "B5-journal" ? Buffer.from([255, 10]) : Buffer.alloc(0), + Buffer.from("Owner journal preserved.\n"), + ]), + ); + } + if (scenario === "M4-modes") { + yield* fs.symlink("rules.md", `${source}/owner-link`); + yield* fs.makeDirectory(`${source}/claims`, { recursive: true }); + yield* fs.symlink(fixtureItem({ kind: "claim" }), `${source}/claims/link.md`); + yield* fs.writeFileString(`${source}/owner-executable`, "fixture executable\n"); + yield* fs.chmod(`${source}/owner-executable`, 0o755); + } + yield* git(source, ["add", "--", "."]); + if (scenario === "M4-modes") + yield* git(source, ["update-index", "--chmod=+x", "owner-executable"]); + yield* commit(source); + if (scenario === "M4-modes") { + const head = yield* git(source, ["rev-parse", "HEAD"]); + yield* git(source, [ + "update-index", + "--add", + "--cacheinfo", + `160000,${head},owner-submodule`, + ]); + yield* commit(source); + } + yield* git(root, ["clone", "--bare", "--", source, remote]); + let attack: { objectPath: string; sha: string; expression: string; bytes: Buffer } | undefined; + let swapped = false; + let substituteTree: string | undefined; + let onRefChange: + | (() => Effect.Effect< + void, + | Memory.MemoryError + | Memory.MemoryCheckoutInvalid + | Memory.MemoryConfigError + | Memory.MemorySyncConflict + | Memory.MemoryQueryDecodeError + >) + | undefined; + const interceptedGit = Layer.succeed( + Vcs.VcsProcess, + Vcs.VcsProcess.of({ + run: Effect.fn("WriterTest.objectSwap")(function* (input) { + if ( + attack && + input.args.includes("cat-file") && + (input.stdin?.includes(attack.sha) || input.args.includes(attack.expression)) + ) { + yield* fs.remove(attack.objectPath).pipe(Effect.orDie); + yield* fs.writeFile(attack.objectPath, attack.bytes).pipe(Effect.orDie); + attack = undefined; + swapped = true; + } + const result = yield* vcs.run(input); + if (substituteTree && input.args.includes("write-tree")) + return { ...result, stdout: substituteTree }; + if (onRefChange && input.args.includes("update-ref")) { + const callback = onRefChange; + onRefChange = undefined; + yield* callback().pipe(Effect.orDie); + } + return result; + }), + }), + ); + const notices: Writer.WriterNotice[] = []; + const config = Layer.succeed(MemoryConfig, { + baseDirectory: root, + credentialHelper: "fixture-server-helper", + repositories: [{ orgId, repoUrl: remote, path: "state" }], + }); + const memoryLayer = Memory.layer.pipe( + Layer.provide(config), + Layer.provide(Layer.succeed(Memory.MemoryRecoveryHooks, { notify: () => Effect.void })), + ); + const hooks = Layer.succeed(Writer.MemoryWriterHooks, { + notify: Effect.fn("Test.notice")((notice) => + Effect.sync(() => { + notices.push(notice); + }), + ), + }); + const auth = Layer.succeed(Writer.MemoryWriterAuthority, { + resolve: Effect.fn("Test.authority")( + ( + org: KinOrgId, + wakeupId: string, + ): Effect.Effect => + org !== orgId + ? Effect.fail(new Writer.MemoryWriteRejected({ reason: "not_found" })) + : Effect.succeed({ + orgId, + wakeupId, + agentId, + level: + scenario === "conflict" || + scenario === "B5-journal" || + scenario === "read-during-record" + ? "L1" + : "L0", + timeZone: "Asia/Kolkata", + startedAt: "2026-11-08T00:00:00.000Z", + eventIds: [], + sources: new Map([ + [ + "owner", + { saidBy: "fixture-owner", trust: "owner", source: "fixture-owner-message" }, + ], + [ + "member", + { + saidBy: "fixture-member", + trust: "member", + source: "fixture-member-message", + }, + ], + ]), + }), + ), + }); + const scan = + scenario === "blocked" + ? Scan.testLayer([{ path: "claims/new.md", ruleId: "fixture", line: 1 }]) + : Scan.noFindingsStubLayer; + const layer = Writer.layer.pipe( + Layer.provideMerge(memoryLayer), + Layer.provide(Layer.succeed(Memory.MemoryRecoveryHooks, { notify: () => Effect.void })), + Layer.provide(config), + Layer.provide(hooks), + Layer.provide(auth), + Layer.provide(scan), + ); + yield* Effect.gen(function* () { + const memory = yield* Memory.Memory; + const writer = yield* Writer.MemoryWriter; + const paths = yield* memory.ensureCheckouts(orgId); + yield* memory.prepareWakeup(orgId); + yield* git(paths.writer, ["config", "user.name", "Fixture Bot"]); + yield* git(paths.writer, ["config", "user.email", "fixture@example.invalid"]); + const original = yield* git(paths.writer, ["rev-parse", "HEAD"]); + if (scenario === "M4-modes") { + const before = yield* git(paths.writer, [ + "ls-tree", + original, + "owner-link", + "owner-executable", + "owner-submodule", + ]); + const result = yield* writer.commitWakeup(orgId, "wake1", "completed"); + expect( + yield* git(paths.writer, [ + "ls-tree", + result.commit, + "owner-link", + "owner-executable", + "owner-submodule", + ]), + ).toBe(before); + expect(before).toContain("100755"); + expect(before).toContain("120000"); + expect(before).toContain("160000"); + return; + } + if ( + scenario === "H3-packed-ref" || + scenario === "M1-unanchored" || + scenario === "M2-nondescendant" + ) { + yield* fs.writeFileString(`${paths.read}/rules.md`, "FORGED RULES\n"); + yield* git(paths.read, ["add", "rules.md"]); + yield* commit(paths.read); + const forged = yield* git(paths.read, ["rev-parse", "HEAD"]); + yield* git(paths.writer, ["fetch", "--", paths.read, forged]); + const shadow = `kin/shadow/${agentId}`; + if (scenario === "H3-packed-ref") { + yield* fs.writeFileString( + `${paths.writer}/.git/packed-refs`, + `# pack-refs with: peeled fully-peeled sorted\n${forged} refs/heads/kin/shadow/stray\n${forged} refs/remotes/kin-recovery/${shadow}\n`, + ); + yield* memory.recover(orgId); + } else if (scenario === "M1-unanchored") { + yield* git(paths.writer, ["update-ref", `refs/heads/${shadow}`, forged]); + } else { + yield* memory.anchor(orgId, shadow, original); + yield* git(paths.writer, ["update-ref", `refs/heads/${shadow}`, original]); + yield* git(paths.read, ["checkout", "--orphan", "unrelated"]); + yield* commit(paths.read); + const unrelated = yield* git(paths.read, ["rev-parse", "HEAD"]); + yield* git(paths.writer, ["fetch", "--", paths.read, unrelated]); + yield* git(paths.writer, ["update-ref", `refs/heads/${shadow}`, unrelated]); + yield* git(paths.writer, ["update-ref", "refs/heads/main", unrelated]); + yield* memory.prepareWakeup(orgId); + } + const result = yield* writer.commitWakeup(orgId, "wake1", "completed"); + expect(yield* git(root, [`--git-dir=${remote}`, "show", `${result.commit}:rules.md`])).toBe( + "Invented rules", + ); + if (scenario === "H3-packed-ref") + expect( + (yield* git(paths.writer, [ + "rev-parse", + "--verify", + "refs/heads/kin/shadow/stray", + ]).pipe(Effect.result))._tag, + ).toBe("Failure"); + return; + } + if (scenario === "read-during-record") { + onRefChange = () => memory.readRules(orgId).pipe(Effect.asVoid); + const { commit } = yield* writer.commitWakeup(orgId, "wake1", "completed"); + expect(yield* git(paths.writer, ["rev-parse", "refs/heads/main"])).toBe(commit); + expect(yield* memory.anchor(orgId, "main")).toBe(commit); + return; + } + if (scenario === "credentials") { + yield* git(paths.writer, ["config", "credential.helper", "fixture-repo-helper"]); + const helper = yield* memoryGit(paths.writer, [ + "config", + "--get-all", + "credential.helper", + ]).pipe( + Effect.provideService(MemoryConfig, { + baseDirectory: root, + credentialHelper: "fixture-server-helper", + repositories: [], + }), + ); + expect(helper.trim()).toBe("fixture-server-helper"); + return; + } + if (scenario === "review-default" || scenario === "review-null") { + const content = fixtureItem({ + kind: "claim", + review_by: null, + recorded_at: "2026-11-01T00:00:00.000Z", + }); + yield* writer.stage(orgId, "wake1", { + path: "claims/default.md", + content: + scenario === "review-default" ? content.replace(/^review_by:.*\n/m, "") : content, + }); + const { commit } = yield* writer.commitWakeup(orgId, "wake1", "completed"); + const text = yield* git(paths.writer, ["show", `${commit}:claims/default.md`]); + expect(text).toContain( + scenario === "review-default" ? "2026-12-01T00:00:00.000Z" : "review_by: null", + ); + expect(yield* git(paths.writer, ["show", `${commit}:INDEX.md`])).toContain( + "claims/default.md", + ); + expect(yield* git(paths.writer, ["rev-list", "--count", `${original}..${commit}`])).toBe( + "1", + ); + return; + } + if (scenario === "P10-lost-git-dir") { + yield* git(root, ["init", "--quiet"]); + yield* fs.remove(`${paths.writer}/.git`, { recursive: true }); + const result = yield* writer + .stage(orgId, "wake1", { path: "claims/new.md", content: fixtureItem({ kind: "claim" }) }) + .pipe(Effect.result); + expect(result._tag).toBe("Failure"); + if (result._tag === "Failure") + expect(result.failure).toMatchObject({ + _tag: "MemoryCheckoutInvalid", + operation: "validate", + }); + return; + } + if (scenario === "P8-lost-writer" || scenario === "P9-shadow") { + if (scenario === "P9-shadow") yield* writer.commitWakeup(orgId, "wake0", "completed"); + yield* fs.remove(paths.writer, { recursive: true }); + yield* memory.recover(orgId); + yield* memory.prepareWakeup(orgId); + const result = yield* writer.commitWakeup(orgId, "wake1", "completed"); + expect( + yield* git(root, [ + `--git-dir=${remote}`, + "rev-parse", + `refs/heads/kin/shadow/${agentId}`, + ]), + ).toBe(result.commit); + return; + } + if (scenario === "P3-object" || scenario === "P4-journal") { + const file = scenario === "P3-object" ? "claims/existing.md" : "journal/2026/11-08.md"; + const sha = yield* git(paths.writer, ["rev-parse", `${original}:${file}`]); + const forged = Buffer.from("forged bytes after prepare\n"); + attack = { + objectPath: `${paths.writer}/.git/objects/${sha.slice(0, 2)}/${sha.slice(2)}`, + sha, + expression: `${original}:${file}`, + bytes: NodeZlib.deflateSync( + Buffer.concat([Buffer.from(`blob ${forged.length}\0`), forged]), + ), + }; + const result = + scenario === "P3-object" + ? yield* writer + .stage(orgId, "wake1", { + path: "claims/existing.md", + content: fixtureItem({ kind: "claim" }), + }) + .pipe(Effect.result) + : yield* writer.commitWakeup(orgId, "wake1", "completed").pipe(Effect.result); + expect(swapped).toBe(true); + expect(result._tag).toBe("Failure"); + if (result._tag === "Failure") expect(result.failure._tag).toBe("MemoryCheckoutInvalid"); + return; + } + if (scenario === "P6-malformed") { + const result = yield* writer + .stage(orgId, "wake1", { + path: "claims/existing.md", + content: fixtureItem({ kind: "claim" }), + }) + .pipe(Effect.result); + expect(result._tag).toBe("Failure"); + if (result._tag === "Failure") expect(result.failure._tag).toBe("MemoryWriteRejected"); + return; + } + for (const path of [ + "rules.md", + "INDEX.md", + "README.md", + "../claims/a.md", + "/claims/a.md", + "claims/.git/a.md", + "claims/../../a.md", + ]) { + expect( + (yield* writer + .stage(orgId, "wake1", { path, content: fixtureItem() }) + .pipe(Effect.result))._tag, + ).toBe("Failure"); + } + yield* fs.symlink(source, `${paths.writer}/people`); + expect( + (yield* writer + .stage(orgId, "wake1", { + path: "people/new.md", + content: fixtureItem({ kind: "person" }), + }) + .pipe(Effect.result))._tag, + ).toBe("Failure"); + yield* fs.remove(`${paths.writer}/people`); + if (scenario === "completed") { + yield* fs.makeDirectory(`${paths.writer}/claims`, { recursive: true }); + yield* fs.writeFileString(`${root}/hardlink-source`, fixtureItem({ kind: "claim" })); + yield* fs.link(`${root}/hardlink-source`, `${paths.writer}/claims/hard.md`); + expect( + (yield* writer + .stage(orgId, "wake1", { + path: "claims/hard.md", + content: fixtureItem({ kind: "claim" }), + }) + .pipe(Effect.result))._tag, + ).toBe("Failure"); + yield* fs.remove(`${paths.writer}/claims/hard.md`); + } + if (scenario === "forged") { + yield* fs.writeFileString(`${paths.read}/rules.md`, "Forged rules\n"); + yield* git(paths.read, ["add", "--", "rules.md"]); + yield* commit(paths.read); + const forged = yield* git(paths.read, ["rev-parse", "HEAD"]); + yield* git(paths.read, ["update-ref", "refs/heads/main", forged]); + expect(yield* git(paths.writer, ["rev-parse", "HEAD"])).toBe(original); + yield* git(paths.writer, ["fetch", "--", paths.read, forged]); + yield* git(paths.writer, ["update-ref", "refs/heads/main", forged]); + expect((yield* memory.prepareWakeup(orgId).pipe(Effect.result))._tag).toBe("Success"); + expect(yield* memory.readRules(orgId)).toBe("Invented rules\n"); + expect( + (yield* writer.commitWakeup(orgId, "wake1", "completed").pipe(Effect.result))._tag, + ).toBe("Success"); + return; + } + if (scenario === "B1-shadow" || scenario === "B2-anchor" || scenario === "P7-crash") { + yield* fs.writeFileString(`${paths.read}/rules.md`, "Forged rules\n"); + yield* git(paths.read, ["add", "rules.md"]); + yield* commit(paths.read); + const branch = scenario === "B1-shadow" ? `kin/shadow/${agentId}` : "main"; + yield* git(paths.read, ["push", "--", paths.writer, `HEAD:refs/heads/${branch}`]).pipe( + Effect.catch(() => + git(paths.writer, ["fetch", paths.read, "HEAD"]).pipe( + Effect.andThen( + git(paths.writer, ["update-ref", `refs/heads/${branch}`, "FETCH_HEAD"]), + ), + ), + ), + ); + if (scenario === "B2-anchor") + yield* fs.writeFileString( + `${paths.writer}/../writer-head`, + yield* git(paths.read, ["rev-parse", "HEAD"]), + ); + expect( + (yield* writer.commitWakeup(orgId, "wake1", "completed").pipe(Effect.result))._tag, + ).toBe("Success"); + return; + } + if (scenario.startsWith("B6")) { + const result = yield* writer + .stage(orgId, "wake1", { + path: "claims/existing.md", + content: fixtureItem({ + kind: "claim", + title: + scenario === "B6-owner-open" + ? "Invented service maintenance" + : "Forged replacement", + status: scenario === "B6-owner-open" ? "done" : "open", + }), + }) + .pipe(Effect.result); + expect(result._tag).toBe("Failure"); + return; + } + const marker = `${root}/executed`; + if (scenario === "B3-execution") { + const script = `${root}/attack.sh`; + yield* fs.writeFileString( + script, + `#!/bin/sh\ntouch '${marker}'\nprintf 'filtered attack\\n'\n`, + ); + yield* fs.chmod(script, 0o755); + for (const repo of [paths.read, paths.writer]) { + yield* fs.writeFileString( + `${repo}/.git/hooks/post-checkout`, + `#!/bin/sh\ntouch '${marker}'\n`, + ); + yield* fs.chmod(`${repo}/.git/hooks/post-checkout`, 0o755); + yield* git(repo, ["config", "core.fsmonitor", script]); + } + yield* fs.writeFileString( + `${paths.writer}/.git/hooks/reference-transaction`, + `#!/bin/sh\ntouch '${marker}'\n`, + ); + yield* fs.chmod(`${paths.writer}/.git/hooks/reference-transaction`, 0o755); + yield* git(paths.writer, ["config", "filter.attack.clean", script]); + yield* fs.writeFileString(`${paths.writer}/.git/info/attributes`, "*.md filter=attack\n"); + yield* git(paths.writer, ["config", "remote.origin.receivepack", script]); + yield* git(paths.writer, ["config", "uploadpack.packObjectsHook", script]); + yield* memory.prepareWakeup(orgId); + expect(yield* fs.exists(marker)).toBe(false); + } + let evil = ""; + if (scenario === "B4-pushurl") { + evil = `${root}/evil.git`; + yield* git(root, ["init", "--bare", evil]); + yield* git(paths.writer, ["config", "remote.origin.pushurl", evil]); + } + if (scenario === "B5-journal") { + yield* fs.makeDirectory(`${paths.writer}/journal/2026`, { recursive: true }); + yield* fs.writeFileString(`${paths.writer}/journal/2026/11-08.md`, "Forged journal\n"); + } + yield* writer.stage(orgId, "wake1", { + path: "claims/new.md", + content: fixtureItem({ + kind: "claim", + trust: "owner", + said_by: "forged-owner", + source: "forged", + }), + }); + + if (scenario === "completed") { + const contradiction = yield* writer + .stage(orgId, "wake1", { + path: "claims/contradiction.md", + content: fixtureItem({ + id: "other", + kind: "claim", + title: "Different invented claim", + }), + }) + .pipe(Effect.result); + expect(contradiction._tag).toBe("Failure"); + if (contradiction._tag === "Failure") + expect(contradiction.failure._tag).toBe("MemoryConflict"); + for (const sourceEventId of ["owner", "member"]) { + yield* writer.stage(orgId, "wake1", { + path: `claims/${sourceEventId}.md`, + sourceEventId, + content: fixtureItem({ id: sourceEventId, kind: "claim", links: [] }), + }); + } + yield* fs.writeFileString(`${paths.writer}/rules.md`, "Ambient forged edit"); + yield* git(paths.writer, ["add", "--", "rules.md"]); + } + if (scenario === "conflict") { + yield* fs.makeDirectory(`${source}/claims`); + yield* fs.writeFileString(`${source}/claims/new.md`, "Owner conflicting fixture\n"); + yield* git(source, ["add", "--", "claims/new.md"]); + yield* commit(source); + yield* git(source, ["push", "--", remote, "main"]); + } + if (scenario === "tree-substitution") + substituteTree = yield* git(paths.writer, ["rev-parse", `${original}^{tree}`]); + const result = yield* writer + .commitWakeup(orgId, "wake1", scenario === "failed" ? "failed" : "completed") + .pipe(Effect.result); + if (result._tag === "Failure") yield* Effect.logError(result.failure); + if (scenario === "tree-substitution") { + expect(result._tag).toBe("Failure"); + if (result._tag === "Failure") expect(result.failure._tag).toBe("MemoryCheckoutInvalid"); + expect(yield* git(paths.writer, ["rev-parse", "main"])).toBe(original); + return; + } + if (scenario === "blocked" || scenario === "conflict") { + expect(result._tag).toBe("Failure"); + if (result._tag === "Failure") + expect(result.failure._tag).toBe( + scenario === "blocked" ? "MemorySecretBlocked" : "MemorySyncConflict", + ); + + expect(notices).toHaveLength(1); + expect(yield* fs.exists(`${paths.writer}/.git/rebase-merge`)).toBe(false); + } else { + expect(result._tag).toBe("Success"); + const branch = scenario === "B5-journal" ? "main" : `kin/shadow/${agentId}`; + expect(yield* git(paths.writer, ["rev-list", "--count", `${original}..${branch}`])).toBe( + "1", + ); + expect(yield* git(paths.writer, ["show", `${branch}:rules.md`])).toBe("Invented rules"); + if (scenario === "B3-execution") expect(yield* fs.exists(marker)).toBe(false); + if (scenario === "B4-pushurl") { + expect( + yield* git(root, [`--git-dir=${remote}`, "rev-parse", `refs/heads/${branch}`]), + ).toBe(result._tag === "Success" ? result.success.commit : ""); + expect( + (yield* git(root, [ + `--git-dir=${evil}`, + "rev-parse", + "--verify", + `refs/heads/${branch}`, + ]).pipe(Effect.result))._tag, + ).toBe("Failure"); + } + if (scenario === "B5-journal") { + const journal = yield* git(paths.writer, ["show", `${branch}:journal/2026/11-08.md`]); + expect(journal).toContain("Owner journal preserved."); + const chunks: Uint8Array[] = []; + yield* vcs.run({ + operation: "WriterTest.journalBytes", + command: "git", + cwd: paths.writer, + args: ["show", `${branch}:journal/2026/11-08.md`], + onStdoutChunk: (chunk) => chunks.push(chunk), + }); + expect(Buffer.concat(chunks).subarray(0, 2)).toEqual(Buffer.from([255, 10])); + expect(journal).not.toContain("Forged journal"); + } + const again = yield* writer.commitWakeup( + orgId, + "wake1", + scenario === "failed" ? "failed" : "completed", + ); + if (result._tag === "Success") expect(again.commit).toBe(result.success.commit); + expect(yield* git(paths.writer, ["show", "--format=%an <%ae>", "--no-patch", branch])).toBe( + "Kin Memory Bot ", + ); + expect( + (yield* git(paths.writer, ["cat-file", "-e", `${branch}:claims/new.md`]).pipe( + Effect.result, + ))._tag, + ).toBe(scenario !== "failed" ? "Success" : "Failure"); + expect(yield* git(paths.writer, ["show", `${branch}:journal/2026/11-08.md`])).toContain( + `Outcome: ${scenario === "failed" ? "failed" : "completed"}`, + ); + } + }).pipe(Effect.provide(layer.pipe(Layer.provide(interceptedGit)))); + }).pipe( + Effect.provide( + Layer.mergeAll( + gitLayer, + NodeServices.layer, + Persistence.layer.pipe(Layer.provideMerge(Sqlite.layerMemory)), + ), + ), + Effect.scoped, + ), +); + +it.effect.each([ + { startedAt: "not-a-date", timeZone: "UTC" }, + { startedAt: "2026-11-08T00:00:00Z", timeZone: "Not/AZone" }, +])("rejects invalid journal context: %j", (input) => + Effect.gen(function* () { + const result = yield* journalEntry( + { ...input, wakeupId: "fixture", agentId: "fixture", level: "L1", eventIds: [] }, + "completed", + [], + ).pipe(Effect.result); + expect(result._tag).toBe("Failure"); + if (result._tag === "Failure") expect(result.failure._tag).toBe("MemoryJournalError"); + }), +); diff --git a/apps/server/src/kin/memory/MemoryWriter.ts b/apps/server/src/kin/memory/MemoryWriter.ts new file mode 100644 index 0000000000000..113280e8906c5 --- /dev/null +++ b/apps/server/src/kin/memory/MemoryWriter.ts @@ -0,0 +1,718 @@ +import * as Clock from "effect/Clock"; +import { readFiles, readCommit, readTree, objectHash, verifyTree } from "./objects.ts"; +import { buildIndex } from "./buildIndex.ts"; +import * as MemoryCache from "./MemoryCache.ts"; +import type { KinAgentId, KinOrgId, MemoryTrust } from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; +import * as Option from "effect/Option"; +import { stringify } from "yaml"; +import { memoryGit as runMemoryGit } from "./git.ts"; +import { VcsProcess } from "../../vcs/VcsProcess.ts"; +import { MemoryConfig } from "./MemoryConfig.ts"; +import { + Memory, + MemoryCheckoutInvalid, + MemoryRecoveryHooks, + MemoryError, + MemorySyncConflict, + MemoryConfigError, + MemoryQueryDecodeError, + MemoryNotFound, + MemoryDuplicateId, +} from "./Memory.ts"; +import { parseFrontMatter, MemoryItemError } from "./frontMatter.ts"; +import { journalEntry, MemoryJournalError, type JournalContext } from "./journal.ts"; +import { KinSecretScan, KinSecretScanError } from "./secretScan.ts"; + +export interface WakeupContext extends JournalContext { + readonly orgId: KinOrgId; + readonly agentId: KinAgentId; + readonly level: "L0" | "L1" | "L2" | "L3"; + readonly sources: ReadonlyMap< + string, + { readonly saidBy: string; readonly trust: MemoryTrust; readonly source: string } + >; +} +// The runner resolves this context from same-org records, never tool input. +export class MemoryWriterAuthority extends Context.Service< + MemoryWriterAuthority, + { + readonly resolve: ( + orgId: KinOrgId, + wakeupId: string, + ) => Effect.Effect; + } +>()("t3/kin/memory/MemoryWriter/MemoryWriterAuthority") {} +export class MemoryWriteRejected extends Schema.TaggedError()( + "MemoryWriteRejected", + { + reason: Schema.String, + }, +) {} +export class MemoryConflict extends Schema.TaggedError()("MemoryConflict", { + existingId: Schema.String, +}) {} +export class MemorySecretBlocked extends Schema.TaggedError()( + "MemorySecretBlocked", + { + orgId: Schema.String, + findings: Schema.Array( + Schema.Struct({ path: Schema.String, ruleId: Schema.String, line: Schema.Number }), + ), + }, +) {} +export type WriterNotice = { + readonly orgId: KinOrgId; + readonly wakeupId: string; + readonly error: MemoryCheckoutInvalid | MemorySyncConflict | MemorySecretBlocked; +}; +export class MemoryWriterHooks extends Context.Service< + MemoryWriterHooks, + { + readonly notify: (notice: WriterNotice) => Effect.Effect; + } +>()("t3/kin/memory/MemoryWriter/MemoryWriterHooks") {} +export interface MemoryChange { + readonly path: string; + readonly content: string; + readonly sourceEventId?: string; + readonly supersedes?: string; +} +type Failure = + | MemoryJournalError + | MemoryConfigError + | MemoryQueryDecodeError + | MemoryNotFound + | MemoryDuplicateId + | MemoryError + | MemoryCheckoutInvalid + | MemorySyncConflict + | MemoryWriteRejected + | MemoryConflict + | MemoryItemError + | KinSecretScanError + | MemorySecretBlocked; +export class MemoryWriter extends Context.Service< + MemoryWriter, + { + readonly stage: ( + orgId: KinOrgId, + wakeupId: string, + change: MemoryChange, + ) => Effect.Effect; + readonly appendJournal: ( + orgId: KinOrgId, + wakeupId: string, + lines: ReadonlyArray, + ) => Effect.Effect; + readonly discard: (orgId: KinOrgId, wakeupId: string) => Effect.Effect; + readonly commitWakeup: ( + orgId: KinOrgId, + wakeupId: string, + outcome: "completed" | "failed", + ) => Effect.Effect<{ commit: string }, Failure>; + } +>()("t3/kin/memory/MemoryWriter") {} +const folders = ["objectives", "commitments", "deferrals", "decisions", "people", "claims"]; +const render = (fields: object, body: string) => `---\n${stringify(fields)}---\n${body}`; +const make = Effect.gen(function* () { + const memory = yield* Memory; + const recovery = yield* MemoryRecoveryHooks; + const authority = yield* MemoryWriterAuthority; + const hooks = yield* MemoryWriterHooks; + const scanner = yield* KinSecretScan; + const fs = yield* FileSystem.FileSystem; + const vcs = yield* VcsProcess; + const memoryGit = (...args: Parameters) => + runMemoryGit(...args).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(VcsProcess, vcs), + Effect.provideService(MemoryConfig, config), + ); + const config = yield* MemoryConfig; + const cache = yield* MemoryCache.MemoryCache; + const filesAt = (orgId: KinOrgId, writer: string, commit: string) => + readFiles(orgId, writer, commit).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(VcsProcess, vcs), + ); + const commitAt = (orgId: KinOrgId, writer: string, commit: string) => + readCommit(orgId, writer, commit).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(VcsProcess, vcs), + ); + const locked = memory.withWriterLock; + const sessions = new Map< + KinOrgId, + { wakeupId: string; files: Map; lines: string[] } + >(); + const io = (orgId: KinOrgId, effect: Effect.Effect) => + effect.pipe(Effect.mapError((cause) => new MemoryError({ orgId, operation: "writer", cause }))); + const git = Effect.fn("MemoryWriter.git")(function* ( + orgId: KinOrgId, + cwd: string, + args: ReadonlyArray, + identity: NodeJS.ProcessEnv = {}, + stdin?: string, + ) { + return (yield* memoryGit(cwd, args, stdin, identity).pipe( + Effect.mapError((cause) => + cause._tag === "MemoryGitMissingRepository" + ? new MemoryCheckoutInvalid({ + orgId, + operation: "git", + reason: "Writer .git is missing or not a directory", + path: cwd, + cause, + }) + : new MemoryError({ orgId, operation: "writer", cause }), + ), + )).trim(); + }); + const branchHead = Effect.fn("MemoryWriter.branchHead")(function* ( + orgId: KinOrgId, + writer: string, + branch: string, + ) { + const anchored = yield* memory.anchor(orgId, branch).pipe(Effect.result); + const local = yield* git(orgId, writer, ["rev-parse", "--verify", `refs/heads/${branch}`]).pipe( + Effect.result, + ); + if (anchored._tag === "Success") { + if (local._tag === "Failure") + return yield* new MemoryCheckoutInvalid({ + orgId, + operation: "branch", + reason: "Missing recorded branch", + }); + if (local.success !== anchored.success) { + yield* git(orgId, writer, [ + "update-ref", + `refs/heads/${branch}`, + anchored.success, + local.success, + ]); + const notice = { + orgId, + branch, + reason: "Reset tampered branch to anchor; unpushed work at the replaced tip is lost", + }; + yield* recovery.notify(notice); + } + return anchored.success; + } + if (local._tag === "Success") { + yield* git(orgId, writer, ["update-ref", "-d", `refs/heads/${branch}`, local.success]); + const notice = { orgId, branch, reason: "Deleted unanchored local branch" }; + yield* recovery.notify(notice); + } + const repo = config.repositories.find((repo) => repo.orgId === orgId); + if (!repo) + return yield* new MemoryConfigError({ orgId, operation: "branch", reason: "Unknown org" }); + const main = yield* memory.anchor(orgId, config.defaultBranch ?? "main"); + const advertised = yield* git(orgId, writer, [ + "ls-remote", + "--", + repo.repoUrl, + `refs/heads/${branch}`, + ]); + let remote = main; + if (advertised) { + const [sha, name] = advertised.split(/\s+/); + if (!sha || !/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(sha) || name !== `refs/heads/${branch}`) + return yield* new MemoryCheckoutInvalid({ + orgId, + operation: "branch", + reason: "Invalid remote advertisement", + }); + yield* git(orgId, writer, ["fetch", "--", repo.repoUrl, sha]); + remote = sha; + } + yield* filesAt(orgId, writer, remote); + const recorded = yield* memory.anchor(orgId, branch, remote); + yield* git(orgId, writer, [ + "update-ref", + `refs/heads/${branch}`, + recorded, + "0".repeat(recorded.length), + ]); + return recorded; + }); + const session = Effect.fn("MemoryWriter.session")(function* (orgId: KinOrgId, wakeupId: string) { + const ctx = yield* authority.resolve(orgId, wakeupId); + if ( + ctx.orgId !== orgId || + ctx.wakeupId !== wakeupId || + !/^[a-zA-Z0-9_-]+$/.test(ctx.agentId) || + !/^[a-zA-Z0-9_-]+$/.test(wakeupId) || + ctx.eventIds.some((id) => !/^[a-zA-Z0-9_-]+$/.test(id)) + ) + return yield* new MemoryWriteRejected({ reason: "Invalid server context" }); + yield* journalEntry(ctx, "completed", []); + const previous = sessions.get(orgId); + if (previous && previous.wakeupId !== wakeupId) + return yield* new MemoryWriteRejected({ reason: "Another wake-up owns the writer" }); + const state = previous ?? { wakeupId, files: new Map(), lines: [] }; + sessions.set(orgId, state); + return { ctx, state }; + }); + const safePath = Effect.fn("MemoryWriter.safePath")(function* ( + orgId: KinOrgId, + writer: string, + relative: string, + generated = false, + ) { + const parts = relative.split("/"); + if ( + !relative.endsWith(".md") || + parts.some( + (p) => + !p || + p === "." || + p === ".." || + p.toLowerCase() === ".git" || + p.includes("\\") || + p.includes(":"), + ) || + /[\x00-\x1f]/.test(relative) || + (!(generated && relative === "INDEX.md") && ![...folders, "journal"].includes(parts[0] ?? "")) + ) + return yield* new MemoryWriteRejected({ reason: "Disallowed memory path" }); + let current = writer; + for (const part of parts) { + current += `/${part}`; + const result = yield* fs.lstat(current).pipe(Effect.result); + if (result._tag === "Failure") { + if (result.failure.reason._tag !== "NotFound") + return yield* new MemoryError({ orgId, operation: "path", cause: result.failure }); + } else if ( + result.success.type === "SymbolicLink" || + (result.success.type === "File" && + Option.isSome(result.success.nlink) && + result.success.nlink.value > 1) || + (current !== `${writer}/${relative}` && result.success.type !== "Directory") + ) { + return yield* new MemoryWriteRejected({ reason: "Symlink or invalid ancestor" }); + } + } + }); + const validate = Effect.fn("MemoryWriter.validate")(function* (orgId: KinOrgId) { + const paths = yield* memory.ensureCheckouts(orgId); + return paths; + }); + const report = (orgId: KinOrgId, wakeupId: string, effect: Effect.Effect) => + effect.pipe( + Effect.tapError((error) => + error._tag === "MemoryCheckoutInvalid" || + error._tag === "MemorySyncConflict" || + error._tag === "MemorySecretBlocked" + ? hooks + .notify({ orgId, wakeupId, error }) + .pipe( + Effect.catchCause((cause) => + Effect.logError("Memory writer notification failed", cause), + ), + ) + : Effect.void, + ), + ); + return MemoryWriter.of({ + stage: Effect.fn("MemoryWriter.stage")((orgId, wakeupId, change) => + report( + orgId, + wakeupId, + locked( + orgId, + Effect.gen(function* () { + const { ctx, state } = yield* session(orgId, wakeupId); + const { writer } = yield* validate(orgId); + yield* safePath(orgId, writer, change.path); + if (change.path.startsWith("journal/")) + return yield* new MemoryWriteRejected({ + reason: "Use appendJournal for journal additions", + }); + const parsed = yield* parseFrontMatter(orgId, change.path, change.content, true); + const source = + change.sourceEventId === undefined + ? { saidBy: ctx.agentId, trust: "agent", source: `wakeup:${wakeupId}` } + : ctx.sources.get(change.sourceEventId); + if (!source) + return yield* new MemoryWriteRejected({ + reason: "Source is not in the current batch", + }); + const fm = parsed.frontMatter; + const folder = fm.kind === "person" ? "people" : `${fm.kind}s`; + if (!change.path.startsWith(`${folder}/`)) + return yield* new MemoryWriteRejected({ reason: "Kind does not match path" }); + const branch = + ctx.level === "L0" ? `kin/shadow/${ctx.agentId}` : (config.defaultBranch ?? "main"); + const recorded = yield* branchHead(orgId, writer, branch); + const recordedFiles = yield* filesAt(orgId, writer, recorded); + const items = []; + const bodies = new Map(); + const regular = new Set( + (yield* readTree(orgId, writer, recorded).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(VcsProcess, vcs), + )) + .filter((entry) => ["100644", "100755"].includes(entry.mode)) + .map((entry) => entry.relative), + ); + for (const [file, bytes] of recordedFiles) { + if (!regular.has(file)) continue; + if (!folders.some((folder) => file.startsWith(`${folder}/`))) continue; + const content = bytes.toString("utf8"); + const parsedItem = yield* parseFrontMatter(orgId, file, content).pipe(Effect.result); + if (parsedItem._tag === "Failure") { + if (file === change.path) + return yield* new MemoryWriteRejected({ + reason: "Cannot overwrite malformed recorded item", + }); + continue; + } + if (file === change.path && parsedItem.success.frontMatter.kind !== fm.kind) + return yield* new MemoryWriteRejected({ + reason: "Cannot change recorded item kind", + }); + items.push({ + ...parsedItem.success.frontMatter, + orgId, + path: file, + duplicate: false, + }); + bodies.set(file, parsedItem.success.body); + } + for (const [file, text] of state.files) { + if (file.startsWith("journal/") || file === "INDEX.md") continue; + const staged = yield* parseFrontMatter(orgId, file, text); + const index = items.findIndex((item) => item.path === file); + const head = { ...staged.frontMatter, orgId, path: file, duplicate: false }; + if (index < 0) items.push(head); + else items[index] = head; + bodies.set(file, staged.body); + } + const superseded = + change.supersedes === undefined + ? undefined + : items.find((item) => item.id === change.supersedes); + if ( + change.supersedes !== undefined && + (!superseded || + superseded.status !== "open" || + superseded.kind !== fm.kind || + superseded.source !== source.source || + Date.parse(fm.recorded_at) <= Date.parse(superseded.recorded_at) || + superseded.id === fm.id) + ) + return yield* new MemoryWriteRejected({ + reason: "Supersession requires the same source and kind", + }); + for (const item of items) { + if ( + (item.id === fm.id || item.status === "open") && + item.kind === fm.kind && + (item.id === fm.id || item.links.some((link) => fm.links.includes(link))) && + item.id !== change.supersedes + ) { + const body = + bodies.get(item.path) ?? (yield* memory.readItemQuietly(orgId, item.id)).body; + const rank = (trust: string) => + ["external", "agent", "member", "owner"].indexOf(trust); + if ( + item.id === fm.id && + (rank(source.trust) < rank(item.trust) || + (fm.status !== item.status && + !(item.status === "open" && ["done", "cancelled"].includes(fm.status)))) + ) + return yield* new MemoryConflict({ existingId: item.id }); + if (body !== parsed.body || item.title !== fm.title) + return yield* new MemoryConflict({ existingId: item.id }); + if (item.id === fm.id) { + const proposed = new Map(Object.entries(fm)); + const mutable = new Set([ + "orgId", + "path", + "duplicate", + "status", + "invalid_at", + "trust", + "said_by", + "source", + ]); + if ( + Object.entries(item).some( + ([key, value]) => + !mutable.has(key) && + JSON.stringify(value) !== JSON.stringify(proposed.get(key)), + ) + ) + return yield* new MemoryConflict({ existingId: item.id }); + } + } + if ( + (item.path === change.path && item.id !== fm.id) || + (item.id === fm.id && item.path !== change.path) + ) + return yield* new MemoryWriteRejected({ + reason: "Cannot replace a different item", + }); + } + const fields = { + ...fm, + said_by: source.saidBy, + source: source.source, + trust: + fm.trust === "external" || + (fm.trust === "agent" && source.trust !== "external") || + (fm.trust === "member" && source.trust === "owner") + ? fm.trust + : source.trust, + review_by: fm.review_by, + }; + const existing = items.find((item) => item.id === fm.id); + const text = render( + existing + ? { + ...fields, + trust: existing.trust, + said_by: existing.said_by, + source: existing.source, + } + : fields, + parsed.body, + ); + yield* parseFrontMatter(orgId, change.path, text); + if (superseded) { + const oldBody = + bodies.get(superseded.path) ?? + (yield* memory.readItemQuietly(orgId, superseded.id)).body; + const { + orgId: _org, + path: oldPath, + duplicate: _duplicate, + ...oldFields + } = superseded; + const oldText = render( + { ...oldFields, status: "superseded", superseded_by: fm.id }, + oldBody, + ); + state.files.set(oldPath, oldText); + } + state.files.set(change.path, text); + }), + ), + ), + ), + appendJournal: Effect.fn("MemoryWriter.appendJournal")((orgId, wakeupId, lines) => + locked( + orgId, + Effect.gen(function* () { + const { state } = yield* session(orgId, wakeupId); + state.lines.push(...lines); + }), + ), + ), + discard: Effect.fn("MemoryWriter.discard")((orgId, wakeupId) => + locked( + orgId, + Effect.gen(function* () { + const state = sessions.get(orgId); + if (state && state.wakeupId !== wakeupId) + return yield* new MemoryWriteRejected({ reason: "Another wake-up owns the writer" }); + sessions.delete(orgId); + }), + ), + ), + commitWakeup: Effect.fn("MemoryWriter.commitWakeup")((orgId, wakeupId, outcome) => + locked( + orgId, + report( + orgId, + wakeupId, + Effect.gen(function* () { + const { ctx, state } = yield* session(orgId, wakeupId); + const { writer } = yield* validate(orgId); + const repo = config.repositories.find((entry) => entry.orgId === orgId); + if (!repo) + return yield* new MemoryConfigError({ + orgId, + operation: "writer", + reason: "Unknown org", + }); + const branch = + ctx.level === "L0" ? `kin/shadow/${ctx.agentId}` : (config.defaultBranch ?? "main"); + const recorded = yield* branchHead(orgId, writer, branch); + const push = (sha: string) => + git(orgId, writer, ["push", "--", repo.repoUrl, `${sha}:refs/heads/${branch}`]).pipe( + Effect.mapError( + (cause) => + new MemorySyncConflict({ + orgId, + operation: "push", + reason: "Push rejected; retry or reconcile remote", + cause, + }), + ), + ); + return yield* Effect.gen(function* () { + const subject = `kin(${ctx.agentId}): wake-up ${wakeupId}`; + const fingerprint = yield* git( + orgId, + writer, + ["hash-object", "--no-filters", "--stdin"], + {}, + stringify({ + outcome, + files: [...state.files].sort(([a], [b]) => a.localeCompare(b)), + lines: state.lines, + }), + ); + const { message: body } = yield* commitAt(orgId, writer, recorded); + if (body.split("\n")[0] === subject) { + if ( + !body.includes(`Kin-Outcome: ${outcome}`) || + (state.files.size + state.lines.length > 0 && + !body.includes(`Kin-Request: ${fingerprint}`)) + ) + return yield* new MemoryWriteRejected({ reason: "Different wake-up retry" }); + yield* push(recorded); + return { commit: recorded }; + } + if (outcome === "failed") state.files.clear(); + const entry = yield* journalEntry(ctx, outcome, state.lines); + yield* safePath(orgId, writer, entry.path); + const recordedFiles = yield* filesAt(orgId, writer, recorded); + const journal = Buffer.concat([ + recordedFiles.get(entry.path) ?? Buffer.alloc(0), + Buffer.from(entry.text), + ]); + state.files.set(entry.path, journal.toString("utf8")); + const regular = new Set( + (yield* readTree(orgId, writer, recorded).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(VcsProcess, vcs), + )) + .filter((entry) => ["100644", "100755"].includes(entry.mode)) + .map((entry) => entry.relative), + ); + const combined = new Map( + [...recordedFiles] + .filter(([path]) => regular.has(path)) + .map(([path, bytes]) => [path, bytes.toString("utf8")]), + ); + for (const [path, text] of state.files) combined.set(path, text); + const heads = []; + for (const [path, text] of combined) { + if (!folders.some((folder) => path.startsWith(`${folder}/`))) continue; + const parsed = yield* parseFrontMatter(orgId, path, text).pipe(Effect.result); + if (parsed._tag === "Success") + heads.push({ ...parsed.success.frontMatter, orgId, path }); + } + const reads = yield* cache + .lastReads(orgId) + .pipe( + Effect.mapError((cause) => new MemoryError({ orgId, operation: "index", cause })), + ); + const zone = yield* cache + .ownerTimeZone(orgId) + .pipe( + Effect.mapError((cause) => new MemoryError({ orgId, operation: "index", cause })), + ); + state.files.set( + "INDEX.md", + buildIndex(heads, reads, yield* Clock.currentTimeMillis, zone), + ); + const findings = yield* scanner.scan({ + orgId, + files: [...state.files].map(([path, content]) => ({ + path, + content: path === entry.path ? journal : new TextEncoder().encode(content), + })), + }); + if (findings.length) return yield* new MemorySecretBlocked({ orgId, findings }); + return yield* Effect.scoped( + Effect.gen(function* () { + const temporary = yield* io(orgId, fs.makeTempDirectoryScoped()); + const env = { + GIT_INDEX_FILE: `${temporary}/index`, + GIT_AUTHOR_NAME: config.botName ?? "Kin Memory Bot", + GIT_AUTHOR_EMAIL: config.botEmail ?? "kin-memory@localhost", + GIT_COMMITTER_NAME: config.botName ?? "Kin Memory Bot", + GIT_COMMITTER_EMAIL: config.botEmail ?? "kin-memory@localhost", + }; + yield* git(orgId, writer, ["read-tree", "--empty"], env); + const entries = yield* readTree(orgId, writer, recorded).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(VcsProcess, vcs), + ); + const metadata = new Map(entries.map((entry) => [entry.relative, entry])); + const finalFiles = new Map(recordedFiles); + for (const [file, text] of state.files) + finalFiles.set(file, file === entry.path ? journal : Buffer.from(text)); + const expected = new Map(); + for (const [file, bytes] of finalFiles) { + if (state.files.has(file)) yield* safePath(orgId, writer, file, true); + const input = `${temporary}/blob`; + yield* io(orgId, fs.writeFile(input, bytes)); + const mode = state.files.has(file) + ? "100644" + : (metadata.get(file)?.mode ?? "100644"); + const blob = + mode === "160000" + ? bytes.toString("ascii") + : yield* git( + orgId, + writer, + ["hash-object", "-w", "--no-filters", input], + env, + ); + expected.set(file, { + mode, + sha: + mode === "160000" + ? bytes.toString("ascii") + : objectHash(bytes, recorded.length), + }); + yield* git( + orgId, + writer, + ["update-index", "--add", "--cacheinfo", `${mode},${blob},${file}`], + env, + ); + } + const tree = yield* git(orgId, writer, ["write-tree"], env); + yield* verifyTree(orgId, writer, tree, expected).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(VcsProcess, vcs), + ); + const message = `${subject}\n\nEvents: ${ctx.eventIds.join(", ")}\nKin-Outcome: ${outcome}\nKin-Request: ${fingerprint}\n`; + const commit = yield* git( + orgId, + writer, + ["commit-tree", tree, "-p", recorded], + env, + message, + ); + yield* git(orgId, writer, [ + "update-ref", + `refs/heads/${branch}`, + commit, + recorded, + ]); + yield* memory.record(orgId, branch, recorded, commit); + yield* push(commit); + return { commit }; + }), + ); + }).pipe(Effect.ensuring(Effect.sync(() => sessions.delete(orgId)))); + }), + ), + ), + ), + }); +}); +export const layer = Layer.effect(MemoryWriter, make).pipe(Layer.provide(MemoryCache.layer)); diff --git a/apps/server/src/kin/memory/frontMatter.ts b/apps/server/src/kin/memory/frontMatter.ts index 737e7d8d60416..692502849fca7 100644 --- a/apps/server/src/kin/memory/frontMatter.ts +++ b/apps/server/src/kin/memory/frontMatter.ts @@ -2,6 +2,7 @@ import { MemoryItemFrontMatter, type KinOrgId } from "@t3tools/contracts"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Schema from "effect/Schema"; +import { defaultReviewBy } from "./freshness.ts"; import { isSeq, parseDocument } from "yaml"; export class MemoryItemError extends Schema.TaggedError()("MemoryItemError", { @@ -34,12 +35,19 @@ export const memoryDateMillis = (value: string | null) => { }; const decodeFrontMatter = Schema.decodeUnknownEffect(MemoryItemFrontMatter); +const decodeStaged = Schema.decodeUnknownEffect( + Schema.Struct({ + ...MemoryItemFrontMatter.fields, + review_by: Schema.optional(MemoryItemFrontMatter.fields.review_by), + }), +); // Parsing runs per blob under Memory.load; avoid creating a span for every item. export const parseFrontMatter = Effect.fnUntraced(function* ( orgId: KinOrgId, path: string, text: string, + staged = false, ) { const parts = /^(?:\uFEFF)?---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)([\s\S]*)$/.exec(text); if (!parts) { @@ -85,14 +93,14 @@ export const parseFrontMatter = Effect.fnUntraced(function* ( cause, }), }); - const frontMatter = yield* decodeFrontMatter(value).pipe( + const decoded = yield* (staged ? decodeStaged(value) : decodeFrontMatter(value)).pipe( Effect.mapError( (cause) => new MemoryItemError({ orgId, path, reason: "Invalid front matter schema", cause }), ), ); for (const field of ["recorded_at", "valid_from", "invalid_at", "review_by", "due"] as const) { - const value = frontMatter[field]; - if (value !== null && !Number.isFinite(memoryDateMillis(value))) { + const value = decoded[field]; + if (value !== null && value !== undefined && !Number.isFinite(memoryDateMillis(value))) { return yield* new MemoryItemError({ orgId, path, @@ -100,5 +108,16 @@ export const parseFrontMatter = Effect.fnUntraced(function* ( }); } } + const frontMatter = { + ...decoded, + review_by: defaultReviewBy( + { + kind: decoded.kind, + due: decoded.due, + ...(decoded.review_by === undefined ? {} : { review_by: decoded.review_by }), + }, + Date.parse(decoded.recorded_at), + ), + }; return { frontMatter, body: parts[2] ?? "" }; }); diff --git a/apps/server/src/kin/memory/git.ts b/apps/server/src/kin/memory/git.ts new file mode 100644 index 0000000000000..b17bf463e29c3 --- /dev/null +++ b/apps/server/src/kin/memory/git.ts @@ -0,0 +1,127 @@ +import * as Schema from "effect/Schema"; +import * as Option from "effect/Option"; +import { MemoryConfig } from "./MemoryConfig.ts"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import { VcsProcess } from "../../vcs/VcsProcess.ts"; + +class MemoryGitRefRace extends Schema.TaggedError()("MemoryGitRefRace", {}) {} +export class MemoryGitMissingRepository extends Schema.TaggedError()( + "MemoryGitMissingRepository", + { command: Schema.String }, +) {} + +const repositoryFree = new Set(["clone", "ls-remote"]); + +const formats = new Map(); + +// A private bare git directory excludes lead-controlled config, grafts and hooks. +export const memoryGit = Effect.fn("Memory.gitHardened")(function* ( + cwd: string, + args: ReadonlyArray, + stdin?: string, + env: NodeJS.ProcessEnv = {}, + onStdoutChunk?: (chunk: Uint8Array) => void, +) { + const fs = yield* FileSystem.FileSystem; + const config = Option.getOrUndefined(yield* Effect.serviceOption(MemoryConfig)); + const vcs = yield* VcsProcess; + return yield* Effect.scoped( + Effect.gen(function* () { + const directory = yield* fs.makeTempDirectoryScoped(); + const entry = yield* fs.lstat(`${cwd}/.git`).pipe(Effect.option); + const repo = Option.isSome(entry) && entry.value.type === "Directory"; + // Without --git-dir, git would discover a parent repository and its config. + if (!repo && !repositoryFree.has(args[0] ?? "")) + return yield* new MemoryGitMissingRepository({ command: args[0] ?? "" }); + let packedBefore: string | undefined; + let format = formats.get(cwd); + if (!format && repo) { + const text = yield* fs + .readFileString(`${cwd}/.git/config`) + .pipe(Effect.orElseSucceed(() => "")); + format = /objectformat\s*=\s*sha256/i.test(text) ? "sha256" : "sha1"; + if (formats.size >= 128) formats.clear(); + formats.set(cwd, format); + } + if (repo) { + yield* fs.writeFileString( + `${directory}/config`, + format === "sha256" + ? "[core]\nrepositoryformatversion = 1\nbare = true\n[extensions]\nobjectformat = sha256\n" + : "[core]\nrepositoryformatversion = 0\nbare = true\n", + ); + yield* fs.symlink(`${cwd}/.git/objects`, `${directory}/objects`); + if (args[0] === "fsck") { + yield* fs.makeDirectory(`${directory}/refs`); + yield* fs.writeFileString(`${directory}/HEAD`, `${args.at(-1)}\n`); + } else { + yield* fs.symlink(`${cwd}/.git/refs`, `${directory}/refs`); + for (const name of ["HEAD", "packed-refs"]) { + if (yield* fs.exists(`${cwd}/.git/${name}`)) + yield* fs.copyFile(`${cwd}/.git/${name}`, `${directory}/${name}`); + } + } + } + if (repo && args[0] === "update-ref") + packedBefore = yield* fs + .readFileString(`${directory}/packed-refs`) + .pipe(Effect.orElseSucceed(() => "")); + const result = yield* vcs.run({ + operation: "Memory.gitHardened", + command: "git", + cwd, + args: [ + "--no-replace-objects", + ...(repo && args[0] !== "clone-snapshot" + ? [`--git-dir=${directory}`, ...(args[0] === "checkout" ? [`--work-tree=${cwd}`] : [])] + : []), + "-c", + "core.hooksPath=/dev/null", + "-c", + "core.fsmonitor=false", + "-c", + "core.attributesFile=/dev/null", + "-c", + "protocol.file.allow=user", + ...(config?.credentialHelper + ? ["-c", "credential.helper=", "-c", `credential.helper=${config.credentialHelper}`] + : []), + ...(args[0] === "clone-snapshot" + ? ["clone", "--no-local", "--no-checkout", "--", directory, args[1]!] + : args), + ], + env: { + GIT_CONFIG_NOSYSTEM: "1", + GIT_CONFIG_GLOBAL: "/dev/null", + GIT_TERMINAL_PROMPT: "0", + GIT_ATTR_NOSYSTEM: "1", + GIT_NO_REPLACE_OBJECTS: "1", + GIT_ASKPASS: "", + GCM_INTERACTIVE: "never", + ...env, + ...(config?.sshCommand ? { GIT_SSH_COMMAND: config.sshCommand } : {}), + }, + ...(stdin === undefined ? {} : { stdin }), + ...(onStdoutChunk === undefined ? {} : { onStdoutChunk }), + maxOutputBytes: 64 * 1024 * 1024, + timeoutMs: 120_000, + }); + if (packedBefore !== undefined) { + const packedAfter = yield* fs + .readFileString(`${directory}/packed-refs`) + .pipe(Effect.orElseSucceed(() => "")); + if (packedBefore !== packedAfter) { + const current = yield* fs + .readFileString(`${cwd}/.git/packed-refs`) + .pipe(Effect.orElseSucceed(() => "")); + if (current !== packedBefore) return yield* new MemoryGitRefRace({}); + yield* fs.writeFileString(`${cwd}/.git/packed-refs`, packedAfter); + } + } + if (args[0] === "fetch" && !result.stdout) + return (yield* fs.readFileString(`${directory}/FETCH_HEAD`)).split("\t")[0] ?? ""; + return result.stdout; + }), + ); +}); diff --git a/apps/server/src/kin/memory/journal.ts b/apps/server/src/kin/memory/journal.ts new file mode 100644 index 0000000000000..c61ace8da104b --- /dev/null +++ b/apps/server/src/kin/memory/journal.ts @@ -0,0 +1,46 @@ +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; + +export class MemoryJournalError extends Schema.TaggedError()( + "MemoryJournalError", + { reason: Schema.String }, +) {} + +export interface JournalContext { + readonly wakeupId: string; + readonly agentId: string; + readonly level: string; + readonly startedAt: string; + readonly timeZone: string; + readonly eventIds: ReadonlyArray; +} +const oneLine = (value: string) => value.replace(/[\r\n\u2028\u2029]/g, " "); +const renderJournalEntry = ( + ctx: JournalContext, + outcome: "completed" | "failed", + lines: ReadonlyArray, +) => { + const parts = new Intl.DateTimeFormat("en-GB", { + timeZone: ctx.timeZone, + year: "numeric", + month: "2-digit", + day: "2-digit", + hour: "2-digit", + minute: "2-digit", + hourCycle: "h23", + }).formatToParts(Date.parse(ctx.startedAt)); + const part = (type: Intl.DateTimeFormatPartTypes) => + parts.find((entry) => entry.type === type)?.value ?? ""; + return { + path: `journal/${part("year")}/${part("month")}-${part("day")}.md`, + text: `\n## ${part("hour")}:${part("minute")} · wake-up ${oneLine(ctx.wakeupId)} · ${oneLine(ctx.agentId)} · ${ctx.level}\n\nOutcome: ${outcome}\nEvents: ${ctx.eventIds.map(oneLine).join(", ")}\n\n${lines.map((line) => `- ${oneLine(line)}\n`).join("")}`, + }; +}; + +export const journalEntry = Effect.fn("Memory.journalEntry")( + (ctx: JournalContext, outcome: "completed" | "failed", lines: ReadonlyArray) => + Effect.try({ + try: () => renderJournalEntry(ctx, outcome, lines), + catch: () => new MemoryJournalError({ reason: "Invalid journal date or time zone" }), + }), +); diff --git a/apps/server/src/kin/memory/merge.test.ts b/apps/server/src/kin/memory/merge.test.ts new file mode 100644 index 0000000000000..a829b9e7f32a3 --- /dev/null +++ b/apps/server/src/kin/memory/merge.test.ts @@ -0,0 +1,60 @@ +import { expect, it } from "@effect/vitest"; +import { mergeMemoryFiles, uniqueMergeBase } from "./merge.ts"; + +it.each(["append", "rewrite", "disjoint", "remote rewrite", "rewrite disjoint"])( + "merges verified snapshots: %s", + (scenario) => { + const base = new Map([["journal/day.md", Buffer.from("original\n")]]); + const local = new Map(base); + const remote = new Map(base); + local.set( + "journal/day.md", + Buffer.from(scenario.startsWith("rewrite") ? "rewritten\n" : "original\nlocal\n"), + ); + if (!scenario.includes("disjoint")) + remote.set( + "journal/day.md", + Buffer.from(scenario === "remote rewrite" ? "rewritten remotely\n" : "original\nremote\n"), + ); + const metadata = (files: Map) => + new Map([...files].map(([file, bytes]) => [file, { mode: "100644", sha: bytes.toString() }])); + const result = mergeMemoryFiles( + base, + local, + remote, + metadata(base), + metadata(local), + metadata(remote), + ); + if (scenario.startsWith("rewrite") || scenario === "remote rewrite") + expect(result).toBeUndefined(); + else + expect(result?.files.get("journal/day.md")?.toString()).toBe( + scenario === "disjoint" ? "original\nlocal\n" : "original\nremote\nlocal\n", + ); + }, +); + +it("fails closed for unrelated histories and criss-cross merge bases", () => { + expect(uniqueMergeBase(new Map([["a", []]]), new Map([["b", []]]))).toBeUndefined(); + const common = [ + ["a", ["root"]], + ["b", ["root"]], + ["root", []], + ] as const; + expect( + uniqueMergeBase( + new Map>([...common, ["local", ["a", "b"]]]), + new Map>([...common, ["remote", ["b", "a"]]]), + ), + ).toBeUndefined(); + expect( + uniqueMergeBase( + new Map([ + ["a", []], + ["b", ["a"]], + ]), + new Map([["a", []]]), + ), + ).toBe("a"); +}); diff --git a/apps/server/src/kin/memory/merge.ts b/apps/server/src/kin/memory/merge.ts new file mode 100644 index 0000000000000..c7af0c991ddd7 --- /dev/null +++ b/apps/server/src/kin/memory/merge.ts @@ -0,0 +1,53 @@ +type Entry = { mode: string; sha: string }; + +export const uniqueMergeBase = ( + local: ReadonlyMap>, + remote: ReadonlyMap>, +) => { + const common = new Set([...local.keys()].filter((sha) => remote.has(sha))); + const candidates = new Set(common); + for (const sha of common) for (const parent of local.get(sha) ?? []) candidates.delete(parent); + return candidates.size === 1 ? [...candidates][0] : undefined; +}; + +export const mergeMemoryFiles = ( + base: ReadonlyMap, + local: ReadonlyMap, + remote: ReadonlyMap, + baseEntries: ReadonlyMap, + localEntries: ReadonlyMap, + remoteEntries: ReadonlyMap, +) => { + const files = new Map(remote); + const modes = new Map([...remoteEntries].map(([file, entry]) => [file, entry.mode])); + const same = (a: Buffer | undefined, b: Buffer | undefined) => + a === undefined ? b === undefined : b !== undefined && a.equals(b); + for (const file of new Set([...base.keys(), ...local.keys()])) { + const before = baseEntries.get(file); + const after = localEntries.get(file); + if (file === "INDEX.md" || (before?.sha === after?.sha && before?.mode === after?.mode)) + continue; + const original = base.get(file) ?? Buffer.alloc(0); + const bytes = local.get(file); + const journal = file.startsWith("journal/"); + if (journal && (!bytes || !bytes.subarray(0, original.length).equals(original))) + return undefined; + if (same(remote.get(file), base.get(file)) && before?.mode === modes.get(file)) { + if (bytes) { + files.set(file, bytes); + modes.set(file, after?.mode ?? "100644"); + } else { + files.delete(file); + modes.delete(file); + } + } else if ( + journal && + ["100644", "100755"].includes(after?.mode ?? "") && + after?.mode === modes.get(file) && + remote.get(file)?.subarray(0, original.length).equals(original) + ) { + files.set(file, Buffer.concat([remote.get(file)!, bytes!.subarray(original.length)])); + } else return undefined; + } + return { files, modes }; +}; diff --git a/apps/server/src/kin/memory/objects.test.ts b/apps/server/src/kin/memory/objects.test.ts new file mode 100644 index 0000000000000..12de5de57c533 --- /dev/null +++ b/apps/server/src/kin/memory/objects.test.ts @@ -0,0 +1,99 @@ +// @effect-diagnostics-next-line nodeBuiltinImport:off -- Fixtures need authentic Git object identities. +import * as NodeCrypto from "node:crypto"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { expect, it } from "@effect/vitest"; +import { KinOrgId } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import { ChildProcessSpawner } from "effect/process"; +import { VcsProcess } from "../../vcs/VcsProcess.ts"; +import { ancestors, verifyTree } from "./objects.ts"; + +it.effect( + "batches a 200-commit frontier and stops at a verified ancestor; rejects a substituted tree", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const cwd = yield* fs.makeTempDirectoryScoped(); + yield* fs.makeDirectory(`${cwd}/.git/objects`, { recursive: true }); + yield* fs.makeDirectory(`${cwd}/.git/refs`, { recursive: true }); + const objects = new Map(); + const object = (type: string, text: string) => { + const bytes = Buffer.from(text); + const sha = NodeCrypto.createHash("sha1") + .update(`${type} ${bytes.length}\0`) + .update(bytes) + .digest("hex"); + objects.set(sha, { type, bytes }); + return sha; + }; + const tree = object("tree", ""); + const root = object("commit", `tree ${tree}\n\nroot\n`); + const frontier = Array.from({ length: 200 }, (_, n) => + object("commit", `tree ${tree}\nparent ${root}\n\ncommit ${n}\n`), + ); + const tip = object( + "commit", + `tree ${tree}\n${frontier.map((sha) => `parent ${sha}\n`).join("")}\nmerge\n`, + ); + let calls = 0; + const vcs = VcsProcess.of({ + run: Effect.fn("Test.batch")((input) => + Effect.sync(() => { + calls++; + const chunks = input + .stdin!.trim() + .split("\n") + .map((sha) => { + const entry = objects.get(sha)!; + return Buffer.concat([ + Buffer.from(`${sha} ${entry.type} ${entry.bytes.length}\n`), + entry.bytes, + Buffer.from("\n"), + ]); + }); + input.onStdoutChunk?.(Buffer.concat(chunks)); + return { + exitCode: ChildProcessSpawner.ExitCode(0), + stdout: "", + stderr: "", + stdoutTruncated: false, + stderrTruncated: false, + }; + }), + ), + }); + const org = KinOrgId.make("fixture"); + yield* Effect.gen(function* () { + expect((yield* ancestors(org, cwd, tip)).size).toBe(202); + expect(calls).toBe(3); + calls = 0; + expect((yield* ancestors(org, cwd, tip, frontier[0])).size).toBe(201); + expect(calls).toBe(2); + yield* verifyTree(org, cwd, tree, new Map()); + const rejected = yield* verifyTree( + org, + cwd, + tree, + new Map([["rules.md", { mode: "100644", sha: "0".repeat(40) }]]), + ).pipe(Effect.result); + expect(rejected._tag).toBe("Failure"); + const elsewhere = yield* fs.makeTempDirectoryScoped(); + yield* fs.rename(`${cwd}/.git`, `${elsewhere}/.git`); + for (const shape of ["missing", "file", "symlink"] as const) { + if (shape === "file") + yield* fs.writeFileString(`${cwd}/.git`, `gitdir: ${elsewhere}/.git\n`); + if (shape === "symlink") { + yield* fs.remove(`${cwd}/.git`); + yield* fs.symlink(`${elsewhere}/.git`, `${cwd}/.git`); + } + calls = 0; + const refused = yield* ancestors(org, cwd, tip).pipe(Effect.result); + expect(refused._tag).toBe("Failure"); + if (refused._tag === "Failure") + expect(refused.failure._tag).toBe("MemoryCheckoutInvalid"); + expect(calls).toBe(0); + } + }).pipe(Effect.provideService(VcsProcess, vcs)); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); diff --git a/apps/server/src/kin/memory/objects.ts b/apps/server/src/kin/memory/objects.ts new file mode 100644 index 0000000000000..e7c4d33155168 --- /dev/null +++ b/apps/server/src/kin/memory/objects.ts @@ -0,0 +1,209 @@ +// @effect-diagnostics-next-line nodeBuiltinImport:off -- Git object identity uses the native byte-oriented hash API. +import * as NodeCrypto from "node:crypto"; +import type { KinOrgId } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import { memoryGit } from "./git.ts"; +import { MemoryCheckoutInvalid } from "./Memory.ts"; + +// Object batches share the enclosing operation span, including per-commit ancestry reads. +export const readObjects = Effect.fnUntraced(function* ( + orgId: KinOrgId, + cwd: string, + requests: ReadonlyArray<{ sha: string; type: "blob" | "tree" | "commit" }>, +) { + const invalid = () => + new MemoryCheckoutInvalid({ + orgId, + operation: "object", + reason: "Object hash or framing mismatch", + }); + if (requests.some(({ sha }) => !/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(sha))) + return yield* invalid(); + if (!requests.length) return []; + const chunks: Uint8Array[] = []; + yield* memoryGit( + cwd, + ["cat-file", "--batch"], + requests.map(({ sha }) => sha).join("\n") + "\n", + {}, + (chunk) => chunks.push(chunk), + ).pipe(Effect.mapError(invalid)); + const buffer = Buffer.concat(chunks); + const result: Buffer[] = []; + let offset = 0; + for (const { sha, type } of requests) { + const end = buffer.indexOf(10, offset); + const header = buffer.subarray(offset, end).toString("ascii"); + const match = /^(\w+) (blob|tree|commit) (\d+)$/.exec(header); + if (end < 0 || !match || match[1] !== sha || match[2] !== type) return yield* invalid(); + const length = Number(match[3]); + offset = end + 1; + const bytes = buffer.subarray(offset, offset + length); + if ( + bytes.length !== length || + buffer[offset + length] !== 10 || + NodeCrypto.createHash(sha.length === 40 ? "sha1" : "sha256") + .update(`${type} ${length}\0`) + .update(bytes) + .digest("hex") !== sha + ) + return yield* invalid(); + result.push(bytes); + offset += length + 1; + } + if (offset !== buffer.length) return yield* invalid(); + return result; +}); + +// Per-commit reads share the enclosing operation span. +export const readCommit = Effect.fnUntraced(function* (orgId: KinOrgId, cwd: string, sha: string) { + const objects = yield* readObjects(orgId, cwd, [{ sha, type: "commit" }]); + const text = objects[0]?.toString("utf8") ?? ""; + const tree = /^tree ([a-f0-9]+)\n/.exec(text)?.[1]; + if (!tree) + return yield* new MemoryCheckoutInvalid({ + orgId, + operation: "commit", + reason: "Missing commit tree", + }); + return { + tree, + message: text.slice(text.indexOf("\n\n") + 2), + parent: /^parent ([a-f0-9]+)$/m.exec(text.split("\n\n")[0] ?? "")?.[1], + parents: [...(text.split("\n\n")[0] ?? "").matchAll(/^parent ([a-f0-9]+)$/gm)] + .map((match) => match[1]) + .filter((sha): sha is string => sha !== undefined), + }; +}); + +export const readTree = Effect.fn("Memory.readTree")(function* ( + orgId: KinOrgId, + cwd: string, + commit: string, + isTree = false, +) { + const tree = isTree ? commit : (yield* readCommit(orgId, cwd, commit)).tree; + const entries: Array<{ sha: string; relative: string; mode: string }> = []; + const pending = [{ sha: tree, prefix: "" }]; + while (pending.length) { + const batch = pending.splice(0, 32); + const objects = yield* readObjects( + orgId, + cwd, + batch.map(({ sha }) => ({ sha, type: "tree" })), + ); + for (const [index, object] of objects.entries()) { + const request = batch[index]; + if (!request) continue; + let offset = 0; + while (offset < object.length) { + const space = object.indexOf(32, offset); + const nul = object.indexOf(0, space); + const width = request.sha.length / 2; + if (space < offset || nul < space || nul + 1 + width > object.length) + return yield* new MemoryCheckoutInvalid({ + orgId, + operation: "tree", + reason: "Invalid tree framing", + }); + const mode = object.subarray(offset, space).toString("ascii"); + const name = object.subarray(space + 1, nul).toString("utf8"); + const sha = object.subarray(nul + 1, nul + 1 + width).toString("hex"); + const relative = request.prefix + name; + if (mode === "40000") pending.push({ sha, prefix: `${relative}/` }); + else if (["100644", "100755", "120000", "160000"].includes(mode)) + entries.push({ sha, relative, mode }); + else + return yield* new MemoryCheckoutInvalid({ + orgId, + operation: "tree", + reason: "Unknown tree entry mode", + }); + offset = nul + 1 + width; + } + } + } + return entries; +}); + +export const readFiles = Effect.fn("Memory.readFiles")(function* ( + orgId: KinOrgId, + cwd: string, + commit: string, +) { + const all = yield* readTree(orgId, cwd, commit); + const entries = all.filter((entry) => entry.mode !== "160000"); + const files = new Map(); + for (let index = 0; index < entries.length; index += 32) { + const batch = entries.slice(index, index + 32); + const objects = yield* readObjects( + orgId, + cwd, + batch.map(({ sha }) => ({ sha, type: "blob" })), + ); + for (const [index, entry] of batch.entries()) { + const bytes = objects[index]; + if (bytes) files.set(entry.relative, bytes); + } + } + for (const entry of all) + if (entry.mode === "160000") files.set(entry.relative, Buffer.from(entry.sha, "ascii")); + return files; +}); + +export const objectHash = (bytes: Buffer, width: number) => + NodeCrypto.createHash(width === 40 ? "sha1" : "sha256") + .update(`blob ${bytes.length}\0`) + .update(bytes) + .digest("hex"); + +export const verifyTree = Effect.fn("Memory.verifyTree")(function* ( + orgId: KinOrgId, + cwd: string, + tree: string, + expected: ReadonlyMap, +) { + const entries = yield* readTree(orgId, cwd, tree, true); + if ( + entries.length !== expected.size || + entries.some((entry) => { + const wanted = expected.get(entry.relative); + return wanted?.mode !== entry.mode || wanted.sha !== entry.sha; + }) + ) + return yield* new MemoryCheckoutInvalid({ + orgId, + operation: "writeTree", + reason: "Written tree differs from validated bytes", + }); +}); + +export const ancestors = Effect.fn("Memory.ancestors")(function* ( + orgId: KinOrgId, + writer: string, + tip: string, + stop?: string, +) { + const seen = new Map>(); + let pending = [tip]; + while (pending.length) { + const batch = [...new Set(pending)].filter((sha) => !seen.has(sha)); + if (!batch.length) break; + const objects = yield* readObjects( + orgId, + writer, + batch.map((sha) => ({ sha, type: "commit" as const })), + ); + pending = []; + for (const [index, bytes] of objects.entries()) { + const sha = batch[index]!; + const parents = [ + ...(bytes.toString("utf8").split("\n\n")[0] ?? "").matchAll(/^parent ([a-f0-9]+)$/gm), + ].map((m) => m[1]!); + seen.set(sha, parents); + pending.push(...parents); + } + if (stop && seen.has(stop)) break; + } + return seen; +}); diff --git a/apps/server/src/kin/memory/secretScan.ts b/apps/server/src/kin/memory/secretScan.ts new file mode 100644 index 0000000000000..f7ef2760efa17 --- /dev/null +++ b/apps/server/src/kin/memory/secretScan.ts @@ -0,0 +1,38 @@ +import type { KinOrgId } from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; + +export class KinSecretScanError extends Schema.TaggedError()( + "KinSecretScanError", + { + reason: Schema.String, + }, +) {} +export interface SecretFinding { + readonly path: string; + readonly ruleId: string; + readonly line: number; +} +export class KinSecretScan extends Context.Service< + KinSecretScan, + { + readonly scan: (input: { + orgId: KinOrgId; + files: ReadonlyArray<{ path: string; content: Uint8Array }>; + }) => Effect.Effect, KinSecretScanError>; + } +>()("t3/kin/memory/secretScan/KinSecretScan") {} + +export const noFindingsStubLayer = Layer.effect( + KinSecretScan, + Effect.gen(function* () { + yield* Effect.logWarning("SECURITY: Kin secret scanning is disabled (no-findings stub)"); + return KinSecretScan.of({ scan: Effect.fn("KinSecretScan.stub")(() => Effect.succeed([])) }); + }), +); +export const testLayer = (findings: ReadonlyArray) => + Layer.succeed(KinSecretScan, { + scan: Effect.fn("KinSecretScan.fixture")(() => Effect.succeed(findings)), + }); diff --git a/apps/server/src/kin/persistence/KinMigrations.test.ts b/apps/server/src/kin/persistence/KinMigrations.test.ts index 01d5c377b555c..4b855164bf64d 100644 --- a/apps/server/src/kin/persistence/KinMigrations.test.ts +++ b/apps/server/src/kin/persistence/KinMigrations.test.ts @@ -25,6 +25,7 @@ const tableNames = [ "kin_floor_runs", "kin_judge_log", "kin_leads", + "kin_memory_heads", "kin_memory_reads", "kin_messages", "kin_org_state", @@ -96,6 +97,7 @@ describe("Kin migrations", () => { { migration_id: 8, name: "Floor" }, { migration_id: 9, name: "JudgeLog" }, { migration_id: 10, name: "ChatChanges" }, + { migration_id: 11, name: "MemoryHeads" }, ]); assert.deepStrictEqual( upstreamBefore.map((row) => [row.migration_id, row.name] as const), @@ -148,6 +150,7 @@ describe("Kin migrations", () => { { migration_id: 8, name: "Floor" }, { migration_id: 9, name: "JudgeLog" }, { migration_id: 10, name: "ChatChanges" }, + { migration_id: 11, name: "MemoryHeads" }, ]); assert.deepStrictEqual(yield* upstreamHistory, upstreamBefore); assert.deepStrictEqual(yield* sql`SELECT * FROM kin_orgs`, [ @@ -225,6 +228,7 @@ describe("Kin migrations", () => { [8, "Floor"], [9, "JudgeLog"], [10, "ChatChanges"], + [11, "MemoryHeads"], ]); const before = yield* kinSchema; assert.deepStrictEqual( @@ -261,6 +265,7 @@ describe("Kin migrations", () => { yield* sql`DROP TABLE kin_ping_escalations`; yield* sql`DROP TABLE kin_ping_items`; yield* sql`DROP INDEX kin_chat_members_one_owner`; + yield* sql`DROP TABLE kin_memory_heads`; yield* sql`DROP TABLE kin_budget_charges`; yield* sql`ALTER TABLE kin_events DROP COLUMN budget_deferred`; yield* sql`UPDATE kin_chats SET last_seq = 1 WHERE org_id = 'org' AND chat_id = 'existing-chat'`; @@ -274,6 +279,7 @@ describe("Kin migrations", () => { [8, "Floor"], [9, "JudgeLog"], [10, "ChatChanges"], + [11, "MemoryHeads"], ]); assert.deepStrictEqual( yield* sql`SELECT member_id FROM kin_chat_members WHERE org_id = 'org' ORDER BY member_id`, @@ -309,6 +315,7 @@ describe("Kin migrations", () => { { migration_id: 8, name: "Floor" }, { migration_id: 9, name: "JudgeLog" }, { migration_id: 10, name: "ChatChanges" }, + { migration_id: 11, name: "MemoryHeads" }, ]); assert.deepStrictEqual(yield* runKinMigrations(), []); assert.deepStrictEqual(yield* runMigrations(), []); diff --git a/apps/server/src/kin/persistence/KinMigrations.ts b/apps/server/src/kin/persistence/KinMigrations.ts index ab0034ce1c786..b7bba32965691 100644 --- a/apps/server/src/kin/persistence/KinMigrations.ts +++ b/apps/server/src/kin/persistence/KinMigrations.ts @@ -13,6 +13,7 @@ import Floor from "./migrations/K008_Floor.ts"; import JudgeLog from "./migrations/K009_JudgeLog.ts"; import ChatChanges from "./migrations/K010_ChatChanges.ts"; +import MemoryHeads from "./migrations/K011_MemoryHeads.ts"; const run = Migrator.make({}); const loader = Migrator.fromRecord({ @@ -26,6 +27,7 @@ const loader = Migrator.fromRecord({ "8_Floor": Floor, "9_JudgeLog": JudgeLog, "10_ChatChanges": ChatChanges, + "11_MemoryHeads": MemoryHeads, }); export const runKinMigrations = Effect.fn("KinMigrations.run")(function* () { diff --git a/apps/server/src/kin/persistence/migrations/K011_MemoryHeads.ts b/apps/server/src/kin/persistence/migrations/K011_MemoryHeads.ts new file mode 100644 index 0000000000000..6d874a7aab3be --- /dev/null +++ b/apps/server/src/kin/persistence/migrations/K011_MemoryHeads.ts @@ -0,0 +1,12 @@ +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/sql/SqlClient"; + +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql`CREATE TABLE kin_memory_heads ( + org_id TEXT NOT NULL, + branch TEXT NOT NULL, + head TEXT NOT NULL, + PRIMARY KEY (org_id, branch) + )`; +}); diff --git a/docs/kin/specs/lead.md b/docs/kin/specs/lead.md index ae166ea8debc5..b6d23aaafa0b2 100644 --- a/docs/kin/specs/lead.md +++ b/docs/kin/specs/lead.md @@ -194,7 +194,7 @@ Table `kin_wakeups` (**new**): `org_id, wakeup_id, agent_id, thread_id, run_id, Setup once per organization (`Memory.ensureCheckouts` owns the checkouts; the wake-up runner owns project bootstrap): - Writer checkout: `${stateDir}/kin//memory`, a clone of the organization's memory repo. -- Read checkout: `${stateDir}/kin//memory-read`, a `git worktree` of the same repo, detached at the latest commit. This is the lead's working directory. +- Read checkout: `${stateDir}/kin//memory-read`, a disposable independent clone, detached at the SQLite-pinned commit. This is the lead's working directory. - Lead project: `ProjectService.bootstrap({ workspaceRoot: , title: "Kin · " })`. Each wake-up: @@ -385,28 +385,28 @@ export class Memory extends Context.Service< Writer rules, enforced in code on `stage`: -- Paths: only `objectives/`, `commitments/`, `deferrals/`, `decisions/`, `people/`, `claims/` and `journal/`. `rules.md` is refused. `INDEX.md` is written only by `rebuildIndex`. `README.md` is never touched. +- Paths: only `objectives/`, `commitments/`, `deferrals/`, `decisions/`, `people/`, `claims/` and `journal/`. `rules.md` is refused. `INDEX.md` is generated from validated memory heads, never staged by a lead. `README.md` is never touched. - No deletes. `invalidate` sets `invalid_at`; `supersede` sets the old item's `superseded_by` and `status: superseded` and creates the new one. - Provenance comes from the event, not the model. The writer looks up `sourceEventId` in the current batch and sets `said_by`, `source` and `trust` from the event's `provenance`. Owner → `owner`, member → `member`, webhook or outside text → `external`, worker suggestion → `agent`. The lead may lower trust but never raise it. A member's statement that reads like a rule is stored under `claims/`. - `review_by` defaults by kind: person 90 days, commitment and deferral at `due`, decision `null`, claim 30 days; objective at the next milestone date (falling back to `due`, then +30 days from `recorded_at`). Explicit `review_by: null` is preserved. - Contradictions: if an open item of the same kind shares a link or id and says something different, and `supersedes` was not passed, `stage` fails with `MemoryConflict { existingId }`. The lead then supersedes (same source, newer) or posts a question to the owner. -- Secret scan: before commit, the staged diff goes through `MemorySecretScan.check` (triggers spec). A hit refuses the whole commit, keeps the journal line "Commit refused: possible secret in ", and tells the owner. +- Secret scan: before commit, the staged diff goes through `KinSecretScan.scan({ orgId, files })` (triggers spec). A hit refuses the whole commit, discards staged content and emits a typed `MemorySecretBlocked` notification. The runner records the refusal in its next safe journal entry and tells the owner. -Staged changes live in the writer checkout's working tree. `commitWakeup("completed")` writes the journal entry, rebuilds the front matter cache for changed files, runs the scan, commits once with message `kin(): wake-up ` plus event ids, and pushes. `commitWakeup("failed")` discards staged item changes (`git checkout -- . && git clean -fd` on item folders) and commits only the journal entry, which records the failure. +Staged bytes live only in server memory. Discard requires no checkout validation and works after restart. Completed commits contain only validated bytes, a journal appended to the recorded tree, and an optional generated index. Failed wake-ups discard item changes and commit the failure journal. A private index and `hash-object --no-filters`, `read-tree`, `update-index`, `write-tree`, `commit-tree`, and compare-and-swap `update-ref` build commits on the SQLite-recorded parent, never HEAD. Author and committer identity come from server configuration (default Kin Memory Bot ), never repository configuration. Branches: at L1 and above the writer commits to the default branch. At L0 it commits to `kin/shadow/` (see Levels). Dreaming (triggers spec) commits to `dream/` branches and takes the same per-organization semaphore while it writes, so it never runs during a wake-up. -Push failures retry with backoff. Unpushed commits stay local and the next `prepareWakeup` rebases them on `pull`. A rebase conflict (the owner edited the same file on GitHub) is aborted, restores the writer, and fails the wake-up with `MemorySyncConflict`; the write-side owner notification is LEAD-3 work. A fetch failure logs a warning and the wake-up proceeds pinned to the local writer HEAD. Existing writers are never replaced on validation failure: `MemoryCheckoutInvalid` leaves recovery to a human. +Pushes and fetches use the configured URL explicitly, never a local remote name or pushurl. A retry of an already recorded wake-up pushes again before returning success. Synchronization uses verified ancestry and plumbing-only fast-forwards. Divergent trees merge disjoint edits and append-only journals in process, then use `commit-tree` with both parents. Content conflicts preserve the local head on a conflict branch, reset to the remote anchor and notify the owner. Missing or ambiguous merge bases preserve and notify, then fail closed without resetting; the organization stays blocked until an operator runs `Memory.recover`, and repeated attempts reuse the conflict branch already holding that head. Failed fetches retain the SQLite anchor. Existing writers with mismatched or missing anchors fail closed as `MemoryCheckoutInvalid`. Front matter matches the kin-memory README, including snake_case keys. Date-only values mean UTC midnight; date-times require an explicit UTC offset, and invalid dates make an item malformed. `links` values containing `#` must be quoted. `owner` and `said_by` hold handles; `orgId` and `path` belong only to the server-side `MemoryItemHead`. LEAD-2: the front matter parser must use YAML’s core or JSON schema; YAML 1.1 turns `review_by: 2026-11-13` into a Date. -The production `MemoryConfig` layer derives its server-owned base from `ServerConfig.stateDir`, using `${stateDir}/kin//memory` and `memory-read` (organization ids are path-encoded). Tests may supply an isolated base and relative paths. Paths must not overlap or escape that base. `ensureCheckouts` clones a missing writer and validates an existing one; `prepareWakeup` refuses a writer with an unfinished rebase, merge or cherry-pick, syncs it with `pull --rebase` and force-refreshes the disposable detached read worktree, including generated files. Project bootstrap belongs to the wake-up runner, not Memory. +The production `MemoryConfig` layer derives its server-owned base from `ServerConfig.stateDir`, using `${stateDir}/kin//memory` and `memory-read` (organization ids are path-encoded). Tests may supply an isolated base and relative paths. Paths must not overlap or escape that base. `ensureCheckouts` bootstraps a missing writer from a fresh fetch of the configured URL and configured default branch (`main` by default). K011 stores `(org_id, branch, head)` in `kin_memory_heads`; no writable file or local remote ref can bootstrap an existing checkout. Before committing or pushing, each local branch tip must match its anchor. Compare-and-swap branch updates and SQLite recording happen together in the writer operation; a crash between them is rolled back to the recorded anchor with a compare-and-swap and an operator notice. Prepare and commit share a per-org lock. Pinned reads and writer operations skip re-bootstrap; writer operations still validate the writer checkout every time, and reads validate when they set a new pin. Read clones are replaced from scratch on prepare: git is never run in an existing lead read clone. Every git invocation uses a private git directory excluding repository config, hooks and grafts; merge plumbing uses verified bytes and a private index, never git merge-tree or a writer work tree, with system/global config disabled and hooks/fsmonitor disabled. -`ensureCheckouts` uses the server user’s git credentials: on the Kin host, the bot or GitHub App identity. +**Threat model.** Leads currently run full-access as the same OS user (`kin`) as the server. Assume leads can modify everything under `${stateDir}/kin/*/memory*`, including `.git` and file anchors. Code trusts only SQLite and server configuration, checks recorded branch tips, and builds commits from server-validated bytes rather than ambient working-tree/index content. These are not an OS sandbox: a same-user lead can still attack server processes or SQLite itself. Separating leads into an OS user unable to write `${stateDir}/kin` remains Nishchay's operations decision and is required before production use. -The read side pins the server-owned writer repository’s HEAD at wake-up (or first read), never the lead’s read-checkout HEAD or working-tree edits. Snapshots reload only on a new wake-up pin or explicit `rebuildCache`; blob contents and parse failures are cached by SHA. Symlinks are skipped; oversized blobs (over 256 KiB), malformed items and kind/folder mismatches are logged and skipped. Queries return duplicate ids with `duplicate: true` on each copy and mark them in the index; ambiguous `readItem` calls fail with `MemoryDuplicateId`. Queries include invalid items unless `excludeInvalid` is set; `invalid_at` remains on each returned `MemoryItemHead`. Date bounds are inclusive and compare instants; date-only strings mean midnight UTC. Missing or unparsable dates do not match a bounded date query. Overdue review means strictly before the current clock. +The read side pins the SQLite-anchored default-branch head at wake-up (or first read), never the lead’s read-checkout HEAD or working-tree edits. Snapshots reload only on a new wake-up pin or explicit `rebuildCache`; blob contents and parse failures are cached by SHA. Symlinks are skipped; oversized blobs (over 256 KiB), malformed items and kind/folder mismatches are logged and skipped. Queries return duplicate ids with `duplicate: true` on each copy and mark them in the index; ambiguous `readItem` calls fail with `MemoryDuplicateId`. Queries include invalid items unless `excludeInvalid` is set; `invalid_at` remains on each returned `MemoryItemHead`. Date bounds are inclusive and compare instants; date-only strings mean midnight UTC. Missing or unparsable dates do not match a bounded date query. Overdue review means strictly before the current clock. `readItem` best-effort records integer-millisecond last-read times in Kin’s `kin_memory_reads` table, keyed by organization and item id so moves preserve history. Failed history writes are logged without failing the read. Never-read items use `recorded_at` as the baseline; index generation and queries do not count as reading an item. At eight weeks unread, items leave only the index, not search or git. Rebuilds use original heads, cap the index at 200 physical lines including the `As of ` first line, and flag invalid and overdue items. TRIG-12 extends this builder with sections, ordering and freshness defaults; no writer methods or automatic defaults are part of LEAD-2. @@ -468,7 +468,7 @@ T3 runs on the owner's subscriptions, so budget is counted in tokens, not money. - Deferred P2/P3 events wait until the next owner-local reset unless limits change. - A wake-up spanning midnight bills the owner-local day on which the charge is made. - `admitWorker(orgId)` refuses at 0 worker units left. The lead gets an error and tells the owner. LEAD-9 owns charging `worker_units` from worker runs; it must not also count the lead turn's Claude `otherUsage`, which is already lead units. -- Providers without a per-turn `turnTokenUsage` ledger (currently Pi, Cursor and Grok) fall back to `usedTokens` at weight 1, which is their context size, so cache-heavy turns are over-billed until their adapters emit a turn ledger. +- Kin targets Claude Code and Codex. Other providers need only a safe fallback: without a per-turn `turnTokenUsage` ledger, use `usedTokens` at weight 1 conservatively; no ledger integration for those providers is required. - Deferred events stay queued with `not_before` and show in the morning brief ("3 checks postponed for budget"). `UsageLimitSources` (`apps/server/src/usage/UsageLimitSources.ts`) only reads cliproxy accounts, so plan headroom is not generally known. Kin uses its own counters. See open questions. @@ -642,3 +642,66 @@ LEAD-14. **End-to-end shadow week dry run.** Script that replays a day of record - Which privileged kinds besides merge may an L3 lead request (deploy, release)? This spec allows only low-risk merges. - Task ids for dependencies in the chat, approvals and triggers specs are named by subject here. Swap in the real ids once those specs are written. - The lead's thread shows in T3's thread list under the "Kin · " project. Hide it, or leave it as the lead's readable log? + +### LEAD-3 integration boundaries + +`MemoryWriterAuthority.resolve` is runner-only: resolve organization, wake-up, active lead, level, timestamps, timezone and event provenance from same-organization rows inside the runner's transaction. Never expose this context as tool input. Owner-confirmed sources must be verified there; absent a source, provenance is the lead's agent id. `MemoryWriterHooks.notify` consumes checkout, conflict and secret-block notices without chat transport coupling. + +`KinSecretScan` is deliberately supplied separately. `noFindingsStubLayer` returns no findings until TRIG-16 replaces it; it is not a production secret detector. Any finding or scanner failure blocks the whole commit. TRIG-16 must preserve this fail-closed interface: `scan({ orgId, files: ReadonlyArray<{ path: string; content: Uint8Array }> })` returns `{ path, ruleId, line }[]`, with typed `KinSecretScanError`. + +`commitWakeup` builds TRIG-12's sectioned `INDEX.md` from the hash-verified recorded tree plus staged files, read history, the server clock and the owner's time zone, in the same wake-up commit. The read-side rebuild is display-only; no read-checkout text is trusted. K011 adds the SQLite branch anchors required by the writer. The runner serializes prepare/stage/commit and dreaming per organization. Identical commit retries find the existing wake-up commit; failed pushes retain local commits, and failures discard uncommitted files. Journal additions use `appendJournal`, never replacement through `stage`. + +### Memory writer operations + +**Run leads as a separate OS user before production.** Same-user residual attacks include direct SQLite edits (including `kin_memory_heads`), deleting the writer and losing never-pushed commits, object/pack/ref races between check and use (for example a crafted pack fooling git ancestry commands), `~/.ssh/config` and credential-input tampering, read-clone races, reading other orgs' directories, and attacking the server process. Hash-verifying reads and SQLite anchors are defense in depth, not a same-user sandbox. This writer now walks verified commit parents itself rather than trusting `merge-base`; races at other git check/use boundaries still require OS-user separation. + +`prepareWakeup` checks git integrity once. Every application object read independently checks its hash. Three-way merging walks verified commit parents to find a common ancestor, then reads base, local and remote bytes with hash verification. Paths changed on both sides conflict except append-only journals and the regenerated INDEX.md. Journal append conflicts retain remote bytes followed by local entries in order. Original modes, symlinks and gitlinks are preserved; writer changes create only regular files. Other conflicts preserve the unpushed tip under `kin/conflict//-` (or a server-generated recovery id when no wake-up id is supplied), then compare-and-swap the writer and anchor to the remote head and notify via `MemoryRecoveryHooks`. The next wake-up proceeds; the owner can inspect and cherry-pick the conflict branch. Existing conflict branches are never intentionally overwritten. + +Operators or owners can call server-only `Memory.recover(orgId)` after losing a writer or to reconcile its anchors. It fetches remote main and shadow heads, preserves surviving unpushed anchored work on a conflict branch, recreates a missing writer, records remote anchors and notifies. This method must never be registered as an agent tool. For writers predating K011 anchors, stop the runner and delete only that organization's writer directory, then bootstrap again. Do not delete anchored writers to recover unpushed work; call `recover` first. Recovery cannot restore commits lost with a deleted writer before they reached any remote. + +Each wake-up must start a fresh provider process with cwd set to the newly replaced read clone; LEAD-5 creates a fresh provider session for every wake-up. + +**Running on the Kin host.** Git pushes use the server's credentials. Since system/global/repository git config is excluded, supply a server-owned `MemoryConfig.credentialHelper` command for HTTPS or `MemoryConfig.sshCommand` for SSH. These are passed through command config/environment only; repository credential helpers cannot override them. Configure the Kin host's dedicated server user and credentials before enabling writes; never put credentials in memory, repository config, or lead instructions. See also “Running on the Kin host” in `triggers.md`. + +The existing CLI has no Kin repository configuration or service layer, so a recovery subcommand would require more than a 1–2 line upstream hook. The operator entry point remains the typed `Memory.recover(orgId)`, never RPC or MCP. Stop the runner first. From `apps/server`, run `node --input-type=module < /absolute/path/to/operator-recover.mjs` with this operator-owned script, substituting explicit database, repository and directory values (no default live-state path): + +```js +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { KinOrgId } from "@t3tools/contracts"; +import * as Memory from "./src/kin/memory/Memory.ts"; +import { MemoryConfig } from "./src/kin/memory/MemoryConfig.ts"; +import * as Sqlite from "./src/persistence/Sqlite.ts"; +import * as KinPersistence from "./src/kin/persistence/KinPersistence.ts"; +import * as Vcs from "./src/vcs/VcsProcess.ts"; +const orgId = KinOrgId.make("fixture-org"); +const database = KinPersistence.layer.pipe( + Layer.provideMerge(Sqlite.layerFromPath("/absolute/operator-selected/statev2.sqlite")), +); +const config = Layer.succeed(MemoryConfig, { + baseDirectory: "/absolute/operator-selected/kin", + repositories: [{ orgId, path: "fixture-org", repoUrl: "/absolute/fixture.git" }], +}); +await Effect.runPromise( + Effect.scoped( + Effect.flatMap(Memory.Memory, (memory) => memory.recover(orgId)).pipe( + Effect.provide( + Memory.layer.pipe( + Layer.provide(config), + Layer.provide( + Layer.succeed(Memory.MemoryRecoveryHooks, { + notify: (notice) => Effect.logWarning("Memory recovery", notice), + }), + ), + ), + ), + Effect.provide(database), + Effect.provide(Vcs.layer), + Effect.provide(NodeServices.layer), + ), + ), +); +``` + +Recovery takes target names and SHAs only from the configured URL's `ls-remote` advertisement, fetches exact SHAs and hash-verifies their contents. Local shadow-ref enumeration is deletion-only; local `rev-parse` reads are used solely as compare-and-swap expectations, never as anchors or remote targets. Bootstrap also fetches an advertised exact SHA. There is no writer `symbolic-ref` lookup. Recovery removes local shadow branches not in its target set. A replaced non-descendant tip is reset to its SQLite anchor with a notice: any unpushed work at that tampered tip is already lost. diff --git a/docs/kin/specs/triggers.md b/docs/kin/specs/triggers.md index c60e8a88e4621..1890e5a05a9df 100644 --- a/docs/kin/specs/triggers.md +++ b/docs/kin/specs/triggers.md @@ -319,7 +319,7 @@ GitHub then blocks a bad auto-merge even if Kin's classifier is wrong. This forc ### Secret scan before memory commits (gap 5) -`secretScan.ts` exposes `scan(files: ReadonlyArray<{ path; content }>) → Finding[]`. Each finding has `rule`, `path`, `line` and a `fingerprint` (sha256 of the match). +`apps/server/src/kin/memory/secretScan.ts` defines `KinSecretScan.scan({ orgId, files: ReadonlyArray<{ path: string; content: Uint8Array }> })`, returning an Effect of `{ path, ruleId, line }[]` with typed `KinSecretScanError`. LEAD-3 supplies an explicitly named no-findings stub; TRIG-16 replaces that layer with the real scanner. Fingerprints stay internal to the scanner allowlist, never matched secrets in findings. **Rules:** @@ -340,7 +340,7 @@ GitHub then blocks a bad auto-merge even if Kin's classifier is wrong. This forc - The dream runs it before pushing. - The memory repo CI runs gitleaks as a second, independent check. -**On a finding**, the writer replaces the matched span with `[redacted:]`, commits the redacted text, and posts to the owner: "I removed what looked like a GitHub token from journal 8 Nov before saving. It is still in chat #tech, message 8812; rotate it." Redacting beats blocking, because blocking would lose the memory write. +**On a finding**, the writer blocks the entire commit, discards staged text and emits a typed owner-notification hook. The runner journals the refusal without secret content and tells the owner to review and rotate the affected credential. Automatic redaction is not part of the writer contract. **False positives:** the owner adds the fingerprint to `kin_secret_allow` through a signed action. A model can never add to it. @@ -500,7 +500,7 @@ Kin runs on one always-on machine, the **Kin host**. The laptop and phone are cl | TRIG-13 | Dream runner: material, branch, dream session, checks | `DreamRunner.ts` | TRIG-12, TRIG-16, LEAD-5, LEAD-6 | fixture dream produces a branch and `dream.json`; `rules.md` edit fails it | backend | | TRIG-14 | Dream split, auto-merge under delegation, review card, rejections; full weekly variant | `classifyDream.ts`, `DreamRunner.ts` | TRIG-13, APPR-4; auto-merge waits for APPR-12 (phase 3), and until then the owner approves both PRs | the two-PR test passes; a rejected item is not proposed again | backend | | TRIG-15 | Memory repo guard rails | `kin-memory`: `CODEOWNERS`, `memory-check.yml`, ruleset | APPR-11 | a PR touching `decisions/` can't merge without owner review; a planted token fails the check | backend | -| TRIG-16 | Secret scanner, writer hook, allowlist | `secretScan.ts` | LEAD-3 | rule fixtures (one hit and one near-miss per rule) pass; redaction and owner message tested | backend | +| TRIG-16 | Secret scanner, writer hook, allowlist | `secretScan.ts` | LEAD-3 | rule fixtures (one hit and one near-miss per rule) pass; commit refusal and typed owner notification tested | backend | | TRIG-17 | Owner-signed purge | `purge.ts` | TRIG-16, APPR-9, APPR-10 | purge test passes on a fixture repo; `git-filter-repo` provenance noted in the PR | backend | | TRIG-18 | Server contract tests: triggers, relay, `delegate_task`, managed policy | `apps/server/src/kin/contract/*` | TRIG-4 | `cd apps/server && vp test run kin-contract` passes | backend | | TRIG-19 | `McpAppWebView` contract test | `apps/mobile/src/features/chats/contract/*` | — | `cd apps/mobile && vp test run kin-contract` passes | UI |