From f056583247ed5f7cf1d92c1e84e14d9a01be5a4b Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Sun, 8 Nov 2026 01:21:58 +0530 Subject: [PATCH 01/11] feat(kin): add memory git writer (LEAD-3) --- apps/server/src/kin/memory/Memory.test.ts | 13 +- apps/server/src/kin/memory/Memory.ts | 48 +- .../src/kin/memory/MemoryWriter.test.ts | 246 ++++++++ apps/server/src/kin/memory/MemoryWriter.ts | 589 ++++++++++++++++++ apps/server/src/kin/memory/journal.ts | 30 + apps/server/src/kin/memory/secretScan.ts | 34 + docs/kin/specs/lead.md | 14 +- docs/kin/specs/triggers.md | 6 +- 8 files changed, 944 insertions(+), 36 deletions(-) create mode 100644 apps/server/src/kin/memory/MemoryWriter.test.ts create mode 100644 apps/server/src/kin/memory/MemoryWriter.ts create mode 100644 apps/server/src/kin/memory/journal.ts create mode 100644 apps/server/src/kin/memory/secretScan.ts diff --git a/apps/server/src/kin/memory/Memory.test.ts b/apps/server/src/kin/memory/Memory.test.ts index cfb90d839e580..3500e503be857 100644 --- a/apps/server/src/kin/memory/Memory.test.ts +++ b/apps/server/src/kin/memory/Memory.test.ts @@ -114,6 +114,7 @@ const fakeRepository = () => { layer, commit: () => { revision++; + return `fixture-head-${revision}`; }, counts: () => ({ treeReads, blobReads }), corruptBatch: () => { @@ -345,7 +346,7 @@ it.effect( expect(moved).toBeDefined(); repository.files.delete("deferrals/def-001.md"); repository.files.set("deferrals/moved.md", moved ?? ""); - repository.commit(); + yield* fs.writeFileString(`${root}/a/writer-head`, repository.commit()); yield* memory.prepareWakeup(orgA); yield* TestClock.adjust("1 week"); yield* memory.rebuildIndex(orgA); @@ -384,7 +385,7 @@ it.effect( }), ); } - repository.commit(); + yield* fs.writeFileString(`${root}/a/writer-head`, repository.commit()); yield* memory.prepareWakeup(orgA); yield* memory.rebuildIndex(orgA); expect((yield* memory.readIndex(orgA)).trimEnd().split("\n")).toHaveLength(200); @@ -402,7 +403,7 @@ it.effect( "# index\nAs of 2027-02-28T00:00:00Z\n", ]) { repository.files.set("INDEX.md", index); - repository.commit(); + yield* fs.writeFileString(`${root}/a/writer-head`, repository.commit()); const result = yield* memory.prepareWakeup(orgA); expect(DateTime.toEpochMillis(result.indexAsOf)).toBe(yield* Clock.currentTimeMillis); } @@ -410,7 +411,7 @@ it.effect( "deferrals/duplicate.md", fixtureItem({ recorded_at: "2027-02-27T00:00:00Z" }), ); - repository.commit(); + yield* fs.writeFileString(`${root}/a/writer-head`, repository.commit()); yield* memory.prepareWakeup(orgA); const duplicates = yield* memory.query(orgA, {}); expect( @@ -427,7 +428,7 @@ it.effect( for (const id of ["fallback-1", "fallback-2"]) { repository.files.set(`deferrals/${id}.md`, fixtureItem({ id })); } - repository.commit(); + yield* fs.writeFileString(`${root}/a/writer-head`, repository.commit()); repository.corruptBatch(); yield* memory.prepareWakeup(orgA); const afterFallback = yield* memory.query(orgA, {}); @@ -663,6 +664,7 @@ it.live("uses filesystem identity for a case-mismatched base without recloning", }).pipe(Effect.provide(memoryLayer(base, source))); yield* fs.writeFileString(`${writer}/keep`, "unpushed work"); const head = yield* commit(writer); + yield* fs.writeFileString(`${writer}/../writer-head`, head); const before = yield* fs.stat(writer); yield* Effect.gen(function* () { const memory = yield* Memory.Memory; @@ -699,6 +701,7 @@ it.live.each(["diverged", "conflict", "network down", "dirty writer", "rebase in yield* git(writer, ["config", "user.email", "fixture@example.invalid"]); yield* fs.writeFileString(`${writer}/rules.md`, "Local invented rules\n"); const local = yield* commit(writer); + yield* fs.writeFileString(`${writer}/../writer-head`, local); yield* fs.writeFileString( `${source}/${scenario === "diverged" || scenario === "network down" || scenario === "dirty writer" ? "remote-only" : "rules.md"}`, "Remote invented edit\n", diff --git a/apps/server/src/kin/memory/Memory.ts b/apps/server/src/kin/memory/Memory.ts index 2a81aa46fe482..304ea8739b40b 100644 --- a/apps/server/src/kin/memory/Memory.ts +++ b/apps/server/src/kin/memory/Memory.ts @@ -2,7 +2,6 @@ import { type KinMemoryItemId, type KinOrgId, MemoryItemHead, - VcsProcessExitError, type MemoryItemKind, type MemoryItemFrontMatter, } from "@t3tools/contracts"; @@ -131,7 +130,6 @@ export class Memory extends Context.Service< >()("t3/kin/memory/Memory") {} const decodeHead = Schema.decodeEffect(MemoryItemHead); -const isVcsProcessExitError = Schema.is(VcsProcessExitError); // Mirrors STATUS_UPSTREAM_REFRESH_ENV in vcs/GitVcsDriverCore.ts. const memoryGitEnv = Object.freeze({ GCM_INTERACTIVE: "never", @@ -315,6 +313,17 @@ const make = Effect.gen(function* () { reason: "Writer is not the repository root", }); yield* git(orgId, paths.writer, ["rev-parse", "--verify", "HEAD^{commit}"]); + const head = (yield* git(orgId, paths.writer, ["rev-parse", "HEAD"])).trim(); + const anchor = path.join(paths.writer, "..", "writer-head"); + const expected = (yield* fs.exists(anchor)) + ? (yield* fs.readFileString(anchor)).trim() + : (yield* git(orgId, paths.writer, ["rev-parse", "refs/remotes/origin/HEAD"])).trim(); + if (head !== expected) + return yield* new MemoryError({ + orgId, + operation: "head", + reason: "Unrecorded writer HEAD", + }); const origin = (yield* git(orgId, paths.writer, ["remote", "get-url", "origin"])).trim(); if (origin !== paths.repo.repoUrl) return yield* new MemoryError({ @@ -339,7 +348,6 @@ const make = Effect.gen(function* () { const ensureCheckouts = Effect.fn("Memory.ensureCheckouts")(function* (orgId: KinOrgId) { const paths = yield* pathsFor(orgId); yield* io(orgId, "mkdir", fs.makeDirectory(paths.root, { recursive: true })); - const gitDirectory = path.join(paths.writer, ".git"); if (yield* io(orgId, "exists", fs.exists(paths.writer))) { yield* validateWriter(orgId, paths); } else { @@ -368,30 +376,12 @@ const make = Effect.gen(function* () { const readExists = yield* io(orgId, "exists", fs.exists(paths.read)); const validRead = readExists && - (yield* git(orgId, paths.read, [ - "rev-parse", - "--path-format=absolute", - "--git-common-dir", - ]).pipe( - Effect.flatMap((common) => sameDirectory(orgId, common.trim(), gitDirectory)), - Effect.flatMap((same) => - same - ? git(orgId, paths.read, ["rev-parse", "--show-toplevel"]).pipe( - Effect.flatMap((root) => sameDirectory(orgId, root.trim(), paths.read)), - ) - : Effect.succeed(false), - ), - Effect.catchTags({ - MemoryError: (error) => - isVcsProcessExitError(error.cause) && error.cause.failureKind === "not-found" - ? Effect.succeed(false) - : Effect.fail(error), - }), - )); + (yield* io(orgId, "exists", fs.exists(path.join(paths.read, ".git")))) && + (yield* io(orgId, "readGit", fs.lstat(path.join(paths.read, ".git")))).type === "Directory"; if (!validRead) { - yield* io(orgId, "recoverWorktree", fs.remove(paths.read, { recursive: true, force: true })); - yield* git(orgId, paths.writer, ["worktree", "prune"]); - yield* git(orgId, paths.writer, ["worktree", "add", "--detach", paths.read, "HEAD"]); + yield* io(orgId, "recoverRead", fs.remove(paths.read, { recursive: true, force: true })); + yield* git(orgId, paths.root, ["clone", "--no-hardlinks", "--", paths.writer, paths.read]); + yield* git(orgId, paths.read, ["remote", "remove", "origin"]); } return { writer: paths.writer, read: paths.read }; }); @@ -769,6 +759,12 @@ const make = Effect.gen(function* () { ); } const head = (yield* git(orgId, paths.writer, ["rev-parse", "HEAD"])).trim(); + yield* io( + orgId, + "recordHead", + fs.writeFileString(path.join(paths.writer, "..", "writer-head"), head), + ); + yield* git(orgId, paths.read, ["fetch", "--", paths.writer, head]); yield* git(orgId, paths.read, ["checkout", "--force", "--detach", head]); yield* git(orgId, paths.read, ["reset", "--hard", head]); yield* git(orgId, paths.read, ["clean", "-fdx"]); diff --git a/apps/server/src/kin/memory/MemoryWriter.test.ts b/apps/server/src/kin/memory/MemoryWriter.test.ts new file mode 100644 index 0000000000000..79401b6244df4 --- /dev/null +++ b/apps/server/src/kin/memory/MemoryWriter.test.ts @@ -0,0 +1,246 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { expect, it } from "@effect/vitest"; +import { KinAgentId, KinOrgId } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Sqlite from "../../persistence/Sqlite.ts"; +import * as Persistence from "../persistence/KinPersistence.ts"; +import * as Vcs from "../../vcs/VcsProcess.ts"; +import * as Memory from "./Memory.ts"; +import { MemoryConfig } from "./MemoryConfig.ts"; +import * as Writer from "./MemoryWriter.ts"; +import * as Scan from "./secretScan.ts"; +import { fixtureItem } from "./__fixtures__/items.ts"; + +const orgId = KinOrgId.make("fixture-org"); +const agentId = KinAgentId.make("fixture-lead"); +const gitLayer = Vcs.layer.pipe(Layer.provide(NodeServices.layer)); +it.live.each(["completed", "failed", "blocked", "forged", "conflict"] as const)( + "writer real git: %s", + (scenario) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const vcs = yield* Vcs.VcsProcess; + const tmp = yield* fs.makeTempDirectoryScoped(); + const root = yield* fs.realPath(tmp); + const source = `${root}/seed`; + const remote = `${root}/remote.git`; + yield* fs.makeDirectory(source); + const git = Effect.fn("WriterTest.git")((cwd: string, args: ReadonlyArray) => + vcs + .run({ operation: "test.writer", command: "git", cwd, args }) + .pipe(Effect.map((r) => r.stdout.trim())), + ); + const commit = (cwd: string) => + git(cwd, [ + "-c", + "user.name=Fixture Bot", + "-c", + "user.email=fixture@example.invalid", + "commit", + "-m", + "fixture", + ]); + yield* git(source, ["init", "-b", "main"]); + yield* fs.writeFileString(`${source}/rules.md`, "Invented rules\n"); + yield* fs.writeFileString(`${source}/INDEX.md`, "As of 2026-11-08T00:00:00.000Z\n"); + yield* git(source, ["add", "--", "."]); + yield* commit(source); + yield* git(root, ["clone", "--bare", "--", source, remote]); + const notices: Writer.WriterNotice[] = []; + const config = Layer.succeed(MemoryConfig, { + baseDirectory: root, + repositories: [{ orgId, repoUrl: remote, path: "state" }], + }); + const memoryLayer = Memory.layer.pipe(Layer.provide(config)); + const hooks = Layer.succeed(Writer.MemoryWriterHooks, { + notify: Effect.fn("Test.notice")((notice) => + Effect.sync(() => { + notices.push(notice); + }), + ), + buildIndex: Effect.fn("Test.index")(() => Effect.succeed(null)), + }); + const auth = Layer.succeed(Writer.MemoryWriterAuthority, { + resolve: Effect.fn("Test.authority")( + ( + org: KinOrgId, + wakeupId: string, + ): Effect.Effect => + org !== orgId + ? Effect.fail(new Writer.MemoryWriteRejected({ reason: "not_found" })) + : Effect.succeed({ + orgId, + wakeupId, + agentId, + level: scenario === "conflict" ? "L1" : "L0", + timeZone: "Asia/Kolkata", + startedAt: "2026-11-08T00:00:00.000Z", + eventIds: [], + sources: new Map([ + [ + "owner", + { saidBy: "fixture-owner", trust: "owner", source: "fixture-owner-message" }, + ], + [ + "member", + { + saidBy: "fixture-member", + trust: "member", + source: "fixture-member-message", + }, + ], + ]), + }), + ), + }); + const scan = + scenario === "blocked" + ? Scan.testLayer([{ path: "claims/new.md", ruleId: "fixture", line: 1 }]) + : Scan.stubLayer; + const layer = Writer.layer.pipe( + Layer.provideMerge(memoryLayer), + Layer.provide(hooks), + Layer.provide(auth), + Layer.provide(scan), + ); + yield* Effect.gen(function* () { + const memory = yield* Memory.Memory; + const writer = yield* Writer.MemoryWriter; + const paths = yield* memory.ensureCheckouts(orgId); + yield* memory.prepareWakeup(orgId); + yield* git(paths.writer, ["config", "user.name", "Fixture Bot"]); + yield* git(paths.writer, ["config", "user.email", "fixture@example.invalid"]); + const original = yield* git(paths.writer, ["rev-parse", "HEAD"]); + for (const path of [ + "rules.md", + "INDEX.md", + "README.md", + "../claims/a.md", + "/claims/a.md", + "claims/.git/a.md", + "claims/../../a.md", + ]) { + expect( + (yield* writer + .stage(orgId, "wake1", { path, content: fixtureItem() }) + .pipe(Effect.result))._tag, + ).toBe("Failure"); + } + yield* fs.symlink(source, `${paths.writer}/claims`); + expect( + (yield* writer + .stage(orgId, "wake1", { + path: "claims/new.md", + content: fixtureItem({ kind: "claim" }), + }) + .pipe(Effect.result))._tag, + ).toBe("Failure"); + yield* fs.remove(`${paths.writer}/claims`); + if (scenario === "forged") { + yield* fs.writeFileString(`${paths.read}/rules.md`, "Forged rules\n"); + yield* git(paths.read, ["add", "--", "rules.md"]); + yield* commit(paths.read); + const forged = yield* git(paths.read, ["rev-parse", "HEAD"]); + yield* git(paths.read, ["update-ref", "refs/heads/main", forged]); + expect(yield* git(paths.writer, ["rev-parse", "HEAD"])).toBe(original); + expect((yield* memory.prepareWakeup(orgId)).head).toBe(original); + yield* git(paths.writer, ["fetch", "--", paths.read, forged]); + yield* git(paths.writer, ["update-ref", "refs/heads/main", forged]); + expect((yield* memory.prepareWakeup(orgId).pipe(Effect.result))._tag).toBe("Failure"); + expect( + (yield* writer.commitWakeup(orgId, "wake1", "completed").pipe(Effect.result))._tag, + ).toBe("Failure"); + expect(notices.at(-1)?.error._tag).toBe("MemoryCheckoutInvalid"); + return; + } + yield* writer.stage(orgId, "wake1", { + path: "claims/new.md", + content: fixtureItem({ + kind: "claim", + trust: "owner", + said_by: "forged-owner", + source: "forged", + }), + }); + expect(yield* fs.readFileString(`${paths.writer}/claims/new.md`)).toContain( + "said_by: fixture-lead", + ); + expect(yield* fs.readFileString(`${paths.writer}/claims/new.md`)).toContain("trust: agent"); + if (scenario === "completed") { + const contradiction = yield* writer + .stage(orgId, "wake1", { + path: "claims/contradiction.md", + content: fixtureItem({ + id: "other", + kind: "claim", + title: "Different invented claim", + }), + }) + .pipe(Effect.result); + expect(contradiction._tag).toBe("Failure"); + if (contradiction._tag === "Failure") + expect(contradiction.failure._tag).toBe("MemoryConflict"); + for (const sourceEventId of ["owner", "member"]) { + yield* writer.stage(orgId, "wake1", { + path: `claims/${sourceEventId}.md`, + sourceEventId, + content: fixtureItem({ id: sourceEventId, kind: "claim", links: [] }), + }); + expect( + yield* fs.readFileString(`${paths.writer}/claims/${sourceEventId}.md`), + ).toContain(`trust: ${sourceEventId}`); + } + yield* fs.writeFileString(`${paths.writer}/rules.md`, "Ambient forged edit"); + yield* git(paths.writer, ["add", "--", "rules.md"]); + } + if (scenario === "conflict") { + yield* fs.makeDirectory(`${source}/claims`); + yield* fs.writeFileString(`${source}/claims/new.md`, "Owner conflicting fixture\n"); + yield* git(source, ["add", "--", "claims/new.md"]); + yield* commit(source); + yield* git(source, ["push", "--", remote, "main"]); + } + const result = yield* writer + .commitWakeup(orgId, "wake1", scenario === "failed" ? "failed" : "completed") + .pipe(Effect.result); + if (scenario === "blocked" || scenario === "conflict") { + expect(result._tag).toBe("Failure"); + if (result._tag === "Failure") + expect(result.failure._tag).toBe( + scenario === "blocked" ? "MemorySecretBlocked" : "MemorySyncConflict", + ); + expect(yield* git(paths.writer, ["status", "--porcelain"])).toBe(""); + expect(notices).toHaveLength(1); + expect(yield* fs.exists(`${paths.writer}/.git/rebase-merge`)).toBe(false); + } else { + expect(result._tag).toBe("Success"); + expect(yield* git(paths.writer, ["rev-list", "--count", `${original}..HEAD`])).toBe("1"); + expect(yield* fs.readFileString(`${paths.writer}/rules.md`)).toBe("Invented rules\n"); + const again = yield* writer.commitWakeup( + orgId, + "wake1", + scenario === "failed" ? "failed" : "completed", + ); + if (result._tag === "Success") expect(again.commit).toBe(result.success.commit); + expect( + yield* git(paths.writer, ["show", "--format=%an <%ae>", "--no-patch", "HEAD"]), + ).toBe("Fixture Bot "); + expect(yield* fs.exists(`${paths.writer}/claims/new.md`)).toBe(scenario === "completed"); + expect(yield* fs.readFileString(`${paths.writer}/journal/2026/11-08.md`)).toContain( + `Outcome: ${scenario}`, + ); + } + }).pipe(Effect.provide(layer)); + }).pipe( + Effect.provide( + Layer.mergeAll( + gitLayer, + NodeServices.layer, + Persistence.layer.pipe(Layer.provideMerge(Sqlite.layerMemory)), + ), + ), + Effect.scoped, + ), +); diff --git a/apps/server/src/kin/memory/MemoryWriter.ts b/apps/server/src/kin/memory/MemoryWriter.ts new file mode 100644 index 0000000000000..f7cf537660003 --- /dev/null +++ b/apps/server/src/kin/memory/MemoryWriter.ts @@ -0,0 +1,589 @@ +import * as DateTime from "effect/DateTime"; +import type { KinAgentId, KinOrgId, MemoryTrust } from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; +import * as Schedule from "effect/Schedule"; +import * as Semaphore from "effect/Semaphore"; +import { stringify } from "yaml"; +import { VcsProcess } from "../../vcs/VcsProcess.ts"; +import { + Memory, + MemoryCheckoutInvalid, + MemoryError, + MemorySyncConflict, + MemoryConfigError, + MemoryQueryDecodeError, + MemoryNotFound, + MemoryDuplicateId, +} from "./Memory.ts"; +import { parseFrontMatter, MemoryItemError } from "./frontMatter.ts"; +import { journalEntry, type JournalContext } from "./journal.ts"; +import { KinSecretScan, KinSecretScanError, type SecretFinding } from "./secretScan.ts"; + +export interface WakeupContext extends JournalContext { + readonly orgId: KinOrgId; + readonly agentId: KinAgentId; + readonly level: "L0" | "L1" | "L2" | "L3"; + readonly sources: ReadonlyMap< + string, + { readonly saidBy: string; readonly trust: MemoryTrust; readonly source: string } + >; +} +// The runner resolves this context from same-org records, never tool input. +export class MemoryWriterAuthority extends Context.Service< + MemoryWriterAuthority, + { + readonly resolve: ( + orgId: KinOrgId, + wakeupId: string, + ) => Effect.Effect; + } +>()("t3/kin/memory/MemoryWriter/MemoryWriterAuthority") {} +export class MemoryWriteRejected extends Schema.TaggedError()( + "MemoryWriteRejected", + { + reason: Schema.String, + }, +) {} +export class MemoryConflict extends Schema.TaggedError()("MemoryConflict", { + existingId: Schema.String, +}) {} +export class MemorySecretBlocked extends Schema.TaggedError()( + "MemorySecretBlocked", + { + orgId: Schema.String, + findings: Schema.Array( + Schema.Struct({ path: Schema.String, ruleId: Schema.String, line: Schema.Number }), + ), + }, +) {} +export type WriterNotice = { + readonly orgId: KinOrgId; + readonly wakeupId: string; + readonly error: MemoryCheckoutInvalid | MemorySyncConflict | MemorySecretBlocked; +}; +export class MemoryWriterHooks extends Context.Service< + MemoryWriterHooks, + { + readonly notify: (notice: WriterNotice) => Effect.Effect; + readonly buildIndex: (input: { + orgId: KinOrgId; + writer: string; + }) => Effect.Effect; + } +>()("t3/kin/memory/MemoryWriter/MemoryWriterHooks") {} +export interface MemoryChange { + readonly path: string; + readonly content: string; + readonly sourceEventId?: string; + readonly supersedes?: string; +} +type Failure = + | MemoryConfigError + | MemoryQueryDecodeError + | MemoryNotFound + | MemoryDuplicateId + | MemoryError + | MemoryCheckoutInvalid + | MemorySyncConflict + | MemoryWriteRejected + | MemoryConflict + | MemoryItemError + | KinSecretScanError + | MemorySecretBlocked; +export class MemoryWriter extends Context.Service< + MemoryWriter, + { + readonly stage: ( + orgId: KinOrgId, + wakeupId: string, + change: MemoryChange, + ) => Effect.Effect; + readonly appendJournal: ( + orgId: KinOrgId, + wakeupId: string, + lines: ReadonlyArray, + ) => Effect.Effect; + readonly discard: (orgId: KinOrgId, wakeupId: string) => Effect.Effect; + readonly commitWakeup: ( + orgId: KinOrgId, + wakeupId: string, + outcome: "completed" | "failed", + ) => Effect.Effect<{ commit: string }, Failure>; + } +>()("t3/kin/memory/MemoryWriter") {} +const folders = ["objectives", "commitments", "deferrals", "decisions", "people", "claims"]; +const render = (fields: object, body: string) => `---\n${stringify(fields)}---\n${body}`; +const make = Effect.gen(function* () { + const memory = yield* Memory; + const authority = yield* MemoryWriterAuthority; + const hooks = yield* MemoryWriterHooks; + const scanner = yield* KinSecretScan; + const fs = yield* FileSystem.FileSystem; + const vcs = yield* VcsProcess; + const locks = new Map(); + const locked = (orgId: KinOrgId, effect: Effect.Effect) => { + let lock = locks.get(orgId); + if (!lock) { + lock = Semaphore.makeUnsafe(1); + locks.set(orgId, lock); + } + return lock.withPermits(1)(effect); + }; + const sessions = new Map< + KinOrgId, + { wakeupId: string; files: Map; lines: string[] } + >(); + const io = (orgId: KinOrgId, effect: Effect.Effect) => + effect.pipe(Effect.mapError((cause) => new MemoryError({ orgId, operation: "writer", cause }))); + const git = Effect.fn("MemoryWriter.git")(function* ( + orgId: KinOrgId, + cwd: string, + args: ReadonlyArray, + identity: NodeJS.ProcessEnv = {}, + stdin?: string, + ) { + const result = yield* io( + orgId, + vcs.run({ + ...(stdin === undefined ? {} : { stdin }), + operation: "MemoryWriter.git", + command: "git", + cwd, + args: ["-c", "core.hooksPath=/dev/null", ...args], + env: { GIT_TERMINAL_PROMPT: "0", GCM_INTERACTIVE: "never", GIT_ASKPASS: "", ...identity }, + }), + ); + return result.stdout.trim(); + }); + const session = Effect.fn("MemoryWriter.session")(function* (orgId: KinOrgId, wakeupId: string) { + const ctx = yield* authority.resolve(orgId, wakeupId); + if ( + ctx.orgId !== orgId || + ctx.wakeupId !== wakeupId || + !/^[a-zA-Z0-9_-]+$/.test(ctx.agentId) || + !/^[a-zA-Z0-9_-]+$/.test(wakeupId) + ) + return yield* new MemoryWriteRejected({ reason: "Invalid server context" }); + const previous = sessions.get(orgId); + if (previous && previous.wakeupId !== wakeupId) + return yield* new MemoryWriteRejected({ reason: "Another wake-up owns the writer" }); + const state = previous ?? { wakeupId, files: new Map(), lines: [] }; + sessions.set(orgId, state); + return { ctx, state }; + }); + const safePath = Effect.fn("MemoryWriter.safePath")(function* ( + orgId: KinOrgId, + writer: string, + relative: string, + generated = false, + ) { + const parts = relative.split("/"); + if ( + !relative.endsWith(".md") || + parts.some( + (p) => + !p || + p === "." || + p === ".." || + p.toLowerCase() === ".git" || + p.includes("\\") || + p.includes(":"), + ) || + /[\x00-\x1f]/.test(relative) || + (!(generated && relative === "INDEX.md") && ![...folders, "journal"].includes(parts[0] ?? "")) + ) + return yield* new MemoryWriteRejected({ reason: "Disallowed memory path" }); + let current = writer; + for (const part of parts) { + current += `/${part}`; + const result = yield* fs.lstat(current).pipe(Effect.result); + if (result._tag === "Failure") { + if (result.failure.reason._tag !== "NotFound") + return yield* new MemoryError({ orgId, operation: "path", cause: result.failure }); + } else if ( + result.success.type === "SymbolicLink" || + (current !== `${writer}/${relative}` && result.success.type !== "Directory") + ) { + return yield* new MemoryWriteRejected({ reason: "Symlink or invalid ancestor" }); + } + } + }); + const validate = Effect.fn("MemoryWriter.validate")(function* (orgId: KinOrgId) { + const paths = yield* memory.ensureCheckouts(orgId); + const head = yield* git(orgId, paths.writer, ["rev-parse", "HEAD"]); + const anchor = `${paths.writer}/../writer-head`; + const expected = (yield* io(orgId, fs.exists(anchor))) + ? (yield* io(orgId, fs.readFileString(anchor))).trim() + : yield* git(orgId, paths.writer, ["rev-parse", "refs/remotes/origin/HEAD"]); + if (expected !== head) + return yield* new MemoryCheckoutInvalid({ + orgId, + operation: "writer", + reason: "Unrecorded writer HEAD", + }); + return paths; + }); + const write = Effect.fn("MemoryWriter.write")(function* ( + orgId: KinOrgId, + writer: string, + relative: string, + text: string, + ) { + yield* safePath(orgId, writer, relative, true); + const target = `${writer}/${relative}`; + yield* io( + orgId, + fs.makeDirectory(target.slice(0, target.lastIndexOf("/")), { recursive: true }), + ); + yield* io(orgId, fs.writeFileString(target, text)); + }); + const clean = Effect.fn("MemoryWriter.clean")(function* (orgId: KinOrgId, writer: string) { + yield* git(orgId, writer, ["reset", "--hard", "HEAD"]); + yield* git(orgId, writer, ["clean", "-fd", "--", ...folders, "journal", "INDEX.md"]); + }); + const report = ( + orgId: KinOrgId, + wakeupId: string, + effect: Effect.Effect<{ commit: string }, Failure>, + ) => + effect.pipe( + Effect.tapError((error) => + error._tag === "MemoryCheckoutInvalid" || + error._tag === "MemorySyncConflict" || + error._tag === "MemorySecretBlocked" + ? hooks.notify({ orgId, wakeupId, error }) + : Effect.void, + ), + ); + return MemoryWriter.of({ + stage: Effect.fn("MemoryWriter.stage")((orgId, wakeupId, change) => + locked( + orgId, + Effect.gen(function* () { + const { ctx, state } = yield* session(orgId, wakeupId); + const { writer } = yield* validate(orgId); + yield* safePath(orgId, writer, change.path); + if (change.path.startsWith("journal/")) + return yield* new MemoryWriteRejected({ + reason: "Use appendJournal for journal additions", + }); + const parsed = yield* parseFrontMatter(orgId, change.path, change.content); + const source = + change.sourceEventId === undefined + ? { saidBy: ctx.agentId, trust: "agent", source: `wakeup:${wakeupId}` } + : ctx.sources.get(change.sourceEventId); + if (!source) + return yield* new MemoryWriteRejected({ reason: "Source is not in the current batch" }); + const fm = parsed.frontMatter; + const folder = fm.kind === "person" ? "people" : `${fm.kind}s`; + if (!change.path.startsWith(`${folder}/`)) + return yield* new MemoryWriteRejected({ reason: "Kind does not match path" }); + const items = [...(yield* memory.query(orgId, {}))]; + const bodies = new Map(); + for (const [file, text] of state.files) { + if (file.startsWith("journal/") || file === "INDEX.md") continue; + const staged = yield* parseFrontMatter(orgId, file, text); + const index = items.findIndex((item) => item.path === file); + const head = { ...staged.frontMatter, orgId, path: file, duplicate: false }; + if (index < 0) items.push(head); + else items[index] = head; + bodies.set(file, staged.body); + } + const superseded = + change.supersedes === undefined + ? undefined + : items.find((item) => item.id === change.supersedes); + if ( + change.supersedes !== undefined && + (!superseded || + superseded.kind !== fm.kind || + superseded.source !== source.source || + Date.parse(fm.recorded_at) <= Date.parse(superseded.recorded_at) || + superseded.id === fm.id) + ) + return yield* new MemoryWriteRejected({ + reason: "Supersession requires the same source and kind", + }); + for (const item of items) { + if ( + item.status === "open" && + item.kind === fm.kind && + (item.id === fm.id || item.links.some((link) => fm.links.includes(link))) && + item.id !== change.supersedes + ) { + const body = bodies.get(item.path) ?? (yield* memory.readItem(orgId, item.id)).body; + if (body !== parsed.body || item.title !== fm.title) + return yield* new MemoryConflict({ existingId: item.id }); + } + if ( + (item.path === change.path && item.id !== fm.id) || + (item.id === fm.id && item.path !== change.path) + ) + return yield* new MemoryWriteRejected({ reason: "Cannot replace a different item" }); + } + const review = + fm.review_by ?? + (fm.kind === "decision" + ? null + : fm.kind === "commitment" || fm.kind === "deferral" + ? fm.due + : DateTime.formatIso( + DateTime.makeUnsafe( + Date.parse(ctx.startedAt) + (fm.kind === "person" ? 90 : 30) * 86400000, + ), + )); + const fields = { + ...fm, + said_by: source.saidBy, + source: source.source, + trust: + fm.trust === "external" || + (fm.trust === "agent" && source.trust !== "external") || + (fm.trust === "member" && source.trust === "owner") + ? fm.trust + : source.trust, + review_by: review, + }; + const text = render(fields, parsed.body); + yield* parseFrontMatter(orgId, change.path, text); + if (superseded) { + const oldBody = + bodies.get(superseded.path) ?? (yield* memory.readItem(orgId, superseded.id)).body; + const { orgId: _org, path: oldPath, duplicate: _duplicate, ...oldFields } = superseded; + const oldText = render( + { ...oldFields, status: "superseded", superseded_by: fm.id }, + oldBody, + ); + yield* write(orgId, writer, oldPath, oldText); + state.files.set(oldPath, oldText); + } + yield* write(orgId, writer, change.path, text); + state.files.set(change.path, text); + }), + ), + ), + appendJournal: Effect.fn("MemoryWriter.appendJournal")((orgId, wakeupId, lines) => + locked( + orgId, + Effect.gen(function* () { + const { state } = yield* session(orgId, wakeupId); + state.lines.push(...lines); + }), + ), + ), + discard: Effect.fn("MemoryWriter.discard")((orgId, wakeupId) => + locked( + orgId, + Effect.gen(function* () { + yield* session(orgId, wakeupId); + const { writer } = yield* validate(orgId); + yield* clean(orgId, writer); + sessions.delete(orgId); + }), + ), + ), + commitWakeup: Effect.fn("MemoryWriter.commitWakeup")((orgId, wakeupId, outcome) => + locked( + orgId, + report( + orgId, + wakeupId, + Effect.gen(function* () { + const { ctx, state } = yield* session(orgId, wakeupId); + const { writer } = yield* validate(orgId); + return yield* Effect.gen(function* () { + const subject = `kin(${ctx.agentId}): wake-up ${wakeupId}`; + const candidates = yield* git(orgId, writer, [ + "log", + "--format=%H%x00%s", + "--fixed-strings", + `--grep=${subject}`, + ]); + const prior = candidates + .split("\n") + .map((line) => line.split("\0")) + .find(([, title]) => title === subject)?.[0]; + const fingerprint = yield* git( + orgId, + writer, + ["hash-object", "--stdin"], + {}, + stringify({ + outcome, + files: [...state.files].sort(([a], [b]) => a.localeCompare(b)), + lines: state.lines, + }), + ); + if (prior) { + const body = yield* git(orgId, writer, [ + "show", + "--format=%B", + "--no-patch", + prior, + ]); + if ( + !body.includes(`Kin-Outcome: ${outcome}`) || + (state.files.size + state.lines.length > 0 && + !body.includes(`Kin-Request: ${fingerprint}`)) + ) + return yield* new MemoryWriteRejected({ + reason: "not_allowed: different wake-up retry", + }); + yield* clean(orgId, writer); + return { commit: prior }; + } + const defaultRef = yield* git(orgId, writer, [ + "symbolic-ref", + "refs/remotes/origin/HEAD", + ]); + const branch = + ctx.level === "L0" + ? `kin/shadow/${ctx.agentId}` + : defaultRef.replace("refs/remotes/origin/", ""); + const current = yield* git(orgId, writer, ["branch", "--show-current"]); + if (current !== branch) { + if (yield* git(orgId, writer, ["status", "--porcelain"])) { + yield* clean(orgId, writer); + } + const exists = yield* git(orgId, writer, [ + "show-ref", + "--verify", + `refs/heads/${branch}`, + ]).pipe(Effect.result); + yield* git( + orgId, + writer, + exists._tag === "Success" ? ["switch", "--", branch] : ["switch", "-c", branch], + ); + yield* io( + orgId, + fs.writeFileString( + `${writer}/../writer-head`, + yield* git(orgId, writer, ["rev-parse", "HEAD"]), + ), + ); + for (const [file, text] of state.files) yield* write(orgId, writer, file, text); + } + if (outcome === "failed") { + yield* clean(orgId, writer); + state.files.clear(); + } + const entry = journalEntry(ctx, outcome, state.lines); + yield* safePath(orgId, writer, entry.path); + const old = (yield* io(orgId, fs.exists(`${writer}/${entry.path}`))) + ? yield* io(orgId, fs.readFileString(`${writer}/${entry.path}`)) + : ""; + state.files.set(entry.path, old + entry.text); + yield* write(orgId, writer, entry.path, old + entry.text); + if (outcome === "completed") { + const index = yield* hooks.buildIndex({ orgId, writer }); + if (index !== null) { + yield* write(orgId, writer, "INDEX.md", index); + state.files.set("INDEX.md", index); + } + } + const findings: ReadonlyArray = yield* scanner.scan({ + orgId, + files: [...state.files].map(([path, content]) => ({ + path, + content: new TextEncoder().encode(content), + })), + }); + if (findings.length) return yield* new MemorySecretBlocked({ orgId, findings }); + // Commit only the server's reconstructed files, never ambient index entries or edits. + yield* clean(orgId, writer); + for (const [file, text] of state.files) yield* write(orgId, writer, file, text); + yield* git(orgId, writer, ["add", "--", ...state.files.keys()]); + const name = yield* git(orgId, writer, ["config", "--get", "user.name"]); + const email = yield* git(orgId, writer, ["config", "--get", "user.email"]); + if (!name || !email) + return yield* new MemoryWriteRejected({ + reason: "Server bot identity is required", + }); + yield* git( + orgId, + writer, + [ + "-c", + `user.name=${name}`, + "-c", + `user.email=${email}`, + "-c", + `committer.name=${name}`, + "-c", + `committer.email=${email}`, + "commit", + `--author=${name} <${email}>`, + "-m", + subject, + "-m", + `Events: ${ctx.eventIds.join(", ")}\nKin-Outcome: ${outcome}\nKin-Request: ${fingerprint}`, + ], + { + GIT_AUTHOR_NAME: name, + GIT_AUTHOR_EMAIL: email, + GIT_COMMITTER_NAME: name, + GIT_COMMITTER_EMAIL: email, + }, + ); + const record = Effect.fn("MemoryWriter.record")(function* () { + const head = yield* git(orgId, writer, ["rev-parse", "HEAD"]); + yield* io(orgId, fs.writeFileString(`${writer}/../writer-head`, head)); + return head; + }); + yield* record(); + yield* git(orgId, writer, [ + "push", + "--set-upstream", + "--", + "origin", + `HEAD:refs/heads/${branch}`, + ]).pipe( + Effect.catch((pushError) => + Effect.gen(function* () { + yield* git(orgId, writer, ["fetch", "origin"]); + const remote = yield* git(orgId, writer, [ + "show-ref", + "--verify", + `refs/remotes/origin/${branch}`, + ]).pipe(Effect.result); + if (remote._tag === "Success") + yield* git(orgId, writer, ["rebase", `refs/remotes/origin/${branch}`]).pipe( + Effect.catch((cause) => + Effect.gen(function* () { + yield* git(orgId, writer, ["rebase", "--abort"]); + return yield* new MemorySyncConflict({ + orgId, + operation: "push", + reason: "Rebase conflicted; writer restored", + cause, + }); + }), + ), + ); + yield* record(); + return yield* pushError; + }), + ), + Effect.retry({ + schedule: Schedule.exponential("100 millis"), + times: 2, + while: (error) => error._tag !== "MemorySyncConflict", + }), + ); + return { commit: yield* record() }; + }).pipe( + Effect.onError(() => clean(orgId, writer).pipe(Effect.orDie)), + Effect.ensuring(Effect.sync(() => sessions.delete(orgId))), + ); + }), + ), + ), + ), + }); +}); +export const layer = Layer.effect(MemoryWriter, make); diff --git a/apps/server/src/kin/memory/journal.ts b/apps/server/src/kin/memory/journal.ts new file mode 100644 index 0000000000000..40c6ae5a60fa7 --- /dev/null +++ b/apps/server/src/kin/memory/journal.ts @@ -0,0 +1,30 @@ +export interface JournalContext { + readonly wakeupId: string; + readonly agentId: string; + readonly level: string; + readonly startedAt: string; + readonly timeZone: string; + readonly eventIds: ReadonlyArray; +} +const oneLine = (value: string) => value.replace(/[\r\n\u2028\u2029]/g, " "); +export const journalEntry = ( + ctx: JournalContext, + outcome: "completed" | "failed", + lines: ReadonlyArray, +) => { + const parts = new Intl.DateTimeFormat("en-GB", { + timeZone: ctx.timeZone, + year: "numeric", + month: "2-digit", + day: "2-digit", + hour: "2-digit", + minute: "2-digit", + hourCycle: "h23", + }).formatToParts(Date.parse(ctx.startedAt)); + const part = (type: Intl.DateTimeFormatPartTypes) => + parts.find((entry) => entry.type === type)?.value ?? ""; + return { + path: `journal/${part("year")}/${part("month")}-${part("day")}.md`, + text: `\n## ${part("hour")}:${part("minute")} · wake-up ${oneLine(ctx.wakeupId)} · ${oneLine(ctx.agentId)} · ${ctx.level}\n\nOutcome: ${outcome}\nEvents: ${ctx.eventIds.map(oneLine).join(", ")}\n\n${lines.map((line) => `- ${oneLine(line)}\n`).join("")}`, + }; +}; diff --git a/apps/server/src/kin/memory/secretScan.ts b/apps/server/src/kin/memory/secretScan.ts new file mode 100644 index 0000000000000..eb9792fe614a8 --- /dev/null +++ b/apps/server/src/kin/memory/secretScan.ts @@ -0,0 +1,34 @@ +import type { KinOrgId } from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; + +export class KinSecretScanError extends Schema.TaggedError()( + "KinSecretScanError", + { + reason: Schema.String, + }, +) {} +export interface SecretFinding { + readonly path: string; + readonly ruleId: string; + readonly line: number; +} +export class KinSecretScan extends Context.Service< + KinSecretScan, + { + readonly scan: (input: { + orgId: KinOrgId; + files: ReadonlyArray<{ path: string; content: Uint8Array }>; + }) => Effect.Effect, KinSecretScanError>; + } +>()("t3/kin/memory/secretScan/KinSecretScan") {} + +export const stubLayer = Layer.succeed(KinSecretScan, { + scan: Effect.fn("KinSecretScan.stub")(() => Effect.succeed([])), +}); +export const testLayer = (findings: ReadonlyArray) => + Layer.succeed(KinSecretScan, { + scan: Effect.fn("KinSecretScan.fixture")(() => Effect.succeed(findings)), + }); diff --git a/docs/kin/specs/lead.md b/docs/kin/specs/lead.md index cd1b7c86552a8..74c6f9d64f36c 100644 --- a/docs/kin/specs/lead.md +++ b/docs/kin/specs/lead.md @@ -374,7 +374,7 @@ Writer rules, enforced in code on `stage`: - Provenance comes from the event, not the model. The writer looks up `sourceEventId` in the current batch and sets `said_by`, `source` and `trust` from the event's `provenance`. Owner → `owner`, member → `member`, webhook or outside text → `external`, worker suggestion → `agent`. The lead may lower trust but never raise it. A member's statement that reads like a rule is stored under `claims/`. - `review_by` defaults by kind: person 90 days, commitment and deferral at `due`, decision `null`, objective and claim 30 days unless given. - Contradictions: if an open item of the same kind shares a link or id and says something different, and `supersedes` was not passed, `stage` fails with `MemoryConflict { existingId }`. The lead then supersedes (same source, newer) or posts a question to the owner. -- Secret scan: before commit, the staged diff goes through `MemorySecretScan.check` (triggers spec). A hit refuses the whole commit, keeps the journal line "Commit refused: possible secret in ", and tells the owner. +- Secret scan: before commit, the staged diff goes through `KinSecretScan.scan({ orgId, files })` (triggers spec). A hit refuses the whole commit, discards staged content and emits a typed `MemorySecretBlocked` notification. The runner records the refusal in its next safe journal entry and tells the owner. Staged changes live in the writer checkout's working tree. `commitWakeup("completed")` writes the journal entry, rebuilds the front matter cache for changed files, runs the scan, commits once with message `kin(): wake-up ` plus event ids, and pushes. `commitWakeup("failed")` discards staged item changes (`git checkout -- . && git clean -fd` on item folders) and commits only the journal entry, which records the failure. @@ -386,10 +386,12 @@ Front matter matches the kin-memory README, including snake_case keys. Date-only LEAD-2: the front matter parser must use YAML’s core or JSON schema; YAML 1.1 turns `review_by: 2026-11-13` into a Date. -The production `MemoryConfig` layer derives its server-owned base from `ServerConfig.stateDir`, using `${stateDir}/kin//memory` and `memory-read` (organization ids are path-encoded). Tests may supply an isolated base and relative paths. Paths must not overlap or escape that base. `ensureCheckouts` clones a missing writer and validates an existing one; `prepareWakeup` refuses a writer with an unfinished rebase, merge or cherry-pick, syncs it with `pull --rebase` and force-refreshes the disposable detached read worktree, including generated files. Project bootstrap belongs to the wake-up runner, not Memory. +The production `MemoryConfig` layer derives its server-owned base from `ServerConfig.stateDir`, using `${stateDir}/kin//memory` and `memory-read` (organization ids are path-encoded). Tests may supply an isolated base and relative paths. Paths must not overlap or escape that base. `ensureCheckouts` clones a missing writer and validates an existing one; `prepareWakeup` refuses a writer with an unfinished rebase, merge or cherry-pick, syncs it with `pull --rebase` and force-refreshes a disposable detached independent read clone (no shared refs or hardlinks), including generated files. Project bootstrap belongs to the wake-up runner, not Memory. `ensureCheckouts` uses the server user’s git credentials: on the Kin host, the bot or GitHub App identity. +LEAD-3 uses an independent read clone, not a git worktree: read-side `update-ref` cannot move writer refs. Before trusting HEAD, validation requires exact equality with the last server-recorded `writer-head` file beside the writer (initially origin HEAD). Successful syncs and commits update that anchor. The dedicated OS-user boundary remains required; this does not protect server files from an agent with the same filesystem authority. + The read side pins the server-owned writer repository’s HEAD at wake-up (or first read), never the lead’s read-checkout HEAD or working-tree edits. Snapshots reload only on a new wake-up pin or explicit `rebuildCache`; blob contents and parse failures are cached by SHA. Symlinks are skipped; oversized blobs (over 256 KiB), malformed items and kind/folder mismatches are logged and skipped. Queries return duplicate ids with `duplicate: true` on each copy and mark them in the index; ambiguous `readItem` calls fail with `MemoryDuplicateId`. Queries include invalid items unless `excludeInvalid` is set; `invalid_at` remains on each returned `MemoryItemHead`. Date bounds are inclusive and compare instants; date-only strings mean midnight UTC. Missing or unparsable dates do not match a bounded date query. Overdue review means strictly before the current clock. `readItem` best-effort records integer-millisecond last-read times in Kin’s `kin_memory_reads` table, keyed by organization and item id so moves preserve history. Failed history writes are logged without failing the read. Never-read items use `recorded_at` as the baseline; index generation and queries do not count as reading an item. At eight weeks unread, items leave only the index, not search or git. Rebuilds use original heads, cap the index at 200 physical lines including the `As of ` first line, and flag invalid and overdue items. TRIG-12 extends this builder with sections, ordering and freshness defaults; no writer methods or automatic defaults are part of LEAD-2. @@ -623,3 +625,11 @@ LEAD-14. **End-to-end shadow week dry run.** Script that replays a day of record - Which privileged kinds besides merge may an L3 lead request (deploy, release)? This spec allows only low-risk merges. - Task ids for dependencies in the chat, approvals and triggers specs are named by subject here. Swap in the real ids once those specs are written. - The lead's thread shows in T3's thread list under the "Kin · " project. Hide it, or leave it as the lead's readable log? + +### LEAD-3 integration boundaries + +`MemoryWriterAuthority.resolve` is runner-only: resolve organization, wake-up, active lead, level, timestamps, timezone and event provenance from same-organization rows inside the runner's transaction. Never expose this context as tool input. Owner-confirmed sources must be verified there; absent a source, provenance is the lead's agent id. `MemoryWriterHooks.notify` consumes checkout, conflict and secret-block notices without chat transport coupling. + +`KinSecretScan` is deliberately supplied separately. `stubLayer` returns no findings until TRIG-16 replaces it; it is not a production secret detector. Any finding or scanner failure blocks the whole commit. TRIG-16 must preserve this fail-closed interface: `scan({ orgId, files: ReadonlyArray<{ path: string; content: Uint8Array }> })` returns `{ path, ruleId, line }[]`, with typed `KinSecretScanError`. + +TRIG-12's sectioned `buildIndex` has not landed on this base. `MemoryWriterHooks.buildIndex` receives the writer tree and returns generated text or null until wired; no read-checkout text is trusted. No migration is needed. The runner serializes prepare/stage/commit and dreaming per organization. Identical commit retries find the existing wake-up commit; failed pushes retain local commits, and failures discard uncommitted files. Journal additions use `appendJournal`, never replacement through `stage`. diff --git a/docs/kin/specs/triggers.md b/docs/kin/specs/triggers.md index 407ece3c76a52..0ea9f308d9f5e 100644 --- a/docs/kin/specs/triggers.md +++ b/docs/kin/specs/triggers.md @@ -313,7 +313,7 @@ GitHub then blocks a bad auto-merge even if Kin's classifier is wrong. This forc ### Secret scan before memory commits (gap 5) -`secretScan.ts` exposes `scan(files: ReadonlyArray<{ path; content }>) → Finding[]`. Each finding has `rule`, `path`, `line` and a `fingerprint` (sha256 of the match). +`apps/server/src/kin/memory/secretScan.ts` defines `KinSecretScan.scan({ orgId, files: ReadonlyArray<{ path: string; content: Uint8Array }> })`, returning an Effect of `{ path, ruleId, line }[]` with typed `KinSecretScanError`. LEAD-3 supplies an explicitly named no-findings stub; TRIG-16 replaces that layer with the real scanner. Fingerprints stay internal to the scanner allowlist, never matched secrets in findings. **Rules:** @@ -334,7 +334,7 @@ GitHub then blocks a bad auto-merge even if Kin's classifier is wrong. This forc - The dream runs it before pushing. - The memory repo CI runs gitleaks as a second, independent check. -**On a finding**, the writer replaces the matched span with `[redacted:]`, commits the redacted text, and posts to the owner: "I removed what looked like a GitHub token from journal 8 Nov before saving. It is still in chat #tech, message 8812; rotate it." Redacting beats blocking, because blocking would lose the memory write. +**On a finding**, the writer blocks the entire commit, discards staged text and emits a typed owner-notification hook. The runner journals the refusal without secret content and tells the owner to review and rotate the affected credential. Automatic redaction is not part of the writer contract. **False positives:** the owner adds the fingerprint to `kin_secret_allow` through a signed action. A model can never add to it. @@ -494,7 +494,7 @@ Kin runs on one always-on machine, the **Kin host**. The laptop and phone are cl | TRIG-13 | Dream runner: material, branch, dream session, checks | `DreamRunner.ts` | TRIG-12, TRIG-16, LEAD-5, LEAD-6 | fixture dream produces a branch and `dream.json`; `rules.md` edit fails it | backend | | TRIG-14 | Dream split, auto-merge under delegation, review card, rejections; full weekly variant | `classifyDream.ts`, `DreamRunner.ts` | TRIG-13, APPR-4; auto-merge waits for APPR-12 (phase 3), and until then the owner approves both PRs | the two-PR test passes; a rejected item is not proposed again | backend | | TRIG-15 | Memory repo guard rails | `kin-memory`: `CODEOWNERS`, `memory-check.yml`, ruleset | APPR-11 | a PR touching `decisions/` can't merge without owner review; a planted token fails the check | backend | -| TRIG-16 | Secret scanner, writer hook, allowlist | `secretScan.ts` | LEAD-3 | rule fixtures (one hit and one near-miss per rule) pass; redaction and owner message tested | backend | +| TRIG-16 | Secret scanner, writer hook, allowlist | `secretScan.ts` | LEAD-3 | rule fixtures (one hit and one near-miss per rule) pass; commit refusal and typed owner notification tested | backend | | TRIG-17 | Owner-signed purge | `purge.ts` | TRIG-16, APPR-9, APPR-10 | purge test passes on a fixture repo; `git-filter-repo` provenance noted in the PR | backend | | TRIG-18 | Server contract tests: triggers, relay, `delegate_task`, managed policy | `apps/server/src/kin/contract/*` | TRIG-4 | `cd apps/server && vp test run kin-contract` passes | backend | | TRIG-19 | `McpAppWebView` contract test | `apps/mobile/src/features/chats/contract/*` | — | `cd apps/mobile && vp test run kin-contract` passes | UI | From df0e0e877e728f170ec8bc091a4784ed18e18228 Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Sun, 8 Nov 2026 02:13:50 +0530 Subject: [PATCH 02/11] fix(kin): anchor memory writes in SQLite and isolate git execution --- apps/server/src/kin/memory/Memory.test.ts | 128 ++-- apps/server/src/kin/memory/Memory.ts | 383 ++++++---- apps/server/src/kin/memory/MemoryConfig.ts | 3 + .../src/kin/memory/MemoryWriter.test.ts | 580 +++++++++------ apps/server/src/kin/memory/MemoryWriter.ts | 684 ++++++++++-------- apps/server/src/kin/memory/git.ts | 68 ++ apps/server/src/kin/memory/journal.ts | 18 +- apps/server/src/kin/memory/secretScan.ts | 10 +- .../src/kin/persistence/KinMigrations.test.ts | 5 + .../src/kin/persistence/KinMigrations.ts | 3 + .../migrations/K005_MemoryHeads.ts | 12 + docs/kin/specs/lead.md | 13 +- 12 files changed, 1177 insertions(+), 730 deletions(-) create mode 100644 apps/server/src/kin/memory/git.ts create mode 100644 apps/server/src/kin/persistence/migrations/K005_MemoryHeads.ts diff --git a/apps/server/src/kin/memory/Memory.test.ts b/apps/server/src/kin/memory/Memory.test.ts index 3500e503be857..b5221a8246834 100644 --- a/apps/server/src/kin/memory/Memory.test.ts +++ b/apps/server/src/kin/memory/Memory.test.ts @@ -31,7 +31,7 @@ const databaseLayer = Layer.effectDiscard( }), ).pipe(Layer.provideMerge(KinPersistence.layer.pipe(Layer.provideMerge(Sqlite.layerMemory)))); -const fakeRepository = () => { +const fakeRepository = (fs: FileSystem.FileSystem) => { const files = new Map(); const hashes = new Map(); const hash = (text: string) => { @@ -49,9 +49,25 @@ const fakeRepository = () => { const layer = Layer.succeed( VcsProcess.VcsProcess, VcsProcess.VcsProcess.of({ - run: Effect.fn("Test.git")(({ args, stdin, cwd, onStdoutChunk }) => { + run: Effect.fn("Test.git")(({ args: rawArgs, stdin, cwd, onStdoutChunk }) => { + const args: string[] = []; + for (let i = 0; i < rawArgs.length; i++) { + const arg = rawArgs[i]; + if (arg === "-c") { + i++; + continue; + } + if ( + arg && + !arg.startsWith("--git-dir=") && + !arg.startsWith("--work-tree=") && + arg !== "--no-replace-objects" + ) + args.push(arg); + } let stdout = ""; switch (args[0]) { + case "fetch": case "rev-parse": stdout = args.includes("--show-toplevel") ? cwd @@ -99,12 +115,17 @@ const fakeRepository = () => { } } onStdoutChunk?.(Buffer.from(stdout)); - return Effect.succeed({ - exitCode: ChildProcessSpawner.ExitCode(0), - stdout, - stderr: "", - stdoutTruncated: false, - stderrTruncated: false, + return Effect.gen(function* () { + if (args[0] === "clone") { + yield* fs.makeDirectory(`${args.at(-1)}/.git`, { recursive: true }).pipe(Effect.orDie); + } + return { + exitCode: ChildProcessSpawner.ExitCode(0), + stdout, + stderr: "", + stdoutTruncated: false, + stderrTruncated: false, + }; }); }), }), @@ -308,7 +329,7 @@ it.effect( const fs = yield* FileSystem.FileSystem; const temporary = yield* fs.makeTempDirectoryScoped(); const root = yield* fs.realPath(temporary); - const repository = fakeRepository(); + const repository = fakeRepository(fs); const messages: string[] = []; const loggerLayer = Logger.layer( [Logger.make(({ message }) => messages.push(String(message)))], @@ -333,6 +354,7 @@ it.effect( repository.files.set("deferrals/broken.md", "---\nid: [\n---\n"); const program = Effect.gen(function* () { const memory = yield* Memory.Memory; + yield* memory.anchor(orgA, "main", "fixture-head-0"); yield* memory.rebuildIndex(orgA); expect(yield* memory.readIndex(orgA)).toContain("As of 2026-11-14T00:00:00.000Z"); expect(yield* memory.readIndex(orgA)).toContain("⚠ review"); @@ -346,7 +368,7 @@ it.effect( expect(moved).toBeDefined(); repository.files.delete("deferrals/def-001.md"); repository.files.set("deferrals/moved.md", moved ?? ""); - yield* fs.writeFileString(`${root}/a/writer-head`, repository.commit()); + yield* memory.record(orgA, "main", yield* memory.anchor(orgA, "main"), repository.commit()); yield* memory.prepareWakeup(orgA); yield* TestClock.adjust("1 week"); yield* memory.rebuildIndex(orgA); @@ -385,7 +407,7 @@ it.effect( }), ); } - yield* fs.writeFileString(`${root}/a/writer-head`, repository.commit()); + yield* memory.record(orgA, "main", yield* memory.anchor(orgA, "main"), repository.commit()); yield* memory.prepareWakeup(orgA); yield* memory.rebuildIndex(orgA); expect((yield* memory.readIndex(orgA)).trimEnd().split("\n")).toHaveLength(200); @@ -403,7 +425,12 @@ it.effect( "# index\nAs of 2027-02-28T00:00:00Z\n", ]) { repository.files.set("INDEX.md", index); - yield* fs.writeFileString(`${root}/a/writer-head`, repository.commit()); + yield* memory.record( + orgA, + "main", + yield* memory.anchor(orgA, "main"), + repository.commit(), + ); const result = yield* memory.prepareWakeup(orgA); expect(DateTime.toEpochMillis(result.indexAsOf)).toBe(yield* Clock.currentTimeMillis); } @@ -411,7 +438,7 @@ it.effect( "deferrals/duplicate.md", fixtureItem({ recorded_at: "2027-02-27T00:00:00Z" }), ); - yield* fs.writeFileString(`${root}/a/writer-head`, repository.commit()); + yield* memory.record(orgA, "main", yield* memory.anchor(orgA, "main"), repository.commit()); yield* memory.prepareWakeup(orgA); const duplicates = yield* memory.query(orgA, {}); expect( @@ -428,7 +455,7 @@ it.effect( for (const id of ["fallback-1", "fallback-2"]) { repository.files.set(`deferrals/${id}.md`, fixtureItem({ id })); } - yield* fs.writeFileString(`${root}/a/writer-head`, repository.commit()); + yield* memory.record(orgA, "main", yield* memory.anchor(orgA, "main"), repository.commit()); repository.corruptBatch(); yield* memory.prepareWakeup(orgA); const afterFallback = yield* memory.query(orgA, {}); @@ -483,7 +510,7 @@ it.effect.each([ })), }), ), - Layer.provide(fakeRepository().layer), + Layer.provide(fakeRepository(fs).layer), ), ), ); @@ -629,7 +656,7 @@ it.live.each([ ), ); -it.live("refuses reads from a writer with the wrong origin after service restart", () => +it.live("ignores lead-controlled origin after service restart", () => Effect.gen(function* () { const { root, source, git } = yield* fixtureRepository(); const base = `${root}/state`; @@ -641,8 +668,7 @@ it.live("refuses reads from a writer with the wrong origin after service restart yield* Effect.gen(function* () { const memory = yield* Memory.Memory; const result = yield* memory.query(orgA, {}).pipe(Effect.result); - expect(result._tag).toBe("Failure"); - if (result._tag === "Failure") expect(result.failure._tag).toBe("MemoryCheckoutInvalid"); + expect(result._tag).toBe("Success"); }).pipe(Effect.provide(memoryLayer(base, source))); }).pipe( Effect.provide(Layer.mergeAll(gitLayer, NodeServices.layer, databaseLayer)), @@ -664,7 +690,10 @@ it.live("uses filesystem identity for a case-mismatched base without recloning", }).pipe(Effect.provide(memoryLayer(base, source))); yield* fs.writeFileString(`${writer}/keep`, "unpushed work"); const head = yield* commit(writer); - yield* fs.writeFileString(`${writer}/../writer-head`, head); + yield* Effect.gen(function* () { + const memory = yield* Memory.Memory; + yield* memory.record(orgA, "main", yield* memory.anchor(orgA, "main"), head); + }).pipe(Effect.provide(memoryLayer(base, source))); const before = yield* fs.stat(writer); yield* Effect.gen(function* () { const memory = yield* Memory.Memory; @@ -701,12 +730,12 @@ it.live.each(["diverged", "conflict", "network down", "dirty writer", "rebase in yield* git(writer, ["config", "user.email", "fixture@example.invalid"]); yield* fs.writeFileString(`${writer}/rules.md`, "Local invented rules\n"); const local = yield* commit(writer); - yield* fs.writeFileString(`${writer}/../writer-head`, local); + yield* memory.record(orgA, "main", initial.head, local); yield* fs.writeFileString( `${source}/${scenario === "diverged" || scenario === "network down" || scenario === "dirty writer" ? "remote-only" : "rules.md"}`, "Remote invented edit\n", ); - const remote = yield* commit(source); + yield* commit(source); if (scenario === "network down") yield* fs.rename(source, `${root}/offline`); if (scenario === "dirty writer") yield* fs.writeFileString(`${writer}/rules.md`, "Uncommitted fixture edit\n"); @@ -716,44 +745,31 @@ it.live.each(["diverged", "conflict", "network down", "dirty writer", "rebase in yield* fs.writeFileString(`${writer}/rules.md`, "Half-resolved fixture edit\n"); } const result = yield* memory.prepareWakeup(orgA).pipe(Effect.result); - if (scenario === "rebase in progress") { - expect(result._tag).toBe("Failure"); - if (result._tag === "Failure") expect(result.failure._tag).toBe("MemoryCheckoutInvalid"); - expect(yield* fs.exists(`${writer}/.git/rebase-merge`)).toBe(true); - expect(yield* fs.readFileString(`${writer}/rules.md`)).toBe( - "Half-resolved fixture edit\n", - ); - } else if (scenario === "dirty writer") { - expect(result._tag).toBe("Failure"); - if (result._tag === "Failure") expect(result.failure._tag).toBe("MemoryError"); - expect(yield* git(writer, ["rev-parse", "HEAD"])).toBe(local); - expect(yield* fs.readFileString(`${writer}/rules.md`)).toBe("Uncommitted fixture edit\n"); - } else if (scenario === "conflict") { - expect(result._tag).toBe("Failure"); - if (result._tag === "Failure") expect(result.failure._tag).toBe("MemorySyncConflict"); - expect(yield* git(writer, ["rev-parse", "HEAD"])).toBe(local); - expect(yield* fs.readFileString(`${writer}/rules.md`)).toBe("Local invented rules\n"); - expect(yield* git(writer, ["status", "--porcelain"])).toBe(""); - expect(yield* fs.exists(`${writer}/.git/rebase-merge`)).toBe(false); - expect(yield* memory.readRules(orgA)).toBe("Invented standing rules.\n"); - } else { + if (scenario === "network down" || scenario === "diverged" || scenario === "dirty writer") { expect(result._tag).toBe("Success"); - if (result._tag !== "Success") return; - expect(result.success.head).not.toBe(initial.head); - expect(yield* memory.readRules(orgA)).toBe("Local invented rules\n"); - if (scenario === "diverged") { - expect(result.success.head).not.toBe(local); - expect(yield* git(writer, ["merge-base", "HEAD", remote])).toBe(remote); - expect(yield* fs.readFileString(`${writer}/remote-only`)).toBe( - "Remote invented edit\n", - ); - } else { - expect(result.success.head).toBe(local); - expect(messages.some((message) => message.includes("using local writer HEAD"))).toBe( - true, - ); + if (result._tag === "Success") { + if (scenario === "network down") expect(result.success.head).toBe(local); + else { + expect(result.success.head).not.toBe(local); + expect(yield* memory.readRules(orgA)).toBe("Local invented rules\n"); + expect(yield* git(writer, ["show", `${result.success.head}:remote-only`])).toBe( + "Remote invented edit", + ); + } } + } else { + expect(result._tag).toBe("Failure"); + if (result._tag === "Failure") + expect(result.failure._tag).toBe( + scenario === "rebase in progress" ? "MemoryCheckoutInvalid" : "MemorySyncConflict", + ); } + if ( + scenario === "conflict" || + scenario === "rebase in progress" || + scenario === "network down" + ) + expect(yield* git(writer, ["rev-parse", "refs/heads/main"])).toBe(local); }).pipe(Effect.provide(Layer.merge(memoryLayer(base, source), logger))); }).pipe( Effect.provide(Layer.mergeAll(gitLayer, NodeServices.layer, databaseLayer)), diff --git a/apps/server/src/kin/memory/Memory.ts b/apps/server/src/kin/memory/Memory.ts index 304ea8739b40b..208bc47b04522 100644 --- a/apps/server/src/kin/memory/Memory.ts +++ b/apps/server/src/kin/memory/Memory.ts @@ -11,13 +11,14 @@ import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; -import * as Option from "effect/Option"; import * as Path from "effect/Path"; import * as Schema from "effect/Schema"; import * as Semaphore from "effect/Semaphore"; -import { isNonRepositoryGitStderr } from "../../vcs/gitErrors.ts"; -import * as VcsProcess from "../../vcs/VcsProcess.ts"; +import * as SqlClient from "effect/sql/SqlClient"; +import { toPersistenceSqlError } from "../../persistence/Errors.ts"; +import { memoryGit as runMemoryGit } from "./git.ts"; +import { VcsProcess } from "../../vcs/VcsProcess.ts"; import * as MemoryCache from "./MemoryCache.ts"; import { MemoryConfig } from "./MemoryConfig.ts"; import { @@ -107,6 +108,21 @@ export interface MemoryQuery { export class Memory extends Context.Service< Memory, { + readonly withWriterLock: ( + orgId: KinOrgId, + effect: Effect.Effect, + ) => Effect.Effect; + readonly anchor: ( + orgId: KinOrgId, + branch: string, + initial?: string, + ) => Effect.Effect; + readonly record: ( + orgId: KinOrgId, + branch: string, + previous: string, + head: string, + ) => Effect.Effect; readonly ensureCheckouts: (orgId: KinOrgId) => Effect.Effect; readonly prepareWakeup: ( orgId: KinOrgId, @@ -130,15 +146,6 @@ export class Memory extends Context.Service< >()("t3/kin/memory/Memory") {} const decodeHead = Schema.decodeEffect(MemoryItemHead); -// Mirrors STATUS_UPSTREAM_REFRESH_ENV in vcs/GitVcsDriverCore.ts. -const memoryGitEnv = Object.freeze({ - GCM_INTERACTIVE: "never", - GIT_ASKPASS: "", - GIT_TERMINAL_PROMPT: "0", - SSH_ASKPASS: "", - SSH_ASKPASS_REQUIRE: "never", -}); - const folders = ["objectives", "commitments", "deferrals", "decisions", "people", "claims"]; const week = 7 * 24 * 60 * 60 * 1000; const oneLine = (value: string) => value.replace(/[\r\n\u2028\u2029]/g, " "); @@ -166,8 +173,92 @@ interface BlobCache { const make = Effect.gen(function* () { const config = yield* MemoryConfig; const fs = yield* FileSystem.FileSystem; + const vcs = yield* VcsProcess; + const memoryGit = (...args: Parameters) => + runMemoryGit(...args).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(VcsProcess, vcs), + ); const path = yield* Path.Path; - const vcs = yield* VcsProcess.VcsProcess; + const sql = yield* SqlClient.SqlClient; + const writerLocks = new Map(); + const withWriterLock = (orgId: KinOrgId, effect: Effect.Effect) => { + let lock = writerLocks.get(orgId); + if (!lock) { + lock = Semaphore.makeUnsafe(1); + writerLocks.set(orgId, lock); + } + return lock.withPermits(1)(effect); + }; + const persistence = (orgId: KinOrgId, effect: Effect.Effect) => + effect.pipe( + Effect.mapError(toPersistenceSqlError("Memory.anchor")), + Effect.mapError((cause) => new MemoryError({ orgId, operation: "anchor", cause })), + ); + const anchor = Effect.fn("Memory.anchor")(function* ( + orgId: KinOrgId, + branch: string, + initial?: string, + ) { + return yield* persistence( + orgId, + sql.withTransaction( + Effect.gen(function* () { + if (initial !== undefined) + yield* sql`INSERT INTO kin_memory_heads (org_id, branch, head) + VALUES (${orgId}, ${branch}, ${initial}) ON CONFLICT DO NOTHING RETURNING head`; + const rows = yield* sql<{ + head: string; + }>`SELECT head FROM kin_memory_heads WHERE org_id = ${orgId} AND branch = ${branch} ORDER BY branch`; + return rows[0]?.head; + }), + ), + ).pipe( + Effect.filterOrFail( + (head): head is string => head !== undefined, + () => + new MemoryCheckoutInvalid({ + orgId, + operation: "anchor", + reason: "Missing SQLite anchor", + }), + ), + ); + }); + const record = Effect.fn("Memory.record")(function* ( + orgId: KinOrgId, + branch: string, + previous: string, + head: string, + ) { + yield* sql + .withTransaction( + Effect.gen(function* () { + const rows = yield* persistence( + orgId, + sql`UPDATE kin_memory_heads SET head = ${head} + WHERE org_id = ${orgId} AND branch = ${branch} AND head = ${previous} RETURNING head`, + ); + if (rows.length !== 1) + return yield* new MemoryCheckoutInvalid({ + orgId, + operation: "record", + reason: "Anchor changed", + }); + }), + ) + .pipe( + Effect.mapError((cause) => + cause._tag === "MemoryCheckoutInvalid" || cause._tag === "MemoryError" + ? cause + : new MemoryError({ + orgId, + operation: "record", + cause: toPersistenceSqlError("Memory.record")(cause), + }), + ), + ); + }); const cache = yield* MemoryCache.MemoryCache; const locks = new Map(); const pinnedHeads = new Map(); @@ -194,31 +285,9 @@ const make = Effect.gen(function* () { stdin?: string, onStdoutChunk?: (chunk: Uint8Array) => void, ) { - const result = yield* vcs - .run({ - operation: "Memory.git", - command: "git", - cwd, - args, - ...(stdin === undefined ? {} : { stdin }), - env: memoryGitEnv, - ...(onStdoutChunk === undefined ? {} : { onStdoutChunk }), - classifyFailure: (stderr) => (isNonRepositoryGitStderr(stderr) ? "not-found" : null), - timeoutMs: 120_000, - maxOutputBytes: 64 * 1024 * 1024, - outputMode: "error", - }) - .pipe( - Effect.tapError((cause) => - Effect.logWarning("Memory git command failed", { - orgId, - command: args[0], - errorTag: cause._tag, - }), - ), - Effect.mapError((cause) => new MemoryError({ orgId, operation: "git", cause })), - ); - return result.stdout; + return yield* memoryGit(cwd, args, stdin, {}, onStdoutChunk).pipe( + Effect.mapError((cause) => new MemoryError({ orgId, operation: "git", cause })), + ); }); const io = (orgId: KinOrgId, operation: string, effect: Effect.Effect) => effect.pipe(Effect.mapError((cause) => new MemoryError({ orgId, operation, cause }))); @@ -281,18 +350,6 @@ const make = Effect.gen(function* () { return { repo, root, writer, read }; }); - const sameDirectory = Effect.fn("Memory.sameDirectory")(function* ( - orgId: KinOrgId, - left: string, - right: string, - ) { - const a = yield* io(orgId, "stat", fs.stat(left)); - const b = yield* io(orgId, "stat", fs.stat(right)); - return ( - a.dev === b.dev && Option.isSome(a.ino) && Option.isSome(b.ino) && a.ino.value === b.ino.value - ); - }); - const validateWriter = Effect.fn("Memory.validateWriter")(function* ( orgId: KinOrgId, paths: { writer: string; repo: { repoUrl: string } }, @@ -305,31 +362,22 @@ const make = Effect.gen(function* () { operation: "source", reason: "Writer .git must be a directory", }); - const root = (yield* git(orgId, paths.writer, ["rev-parse", "--show-toplevel"])).trim(); - if (!(yield* sameDirectory(orgId, root, paths.writer))) - return yield* new MemoryError({ - orgId, - operation: "source", - reason: "Writer is not the repository root", - }); - yield* git(orgId, paths.writer, ["rev-parse", "--verify", "HEAD^{commit}"]); - const head = (yield* git(orgId, paths.writer, ["rev-parse", "HEAD"])).trim(); - const anchor = path.join(paths.writer, "..", "writer-head"); - const expected = (yield* fs.exists(anchor)) - ? (yield* fs.readFileString(anchor)).trim() - : (yield* git(orgId, paths.writer, ["rev-parse", "refs/remotes/origin/HEAD"])).trim(); + const branch = config.defaultBranch ?? "main"; + const expected = yield* anchor(orgId, branch); + yield* git(orgId, paths.writer, [ + "fsck", + "--full", + "--strict", + "--no-reflogs", + "--no-references", + expected, + ]); + const head = (yield* git(orgId, paths.writer, ["rev-parse", `refs/heads/${branch}`])).trim(); if (head !== expected) - return yield* new MemoryError({ + return yield* new MemoryCheckoutInvalid({ orgId, operation: "head", - reason: "Unrecorded writer HEAD", - }); - const origin = (yield* git(orgId, paths.writer, ["remote", "get-url", "origin"])).trim(); - if (origin !== paths.repo.repoUrl) - return yield* new MemoryError({ - orgId, - operation: "origin", - reason: "Repository origin does not match configuration", + reason: "Unrecorded branch tip", }); }).pipe( Effect.mapError( @@ -367,30 +415,39 @@ const make = Effect.gen(function* () { directory: paths.root, prefix: "memory.clone-", }); - yield* git(orgId, paths.root, ["clone", "--", paths.repo.repoUrl, temporary]); - yield* validateWriter(orgId, { ...paths, writer: temporary }); + yield* git(orgId, paths.root, [ + "clone", + "--branch", + config.defaultBranch ?? "main", + "--", + paths.repo.repoUrl, + temporary, + ]); + const branch = config.defaultBranch ?? "main"; + const head = (yield* git(orgId, temporary, [ + "fetch", + "--", + paths.repo.repoUrl, + `refs/heads/${branch}`, + ])).trim(); + const expected = yield* anchor(orgId, branch, head); + if (expected !== head) + return yield* new MemoryCheckoutInvalid({ + orgId, + operation: "bootstrap", + reason: "Existing anchor differs", + }); yield* fs.rename(temporary, paths.writer); }).pipe(Effect.scoped), ); } - const readExists = yield* io(orgId, "exists", fs.exists(paths.read)); - const validRead = - readExists && - (yield* io(orgId, "exists", fs.exists(path.join(paths.read, ".git")))) && - (yield* io(orgId, "readGit", fs.lstat(path.join(paths.read, ".git")))).type === "Directory"; - if (!validRead) { - yield* io(orgId, "recoverRead", fs.remove(paths.read, { recursive: true, force: true })); - yield* git(orgId, paths.root, ["clone", "--no-hardlinks", "--", paths.writer, paths.read]); - yield* git(orgId, paths.read, ["remote", "remove", "origin"]); - } return { writer: paths.writer, read: paths.read }; }); const load = Effect.fn("Memory.load")(function* (orgId: KinOrgId, force = false) { const paths = yield* pathsFor(orgId); yield* validateWriter(orgId, paths); - const head = - pinnedHeads.get(orgId) ?? (yield* git(orgId, paths.writer, ["rev-parse", "HEAD"])).trim(); + const head = pinnedHeads.get(orgId) ?? (yield* anchor(orgId, config.defaultBranch ?? "main")); pinnedHeads.set(orgId, head); const previous = snapshots.get(orgId); if (!force && previous?.head === head) return previous; @@ -711,63 +768,116 @@ const make = Effect.gen(function* () { const prepare = Effect.fn("Memory.prepare")(function* (orgId: KinOrgId) { const paths = yield* ensureCheckouts(orgId); for (const marker of ["rebase-merge", "rebase-apply", "MERGE_HEAD", "CHERRY_PICK_HEAD"]) { - if (yield* io(orgId, "sync", fs.exists(path.join(paths.writer, ".git", marker)))) + if (yield* io(orgId, "sync", fs.exists(`${paths.writer}/.git/${marker}`))) return yield* new MemoryCheckoutInvalid({ orgId, operation: "sync", - path: paths.writer, - reason: `Writer has an unfinished git operation (${marker}); manual recovery required`, + reason: "Unfinished Git operation", }); } - const fetched = yield* git(orgId, paths.writer, ["fetch", "origin"]).pipe(Effect.result); - if (fetched._tag === "Failure") { - yield* Effect.logWarning("Memory sync unavailable; using local writer HEAD", { orgId }); - } else { - yield* git(orgId, paths.writer, ["-c", "rebase.autoStash=false", "pull", "--rebase"]).pipe( - Effect.catch((cause) => - Effect.gen(function* () { - const rebasing = yield* io( - orgId, - "sync", - fs.exists(path.join(paths.writer, ".git", "rebase-merge")), - ); - const applying = yield* io( - orgId, - "sync", - fs.exists(path.join(paths.writer, ".git", "rebase-apply")), - ); - if (!rebasing && !applying) { - const reachable = yield* git(orgId, paths.writer, ["fetch", "origin"]).pipe( - Effect.result, - ); - if (reachable._tag === "Success") return yield* cause; - yield* Effect.logWarning("Memory pull unavailable; using local writer HEAD", { - orgId, - }); - return; + const { repo } = yield* pathsFor(orgId); + const branch = config.defaultBranch ?? "main"; + let head = yield* anchor(orgId, branch); + const fetched = yield* git(orgId, paths.writer, [ + "fetch", + "--", + repo.repoUrl, + `refs/heads/${branch}`, + ]).pipe(Effect.result); + if (fetched._tag === "Success") { + const remote = fetched.success.trim(); + if (remote !== head) { + const ff = yield* git(orgId, paths.writer, [ + "merge-base", + "--is-ancestor", + head, + remote, + ]).pipe(Effect.result); + let next = remote; + if (ff._tag === "Failure") { + const ahead = yield* git(orgId, paths.writer, [ + "merge-base", + "--is-ancestor", + remote, + head, + ]).pipe(Effect.result); + if (ahead._tag === "Success") next = head; + else { + const commits = (yield* git(orgId, paths.writer, [ + "rev-list", + "--reverse", + "--first-parent", + `${remote}..${head}`, + ])) + .trim() + .split("\n") + .filter(Boolean); + for (const commit of commits) { + const parent = (yield* git(orgId, paths.writer, ["rev-parse", `${commit}^`])).trim(); + const merged = yield* git(orgId, paths.writer, [ + "merge-tree", + "--write-tree", + `--merge-base=${parent}`, + next, + commit, + ]).pipe(Effect.result); + if (merged._tag === "Failure") + return yield* new MemorySyncConflict({ + orgId, + operation: "sync", + reason: "Plumbing rebase conflicted; branch unchanged", + }); + const tree = merged.success.trim().split("\n")[0]; + if (!tree || !/^[0-9a-f]{40,64}$/.test(tree)) + return yield* new MemorySyncConflict({ + orgId, + operation: "sync", + reason: "Invalid merge tree", + }); + const message = yield* git(orgId, paths.writer, [ + "show", + "--format=%B", + "--no-patch", + commit, + ]); + next = (yield* memoryGit(paths.writer, ["commit-tree", tree, "-p", next], message, { + GIT_AUTHOR_NAME: config.botName ?? "Kin Memory Bot", + GIT_AUTHOR_EMAIL: config.botEmail ?? "kin-memory@localhost", + GIT_COMMITTER_NAME: config.botName ?? "Kin Memory Bot", + GIT_COMMITTER_EMAIL: config.botEmail ?? "kin-memory@localhost", + }).pipe( + Effect.mapError((cause) => new MemoryError({ orgId, operation: "rebase", cause })), + )).trim(); } - yield* git(orgId, paths.writer, ["rebase", "--abort"]); - return yield* new MemorySyncConflict({ - orgId, - operation: "sync", - path: paths.writer, - reason: "Rebase conflicted; writer restored", - cause, - }); - }), - ), - ); + } + } + if (next !== head) { + yield* git(orgId, paths.writer, ["update-ref", `refs/heads/${branch}`, next, head]); + yield* record(orgId, branch, head, next); + head = next; + } + } } - const head = (yield* git(orgId, paths.writer, ["rev-parse", "HEAD"])).trim(); yield* io( orgId, - "recordHead", - fs.writeFileString(path.join(paths.writer, "..", "writer-head"), head), + "readSnapshot", + Effect.scoped( + Effect.gen(function* () { + const temporary = yield* fs.makeTempDirectoryScoped(); + yield* git(orgId, temporary, [ + "clone", + "--no-hardlinks", + "--no-checkout", + "--", + paths.writer, + `${temporary}/read`, + ]); + yield* git(orgId, `${temporary}/read`, ["checkout", "--detach", head]); + yield* fs.remove(paths.read, { recursive: true, force: true }); + yield* fs.rename(`${temporary}/read`, paths.read); + }), + ), ); - yield* git(orgId, paths.read, ["fetch", "--", paths.writer, head]); - yield* git(orgId, paths.read, ["checkout", "--force", "--detach", head]); - yield* git(orgId, paths.read, ["reset", "--hard", head]); - yield* git(orgId, paths.read, ["clean", "-fdx"]); pinnedHeads.set(orgId, head); const index = yield* readFile(orgId, "INDEX.md").pipe(Effect.orElseSucceed(() => "")); const asOf = /^As of ([^\r\n]+)(?:\r?\n|$)/.exec(index)?.[1]; @@ -788,10 +898,15 @@ const make = Effect.gen(function* () { }); return Memory.of({ + withWriterLock, + anchor, + record, ensureCheckouts: Effect.fn("Memory.ensureCheckoutsLocked")((orgId) => locked(orgId, ensureCheckouts(orgId)), ), - prepareWakeup: Effect.fn("Memory.prepareWakeup")((orgId) => locked(orgId, prepare(orgId))), + prepareWakeup: Effect.fn("Memory.prepareWakeup")((orgId) => + withWriterLock(orgId, locked(orgId, prepare(orgId))), + ), readIndex: Effect.fn("Memory.readIndex")((orgId) => locked(orgId, readFile(orgId, "INDEX.md"))), readRules: Effect.fn("Memory.readRules")((orgId) => locked(orgId, readFile(orgId, "rules.md"))), query: Effect.fn("Memory.queryLocked")((orgId, q) => locked(orgId, query(orgId, q))), diff --git a/apps/server/src/kin/memory/MemoryConfig.ts b/apps/server/src/kin/memory/MemoryConfig.ts index a8eea522ea190..fc61f4967a950 100644 --- a/apps/server/src/kin/memory/MemoryConfig.ts +++ b/apps/server/src/kin/memory/MemoryConfig.ts @@ -10,6 +10,9 @@ export class MemoryConfig extends Context.Service< MemoryConfig, { readonly baseDirectory: string; + readonly defaultBranch?: string; + readonly botName?: string; + readonly botEmail?: string; readonly repositories: ReadonlyArray<{ readonly orgId: KinOrgId; readonly repoUrl: string; diff --git a/apps/server/src/kin/memory/MemoryWriter.test.ts b/apps/server/src/kin/memory/MemoryWriter.test.ts index 79401b6244df4..98e1d57118e01 100644 --- a/apps/server/src/kin/memory/MemoryWriter.test.ts +++ b/apps/server/src/kin/memory/MemoryWriter.test.ts @@ -11,236 +11,394 @@ import * as Memory from "./Memory.ts"; import { MemoryConfig } from "./MemoryConfig.ts"; import * as Writer from "./MemoryWriter.ts"; import * as Scan from "./secretScan.ts"; +import { journalEntry } from "./journal.ts"; import { fixtureItem } from "./__fixtures__/items.ts"; const orgId = KinOrgId.make("fixture-org"); const agentId = KinAgentId.make("fixture-lead"); const gitLayer = Vcs.layer.pipe(Layer.provide(NodeServices.layer)); -it.live.each(["completed", "failed", "blocked", "forged", "conflict"] as const)( - "writer real git: %s", - (scenario) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const vcs = yield* Vcs.VcsProcess; - const tmp = yield* fs.makeTempDirectoryScoped(); - const root = yield* fs.realPath(tmp); - const source = `${root}/seed`; - const remote = `${root}/remote.git`; - yield* fs.makeDirectory(source); - const git = Effect.fn("WriterTest.git")((cwd: string, args: ReadonlyArray) => - vcs - .run({ operation: "test.writer", command: "git", cwd, args }) - .pipe(Effect.map((r) => r.stdout.trim())), +it.live.each([ + "completed", + "failed", + "blocked", + "forged", + "conflict", + "B1-shadow", + "B2-anchor", + "B3-execution", + "B4-pushurl", + "B5-journal", + "B6-owner-open", + "B6-owner-closed", + "B6-agent-own", + "B7-identity", +] as const)("writer real git: %s", (scenario) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const vcs = yield* Vcs.VcsProcess; + const tmp = yield* fs.makeTempDirectoryScoped(); + const root = yield* fs.realPath(tmp); + const source = `${root}/seed`; + const remote = `${root}/remote.git`; + yield* fs.makeDirectory(source); + const git = Effect.fn("WriterTest.git")((cwd: string, args: ReadonlyArray) => + vcs + .run({ operation: "test.writer", command: "git", cwd, args }) + .pipe(Effect.map((r) => r.stdout.trim())), + ); + const commit = (cwd: string) => + git(cwd, [ + "-c", + "user.name=Fixture Bot", + "-c", + "user.email=fixture@example.invalid", + "commit", + "-m", + "fixture", + ]); + yield* git(source, ["init", "-b", "main"]); + yield* fs.writeFileString(`${source}/rules.md`, "Invented rules\n"); + yield* fs.writeFileString(`${source}/INDEX.md`, "As of 2026-11-08T00:00:00.000Z\n"); + if (scenario.startsWith("B6")) { + yield* fs.makeDirectory(`${source}/claims`); + yield* fs.writeFileString( + `${source}/claims/existing.md`, + fixtureItem({ + kind: "claim", + status: scenario === "B6-owner-open" ? "open" : "done", + trust: scenario === "B6-agent-own" ? "agent" : "owner", + said_by: scenario === "B6-agent-own" ? agentId : "fixture-owner", + }), ); - const commit = (cwd: string) => - git(cwd, [ - "-c", - "user.name=Fixture Bot", - "-c", - "user.email=fixture@example.invalid", - "commit", - "-m", - "fixture", - ]); - yield* git(source, ["init", "-b", "main"]); - yield* fs.writeFileString(`${source}/rules.md`, "Invented rules\n"); - yield* fs.writeFileString(`${source}/INDEX.md`, "As of 2026-11-08T00:00:00.000Z\n"); - yield* git(source, ["add", "--", "."]); - yield* commit(source); - yield* git(root, ["clone", "--bare", "--", source, remote]); - const notices: Writer.WriterNotice[] = []; - const config = Layer.succeed(MemoryConfig, { - baseDirectory: root, - repositories: [{ orgId, repoUrl: remote, path: "state" }], - }); - const memoryLayer = Memory.layer.pipe(Layer.provide(config)); - const hooks = Layer.succeed(Writer.MemoryWriterHooks, { - notify: Effect.fn("Test.notice")((notice) => - Effect.sync(() => { - notices.push(notice); - }), - ), - buildIndex: Effect.fn("Test.index")(() => Effect.succeed(null)), - }); - const auth = Layer.succeed(Writer.MemoryWriterAuthority, { - resolve: Effect.fn("Test.authority")( - ( - org: KinOrgId, - wakeupId: string, - ): Effect.Effect => - org !== orgId - ? Effect.fail(new Writer.MemoryWriteRejected({ reason: "not_found" })) - : Effect.succeed({ - orgId, - wakeupId, - agentId, - level: scenario === "conflict" ? "L1" : "L0", - timeZone: "Asia/Kolkata", - startedAt: "2026-11-08T00:00:00.000Z", - eventIds: [], - sources: new Map([ - [ - "owner", - { saidBy: "fixture-owner", trust: "owner", source: "fixture-owner-message" }, - ], - [ - "member", - { - saidBy: "fixture-member", - trust: "member", - source: "fixture-member-message", - }, - ], - ]), - }), - ), - }); - const scan = - scenario === "blocked" - ? Scan.testLayer([{ path: "claims/new.md", ruleId: "fixture", line: 1 }]) - : Scan.stubLayer; - const layer = Writer.layer.pipe( - Layer.provideMerge(memoryLayer), - Layer.provide(hooks), - Layer.provide(auth), - Layer.provide(scan), - ); - yield* Effect.gen(function* () { - const memory = yield* Memory.Memory; - const writer = yield* Writer.MemoryWriter; - const paths = yield* memory.ensureCheckouts(orgId); - yield* memory.prepareWakeup(orgId); - yield* git(paths.writer, ["config", "user.name", "Fixture Bot"]); - yield* git(paths.writer, ["config", "user.email", "fixture@example.invalid"]); - const original = yield* git(paths.writer, ["rev-parse", "HEAD"]); - for (const path of [ - "rules.md", - "INDEX.md", - "README.md", - "../claims/a.md", - "/claims/a.md", - "claims/.git/a.md", - "claims/../../a.md", - ]) { - expect( - (yield* writer - .stage(orgId, "wake1", { path, content: fixtureItem() }) - .pipe(Effect.result))._tag, - ).toBe("Failure"); - } - yield* fs.symlink(source, `${paths.writer}/claims`); + } + if (scenario === "B5-journal") { + yield* fs.makeDirectory(`${source}/journal/2026`, { recursive: true }); + yield* fs.writeFileString(`${source}/journal/2026/11-08.md`, "Owner journal preserved.\n"); + } + yield* git(source, ["add", "--", "."]); + yield* commit(source); + yield* git(root, ["clone", "--bare", "--", source, remote]); + const notices: Writer.WriterNotice[] = []; + const config = Layer.succeed(MemoryConfig, { + baseDirectory: root, + repositories: [{ orgId, repoUrl: remote, path: "state" }], + }); + const memoryLayer = Memory.layer.pipe(Layer.provide(config)); + const hooks = Layer.succeed(Writer.MemoryWriterHooks, { + notify: Effect.fn("Test.notice")((notice) => + Effect.sync(() => { + notices.push(notice); + }), + ), + buildIndex: Effect.fn("Test.index")(() => Effect.succeed(null)), + }); + const auth = Layer.succeed(Writer.MemoryWriterAuthority, { + resolve: Effect.fn("Test.authority")( + ( + org: KinOrgId, + wakeupId: string, + ): Effect.Effect => + org !== orgId + ? Effect.fail(new Writer.MemoryWriteRejected({ reason: "not_found" })) + : Effect.succeed({ + orgId, + wakeupId, + agentId, + level: scenario === "conflict" || scenario === "B5-journal" ? "L1" : "L0", + timeZone: "Asia/Kolkata", + startedAt: "2026-11-08T00:00:00.000Z", + eventIds: [], + sources: new Map([ + [ + "owner", + { saidBy: "fixture-owner", trust: "owner", source: "fixture-owner-message" }, + ], + [ + "member", + { + saidBy: "fixture-member", + trust: "member", + source: "fixture-member-message", + }, + ], + ]), + }), + ), + }); + const scan = + scenario === "blocked" + ? Scan.testLayer([{ path: "claims/new.md", ruleId: "fixture", line: 1 }]) + : Scan.noFindingsStubLayer; + const layer = Writer.layer.pipe( + Layer.provideMerge(memoryLayer), + Layer.provide(config), + Layer.provide(hooks), + Layer.provide(auth), + Layer.provide(scan), + ); + yield* Effect.gen(function* () { + const memory = yield* Memory.Memory; + const writer = yield* Writer.MemoryWriter; + const paths = yield* memory.ensureCheckouts(orgId); + yield* memory.prepareWakeup(orgId); + yield* git(paths.writer, ["config", "user.name", "Fixture Bot"]); + yield* git(paths.writer, ["config", "user.email", "fixture@example.invalid"]); + const original = yield* git(paths.writer, ["rev-parse", "HEAD"]); + for (const path of [ + "rules.md", + "INDEX.md", + "README.md", + "../claims/a.md", + "/claims/a.md", + "claims/.git/a.md", + "claims/../../a.md", + ]) { + expect( + (yield* writer + .stage(orgId, "wake1", { path, content: fixtureItem() }) + .pipe(Effect.result))._tag, + ).toBe("Failure"); + } + yield* fs.symlink(source, `${paths.writer}/people`); + expect( + (yield* writer + .stage(orgId, "wake1", { + path: "people/new.md", + content: fixtureItem({ kind: "person" }), + }) + .pipe(Effect.result))._tag, + ).toBe("Failure"); + yield* fs.remove(`${paths.writer}/people`); + if (scenario === "completed") { + yield* fs.makeDirectory(`${paths.writer}/claims`, { recursive: true }); + yield* fs.writeFileString(`${root}/hardlink-source`, fixtureItem({ kind: "claim" })); + yield* fs.link(`${root}/hardlink-source`, `${paths.writer}/claims/hard.md`); expect( (yield* writer .stage(orgId, "wake1", { - path: "claims/new.md", + path: "claims/hard.md", content: fixtureItem({ kind: "claim" }), }) .pipe(Effect.result))._tag, ).toBe("Failure"); - yield* fs.remove(`${paths.writer}/claims`); - if (scenario === "forged") { - yield* fs.writeFileString(`${paths.read}/rules.md`, "Forged rules\n"); - yield* git(paths.read, ["add", "--", "rules.md"]); - yield* commit(paths.read); - const forged = yield* git(paths.read, ["rev-parse", "HEAD"]); - yield* git(paths.read, ["update-ref", "refs/heads/main", forged]); - expect(yield* git(paths.writer, ["rev-parse", "HEAD"])).toBe(original); - expect((yield* memory.prepareWakeup(orgId)).head).toBe(original); - yield* git(paths.writer, ["fetch", "--", paths.read, forged]); - yield* git(paths.writer, ["update-ref", "refs/heads/main", forged]); - expect((yield* memory.prepareWakeup(orgId).pipe(Effect.result))._tag).toBe("Failure"); - expect( - (yield* writer.commitWakeup(orgId, "wake1", "completed").pipe(Effect.result))._tag, - ).toBe("Failure"); - expect(notices.at(-1)?.error._tag).toBe("MemoryCheckoutInvalid"); - return; - } - yield* writer.stage(orgId, "wake1", { - path: "claims/new.md", - content: fixtureItem({ - kind: "claim", - trust: "owner", - said_by: "forged-owner", - source: "forged", - }), - }); - expect(yield* fs.readFileString(`${paths.writer}/claims/new.md`)).toContain( - "said_by: fixture-lead", + yield* fs.remove(`${paths.writer}/claims/hard.md`); + } + if (scenario === "forged") { + yield* fs.writeFileString(`${paths.read}/rules.md`, "Forged rules\n"); + yield* git(paths.read, ["add", "--", "rules.md"]); + yield* commit(paths.read); + const forged = yield* git(paths.read, ["rev-parse", "HEAD"]); + yield* git(paths.read, ["update-ref", "refs/heads/main", forged]); + expect(yield* git(paths.writer, ["rev-parse", "HEAD"])).toBe(original); + yield* git(paths.writer, ["fetch", "--", paths.read, forged]); + yield* git(paths.writer, ["update-ref", "refs/heads/main", forged]); + expect((yield* memory.prepareWakeup(orgId).pipe(Effect.result))._tag).toBe("Failure"); + expect( + (yield* writer.commitWakeup(orgId, "wake1", "completed").pipe(Effect.result))._tag, + ).toBe("Failure"); + expect(notices.at(-1)?.error._tag).toBe("MemoryCheckoutInvalid"); + return; + } + if (scenario === "B1-shadow" || scenario === "B2-anchor") { + yield* fs.writeFileString(`${paths.read}/rules.md`, "Forged rules\n"); + yield* git(paths.read, ["add", "rules.md"]); + yield* commit(paths.read); + const branch = scenario === "B1-shadow" ? `kin/shadow/${agentId}` : "main"; + yield* git(paths.read, ["push", "--", paths.writer, `HEAD:refs/heads/${branch}`]).pipe( + Effect.catch(() => + git(paths.writer, ["fetch", paths.read, "HEAD"]).pipe( + Effect.andThen( + git(paths.writer, ["update-ref", `refs/heads/${branch}`, "FETCH_HEAD"]), + ), + ), + ), ); - expect(yield* fs.readFileString(`${paths.writer}/claims/new.md`)).toContain("trust: agent"); - if (scenario === "completed") { - const contradiction = yield* writer - .stage(orgId, "wake1", { - path: "claims/contradiction.md", - content: fixtureItem({ - id: "other", - kind: "claim", - title: "Different invented claim", - }), - }) - .pipe(Effect.result); - expect(contradiction._tag).toBe("Failure"); - if (contradiction._tag === "Failure") - expect(contradiction.failure._tag).toBe("MemoryConflict"); - for (const sourceEventId of ["owner", "member"]) { - yield* writer.stage(orgId, "wake1", { - path: `claims/${sourceEventId}.md`, - sourceEventId, - content: fixtureItem({ id: sourceEventId, kind: "claim", links: [] }), - }); - expect( - yield* fs.readFileString(`${paths.writer}/claims/${sourceEventId}.md`), - ).toContain(`trust: ${sourceEventId}`); - } - yield* fs.writeFileString(`${paths.writer}/rules.md`, "Ambient forged edit"); - yield* git(paths.writer, ["add", "--", "rules.md"]); - } - if (scenario === "conflict") { - yield* fs.makeDirectory(`${source}/claims`); - yield* fs.writeFileString(`${source}/claims/new.md`, "Owner conflicting fixture\n"); - yield* git(source, ["add", "--", "claims/new.md"]); - yield* commit(source); - yield* git(source, ["push", "--", remote, "main"]); - } + if (scenario === "B2-anchor") + yield* fs.writeFileString( + `${paths.writer}/../writer-head`, + yield* git(paths.read, ["rev-parse", "HEAD"]), + ); + expect( + (yield* writer.commitWakeup(orgId, "wake1", "completed").pipe(Effect.result))._tag, + ).toBe("Failure"); + return; + } + if (scenario.startsWith("B6")) { const result = yield* writer - .commitWakeup(orgId, "wake1", scenario === "failed" ? "failed" : "completed") + .stage(orgId, "wake1", { + path: "claims/existing.md", + content: fixtureItem({ + kind: "claim", + title: + scenario === "B6-owner-open" + ? "Invented service maintenance" + : "Forged replacement", + status: scenario === "B6-owner-open" ? "done" : "open", + }), + }) .pipe(Effect.result); - if (scenario === "blocked" || scenario === "conflict") { - expect(result._tag).toBe("Failure"); - if (result._tag === "Failure") - expect(result.failure._tag).toBe( - scenario === "blocked" ? "MemorySecretBlocked" : "MemorySyncConflict", - ); - expect(yield* git(paths.writer, ["status", "--porcelain"])).toBe(""); - expect(notices).toHaveLength(1); - expect(yield* fs.exists(`${paths.writer}/.git/rebase-merge`)).toBe(false); - } else { - expect(result._tag).toBe("Success"); - expect(yield* git(paths.writer, ["rev-list", "--count", `${original}..HEAD`])).toBe("1"); - expect(yield* fs.readFileString(`${paths.writer}/rules.md`)).toBe("Invented rules\n"); - const again = yield* writer.commitWakeup( - orgId, - "wake1", - scenario === "failed" ? "failed" : "completed", + expect(result._tag).toBe("Failure"); + return; + } + const marker = `${root}/executed`; + if (scenario === "B3-execution") { + const script = `${root}/attack.sh`; + yield* fs.writeFileString( + script, + `#!/bin/sh\ntouch '${marker}'\nprintf 'filtered attack\\n'\n`, + ); + yield* fs.chmod(script, 0o755); + for (const repo of [paths.read, paths.writer]) { + yield* fs.writeFileString( + `${repo}/.git/hooks/post-checkout`, + `#!/bin/sh\ntouch '${marker}'\n`, ); - if (result._tag === "Success") expect(again.commit).toBe(result.success.commit); - expect( - yield* git(paths.writer, ["show", "--format=%an <%ae>", "--no-patch", "HEAD"]), - ).toBe("Fixture Bot "); - expect(yield* fs.exists(`${paths.writer}/claims/new.md`)).toBe(scenario === "completed"); - expect(yield* fs.readFileString(`${paths.writer}/journal/2026/11-08.md`)).toContain( - `Outcome: ${scenario}`, + yield* fs.chmod(`${repo}/.git/hooks/post-checkout`, 0o755); + yield* git(repo, ["config", "core.fsmonitor", script]); + } + yield* fs.writeFileString( + `${paths.writer}/.git/hooks/reference-transaction`, + `#!/bin/sh\ntouch '${marker}'\n`, + ); + yield* fs.chmod(`${paths.writer}/.git/hooks/reference-transaction`, 0o755); + yield* git(paths.writer, ["config", "filter.attack.clean", script]); + yield* fs.writeFileString(`${paths.writer}/.git/info/attributes`, "*.md filter=attack\n"); + yield* git(paths.writer, ["config", "remote.origin.receivepack", script]); + yield* memory.prepareWakeup(orgId); + expect(yield* fs.exists(marker)).toBe(false); + } + let evil = ""; + if (scenario === "B4-pushurl") { + evil = `${root}/evil.git`; + yield* git(root, ["init", "--bare", evil]); + yield* git(paths.writer, ["config", "remote.origin.pushurl", evil]); + } + if (scenario === "B5-journal") { + yield* fs.makeDirectory(`${paths.writer}/journal/2026`, { recursive: true }); + yield* fs.writeFileString(`${paths.writer}/journal/2026/11-08.md`, "Forged journal\n"); + } + yield* writer.stage(orgId, "wake1", { + path: "claims/new.md", + content: fixtureItem({ + kind: "claim", + trust: "owner", + said_by: "forged-owner", + source: "forged", + }), + }); + + if (scenario === "completed") { + const contradiction = yield* writer + .stage(orgId, "wake1", { + path: "claims/contradiction.md", + content: fixtureItem({ + id: "other", + kind: "claim", + title: "Different invented claim", + }), + }) + .pipe(Effect.result); + expect(contradiction._tag).toBe("Failure"); + if (contradiction._tag === "Failure") + expect(contradiction.failure._tag).toBe("MemoryConflict"); + for (const sourceEventId of ["owner", "member"]) { + yield* writer.stage(orgId, "wake1", { + path: `claims/${sourceEventId}.md`, + sourceEventId, + content: fixtureItem({ id: sourceEventId, kind: "claim", links: [] }), + }); + } + yield* fs.writeFileString(`${paths.writer}/rules.md`, "Ambient forged edit"); + yield* git(paths.writer, ["add", "--", "rules.md"]); + } + if (scenario === "conflict") { + yield* fs.makeDirectory(`${source}/claims`); + yield* fs.writeFileString(`${source}/claims/new.md`, "Owner conflicting fixture\n"); + yield* git(source, ["add", "--", "claims/new.md"]); + yield* commit(source); + yield* git(source, ["push", "--", remote, "main"]); + } + const result = yield* writer + .commitWakeup(orgId, "wake1", scenario === "failed" ? "failed" : "completed") + .pipe(Effect.result); + if (result._tag === "Failure") yield* Effect.logError(result.failure); + if (scenario === "blocked" || scenario === "conflict") { + expect(result._tag).toBe("Failure"); + if (result._tag === "Failure") + expect(result.failure._tag).toBe( + scenario === "blocked" ? "MemorySecretBlocked" : "MemorySyncConflict", ); + + expect(notices).toHaveLength(1); + expect(yield* fs.exists(`${paths.writer}/.git/rebase-merge`)).toBe(false); + } else { + expect(result._tag).toBe("Success"); + const branch = scenario === "B5-journal" ? "main" : `kin/shadow/${agentId}`; + expect(yield* git(paths.writer, ["rev-list", "--count", `${original}..${branch}`])).toBe( + "1", + ); + expect(yield* git(paths.writer, ["show", `${branch}:rules.md`])).toBe("Invented rules"); + if (scenario === "B3-execution") expect(yield* fs.exists(marker)).toBe(false); + if (scenario === "B4-pushurl") { + expect( + yield* git(root, [`--git-dir=${remote}`, "rev-parse", `refs/heads/${branch}`]), + ).toBe(result._tag === "Success" ? result.success.commit : ""); + expect( + (yield* git(root, [ + `--git-dir=${evil}`, + "rev-parse", + "--verify", + `refs/heads/${branch}`, + ]).pipe(Effect.result))._tag, + ).toBe("Failure"); + } + if (scenario === "B5-journal") { + const journal = yield* git(paths.writer, ["show", `${branch}:journal/2026/11-08.md`]); + expect(journal).toContain("Owner journal preserved."); + expect(journal).not.toContain("Forged journal"); } - }).pipe(Effect.provide(layer)); - }).pipe( - Effect.provide( - Layer.mergeAll( - gitLayer, - NodeServices.layer, - Persistence.layer.pipe(Layer.provideMerge(Sqlite.layerMemory)), - ), + const again = yield* writer.commitWakeup( + orgId, + "wake1", + scenario === "failed" ? "failed" : "completed", + ); + if (result._tag === "Success") expect(again.commit).toBe(result.success.commit); + expect(yield* git(paths.writer, ["show", "--format=%an <%ae>", "--no-patch", branch])).toBe( + "Kin Memory Bot ", + ); + expect( + (yield* git(paths.writer, ["cat-file", "-e", `${branch}:claims/new.md`]).pipe( + Effect.result, + ))._tag, + ).toBe(scenario !== "failed" ? "Success" : "Failure"); + expect(yield* git(paths.writer, ["show", `${branch}:journal/2026/11-08.md`])).toContain( + `Outcome: ${scenario === "failed" ? "failed" : "completed"}`, + ); + } + }).pipe(Effect.provide(layer)); + }).pipe( + Effect.provide( + Layer.mergeAll( + gitLayer, + NodeServices.layer, + Persistence.layer.pipe(Layer.provideMerge(Sqlite.layerMemory)), ), - Effect.scoped, ), + Effect.scoped, + ), +); + +it.effect.each([ + { startedAt: "not-a-date", timeZone: "UTC" }, + { startedAt: "2026-11-08T00:00:00Z", timeZone: "Not/AZone" }, +])("rejects invalid journal context: %j", (input) => + Effect.gen(function* () { + const result = yield* journalEntry( + { ...input, wakeupId: "fixture", agentId: "fixture", level: "L1", eventIds: [] }, + "completed", + [], + ).pipe(Effect.result); + expect(result._tag).toBe("Failure"); + if (result._tag === "Failure") expect(result.failure._tag).toBe("MemoryJournalError"); + }), ); diff --git a/apps/server/src/kin/memory/MemoryWriter.ts b/apps/server/src/kin/memory/MemoryWriter.ts index f7cf537660003..0734876e435a8 100644 --- a/apps/server/src/kin/memory/MemoryWriter.ts +++ b/apps/server/src/kin/memory/MemoryWriter.ts @@ -5,10 +5,11 @@ import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Schema from "effect/Schema"; -import * as Schedule from "effect/Schedule"; -import * as Semaphore from "effect/Semaphore"; +import * as Option from "effect/Option"; import { stringify } from "yaml"; +import { memoryGit as runMemoryGit } from "./git.ts"; import { VcsProcess } from "../../vcs/VcsProcess.ts"; +import { MemoryConfig } from "./MemoryConfig.ts"; import { Memory, MemoryCheckoutInvalid, @@ -20,8 +21,8 @@ import { MemoryDuplicateId, } from "./Memory.ts"; import { parseFrontMatter, MemoryItemError } from "./frontMatter.ts"; -import { journalEntry, type JournalContext } from "./journal.ts"; -import { KinSecretScan, KinSecretScanError, type SecretFinding } from "./secretScan.ts"; +import { journalEntry, MemoryJournalError, type JournalContext } from "./journal.ts"; +import { KinSecretScan, KinSecretScanError } from "./secretScan.ts"; export interface WakeupContext extends JournalContext { readonly orgId: KinOrgId; @@ -72,6 +73,7 @@ export class MemoryWriterHooks extends Context.Service< readonly buildIndex: (input: { orgId: KinOrgId; writer: string; + recorded: string; }) => Effect.Effect; } >()("t3/kin/memory/MemoryWriter/MemoryWriterHooks") {} @@ -82,6 +84,7 @@ export interface MemoryChange { readonly supersedes?: string; } type Failure = + | MemoryJournalError | MemoryConfigError | MemoryQueryDecodeError | MemoryNotFound @@ -124,20 +127,18 @@ const make = Effect.gen(function* () { const scanner = yield* KinSecretScan; const fs = yield* FileSystem.FileSystem; const vcs = yield* VcsProcess; - const locks = new Map(); - const locked = (orgId: KinOrgId, effect: Effect.Effect) => { - let lock = locks.get(orgId); - if (!lock) { - lock = Semaphore.makeUnsafe(1); - locks.set(orgId, lock); - } - return lock.withPermits(1)(effect); - }; + const memoryGit = (...args: Parameters) => + runMemoryGit(...args).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(VcsProcess, vcs), + ); + const config = yield* MemoryConfig; + const locked = memory.withWriterLock; const sessions = new Map< KinOrgId, { wakeupId: string; files: Map; lines: string[] } >(); - const io = (orgId: KinOrgId, effect: Effect.Effect) => + const io = (orgId: KinOrgId, effect: Effect.Effect) => effect.pipe(Effect.mapError((cause) => new MemoryError({ orgId, operation: "writer", cause }))); const git = Effect.fn("MemoryWriter.git")(function* ( orgId: KinOrgId, @@ -146,18 +147,7 @@ const make = Effect.gen(function* () { identity: NodeJS.ProcessEnv = {}, stdin?: string, ) { - const result = yield* io( - orgId, - vcs.run({ - ...(stdin === undefined ? {} : { stdin }), - operation: "MemoryWriter.git", - command: "git", - cwd, - args: ["-c", "core.hooksPath=/dev/null", ...args], - env: { GIT_TERMINAL_PROMPT: "0", GCM_INTERACTIVE: "never", GIT_ASKPASS: "", ...identity }, - }), - ); - return result.stdout.trim(); + return (yield* io(orgId, memoryGit(cwd, args, stdin, identity))).trim(); }); const session = Effect.fn("MemoryWriter.session")(function* (orgId: KinOrgId, wakeupId: string) { const ctx = yield* authority.resolve(orgId, wakeupId); @@ -165,7 +155,8 @@ const make = Effect.gen(function* () { ctx.orgId !== orgId || ctx.wakeupId !== wakeupId || !/^[a-zA-Z0-9_-]+$/.test(ctx.agentId) || - !/^[a-zA-Z0-9_-]+$/.test(wakeupId) + !/^[a-zA-Z0-9_-]+$/.test(wakeupId) || + ctx.eventIds.some((id) => !/^[a-zA-Z0-9_-]+$/.test(id)) ) return yield* new MemoryWriteRejected({ reason: "Invalid server context" }); const previous = sessions.get(orgId); @@ -206,6 +197,9 @@ const make = Effect.gen(function* () { return yield* new MemoryError({ orgId, operation: "path", cause: result.failure }); } else if ( result.success.type === "SymbolicLink" || + (result.success.type === "File" && + Option.isSome(result.success.nlink) && + result.success.nlink.value > 1) || (current !== `${writer}/${relative}` && result.success.type !== "Directory") ) { return yield* new MemoryWriteRejected({ reason: "Symlink or invalid ancestor" }); @@ -214,156 +208,228 @@ const make = Effect.gen(function* () { }); const validate = Effect.fn("MemoryWriter.validate")(function* (orgId: KinOrgId) { const paths = yield* memory.ensureCheckouts(orgId); - const head = yield* git(orgId, paths.writer, ["rev-parse", "HEAD"]); - const anchor = `${paths.writer}/../writer-head`; - const expected = (yield* io(orgId, fs.exists(anchor))) - ? (yield* io(orgId, fs.readFileString(anchor))).trim() - : yield* git(orgId, paths.writer, ["rev-parse", "refs/remotes/origin/HEAD"]); - if (expected !== head) - return yield* new MemoryCheckoutInvalid({ - orgId, - operation: "writer", - reason: "Unrecorded writer HEAD", - }); return paths; }); - const write = Effect.fn("MemoryWriter.write")(function* ( - orgId: KinOrgId, - writer: string, - relative: string, - text: string, - ) { - yield* safePath(orgId, writer, relative, true); - const target = `${writer}/${relative}`; - yield* io( - orgId, - fs.makeDirectory(target.slice(0, target.lastIndexOf("/")), { recursive: true }), - ); - yield* io(orgId, fs.writeFileString(target, text)); - }); - const clean = Effect.fn("MemoryWriter.clean")(function* (orgId: KinOrgId, writer: string) { - yield* git(orgId, writer, ["reset", "--hard", "HEAD"]); - yield* git(orgId, writer, ["clean", "-fd", "--", ...folders, "journal", "INDEX.md"]); - }); - const report = ( - orgId: KinOrgId, - wakeupId: string, - effect: Effect.Effect<{ commit: string }, Failure>, - ) => + const report = (orgId: KinOrgId, wakeupId: string, effect: Effect.Effect) => effect.pipe( Effect.tapError((error) => error._tag === "MemoryCheckoutInvalid" || error._tag === "MemorySyncConflict" || error._tag === "MemorySecretBlocked" - ? hooks.notify({ orgId, wakeupId, error }) + ? hooks + .notify({ orgId, wakeupId, error }) + .pipe( + Effect.catchCause((cause) => + Effect.logError("Memory writer notification failed", cause), + ), + ) : Effect.void, ), ); return MemoryWriter.of({ stage: Effect.fn("MemoryWriter.stage")((orgId, wakeupId, change) => - locked( + report( orgId, - Effect.gen(function* () { - const { ctx, state } = yield* session(orgId, wakeupId); - const { writer } = yield* validate(orgId); - yield* safePath(orgId, writer, change.path); - if (change.path.startsWith("journal/")) - return yield* new MemoryWriteRejected({ - reason: "Use appendJournal for journal additions", - }); - const parsed = yield* parseFrontMatter(orgId, change.path, change.content); - const source = - change.sourceEventId === undefined - ? { saidBy: ctx.agentId, trust: "agent", source: `wakeup:${wakeupId}` } - : ctx.sources.get(change.sourceEventId); - if (!source) - return yield* new MemoryWriteRejected({ reason: "Source is not in the current batch" }); - const fm = parsed.frontMatter; - const folder = fm.kind === "person" ? "people" : `${fm.kind}s`; - if (!change.path.startsWith(`${folder}/`)) - return yield* new MemoryWriteRejected({ reason: "Kind does not match path" }); - const items = [...(yield* memory.query(orgId, {}))]; - const bodies = new Map(); - for (const [file, text] of state.files) { - if (file.startsWith("journal/") || file === "INDEX.md") continue; - const staged = yield* parseFrontMatter(orgId, file, text); - const index = items.findIndex((item) => item.path === file); - const head = { ...staged.frontMatter, orgId, path: file, duplicate: false }; - if (index < 0) items.push(head); - else items[index] = head; - bodies.set(file, staged.body); - } - const superseded = - change.supersedes === undefined - ? undefined - : items.find((item) => item.id === change.supersedes); - if ( - change.supersedes !== undefined && - (!superseded || - superseded.kind !== fm.kind || - superseded.source !== source.source || - Date.parse(fm.recorded_at) <= Date.parse(superseded.recorded_at) || - superseded.id === fm.id) - ) - return yield* new MemoryWriteRejected({ - reason: "Supersession requires the same source and kind", - }); - for (const item of items) { + wakeupId, + locked( + orgId, + Effect.gen(function* () { + const { ctx, state } = yield* session(orgId, wakeupId); + const { writer } = yield* validate(orgId); + yield* safePath(orgId, writer, change.path); + if (change.path.startsWith("journal/")) + return yield* new MemoryWriteRejected({ + reason: "Use appendJournal for journal additions", + }); + const parsed = yield* parseFrontMatter(orgId, change.path, change.content); + const source = + change.sourceEventId === undefined + ? { saidBy: ctx.agentId, trust: "agent", source: `wakeup:${wakeupId}` } + : ctx.sources.get(change.sourceEventId); + if (!source) + return yield* new MemoryWriteRejected({ + reason: "Source is not in the current batch", + }); + const fm = parsed.frontMatter; + const folder = fm.kind === "person" ? "people" : `${fm.kind}s`; + if (!change.path.startsWith(`${folder}/`)) + return yield* new MemoryWriteRejected({ reason: "Kind does not match path" }); + const branch = + ctx.level === "L0" ? `kin/shadow/${ctx.agentId}` : (config.defaultBranch ?? "main"); + const anchored = yield* memory.anchor(orgId, branch).pipe(Effect.result); + const local = yield* git(orgId, writer, [ + "rev-parse", + "--verify", + `refs/heads/${branch}`, + ]).pipe(Effect.result); if ( - item.status === "open" && - item.kind === fm.kind && - (item.id === fm.id || item.links.some((link) => fm.links.includes(link))) && - item.id !== change.supersedes - ) { - const body = bodies.get(item.path) ?? (yield* memory.readItem(orgId, item.id)).body; - if (body !== parsed.body || item.title !== fm.title) - return yield* new MemoryConflict({ existingId: item.id }); + (anchored._tag === "Success" && + (local._tag === "Failure" || local.success !== anchored.success)) || + (anchored._tag === "Failure" && local._tag === "Success") + ) + return yield* new MemoryCheckoutInvalid({ + orgId, + operation: "stage", + reason: "Unrecorded branch tip", + }); + const recorded = + anchored._tag === "Success" + ? anchored.success + : yield* memory.anchor(orgId, config.defaultBranch ?? "main"); + const tree = yield* git(orgId, writer, [ + "ls-tree", + "-r", + "--name-only", + "-z", + recorded, + "--", + ...folders, + ]); + const items = []; + const bodies = new Map(); + for (const file of tree.split("\0").filter(Boolean)) { + const content = yield* io( + orgId, + memoryGit(writer, ["cat-file", "blob", `${recorded}:${file}`]), + ); + const parsedItem = yield* parseFrontMatter(orgId, file, content).pipe(Effect.result); + if (parsedItem._tag === "Failure") continue; + items.push({ + ...parsedItem.success.frontMatter, + orgId, + path: file, + duplicate: false, + }); + bodies.set(file, parsedItem.success.body); } + for (const [file, text] of state.files) { + if (file.startsWith("journal/") || file === "INDEX.md") continue; + const staged = yield* parseFrontMatter(orgId, file, text); + const index = items.findIndex((item) => item.path === file); + const head = { ...staged.frontMatter, orgId, path: file, duplicate: false }; + if (index < 0) items.push(head); + else items[index] = head; + bodies.set(file, staged.body); + } + const superseded = + change.supersedes === undefined + ? undefined + : items.find((item) => item.id === change.supersedes); if ( - (item.path === change.path && item.id !== fm.id) || - (item.id === fm.id && item.path !== change.path) + change.supersedes !== undefined && + (!superseded || + superseded.kind !== fm.kind || + superseded.source !== source.source || + Date.parse(fm.recorded_at) <= Date.parse(superseded.recorded_at) || + superseded.id === fm.id) ) - return yield* new MemoryWriteRejected({ reason: "Cannot replace a different item" }); - } - const review = - fm.review_by ?? - (fm.kind === "decision" - ? null - : fm.kind === "commitment" || fm.kind === "deferral" - ? fm.due - : DateTime.formatIso( - DateTime.makeUnsafe( - Date.parse(ctx.startedAt) + (fm.kind === "person" ? 90 : 30) * 86400000, - ), - )); - const fields = { - ...fm, - said_by: source.saidBy, - source: source.source, - trust: - fm.trust === "external" || - (fm.trust === "agent" && source.trust !== "external") || - (fm.trust === "member" && source.trust === "owner") - ? fm.trust - : source.trust, - review_by: review, - }; - const text = render(fields, parsed.body); - yield* parseFrontMatter(orgId, change.path, text); - if (superseded) { - const oldBody = - bodies.get(superseded.path) ?? (yield* memory.readItem(orgId, superseded.id)).body; - const { orgId: _org, path: oldPath, duplicate: _duplicate, ...oldFields } = superseded; - const oldText = render( - { ...oldFields, status: "superseded", superseded_by: fm.id }, - oldBody, + return yield* new MemoryWriteRejected({ + reason: "Supersession requires the same source and kind", + }); + for (const item of items) { + if ( + (item.id === fm.id || item.status === "open") && + item.kind === fm.kind && + (item.id === fm.id || item.links.some((link) => fm.links.includes(link))) && + item.id !== change.supersedes + ) { + const body = bodies.get(item.path) ?? (yield* memory.readItem(orgId, item.id)).body; + const rank = (trust: string) => + ["external", "agent", "member", "owner"].indexOf(trust); + if ( + item.id === fm.id && + (rank(source.trust) < rank(item.trust) || + (fm.status !== item.status && + !(item.status === "open" && ["done", "cancelled"].includes(fm.status)))) + ) + return yield* new MemoryConflict({ existingId: item.id }); + if (body !== parsed.body || item.title !== fm.title) + return yield* new MemoryConflict({ existingId: item.id }); + if (item.id === fm.id) { + const proposed = new Map(Object.entries(fm)); + const mutable = new Set([ + "orgId", + "path", + "duplicate", + "status", + "invalid_at", + "trust", + "said_by", + "source", + ]); + if ( + Object.entries(item).some( + ([key, value]) => + !mutable.has(key) && + JSON.stringify(value) !== JSON.stringify(proposed.get(key)), + ) + ) + return yield* new MemoryConflict({ existingId: item.id }); + } + } + if ( + (item.path === change.path && item.id !== fm.id) || + (item.id === fm.id && item.path !== change.path) + ) + return yield* new MemoryWriteRejected({ + reason: "Cannot replace a different item", + }); + } + const review = + fm.review_by !== undefined + ? fm.review_by + : fm.kind === "decision" + ? null + : fm.kind === "commitment" || fm.kind === "deferral" + ? fm.due + : DateTime.formatIso( + DateTime.makeUnsafe( + Date.parse(ctx.startedAt) + (fm.kind === "person" ? 90 : 30) * 86400000, + ), + ); + const fields = { + ...fm, + said_by: source.saidBy, + source: source.source, + trust: + fm.trust === "external" || + (fm.trust === "agent" && source.trust !== "external") || + (fm.trust === "member" && source.trust === "owner") + ? fm.trust + : source.trust, + review_by: review, + }; + const existing = items.find((item) => item.id === fm.id); + const text = render( + existing + ? { + ...fields, + trust: existing.trust, + said_by: existing.said_by, + source: existing.source, + } + : fields, + parsed.body, ); - yield* write(orgId, writer, oldPath, oldText); - state.files.set(oldPath, oldText); - } - yield* write(orgId, writer, change.path, text); - state.files.set(change.path, text); - }), + yield* parseFrontMatter(orgId, change.path, text); + if (superseded) { + const oldBody = + bodies.get(superseded.path) ?? (yield* memory.readItem(orgId, superseded.id)).body; + const { + orgId: _org, + path: oldPath, + duplicate: _duplicate, + ...oldFields + } = superseded; + const oldText = render( + { ...oldFields, status: "superseded", superseded_by: fm.id }, + oldBody, + ); + state.files.set(oldPath, oldText); + } + state.files.set(change.path, text); + }), + ), ), ), appendJournal: Effect.fn("MemoryWriter.appendJournal")((orgId, wakeupId, lines) => @@ -379,9 +445,9 @@ const make = Effect.gen(function* () { locked( orgId, Effect.gen(function* () { - yield* session(orgId, wakeupId); - const { writer } = yield* validate(orgId); - yield* clean(orgId, writer); + const state = sessions.get(orgId); + if (state && state.wakeupId !== wakeupId) + return yield* new MemoryWriteRejected({ reason: "Another wake-up owns the writer" }); sessions.delete(orgId); }), ), @@ -395,22 +461,75 @@ const make = Effect.gen(function* () { Effect.gen(function* () { const { ctx, state } = yield* session(orgId, wakeupId); const { writer } = yield* validate(orgId); + const repo = config.repositories.find((entry) => entry.orgId === orgId); + if (!repo) + return yield* new MemoryConfigError({ + orgId, + operation: "writer", + reason: "Unknown org", + }); + const branch = + ctx.level === "L0" ? `kin/shadow/${ctx.agentId}` : (config.defaultBranch ?? "main"); + const main = yield* memory.anchor(orgId, config.defaultBranch ?? "main"); + const anchored = yield* memory.anchor(orgId, branch).pipe(Effect.result); + let recorded: string; + if (anchored._tag === "Failure") { + const exists = yield* git(orgId, writer, [ + "rev-parse", + "--verify", + `refs/heads/${branch}`, + ]).pipe(Effect.result); + if (exists._tag === "Success") + return yield* new MemoryCheckoutInvalid({ + orgId, + operation: "branch", + reason: "Unanchored local branch", + }); + const fresh = yield* git(orgId, writer, [ + "fetch", + "--", + repo.repoUrl, + `refs/heads/${config.defaultBranch ?? "main"}`, + ]); + if (fresh !== main) + return yield* new MemoryCheckoutInvalid({ + orgId, + operation: "bootstrap", + reason: "Prepare required before branch creation", + }); + recorded = yield* memory.anchor(orgId, branch, fresh); + yield* git(orgId, writer, [ + "update-ref", + `refs/heads/${branch}`, + recorded, + "0000000000000000000000000000000000000000", + ]); + } else recorded = anchored.success; + const tip = yield* git(orgId, writer, ["rev-parse", `refs/heads/${branch}`]); + if (tip !== recorded) + return yield* new MemoryCheckoutInvalid({ + orgId, + operation: "branch", + reason: "Unrecorded branch tip", + }); + const push = (sha: string) => + git(orgId, writer, ["push", "--", repo.repoUrl, `${sha}:refs/heads/${branch}`]).pipe( + Effect.mapError( + (cause) => + new MemorySyncConflict({ + orgId, + operation: "push", + reason: "Push rejected; retry or reconcile remote", + cause, + }), + ), + ); return yield* Effect.gen(function* () { const subject = `kin(${ctx.agentId}): wake-up ${wakeupId}`; - const candidates = yield* git(orgId, writer, [ - "log", - "--format=%H%x00%s", - "--fixed-strings", - `--grep=${subject}`, - ]); - const prior = candidates - .split("\n") - .map((line) => line.split("\0")) - .find(([, title]) => title === subject)?.[0]; const fingerprint = yield* git( orgId, writer, - ["hash-object", "--stdin"], + ["hash-object", "--no-filters", "--stdin"], {}, stringify({ outcome, @@ -418,75 +537,42 @@ const make = Effect.gen(function* () { lines: state.lines, }), ); - if (prior) { - const body = yield* git(orgId, writer, [ - "show", - "--format=%B", - "--no-patch", - prior, - ]); + const body = yield* git(orgId, writer, [ + "show", + "--format=%B", + "--no-patch", + recorded, + ]); + if (body.split("\n")[0] === subject) { if ( !body.includes(`Kin-Outcome: ${outcome}`) || (state.files.size + state.lines.length > 0 && !body.includes(`Kin-Request: ${fingerprint}`)) ) - return yield* new MemoryWriteRejected({ - reason: "not_allowed: different wake-up retry", - }); - yield* clean(orgId, writer); - return { commit: prior }; + return yield* new MemoryWriteRejected({ reason: "Different wake-up retry" }); + yield* push(recorded); + return { commit: recorded }; } - const defaultRef = yield* git(orgId, writer, [ - "symbolic-ref", - "refs/remotes/origin/HEAD", - ]); - const branch = - ctx.level === "L0" - ? `kin/shadow/${ctx.agentId}` - : defaultRef.replace("refs/remotes/origin/", ""); - const current = yield* git(orgId, writer, ["branch", "--show-current"]); - if (current !== branch) { - if (yield* git(orgId, writer, ["status", "--porcelain"])) { - yield* clean(orgId, writer); - } - const exists = yield* git(orgId, writer, [ - "show-ref", - "--verify", - `refs/heads/${branch}`, - ]).pipe(Effect.result); - yield* git( - orgId, - writer, - exists._tag === "Success" ? ["switch", "--", branch] : ["switch", "-c", branch], - ); - yield* io( - orgId, - fs.writeFileString( - `${writer}/../writer-head`, - yield* git(orgId, writer, ["rev-parse", "HEAD"]), - ), - ); - for (const [file, text] of state.files) yield* write(orgId, writer, file, text); - } - if (outcome === "failed") { - yield* clean(orgId, writer); - state.files.clear(); - } - const entry = journalEntry(ctx, outcome, state.lines); + if (outcome === "failed") state.files.clear(); + const entry = yield* journalEntry(ctx, outcome, state.lines); yield* safePath(orgId, writer, entry.path); - const old = (yield* io(orgId, fs.exists(`${writer}/${entry.path}`))) - ? yield* io(orgId, fs.readFileString(`${writer}/${entry.path}`)) - : ""; + const oldResult = yield* git(orgId, writer, [ + "cat-file", + "blob", + `${recorded}:${entry.path}`, + ]).pipe(Effect.result); + // Preserve journal bytes; the general git wrapper trims command output. + const old = + oldResult._tag === "Success" + ? yield* io( + orgId, + memoryGit(writer, ["cat-file", "blob", `${recorded}:${entry.path}`]), + ) + : ""; state.files.set(entry.path, old + entry.text); - yield* write(orgId, writer, entry.path, old + entry.text); - if (outcome === "completed") { - const index = yield* hooks.buildIndex({ orgId, writer }); - if (index !== null) { - yield* write(orgId, writer, "INDEX.md", index); - state.files.set("INDEX.md", index); - } - } - const findings: ReadonlyArray = yield* scanner.scan({ + const index = yield* hooks.buildIndex({ orgId, writer, recorded }); + if (index !== null) state.files.set("INDEX.md", index); + const findings = yield* scanner.scan({ orgId, files: [...state.files].map(([path, content]) => ({ path, @@ -494,92 +580,54 @@ const make = Effect.gen(function* () { })), }); if (findings.length) return yield* new MemorySecretBlocked({ orgId, findings }); - // Commit only the server's reconstructed files, never ambient index entries or edits. - yield* clean(orgId, writer); - for (const [file, text] of state.files) yield* write(orgId, writer, file, text); - yield* git(orgId, writer, ["add", "--", ...state.files.keys()]); - const name = yield* git(orgId, writer, ["config", "--get", "user.name"]); - const email = yield* git(orgId, writer, ["config", "--get", "user.email"]); - if (!name || !email) - return yield* new MemoryWriteRejected({ - reason: "Server bot identity is required", - }); - yield* git( - orgId, - writer, - [ - "-c", - `user.name=${name}`, - "-c", - `user.email=${email}`, - "-c", - `committer.name=${name}`, - "-c", - `committer.email=${email}`, - "commit", - `--author=${name} <${email}>`, - "-m", - subject, - "-m", - `Events: ${ctx.eventIds.join(", ")}\nKin-Outcome: ${outcome}\nKin-Request: ${fingerprint}`, - ], - { - GIT_AUTHOR_NAME: name, - GIT_AUTHOR_EMAIL: email, - GIT_COMMITTER_NAME: name, - GIT_COMMITTER_EMAIL: email, - }, - ); - const record = Effect.fn("MemoryWriter.record")(function* () { - const head = yield* git(orgId, writer, ["rev-parse", "HEAD"]); - yield* io(orgId, fs.writeFileString(`${writer}/../writer-head`, head)); - return head; - }); - yield* record(); - yield* git(orgId, writer, [ - "push", - "--set-upstream", - "--", - "origin", - `HEAD:refs/heads/${branch}`, - ]).pipe( - Effect.catch((pushError) => - Effect.gen(function* () { - yield* git(orgId, writer, ["fetch", "origin"]); - const remote = yield* git(orgId, writer, [ - "show-ref", - "--verify", - `refs/remotes/origin/${branch}`, - ]).pipe(Effect.result); - if (remote._tag === "Success") - yield* git(orgId, writer, ["rebase", `refs/remotes/origin/${branch}`]).pipe( - Effect.catch((cause) => - Effect.gen(function* () { - yield* git(orgId, writer, ["rebase", "--abort"]); - return yield* new MemorySyncConflict({ - orgId, - operation: "push", - reason: "Rebase conflicted; writer restored", - cause, - }); - }), - ), - ); - yield* record(); - return yield* pushError; - }), - ), - Effect.retry({ - schedule: Schedule.exponential("100 millis"), - times: 2, - while: (error) => error._tag !== "MemorySyncConflict", + return yield* Effect.scoped( + Effect.gen(function* () { + const temporary = yield* io(orgId, fs.makeTempDirectoryScoped()); + const env = { + GIT_INDEX_FILE: `${temporary}/index`, + GIT_AUTHOR_NAME: config.botName ?? "Kin Memory Bot", + GIT_AUTHOR_EMAIL: config.botEmail ?? "kin-memory@localhost", + GIT_COMMITTER_NAME: config.botName ?? "Kin Memory Bot", + GIT_COMMITTER_EMAIL: config.botEmail ?? "kin-memory@localhost", + }; + yield* git(orgId, writer, ["read-tree", recorded], env); + for (const [file, text] of state.files) { + yield* safePath(orgId, writer, file, true); + const blob = yield* git( + orgId, + writer, + ["hash-object", "-w", "--no-filters", "--stdin"], + env, + text, + ); + yield* git( + orgId, + writer, + ["update-index", "--add", "--cacheinfo", `100644,${blob},${file}`], + env, + ); + } + const tree = yield* git(orgId, writer, ["write-tree"], env); + const message = `${subject}\n\nEvents: ${ctx.eventIds.join(", ")}\nKin-Outcome: ${outcome}\nKin-Request: ${fingerprint}\n`; + const commit = yield* git( + orgId, + writer, + ["commit-tree", tree, "-p", recorded], + env, + message, + ); + yield* git(orgId, writer, [ + "update-ref", + `refs/heads/${branch}`, + commit, + recorded, + ]); + yield* memory.record(orgId, branch, recorded, commit); + yield* push(commit); + return { commit }; }), ); - return { commit: yield* record() }; - }).pipe( - Effect.onError(() => clean(orgId, writer).pipe(Effect.orDie)), - Effect.ensuring(Effect.sync(() => sessions.delete(orgId))), - ); + }).pipe(Effect.ensuring(Effect.sync(() => sessions.delete(orgId)))); }), ), ), diff --git a/apps/server/src/kin/memory/git.ts b/apps/server/src/kin/memory/git.ts new file mode 100644 index 0000000000000..f89a5dcebf309 --- /dev/null +++ b/apps/server/src/kin/memory/git.ts @@ -0,0 +1,68 @@ +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import { VcsProcess } from "../../vcs/VcsProcess.ts"; + +// A private git directory excludes lead-controlled config, attributes, grafts and hooks. +export const memoryGit = Effect.fn("Memory.gitHardened")(function* ( + cwd: string, + args: ReadonlyArray, + stdin?: string, + env: NodeJS.ProcessEnv = {}, + onStdoutChunk?: (chunk: Uint8Array) => void, +) { + const fs = yield* FileSystem.FileSystem; + const vcs = yield* VcsProcess; + return yield* Effect.scoped( + Effect.gen(function* () { + const directory = yield* fs.makeTempDirectoryScoped(); + const repo = yield* fs.exists(`${cwd}/.git`); + if (repo) { + yield* fs.writeFileString( + `${directory}/config`, + "[core]\nrepositoryformatversion = 0\nbare = false\n", + ); + for (const name of ["objects", "refs"]) + yield* fs.symlink(`${cwd}/.git/${name}`, `${directory}/${name}`); + for (const name of ["HEAD", "packed-refs"]) { + if (yield* fs.exists(`${cwd}/.git/${name}`)) + yield* fs.copyFile(`${cwd}/.git/${name}`, `${directory}/${name}`); + } + } + const result = yield* vcs.run({ + operation: "Memory.gitHardened", + command: "git", + cwd, + args: [ + "--no-replace-objects", + ...(repo ? [`--git-dir=${directory}`, `--work-tree=${cwd}`] : []), + "-c", + "core.hooksPath=/dev/null", + "-c", + "core.fsmonitor=false", + "-c", + "core.attributesFile=/dev/null", + "-c", + "protocol.file.allow=user", + ...args, + ], + env: { + GIT_CONFIG_NOSYSTEM: "1", + GIT_CONFIG_GLOBAL: "/dev/null", + GIT_TERMINAL_PROMPT: "0", + GIT_ATTR_NOSYSTEM: "1", + GIT_NO_REPLACE_OBJECTS: "1", + GIT_ASKPASS: "", + GCM_INTERACTIVE: "never", + ...env, + }, + ...(stdin === undefined ? {} : { stdin }), + ...(onStdoutChunk === undefined ? {} : { onStdoutChunk }), + maxOutputBytes: 64 * 1024 * 1024, + timeoutMs: 120_000, + }); + if (args[0] === "fetch" && !result.stdout) + return (yield* fs.readFileString(`${directory}/FETCH_HEAD`)).split("\t")[0] ?? ""; + return result.stdout; + }), + ); +}); diff --git a/apps/server/src/kin/memory/journal.ts b/apps/server/src/kin/memory/journal.ts index 40c6ae5a60fa7..c61ace8da104b 100644 --- a/apps/server/src/kin/memory/journal.ts +++ b/apps/server/src/kin/memory/journal.ts @@ -1,3 +1,11 @@ +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; + +export class MemoryJournalError extends Schema.TaggedError()( + "MemoryJournalError", + { reason: Schema.String }, +) {} + export interface JournalContext { readonly wakeupId: string; readonly agentId: string; @@ -7,7 +15,7 @@ export interface JournalContext { readonly eventIds: ReadonlyArray; } const oneLine = (value: string) => value.replace(/[\r\n\u2028\u2029]/g, " "); -export const journalEntry = ( +const renderJournalEntry = ( ctx: JournalContext, outcome: "completed" | "failed", lines: ReadonlyArray, @@ -28,3 +36,11 @@ export const journalEntry = ( text: `\n## ${part("hour")}:${part("minute")} · wake-up ${oneLine(ctx.wakeupId)} · ${oneLine(ctx.agentId)} · ${ctx.level}\n\nOutcome: ${outcome}\nEvents: ${ctx.eventIds.map(oneLine).join(", ")}\n\n${lines.map((line) => `- ${oneLine(line)}\n`).join("")}`, }; }; + +export const journalEntry = Effect.fn("Memory.journalEntry")( + (ctx: JournalContext, outcome: "completed" | "failed", lines: ReadonlyArray) => + Effect.try({ + try: () => renderJournalEntry(ctx, outcome, lines), + catch: () => new MemoryJournalError({ reason: "Invalid journal date or time zone" }), + }), +); diff --git a/apps/server/src/kin/memory/secretScan.ts b/apps/server/src/kin/memory/secretScan.ts index eb9792fe614a8..f7ef2760efa17 100644 --- a/apps/server/src/kin/memory/secretScan.ts +++ b/apps/server/src/kin/memory/secretScan.ts @@ -25,9 +25,13 @@ export class KinSecretScan extends Context.Service< } >()("t3/kin/memory/secretScan/KinSecretScan") {} -export const stubLayer = Layer.succeed(KinSecretScan, { - scan: Effect.fn("KinSecretScan.stub")(() => Effect.succeed([])), -}); +export const noFindingsStubLayer = Layer.effect( + KinSecretScan, + Effect.gen(function* () { + yield* Effect.logWarning("SECURITY: Kin secret scanning is disabled (no-findings stub)"); + return KinSecretScan.of({ scan: Effect.fn("KinSecretScan.stub")(() => Effect.succeed([])) }); + }), +); export const testLayer = (findings: ReadonlyArray) => Layer.succeed(KinSecretScan, { scan: Effect.fn("KinSecretScan.fixture")(() => Effect.succeed(findings)), diff --git a/apps/server/src/kin/persistence/KinMigrations.test.ts b/apps/server/src/kin/persistence/KinMigrations.test.ts index 8427941cde643..3d99ca430e786 100644 --- a/apps/server/src/kin/persistence/KinMigrations.test.ts +++ b/apps/server/src/kin/persistence/KinMigrations.test.ts @@ -20,6 +20,7 @@ const tableNames = [ "kin_cursor", "kin_events", "kin_leads", + "kin_memory_heads", "kin_memory_reads", "kin_messages", "kin_org_state", @@ -82,6 +83,7 @@ describe("Kin migrations", () => { { migration_id: 2, name: "Lead" }, { migration_id: 3, name: "Triggers" }, { migration_id: 4, name: "MemoryReads" }, + { migration_id: 5, name: "MemoryHeads" }, ]); assert.deepStrictEqual( upstreamBefore.map((row) => [row.migration_id, row.name] as const), @@ -125,6 +127,7 @@ describe("Kin migrations", () => { { migration_id: 2, name: "Lead" }, { migration_id: 3, name: "Triggers" }, { migration_id: 4, name: "MemoryReads" }, + { migration_id: 5, name: "MemoryHeads" }, ]); assert.deepStrictEqual(yield* upstreamHistory, upstreamBefore); assert.deepStrictEqual(yield* sql`SELECT * FROM kin_orgs`, [ @@ -189,6 +192,7 @@ describe("Kin migrations", () => { [2, "Lead"], [3, "Triggers"], [4, "MemoryReads"], + [5, "MemoryHeads"], ]); const before = yield* kinSchema; assert.deepStrictEqual( @@ -211,6 +215,7 @@ describe("Kin migrations", () => { { migration_id: 2, name: "Lead" }, { migration_id: 3, name: "Triggers" }, { migration_id: 4, name: "MemoryReads" }, + { migration_id: 5, name: "MemoryHeads" }, ]); assert.deepStrictEqual(yield* runKinMigrations(), []); assert.deepStrictEqual(yield* runMigrations(), []); diff --git a/apps/server/src/kin/persistence/KinMigrations.ts b/apps/server/src/kin/persistence/KinMigrations.ts index 4d6272492bffc..d6d2b4be0361d 100644 --- a/apps/server/src/kin/persistence/KinMigrations.ts +++ b/apps/server/src/kin/persistence/KinMigrations.ts @@ -6,12 +6,15 @@ import Lead from "./migrations/K002_Lead.ts"; import Triggers from "./migrations/K003_Triggers.ts"; import MemoryReads from "./migrations/K004_MemoryReads.ts"; +import MemoryHeads from "./migrations/K005_MemoryHeads.ts"; + const run = Migrator.make({}); const loader = Migrator.fromRecord({ "1_ChatCore": ChatCore, "2_Lead": Lead, "3_Triggers": Triggers, "4_MemoryReads": MemoryReads, + "5_MemoryHeads": MemoryHeads, }); export const runKinMigrations = Effect.fn("KinMigrations.run")(function* () { diff --git a/apps/server/src/kin/persistence/migrations/K005_MemoryHeads.ts b/apps/server/src/kin/persistence/migrations/K005_MemoryHeads.ts new file mode 100644 index 0000000000000..6d874a7aab3be --- /dev/null +++ b/apps/server/src/kin/persistence/migrations/K005_MemoryHeads.ts @@ -0,0 +1,12 @@ +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/sql/SqlClient"; + +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql`CREATE TABLE kin_memory_heads ( + org_id TEXT NOT NULL, + branch TEXT NOT NULL, + head TEXT NOT NULL, + PRIMARY KEY (org_id, branch) + )`; +}); diff --git a/docs/kin/specs/lead.md b/docs/kin/specs/lead.md index 74c6f9d64f36c..175340883fd01 100644 --- a/docs/kin/specs/lead.md +++ b/docs/kin/specs/lead.md @@ -192,7 +192,7 @@ Table `kin_wakeups` (**new**): `org_id, wakeup_id, agent_id, thread_id, run_id, Setup once per organization (`Memory.ensureCheckouts` owns the checkouts; the wake-up runner owns project bootstrap): - Writer checkout: `${stateDir}/kin//memory`, a clone of the organization's memory repo. -- Read checkout: `${stateDir}/kin//memory-read`, a `git worktree` of the same repo, detached at the latest commit. This is the lead's working directory. +- Read checkout: `${stateDir}/kin//memory-read`, a disposable independent clone, detached at the SQLite-pinned commit. This is the lead's working directory. - Lead project: `ProjectService.bootstrap({ workspaceRoot: , title: "Kin · " })`. Each wake-up: @@ -373,24 +373,23 @@ Writer rules, enforced in code on `stage`: - No deletes. `invalidate` sets `invalid_at`; `supersede` sets the old item's `superseded_by` and `status: superseded` and creates the new one. - Provenance comes from the event, not the model. The writer looks up `sourceEventId` in the current batch and sets `said_by`, `source` and `trust` from the event's `provenance`. Owner → `owner`, member → `member`, webhook or outside text → `external`, worker suggestion → `agent`. The lead may lower trust but never raise it. A member's statement that reads like a rule is stored under `claims/`. - `review_by` defaults by kind: person 90 days, commitment and deferral at `due`, decision `null`, objective and claim 30 days unless given. +- Existing item ids are immutable in title and body, whether open, closed, or agent-authored. Only open-to-done/cancelled status transitions from a source of at least the existing trust are permitted in place; provenance can never be downgraded or reassigned. Reopening requires a new, proper superseding item. - Contradictions: if an open item of the same kind shares a link or id and says something different, and `supersedes` was not passed, `stage` fails with `MemoryConflict { existingId }`. The lead then supersedes (same source, newer) or posts a question to the owner. - Secret scan: before commit, the staged diff goes through `KinSecretScan.scan({ orgId, files })` (triggers spec). A hit refuses the whole commit, discards staged content and emits a typed `MemorySecretBlocked` notification. The runner records the refusal in its next safe journal entry and tells the owner. -Staged changes live in the writer checkout's working tree. `commitWakeup("completed")` writes the journal entry, rebuilds the front matter cache for changed files, runs the scan, commits once with message `kin(): wake-up ` plus event ids, and pushes. `commitWakeup("failed")` discards staged item changes (`git checkout -- . && git clean -fd` on item folders) and commits only the journal entry, which records the failure. +Staged bytes live only in server memory. Discard requires no checkout validation and works after restart. Completed commits contain only validated bytes, a journal appended to the recorded tree, and an optional generated index. Failed wake-ups discard item changes and commit the failure journal. A private index and `hash-object --no-filters`, `read-tree`, `update-index`, `write-tree`, `commit-tree`, and compare-and-swap `update-ref` build commits on the SQLite-recorded parent, never HEAD. Author and committer identity come from server configuration (default Kin Memory Bot ), never repository configuration. Branches: at L1 and above the writer commits to the default branch. At L0 it commits to `kin/shadow/` (see Levels). Dreaming (triggers spec) commits to `dream/` branches and takes the same per-organization semaphore while it writes, so it never runs during a wake-up. -Push failures retry with backoff. Unpushed commits stay local and the next `prepareWakeup` rebases them on `pull`. A rebase conflict (the owner edited the same file on GitHub) is aborted, restores the writer, and fails the wake-up with `MemorySyncConflict`; the write-side owner notification is LEAD-3 work. A fetch failure logs a warning and the wake-up proceeds pinned to the local writer HEAD. Existing writers are never replaced on validation failure: `MemoryCheckoutInvalid` leaves recovery to a human. +Pushes and fetches use the configured URL explicitly, never a local remote name or pushurl. A retry of an already recorded wake-up pushes again before returning success. Synchronization uses plumbing-only fast-forwards and replays divergent commits using `merge-tree` and `commit-tree`; conflicts leave the branch unchanged and fail with `MemorySyncConflict`. Failed fetches retain the SQLite anchor. Existing writers with mismatched or missing anchors fail closed as `MemoryCheckoutInvalid`. Front matter matches the kin-memory README, including snake_case keys. Date-only values mean UTC midnight; date-times require an explicit UTC offset, and invalid dates make an item malformed. `links` values containing `#` must be quoted. `owner` and `said_by` hold handles; `orgId` and `path` belong only to the server-side `MemoryItemHead`. LEAD-2: the front matter parser must use YAML’s core or JSON schema; YAML 1.1 turns `review_by: 2026-11-13` into a Date. -The production `MemoryConfig` layer derives its server-owned base from `ServerConfig.stateDir`, using `${stateDir}/kin//memory` and `memory-read` (organization ids are path-encoded). Tests may supply an isolated base and relative paths. Paths must not overlap or escape that base. `ensureCheckouts` clones a missing writer and validates an existing one; `prepareWakeup` refuses a writer with an unfinished rebase, merge or cherry-pick, syncs it with `pull --rebase` and force-refreshes a disposable detached independent read clone (no shared refs or hardlinks), including generated files. Project bootstrap belongs to the wake-up runner, not Memory. +The production `MemoryConfig` layer derives its server-owned base from `ServerConfig.stateDir`, using `${stateDir}/kin//memory` and `memory-read` (organization ids are path-encoded). Tests may supply an isolated base and relative paths. Paths must not overlap or escape that base. `ensureCheckouts` bootstraps a missing writer from a fresh fetch of the configured URL and configured default branch (`main` by default). K005 stores `(org_id, branch, head)` in `kin_memory_heads`; no writable file or local remote ref can bootstrap an existing checkout. Before committing or pushing, each local branch tip must match its anchor. Compare-and-swap branch updates and SQLite recording happen together in the writer operation; a crash between them fails closed and requires recovery. Prepare and commit share a per-org lock. Read clones are replaced from scratch on prepare: git is never run in an existing lead read clone. Every git invocation uses a private git directory excluding repository config, hooks, attributes and grafts, with system/global config disabled and hooks/fsmonitor disabled. -`ensureCheckouts` uses the server user’s git credentials: on the Kin host, the bot or GitHub App identity. - -LEAD-3 uses an independent read clone, not a git worktree: read-side `update-ref` cannot move writer refs. Before trusting HEAD, validation requires exact equality with the last server-recorded `writer-head` file beside the writer (initially origin HEAD). Successful syncs and commits update that anchor. The dedicated OS-user boundary remains required; this does not protect server files from an agent with the same filesystem authority. +**Threat model.** Leads currently run full-access as the same OS user (`kin`) as the server. Assume leads can modify everything under `${stateDir}/kin/*/memory*`, including `.git` and file anchors. Code trusts only SQLite and server configuration, checks recorded branch tips, and builds commits from server-validated bytes rather than ambient working-tree/index content. These are not an OS sandbox: a same-user lead can still attack server processes or SQLite itself. Separating leads into an OS user unable to write `${stateDir}/kin` remains Nishchay's operations decision and is recommended before multi-org use. The read side pins the server-owned writer repository’s HEAD at wake-up (or first read), never the lead’s read-checkout HEAD or working-tree edits. Snapshots reload only on a new wake-up pin or explicit `rebuildCache`; blob contents and parse failures are cached by SHA. Symlinks are skipped; oversized blobs (over 256 KiB), malformed items and kind/folder mismatches are logged and skipped. Queries return duplicate ids with `duplicate: true` on each copy and mark them in the index; ambiguous `readItem` calls fail with `MemoryDuplicateId`. Queries include invalid items unless `excludeInvalid` is set; `invalid_at` remains on each returned `MemoryItemHead`. Date bounds are inclusive and compare instants; date-only strings mean midnight UTC. Missing or unparsable dates do not match a bounded date query. Overdue review means strictly before the current clock. From 5b4250c3e257abf29fca7a3ab773a8a7f4812c49 Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Sun, 8 Nov 2026 02:15:31 +0530 Subject: [PATCH 03/11] fix(kin): verify object integrity in an isolated reference database --- apps/server/src/kin/memory/Memory.ts | 9 +-------- apps/server/src/kin/memory/MemoryWriter.ts | 1 + apps/server/src/kin/memory/git.ts | 15 ++++++++++----- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/apps/server/src/kin/memory/Memory.ts b/apps/server/src/kin/memory/Memory.ts index 208bc47b04522..4e187af2e6d87 100644 --- a/apps/server/src/kin/memory/Memory.ts +++ b/apps/server/src/kin/memory/Memory.ts @@ -364,14 +364,7 @@ const make = Effect.gen(function* () { }); const branch = config.defaultBranch ?? "main"; const expected = yield* anchor(orgId, branch); - yield* git(orgId, paths.writer, [ - "fsck", - "--full", - "--strict", - "--no-reflogs", - "--no-references", - expected, - ]); + yield* git(orgId, paths.writer, ["fsck", "--full", "--strict", "--no-reflogs", expected]); const head = (yield* git(orgId, paths.writer, ["rev-parse", `refs/heads/${branch}`])).trim(); if (head !== expected) return yield* new MemoryCheckoutInvalid({ diff --git a/apps/server/src/kin/memory/MemoryWriter.ts b/apps/server/src/kin/memory/MemoryWriter.ts index 0734876e435a8..e6f161e433ac7 100644 --- a/apps/server/src/kin/memory/MemoryWriter.ts +++ b/apps/server/src/kin/memory/MemoryWriter.ts @@ -159,6 +159,7 @@ const make = Effect.gen(function* () { ctx.eventIds.some((id) => !/^[a-zA-Z0-9_-]+$/.test(id)) ) return yield* new MemoryWriteRejected({ reason: "Invalid server context" }); + yield* journalEntry(ctx, "completed", []); const previous = sessions.get(orgId); if (previous && previous.wakeupId !== wakeupId) return yield* new MemoryWriteRejected({ reason: "Another wake-up owns the writer" }); diff --git a/apps/server/src/kin/memory/git.ts b/apps/server/src/kin/memory/git.ts index f89a5dcebf309..07befb3f9f9a7 100644 --- a/apps/server/src/kin/memory/git.ts +++ b/apps/server/src/kin/memory/git.ts @@ -21,11 +21,16 @@ export const memoryGit = Effect.fn("Memory.gitHardened")(function* ( `${directory}/config`, "[core]\nrepositoryformatversion = 0\nbare = false\n", ); - for (const name of ["objects", "refs"]) - yield* fs.symlink(`${cwd}/.git/${name}`, `${directory}/${name}`); - for (const name of ["HEAD", "packed-refs"]) { - if (yield* fs.exists(`${cwd}/.git/${name}`)) - yield* fs.copyFile(`${cwd}/.git/${name}`, `${directory}/${name}`); + yield* fs.symlink(`${cwd}/.git/objects`, `${directory}/objects`); + if (args[0] === "fsck") { + yield* fs.makeDirectory(`${directory}/refs`); + yield* fs.writeFileString(`${directory}/HEAD`, `${args.at(-1)}\n`); + } else { + yield* fs.symlink(`${cwd}/.git/refs`, `${directory}/refs`); + for (const name of ["HEAD", "packed-refs"]) { + if (yield* fs.exists(`${cwd}/.git/${name}`)) + yield* fs.copyFile(`${cwd}/.git/${name}`, `${directory}/${name}`); + } } } const result = yield* vcs.run({ From 95f0a211a2877ce423051f521af788cb17d092ee Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Sun, 8 Nov 2026 03:12:08 +0530 Subject: [PATCH 04/11] fix(kin): verify memory reads and recover anchored writers --- apps/server/src/kin/memory/Memory.test.ts | 279 ++++++---- apps/server/src/kin/memory/Memory.ts | 487 ++++++++++++++---- apps/server/src/kin/memory/MemoryConfig.ts | 2 + .../src/kin/memory/MemoryWriter.test.ts | 180 ++++++- apps/server/src/kin/memory/MemoryWriter.ts | 266 +++++----- apps/server/src/kin/memory/frontMatter.ts | 25 +- apps/server/src/kin/memory/git.ts | 40 +- apps/server/src/kin/memory/objects.ts | 140 +++++ docs/kin/specs/lead.md | 22 +- 9 files changed, 1065 insertions(+), 376 deletions(-) create mode 100644 apps/server/src/kin/memory/objects.ts diff --git a/apps/server/src/kin/memory/Memory.test.ts b/apps/server/src/kin/memory/Memory.test.ts index 5ad777d275eff..fc8e4504a626a 100644 --- a/apps/server/src/kin/memory/Memory.test.ts +++ b/apps/server/src/kin/memory/Memory.test.ts @@ -1,3 +1,5 @@ +// @effect-diagnostics-next-line nodeBuiltinImport:off -- Git object identity uses the native byte-oriented hash API. +import * as NodeCrypto from "node:crypto"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { expect, it } from "@effect/vitest"; import { KinMemoryItemId, KinOrgId } from "@t3tools/contracts"; @@ -33,15 +35,41 @@ const databaseLayer = Layer.effectDiscard( const fakeRepository = (fs: FileSystem.FileSystem) => { const files = new Map(); - const hashes = new Map(); - const hash = (text: string) => { - let sha = hashes.get(text); - if (!sha) { - sha = (hashes.size + 1).toString(16).padStart(40, "0"); - hashes.set(text, sha); - } + const objects = new Map(); + const object = (type: string, bytes: Buffer) => { + const sha = NodeCrypto.createHash("sha1") + .update(`${type} ${bytes.length}\0`) + .update(bytes) + .digest("hex"); + objects.set(sha, { type, bytes }); return sha; }; + const head = () => { + const tree = (prefix: string): string => { + const names = [ + ...new Set( + [...files.keys()] + .filter((name) => name.startsWith(prefix)) + .map((name) => name.slice(prefix.length).split("/")[0] ?? ""), + ), + ].sort(); + return object( + "tree", + Buffer.concat( + names.map((name) => { + const text = files.get(prefix + name); + const sha = + text === undefined ? tree(prefix + name + "/") : object("blob", Buffer.from(text)); + return Buffer.concat([ + Buffer.from(`${text === undefined ? "40000" : "100644"} ${name}\0`), + Buffer.from(sha, "hex"), + ]); + }), + ), + ); + }; + return object("commit", Buffer.from(`tree ${tree("")}\n\nfixture ${revision}\n`)); + }; let treeReads = 0; let blobReads = 0; let revision = 0; @@ -73,51 +101,37 @@ const fakeRepository = (fs: FileSystem.FileSystem) => { ? cwd : args.includes("--git-common-dir") ? cwd.replace(/memory-read$/, "memory/.git") - : `fixture-head-${revision}`; + : head(); break; case "remote": stdout = "unused"; break; - case "ls-tree": { - treeReads++; - stdout = [...files] - .sort(([a], [b]) => a.localeCompare(b)) - .map( - ([file, text]) => `100644 blob ${hash(text)} ${Buffer.byteLength(text)}\t${file}\0`, - ) - .join(""); - break; - } case "cat-file": { - blobReads++; - if (args[1] === "blob") { - stdout = - [...files.values()].find( - (value) => hash(value) === args[2] && value !== "unreadable fixture", - ) ?? ""; - break; - } - stdout = (stdin?.trim().split("\n") ?? []) - .map((sha) => { - const text = [...files.values()].find( - (value) => hash(value) === sha && value !== "unreadable fixture", - ); - return text === undefined - ? `${sha} missing\n` - : `${sha} blob ${Buffer.byteLength(text)}\n${text}\n`; - }) - .join(""); + const chunks = (stdin?.trim().split("\n") ?? []).map((sha) => { + const entry = objects.get(sha); + if (!entry) return Buffer.from(`${sha} missing\n`); + if (entry.type === "blob") blobReads++; + if (entry.type === "tree") treeReads++; + return Buffer.concat([ + Buffer.from(`${sha} ${entry.type} ${entry.bytes.length}\n`), + entry.bytes, + Buffer.from("\n"), + ]); + }); if (corruptNextBatch) { - stdout = "invalid header\n" + stdout; + chunks.unshift(Buffer.from("invalid header\n")); corruptNextBatch = false; } + onStdoutChunk?.(Buffer.concat(chunks)); break; } } onStdoutChunk?.(Buffer.from(stdout)); return Effect.gen(function* () { if (args[0] === "clone") { - yield* fs.makeDirectory(`${args.at(-1)}/.git`, { recursive: true }).pipe(Effect.orDie); + yield* fs + .makeDirectory(`${args.at(-1)}/.git/objects`, { recursive: true }) + .pipe(Effect.orDie); } return { exitCode: ChildProcessSpawner.ExitCode(0), @@ -132,10 +146,11 @@ const fakeRepository = (fs: FileSystem.FileSystem) => { ); return { files, + head, layer, commit: () => { revision++; - return `fixture-head-${revision}`; + return head(); }, counts: () => ({ treeReads, blobReads }), corruptBatch: () => { @@ -337,7 +352,8 @@ it.effect( ); const checkout = `${root}/a/memory-read`; yield* fs.makeDirectory(`${checkout}/deferrals`, { recursive: true }); - yield* fs.makeDirectory(`${root}/a/memory/.git`, { recursive: true }); + yield* fs.makeDirectory(`${root}/a/memory/.git/objects`, { recursive: true }); + yield* fs.makeDirectory(`${root}/a/memory/.git/refs`, { recursive: true }); repository.files.set( "deferrals/def-001.md", fixtureItem({ recorded_at: "2026-11-14T00:00:00.000Z" }), @@ -360,7 +376,7 @@ it.effect( VALUES (${orgA}, 'Asia/Kolkata', '08:30', '18:30', '02:00', 0, 5, '12:00', '{"from":"22:00","to":"07:00"}') `; const memory = yield* Memory.Memory; - yield* memory.anchor(orgA, "main", "fixture-head-0"); + yield* memory.anchor(orgA, "main", repository.head()); yield* memory.rebuildIndex(orgA); expect(yield* memory.readIndex(orgA)).not.toContain("## Due in 7 days"); expect(yield* memory.readIndex(orgA)).toContain("As of 2026-11-14T00:00:00.000Z"); @@ -399,8 +415,8 @@ it.effect( expect(yield* memory.query(orgA, {})).toHaveLength(2); const beforeRebuild = repository.counts(); yield* memory.rebuildCache(orgA); - expect(repository.counts().treeReads).toBe(beforeRebuild.treeReads + 1); - expect(repository.counts().blobReads).toBe(beforeRebuild.blobReads + 1); + expect(repository.counts().treeReads).toBe(beforeRebuild.treeReads + 2); + expect(repository.counts().blobReads).toBe(beforeRebuild.blobReads); yield* memory.readItem(orgA, itemId); yield* memory.rebuildIndex(orgA); expect(yield* memory.readIndex(orgA)).toContain("Invented service maintenance"); @@ -464,6 +480,7 @@ it.effect( } yield* memory.record(orgA, "main", yield* memory.anchor(orgA, "main"), repository.commit()); repository.corruptBatch(); + expect((yield* memory.prepareWakeup(orgA).pipe(Effect.result))._tag).toBe("Failure"); yield* memory.prepareWakeup(orgA); const afterFallback = yield* memory.query(orgA, {}); expect(afterFallback.filter((item) => !item.id.startsWith("fallback-"))).toEqual( @@ -474,14 +491,14 @@ it.effect( expect(messages.filter((message) => message.includes("Duplicate memory id"))).toHaveLength( 1, ); - expect(messages.some((message) => message.includes("batch framing failed"))).toBe(true); + const sql = yield* SqlClient.SqlClient; yield* sql`UPDATE kin_trigger_settings SET time_zone = 'UTC' WHERE org_id = ${orgA}`; - repository.commit(); + yield* memory.record(orgA, "main", yield* memory.anchor(orgA, "main"), repository.commit()); yield* memory.prepareWakeup(orgA); const utcIndex = yield* memory.readIndex(orgA); yield* sql`UPDATE kin_trigger_settings SET time_zone = 'Invented/Invalid' WHERE org_id = ${orgA}`; - repository.commit(); + yield* memory.record(orgA, "main", yield* memory.anchor(orgA, "main"), repository.commit()); yield* memory.prepareWakeup(orgA); expect(yield* memory.readIndex(orgA)).toContain("As of "); expect(yield* memory.readIndex(orgA)).toBe(utcIndex); @@ -490,7 +507,7 @@ it.effect( ); yield* sql`DROP TABLE kin_trigger_settings`; yield* sql`DROP TABLE kin_memory_reads`; - repository.commit(); + yield* memory.record(orgA, "main", yield* memory.anchor(orgA, "main"), repository.commit()); yield* memory.prepareWakeup(orgA); expect(yield* memory.readIndex(orgA)).toContain("As of "); expect( @@ -504,7 +521,7 @@ it.effect( ); expect( messages.filter((message) => message.includes("Skipping malformed memory item")), - ).toHaveLength(2); + ).toHaveLength(4); }).pipe(Effect.provide(Layer.merge(layer, loggerLayer))); }).pipe(Effect.provide(Layer.merge(NodeServices.layer, databaseLayer)), Effect.scoped), ); @@ -740,68 +757,112 @@ it.live("uses filesystem identity for a case-mismatched base without recloning", ), ); -it.live.each(["diverged", "conflict", "network down", "dirty writer", "rebase in progress"])( - "syncs the writer with $0", - (scenario) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const { root, source, git, commit } = yield* fixtureRepository(); - const base = `${root}/state`; - const messages: string[] = []; - const logger = Logger.layer([Logger.make(({ message }) => messages.push(String(message)))], { - mergeWithExisting: false, - }); - yield* Effect.gen(function* () { - const memory = yield* Memory.Memory; - const initial = yield* memory.prepareWakeup(orgA); - const { writer } = yield* memory.ensureCheckouts(orgA); - yield* git(writer, ["config", "user.name", "Fixture"]); - yield* git(writer, ["config", "user.email", "fixture@example.invalid"]); - yield* fs.writeFileString(`${writer}/rules.md`, "Local invented rules\n"); - const local = yield* commit(writer); - yield* memory.record(orgA, "main", initial.head, local); - yield* fs.writeFileString( - `${source}/${scenario === "diverged" || scenario === "network down" || scenario === "dirty writer" ? "remote-only" : "rules.md"}`, - "Remote invented edit\n", - ); - yield* commit(source); - if (scenario === "network down") yield* fs.rename(source, `${root}/offline`); - if (scenario === "dirty writer") - yield* fs.writeFileString(`${writer}/rules.md`, "Uncommitted fixture edit\n"); - if (scenario === "rebase in progress") { - yield* git(writer, ["fetch", "origin"]); - yield* git(writer, ["rebase", "@{u}"]).pipe(Effect.ignore); - yield* fs.writeFileString(`${writer}/rules.md`, "Half-resolved fixture edit\n"); - } - const result = yield* memory.prepareWakeup(orgA).pipe(Effect.result); - if (scenario === "network down" || scenario === "diverged" || scenario === "dirty writer") { - expect(result._tag).toBe("Success"); - if (result._tag === "Success") { - if (scenario === "network down") expect(result.success.head).toBe(local); - else { - expect(result.success.head).not.toBe(local); - expect(yield* memory.readRules(orgA)).toBe("Local invented rules\n"); - expect(yield* git(writer, ["show", `${result.success.head}:remote-only`])).toBe( - "Remote invented edit", - ); - } - } - } else { - expect(result._tag).toBe("Failure"); - if (result._tag === "Failure") - expect(result.failure._tag).toBe( - scenario === "rebase in progress" ? "MemoryCheckoutInvalid" : "MemorySyncConflict", +it.live.each([ + "diverged", + "conflict", + "network down", + "dirty writer", + "rebase in progress", + "journal conflict", +])("syncs the writer with $0", (scenario) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const { root, source, git, commit } = yield* fixtureRepository(); + const base = `${root}/state`; + const remote = `${root}/sync-remote.git`; + yield* git(root, ["clone", "--bare", "--", source, remote]); + const messages: string[] = []; + const logger = Logger.layer([Logger.make(({ message }) => messages.push(String(message)))], { + mergeWithExisting: false, + }); + yield* Effect.gen(function* () { + const memory = yield* Memory.Memory; + const initial = yield* memory.prepareWakeup(orgA); + const { writer } = yield* memory.ensureCheckouts(orgA); + yield* git(writer, ["config", "user.name", "Fixture"]); + yield* git(writer, ["config", "user.email", "fixture@example.invalid"]); + if (scenario === "journal conflict") { + yield* fs.makeDirectory(`${writer}/journal`, { recursive: true }); + yield* fs.makeDirectory(`${source}/journal`, { recursive: true }); + yield* fs.writeFileString(`${writer}/journal/day.md`, "local entry\n"); + yield* fs.writeFileString(`${writer}/INDEX.md`, "local generated index\n"); + } else yield* fs.writeFileString(`${writer}/rules.md`, "Local invented rules\n"); + const local = yield* commit(writer); + yield* memory.record(orgA, "main", initial.head, local); + yield* fs.writeFileString( + `${source}/${scenario === "diverged" || scenario === "network down" || scenario === "dirty writer" ? "remote-only" : "rules.md"}`, + "Remote invented edit\n", + ); + if (scenario === "journal conflict") { + yield* fs.writeFileString(`${source}/journal/day.md`, "remote entry\n"); + yield* fs.writeFileString(`${source}/INDEX.md`, "remote generated index\n"); + } + yield* commit(source); + yield* git(source, ["push", "--", remote, "main"]); + if (scenario === "network down") yield* fs.rename(remote, `${root}/offline`); + if (scenario === "dirty writer") + yield* fs.writeFileString(`${writer}/rules.md`, "Uncommitted fixture edit\n"); + if (scenario === "rebase in progress") { + yield* git(writer, ["fetch", "origin"]); + yield* git(writer, ["rebase", "@{u}"]).pipe(Effect.ignore); + yield* fs.writeFileString(`${writer}/rules.md`, "Half-resolved fixture edit\n"); + } + const result = yield* memory.prepareWakeup(orgA).pipe(Effect.result); + if (scenario === "network down" || scenario === "diverged" || scenario === "dirty writer") { + expect(result._tag).toBe("Success"); + if (result._tag === "Success") { + if (scenario === "network down") expect(result.success.head).toBe(local); + else { + expect(result.success.head).not.toBe(local); + expect(yield* memory.readRules(orgA)).toBe("Local invented rules\n"); + expect(yield* git(writer, ["show", `${result.success.head}:remote-only`])).toBe( + "Remote invented edit", ); + } } - if ( - scenario === "conflict" || - scenario === "rebase in progress" || - scenario === "network down" - ) - expect(yield* git(writer, ["rev-parse", "refs/heads/main"])).toBe(local); - }).pipe(Effect.provide(Layer.merge(memoryLayer(base, source), logger))); + } else if (scenario === "journal conflict") { + expect(result._tag).toBe("Success"); + if (result._tag === "Success") + expect(yield* git(writer, ["show", `${result.success.head}:journal/day.md`])).toBe( + "remote entry\nlocal entry", + ); + } else if (scenario === "conflict") { + expect(result._tag).toBe("Success"); + const backup = (yield* git(root, [ + `--git-dir=${remote}`, + "for-each-ref", + "--format=%(objectname)", + `refs/heads/kin/conflict/${orgA}/`, + ])).trim(); + expect(backup).toBe(local); + expect((yield* memory.prepareWakeup(orgA)).head).toBe( + yield* git(source, ["rev-parse", "HEAD"]), + ); + } else { + expect(result._tag).toBe("Failure"); + if (result._tag === "Failure") expect(result.failure._tag).toBe("MemoryCheckoutInvalid"); + } + if (scenario === "rebase in progress" || scenario === "network down") + expect(yield* git(writer, ["rev-parse", "refs/heads/main"])).toBe(local); }).pipe( - Effect.provide(Layer.mergeAll(gitLayer, NodeServices.layer, databaseLayer)), - Effect.scoped, - ), + Effect.provide( + Layer.merge( + memoryLayer(base, remote).pipe( + Layer.provide( + Layer.succeed(Memory.MemoryRecoveryHooks, { + notify: (notice) => + Effect.sync(() => { + messages.push(JSON.stringify(notice)); + }), + }), + ), + ), + logger, + ), + ), + ); + }).pipe( + Effect.provide(Layer.mergeAll(gitLayer, NodeServices.layer, databaseLayer)), + Effect.scoped, + ), ); diff --git a/apps/server/src/kin/memory/Memory.ts b/apps/server/src/kin/memory/Memory.ts index 531290c88f259..eada248c87100 100644 --- a/apps/server/src/kin/memory/Memory.ts +++ b/apps/server/src/kin/memory/Memory.ts @@ -1,3 +1,5 @@ +import * as Option from "effect/Option"; +import { readTree, readObjects, readCommit, readFiles } from "./objects.ts"; import { type KinMemoryItemId, type KinOrgId, @@ -106,6 +108,18 @@ export interface MemoryQuery { readonly limit?: number; } +export class MemoryRecoveryHooks extends Context.Service< + MemoryRecoveryHooks, + { + readonly notify: (notice: { + orgId: KinOrgId; + branch: string; + reason: string; + conflictBranch?: string; + }) => Effect.Effect; + } +>()("t3/kin/memory/Memory/MemoryRecoveryHooks") {} + export class Memory extends Context.Service< Memory, { @@ -124,9 +138,11 @@ export class Memory extends Context.Service< previous: string, head: string, ) => Effect.Effect; + readonly recover: (orgId: KinOrgId) => Effect.Effect; readonly ensureCheckouts: (orgId: KinOrgId) => Effect.Effect; readonly prepareWakeup: ( orgId: KinOrgId, + wakeupId?: string, ) => Effect.Effect<{ head: string; indexAsOf: DateTime.Utc }, MemoryFailure>; readonly readIndex: (orgId: KinOrgId) => Effect.Effect; readonly readRules: (orgId: KinOrgId) => Effect.Effect; @@ -172,11 +188,20 @@ interface BlobCache { const make = Effect.gen(function* () { const config = yield* MemoryConfig; const fs = yield* FileSystem.FileSystem; + const recoveryHooks = Option.getOrUndefined(yield* Effect.serviceOption(MemoryRecoveryHooks)); + const notify = (notice: { + orgId: KinOrgId; + branch: string; + reason: string; + conflictBranch?: string; + }) => + recoveryHooks ? recoveryHooks.notify(notice) : Effect.logWarning("Memory recovery", notice); const vcs = yield* VcsProcess; const memoryGit = (...args: Parameters) => runMemoryGit(...args).pipe( Effect.provideService(FileSystem.FileSystem, fs), Effect.provideService(VcsProcess, vcs), + Effect.provideService(MemoryConfig, config), ); const path = yield* Path.Path; const sql = yield* SqlClient.SqlClient; @@ -288,7 +313,7 @@ const make = Effect.gen(function* () { Effect.mapError((cause) => new MemoryError({ orgId, operation: "git", cause })), ); }); - const io = (orgId: KinOrgId, operation: string, effect: Effect.Effect) => + const io = (orgId: KinOrgId, operation: string, effect: Effect.Effect) => effect.pipe(Effect.mapError((cause) => new MemoryError({ orgId, operation, cause }))); const pathsFor = Effect.fn("Memory.pathsFor")(function* (orgId: KinOrgId) { @@ -363,14 +388,24 @@ const make = Effect.gen(function* () { }); const branch = config.defaultBranch ?? "main"; const expected = yield* anchor(orgId, branch); - yield* git(orgId, paths.writer, ["fsck", "--full", "--strict", "--no-reflogs", expected]); + for (const name of ["objects", "refs"]) { + if ((yield* fs.lstat(`${paths.writer}/.git/${name}`)).type !== "Directory") + return yield* new MemoryCheckoutInvalid({ + orgId, + operation: "validate", + reason: "Invalid git directory", + }); + } const head = (yield* git(orgId, paths.writer, ["rev-parse", `refs/heads/${branch}`])).trim(); - if (head !== expected) - return yield* new MemoryCheckoutInvalid({ - orgId, - operation: "head", - reason: "Unrecorded branch tip", - }); + if (head !== expected) { + yield* readCommit(orgId, paths.writer, head).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(VcsProcess, vcs), + ); + yield* git(orgId, paths.writer, ["merge-base", "--is-ancestor", expected, head]); + yield* git(orgId, paths.writer, ["update-ref", `refs/heads/${branch}`, expected, head]); + yield* notify({ orgId, branch, reason: "Rolled back unrecorded branch tip" }); + } }).pipe( Effect.mapError( (cause) => @@ -450,25 +485,18 @@ const make = Effect.gen(function* () { }; blobCaches.set(orgId, blobCache); const { blobs, parsedBlobs, mismatchWarnings } = blobCache; - const tree = yield* git(orgId, paths.writer, [ - "ls-tree", - "-r", - "-l", - "-z", - head, - "--", - ...folders, - "rules.md", - "INDEX.md", - ]); - const entries = tree.split("\0").flatMap((entry) => { - const match = /^(\d+) (\S+) ([0-9a-f]+) +([0-9]+)\t([\s\S]+)$/.exec(entry); - if (!match) return []; - const [, mode, type, sha, sizeText, relative] = match; - if (!sha || !relative || mode === "120000" || type !== "blob" || !relative.endsWith(".md")) - return []; - return [{ sha, relative, size: Number(sizeText) }]; - }); + const verifiedEntries = yield* readTree(orgId, paths.writer, head).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(VcsProcess, vcs), + ); + const entries = verifiedEntries + .filter( + ({ relative }) => + relative.endsWith(".md") && + (folders.some((folder) => relative.startsWith(`${folder}/`)) || + ["rules.md", "INDEX.md"].includes(relative)), + ) + .map((entry) => ({ ...entry, size: 0 })); const pending = [ ...new Map( entries @@ -504,67 +532,18 @@ const make = Effect.gen(function* () { }); for (let index = 0; index < pending.length; index += 32) { const batch = pending.slice(index, index + 32); - const chunks: Uint8Array[] = []; - const result = yield* git( + const objects = yield* readObjects( orgId, paths.writer, - ["cat-file", "--batch"], - batch.map((entry) => entry.sha).join("\n") + "\n", - (chunk) => chunks.push(chunk), - ).pipe(Effect.result); - const buffer = Buffer.concat(chunks); - let offset = 0; - let fallback = result._tag === "Failure"; - for (const entry of batch) { - if (!fallback) { - const end = buffer.indexOf(10, offset); - const header = end < 0 ? "" : buffer.subarray(offset, end).toString("ascii"); - offset = end + 1; - if (header === `${entry.sha} missing`) continue; - if ( - end < 0 || - header !== `${entry.sha} blob ${entry.size}` || - buffer.length < offset + entry.size + 1 || - buffer[offset + entry.size] !== 10 - ) { - fallback = true; - const error = new MemoryError({ - orgId, - operation: "batch", - reason: "Invalid git blob batch framing", - }); - yield* Effect.logWarning( - "Memory blob batch framing failed; reading remaining blobs individually", - { - orgId, - errorTag: error._tag, - path: entry.relative, - }, - ); - } else { - yield* decodeBlob(entry, buffer.subarray(offset, offset + entry.size)); - offset += entry.size + 1; - continue; - } - } - const singleChunks: Uint8Array[] = []; - const single = yield* git( - orgId, - paths.writer, - ["cat-file", "blob", entry.sha], - undefined, - (chunk) => singleChunks.push(chunk), - ).pipe(Effect.result); - if (single._tag === "Success") { - const bytes = Buffer.concat(singleChunks); - if (bytes.length === entry.size) yield* decodeBlob(entry, bytes); - else - yield* Effect.logWarning("Skipping invalid memory blob length", { - orgId, - path: entry.relative, - errorTag: "MemoryError", - }); - } + batch.map(({ sha }) => ({ sha, type: "blob" })), + ).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(VcsProcess, vcs), + ); + for (const [index, entry] of batch.entries()) { + const bytes = objects[index]; + if (bytes && bytes.length <= maxBlobBytes) yield* decodeBlob(entry, bytes); + else blobs.set(entry.sha, undefined); } } const items: Array = []; @@ -758,8 +737,153 @@ const make = Effect.gen(function* () { return text; }); - const prepare = Effect.fn("Memory.prepare")(function* (orgId: KinOrgId) { + const preserve = Effect.fn("Memory.preserve")(function* ( + orgId: KinOrgId, + writer: string, + url: string, + head: string, + wakeupId?: string, + ) { + const now = yield* Clock.currentTimeMillis; + const id = wakeupId ?? `recovery-${now}-${head.slice(0, 12)}`; + if (!/^[a-zA-Z0-9_-]+$/.test(id)) + return yield* new MemoryCheckoutInvalid({ + orgId, + operation: "preserve", + reason: "Invalid wake-up id", + }); + const branch = `kin/conflict/${orgId}/${id}`; + if ((yield* git(orgId, writer, ["ls-remote", "--", url, `refs/heads/${branch}`])).trim()) + return yield* new MemorySyncConflict({ + orgId, + operation: "preserve", + reason: "Conflict branch already exists", + }); + yield* git(orgId, writer, ["push", "--", url, `${head}:refs/heads/${branch}`]); + return branch; + }); + + const recover = Effect.fn("Memory.recover")(function* (orgId: KinOrgId) { + const paths = yield* pathsFor(orgId); + yield* io(orgId, "mkdir", fs.makeDirectory(paths.root, { recursive: true })); + const exists = yield* io(orgId, "exists", fs.exists(paths.writer)); + if (!exists) + yield* git(orgId, paths.root, [ + "clone", + "--no-checkout", + "--", + paths.repo.repoUrl, + paths.writer, + ]); + for (const name of [".git", ".git/objects", ".git/refs"]) { + const stat = yield* io(orgId, "recoverStructure", fs.lstat(`${paths.writer}/${name}`)); + if (stat.type !== "Directory") + return yield* new MemoryCheckoutInvalid({ + orgId, + operation: "recover", + reason: "Invalid writer directory", + }); + } + yield* git(orgId, paths.writer, [ + "fetch", + "--prune", + "--", + paths.repo.repoUrl, + "+refs/heads/*:refs/remotes/kin-recovery/*", + ]); + const refs = (yield* git(orgId, paths.writer, [ + "for-each-ref", + "--format=%(refname:strip=3) %(objectname)", + "refs/remotes/kin-recovery/", + ])) + .trim() + .split("\n") + .filter(Boolean); + const targets = new Map(); + for (const ref of refs) { + const [branch, head] = ref.split(" "); + if ( + branch && + head && + (branch === (config.defaultBranch ?? "main") || branch.startsWith("kin/shadow/")) + ) + targets.set(branch, head); + } + const main = targets.get(config.defaultBranch ?? "main"); + if (!main) + return yield* new MemoryCheckoutInvalid({ + orgId, + operation: "recover", + reason: "Missing remote main branch", + }); + const rows = yield* persistence( + orgId, + sql<{ + branch: string; + head: string; + }>`SELECT branch, head FROM kin_memory_heads WHERE org_id = ${orgId} ORDER BY branch`, + ); + for (const row of rows) + if (row.branch.startsWith("kin/shadow/") && !targets.has(row.branch)) + targets.set(row.branch, main); + const preserved = new Set(); + for (const [branch, head] of targets) { + yield* readFiles(orgId, paths.writer, head).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(VcsProcess, vcs), + ); + const old = yield* anchor(orgId, branch, head); + if (old !== head && exists) { + const local = yield* readCommit(orgId, paths.writer, old).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(VcsProcess, vcs), + Effect.result, + ); + const present = yield* git(orgId, paths.writer, [ + "merge-base", + "--is-ancestor", + old, + head, + ]).pipe(Effect.result); + if (local._tag === "Success" && present._tag === "Failure" && !preserved.has(old)) { + const conflictBranch = yield* preserve(orgId, paths.writer, paths.repo.repoUrl, old); + preserved.add(old); + yield* notify({ + orgId, + branch, + reason: "Preserved old anchor before recovery", + conflictBranch, + }); + } + } + const local = yield* git(orgId, paths.writer, [ + "rev-parse", + "--verify", + `refs/heads/${branch}`, + ]).pipe(Effect.result); + yield* git(orgId, paths.writer, [ + "update-ref", + `refs/heads/${branch}`, + head, + local._tag === "Success" ? local.success.trim() : "0".repeat(head.length), + ]); + if (old !== head) yield* record(orgId, branch, old, head); + yield* notify({ orgId, branch, reason: "Recovered from remote head" }); + } + pinnedHeads.delete(orgId); + snapshots.delete(orgId); + blobCaches.delete(orgId); + }); + + const prepare = Effect.fn("Memory.prepare")(function* (orgId: KinOrgId, wakeupId?: string) { const paths = yield* ensureCheckouts(orgId); + yield* git(orgId, paths.writer, [ + "fsck", + "--full", + "--strict", + "--no-reflogs", + yield* anchor(orgId, config.defaultBranch ?? "main"), + ]); for (const marker of ["rebase-merge", "rebase-apply", "MERGE_HEAD", "CHERRY_PICK_HEAD"]) { if (yield* io(orgId, "sync", fs.exists(`${paths.writer}/.git/${marker}`))) return yield* new MemoryCheckoutInvalid({ @@ -796,17 +920,36 @@ const make = Effect.gen(function* () { ]).pipe(Effect.result); if (ahead._tag === "Success") next = head; else { - const commits = (yield* git(orgId, paths.writer, [ - "rev-list", - "--reverse", - "--first-parent", - `${remote}..${head}`, - ])) - .trim() - .split("\n") - .filter(Boolean); + const base = (yield* git(orgId, paths.writer, ["merge-base", remote, head])).trim(); + const commits: string[] = []; + let cursor: string | undefined = head; + while (cursor !== base) { + if (!cursor || commits.includes(cursor)) + return yield* new MemoryCheckoutInvalid({ + orgId, + operation: "rebase", + reason: "Invalid commit ancestry", + }); + const sha: string = cursor; + const verified = yield* readCommit(orgId, paths.writer, sha).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(VcsProcess, vcs), + ); + commits.unshift(sha); + cursor = verified.parent; + } for (const commit of commits) { - const parent = (yield* git(orgId, paths.writer, ["rev-parse", `${commit}^`])).trim(); + const verified = yield* readCommit(orgId, paths.writer, commit).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(VcsProcess, vcs), + ); + const parent = verified.parent; + if (!parent) + return yield* new MemoryCheckoutInvalid({ + orgId, + operation: "rebase", + reason: "Missing parent", + }); const merged = yield* git(orgId, paths.writer, [ "merge-tree", "--write-tree", @@ -814,25 +957,131 @@ const make = Effect.gen(function* () { next, commit, ]).pipe(Effect.result); - if (merged._tag === "Failure") - return yield* new MemorySyncConflict({ - orgId, - operation: "sync", - reason: "Plumbing rebase conflicted; branch unchanged", - }); - const tree = merged.success.trim().split("\n")[0]; + let tree: string | undefined; + if (merged._tag === "Failure") { + const read = (sha: string) => + readFiles(orgId, paths.writer, sha).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(VcsProcess, vcs), + ); + const base = yield* read(parent); + const local = yield* read(commit); + const remoteFiles = yield* read(next); + const changed = [...new Set([...base.keys(), ...local.keys()])].filter( + (file) => !base.get(file)?.equals(local.get(file) ?? Buffer.alloc(0)), + ); + const same = (left: Buffer | undefined, right: Buffer | undefined) => + left === undefined + ? right === undefined + : right !== undefined && left.equals(right); + const journalOnly = changed.every( + (file) => + file === "INDEX.md" || + same(remoteFiles.get(file), base.get(file)) || + same(remoteFiles.get(file), local.get(file)) || + (file.startsWith("journal/") && + local + .get(file) + ?.subarray(0, base.get(file)?.length ?? 0) + .equals(base.get(file) ?? Buffer.alloc(0)) && + remoteFiles + .get(file) + ?.subarray(0, base.get(file)?.length ?? 0) + .equals(base.get(file) ?? Buffer.alloc(0))), + ); + if (!journalOnly) { + const conflictBranch = yield* preserve( + orgId, + paths.writer, + repo.repoUrl, + head, + wakeupId, + ); + next = remote; + yield* notify({ + orgId, + branch, + reason: "Content conflict preserved", + conflictBranch, + }); + break; + } + for (const file of changed) { + if (file === "INDEX.md" || same(remoteFiles.get(file), local.get(file))) continue; + const bytes = local.get(file); + if (same(remoteFiles.get(file), base.get(file))) { + if (bytes) remoteFiles.set(file, bytes); + else remoteFiles.delete(file); + } else + remoteFiles.set( + file, + Buffer.concat([ + remoteFiles.get(file) ?? Buffer.alloc(0), + bytes?.subarray(base.get(file)?.length ?? 0) ?? Buffer.alloc(0), + ]), + ); + } + const heads = []; + for (const [file, bytes] of remoteFiles) { + if (!folders.some((folder) => file.startsWith(`${folder}/`))) continue; + const parsed = yield* parseFrontMatter(orgId, file, bytes.toString("utf8")).pipe( + Effect.result, + ); + if (parsed._tag === "Success") + heads.push({ ...parsed.success.frontMatter, orgId, path: file }); + } + const reads = yield* io(orgId, "mergeIndex", cache.lastReads(orgId)); + const zone = yield* io(orgId, "mergeIndex", cache.ownerTimeZone(orgId)); + remoteFiles.set( + "INDEX.md", + Buffer.from(buildIndex(heads, reads, yield* Clock.currentTimeMillis, zone)), + ); + tree = yield* Effect.scoped( + Effect.gen(function* () { + const temp = yield* io(orgId, "journalMerge", fs.makeTempDirectoryScoped()); + const env = { GIT_INDEX_FILE: `${temp}/index` }; + yield* memoryGit(paths.writer, ["read-tree", "--empty"], undefined, env).pipe( + Effect.mapError( + (cause) => new MemoryError({ orgId, operation: "merge", cause }), + ), + ); + for (const [file, bytes] of remoteFiles) { + yield* io(orgId, "mergeBlob", fs.writeFile(`${temp}/blob`, bytes)); + const blob = (yield* git(orgId, paths.writer, [ + "hash-object", + "-w", + "--no-filters", + `${temp}/blob`, + ])).trim(); + yield* memoryGit( + paths.writer, + ["update-index", "--add", "--cacheinfo", `100644,${blob},${file}`], + undefined, + env, + ).pipe( + Effect.mapError( + (cause) => new MemoryError({ orgId, operation: "merge", cause }), + ), + ); + } + return (yield* memoryGit(paths.writer, ["write-tree"], undefined, env).pipe( + Effect.mapError( + (cause) => new MemoryError({ orgId, operation: "merge", cause }), + ), + )).trim(); + }), + ); + } else tree = merged.success.trim().split("\n")[0]; if (!tree || !/^[0-9a-f]{40,64}$/.test(tree)) return yield* new MemorySyncConflict({ orgId, operation: "sync", reason: "Invalid merge tree", }); - const message = yield* git(orgId, paths.writer, [ - "show", - "--format=%B", - "--no-patch", - commit, - ]); + const { message } = yield* readCommit(orgId, paths.writer, commit).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(VcsProcess, vcs), + ); next = (yield* memoryGit(paths.writer, ["commit-tree", tree, "-p", next], message, { GIT_AUTHOR_NAME: config.botName ?? "Kin Memory Bot", GIT_AUTHOR_EMAIL: config.botEmail ?? "kin-memory@localhost", @@ -845,6 +1094,13 @@ const make = Effect.gen(function* () { } } if (next !== head) { + if (next !== remote) + yield* git(orgId, paths.writer, [ + "push", + "--", + repo.repoUrl, + `${next}:refs/heads/${branch}`, + ]); yield* git(orgId, paths.writer, ["update-ref", `refs/heads/${branch}`, next, head]); yield* record(orgId, branch, head, next); head = next; @@ -872,7 +1128,9 @@ const make = Effect.gen(function* () { ), ); pinnedHeads.set(orgId, head); - const index = yield* readFile(orgId, "INDEX.md").pipe(Effect.orElseSucceed(() => "")); + const index = yield* readFile(orgId, "INDEX.md").pipe( + Effect.catchTags({ MemoryError: () => Effect.succeed("") }), + ); const asOf = /^As of ([^\r\n]+)(?:\r?\n|$)/.exec(index)?.[1]; const now = yield* Clock.currentTimeMillis; if ( @@ -892,13 +1150,16 @@ const make = Effect.gen(function* () { return Memory.of({ withWriterLock, + recover: Effect.fn("Memory.recoverLocked")((orgId) => + withWriterLock(orgId, locked(orgId, recover(orgId))), + ), anchor, record, ensureCheckouts: Effect.fn("Memory.ensureCheckoutsLocked")((orgId) => locked(orgId, ensureCheckouts(orgId)), ), - prepareWakeup: Effect.fn("Memory.prepareWakeup")((orgId) => - withWriterLock(orgId, locked(orgId, prepare(orgId))), + prepareWakeup: Effect.fn("Memory.prepareWakeup")((orgId, wakeupId) => + withWriterLock(orgId, locked(orgId, prepare(orgId, wakeupId))), ), readIndex: Effect.fn("Memory.readIndex")((orgId) => locked(orgId, readFile(orgId, "INDEX.md"))), readRules: Effect.fn("Memory.readRules")((orgId) => locked(orgId, readFile(orgId, "rules.md"))), diff --git a/apps/server/src/kin/memory/MemoryConfig.ts b/apps/server/src/kin/memory/MemoryConfig.ts index fc61f4967a950..045cee1547640 100644 --- a/apps/server/src/kin/memory/MemoryConfig.ts +++ b/apps/server/src/kin/memory/MemoryConfig.ts @@ -12,6 +12,8 @@ export class MemoryConfig extends Context.Service< readonly baseDirectory: string; readonly defaultBranch?: string; readonly botName?: string; + readonly credentialHelper?: string; + readonly sshCommand?: string; readonly botEmail?: string; readonly repositories: ReadonlyArray<{ readonly orgId: KinOrgId; diff --git a/apps/server/src/kin/memory/MemoryWriter.test.ts b/apps/server/src/kin/memory/MemoryWriter.test.ts index 98e1d57118e01..c1cb0db2270b2 100644 --- a/apps/server/src/kin/memory/MemoryWriter.test.ts +++ b/apps/server/src/kin/memory/MemoryWriter.test.ts @@ -1,3 +1,5 @@ +import { memoryGit } from "./git.ts"; +import * as NodeZlib from "node:zlib"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { expect, it } from "@effect/vitest"; import { KinAgentId, KinOrgId } from "@t3tools/contracts"; @@ -32,6 +34,16 @@ it.live.each([ "B6-owner-closed", "B6-agent-own", "B7-identity", + "P3-object", + "P4-journal", + "P6-malformed", + "P7-crash", + "P8-lost-writer", + "P9-shadow", + "sha256", + "review-default", + "review-null", + "credentials", ] as const)("writer real git: %s", (scenario) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -56,31 +68,65 @@ it.live.each([ "-m", "fixture", ]); - yield* git(source, ["init", "-b", "main"]); + yield* git(source, [ + "init", + "-b", + "main", + ...(scenario === "sha256" ? ["--object-format=sha256"] : []), + ]); yield* fs.writeFileString(`${source}/rules.md`, "Invented rules\n"); yield* fs.writeFileString(`${source}/INDEX.md`, "As of 2026-11-08T00:00:00.000Z\n"); - if (scenario.startsWith("B6")) { + if (scenario.startsWith("B6") || scenario === "P3-object" || scenario === "P6-malformed") { yield* fs.makeDirectory(`${source}/claims`); yield* fs.writeFileString( `${source}/claims/existing.md`, - fixtureItem({ - kind: "claim", - status: scenario === "B6-owner-open" ? "open" : "done", - trust: scenario === "B6-agent-own" ? "agent" : "owner", - said_by: scenario === "B6-agent-own" ? agentId : "fixture-owner", - }), + scenario === "P6-malformed" + ? "---\nid: [\n---\n" + : fixtureItem({ + kind: "claim", + status: scenario === "B6-owner-open" ? "open" : "done", + trust: scenario === "B6-agent-own" ? "agent" : "owner", + said_by: scenario === "B6-agent-own" ? agentId : "fixture-owner", + }), ); } - if (scenario === "B5-journal") { + if (scenario === "B5-journal" || scenario === "P4-journal") { yield* fs.makeDirectory(`${source}/journal/2026`, { recursive: true }); - yield* fs.writeFileString(`${source}/journal/2026/11-08.md`, "Owner journal preserved.\n"); + yield* fs.writeFile( + `${source}/journal/2026/11-08.md`, + Buffer.concat([ + scenario === "B5-journal" ? Buffer.from([255, 10]) : Buffer.alloc(0), + Buffer.from("Owner journal preserved.\n"), + ]), + ); } yield* git(source, ["add", "--", "."]); yield* commit(source); yield* git(root, ["clone", "--bare", "--", source, remote]); + let attack: { objectPath: string; sha: string; expression: string; bytes: Buffer } | undefined; + let swapped = false; + const interceptedGit = Layer.succeed( + Vcs.VcsProcess, + Vcs.VcsProcess.of({ + run: Effect.fn("WriterTest.objectSwap")(function* (input) { + if ( + attack && + input.args.includes("cat-file") && + (input.stdin?.includes(attack.sha) || input.args.includes(attack.expression)) + ) { + yield* fs.remove(attack.objectPath).pipe(Effect.orDie); + yield* fs.writeFile(attack.objectPath, attack.bytes).pipe(Effect.orDie); + attack = undefined; + swapped = true; + } + return yield* vcs.run(input); + }), + }), + ); const notices: Writer.WriterNotice[] = []; const config = Layer.succeed(MemoryConfig, { baseDirectory: root, + credentialHelper: "fixture-server-helper", repositories: [{ orgId, repoUrl: remote, path: "state" }], }); const memoryLayer = Memory.layer.pipe(Layer.provide(config)); @@ -90,7 +136,6 @@ it.live.each([ notices.push(notice); }), ), - buildIndex: Effect.fn("Test.index")(() => Effect.succeed(null)), }); const auth = Layer.succeed(Writer.MemoryWriterAuthority, { resolve: Effect.fn("Test.authority")( @@ -144,6 +189,98 @@ it.live.each([ yield* git(paths.writer, ["config", "user.name", "Fixture Bot"]); yield* git(paths.writer, ["config", "user.email", "fixture@example.invalid"]); const original = yield* git(paths.writer, ["rev-parse", "HEAD"]); + if (scenario === "credentials") { + yield* git(paths.writer, ["config", "credential.helper", "fixture-repo-helper"]); + const helper = yield* memoryGit(paths.writer, [ + "config", + "--get-all", + "credential.helper", + ]).pipe( + Effect.provideService(MemoryConfig, { + baseDirectory: root, + credentialHelper: "fixture-server-helper", + repositories: [], + }), + ); + expect(helper.trim()).toBe("fixture-server-helper"); + return; + } + if (scenario === "review-default" || scenario === "review-null") { + const content = fixtureItem({ + kind: "claim", + review_by: null, + recorded_at: "2026-11-01T00:00:00.000Z", + }); + yield* writer.stage(orgId, "wake1", { + path: "claims/default.md", + content: + scenario === "review-default" ? content.replace(/^review_by:.*\n/m, "") : content, + }); + const { commit } = yield* writer.commitWakeup(orgId, "wake1", "completed"); + const text = yield* git(paths.writer, ["show", `${commit}:claims/default.md`]); + expect(text).toContain( + scenario === "review-default" ? "2026-12-01T00:00:00.000Z" : "review_by: null", + ); + expect(yield* git(paths.writer, ["show", `${commit}:INDEX.md`])).toContain( + "claims/default.md", + ); + expect(yield* git(paths.writer, ["rev-list", "--count", `${original}..${commit}`])).toBe( + "1", + ); + return; + } + if (scenario === "P8-lost-writer" || scenario === "P9-shadow") { + if (scenario === "P9-shadow") yield* writer.commitWakeup(orgId, "wake0", "completed"); + yield* fs.remove(paths.writer, { recursive: true }); + yield* memory.recover(orgId); + yield* memory.prepareWakeup(orgId); + const result = yield* writer.commitWakeup(orgId, "wake1", "completed"); + expect( + yield* git(root, [ + `--git-dir=${remote}`, + "rev-parse", + `refs/heads/kin/shadow/${agentId}`, + ]), + ).toBe(result.commit); + return; + } + if (scenario === "P3-object" || scenario === "P4-journal") { + const file = scenario === "P3-object" ? "claims/existing.md" : "journal/2026/11-08.md"; + const sha = yield* git(paths.writer, ["rev-parse", `${original}:${file}`]); + const forged = Buffer.from("forged bytes after prepare\n"); + attack = { + objectPath: `${paths.writer}/.git/objects/${sha.slice(0, 2)}/${sha.slice(2)}`, + sha, + expression: `${original}:${file}`, + bytes: NodeZlib.deflateSync( + Buffer.concat([Buffer.from(`blob ${forged.length}\0`), forged]), + ), + }; + const result = + scenario === "P3-object" + ? yield* writer + .stage(orgId, "wake1", { + path: "claims/existing.md", + content: fixtureItem({ kind: "claim" }), + }) + .pipe(Effect.result) + : yield* writer.commitWakeup(orgId, "wake1", "completed").pipe(Effect.result); + expect(swapped).toBe(true); + expect(result._tag).toBe("Failure"); + if (result._tag === "Failure") expect(result.failure._tag).toBe("MemoryCheckoutInvalid"); + return; + } + if (scenario === "P6-malformed") { + const result = yield* writer + .stage(orgId, "wake1", { + path: "claims/existing.md", + content: fixtureItem({ kind: "claim" }), + }) + .pipe(Effect.result); + expect(result._tag).toBe("Failure"); + if (result._tag === "Failure") expect(result.failure._tag).toBe("MemoryWriteRejected"); + return; + } for (const path of [ "rules.md", "INDEX.md", @@ -192,14 +329,14 @@ it.live.each([ expect(yield* git(paths.writer, ["rev-parse", "HEAD"])).toBe(original); yield* git(paths.writer, ["fetch", "--", paths.read, forged]); yield* git(paths.writer, ["update-ref", "refs/heads/main", forged]); - expect((yield* memory.prepareWakeup(orgId).pipe(Effect.result))._tag).toBe("Failure"); + expect((yield* memory.prepareWakeup(orgId).pipe(Effect.result))._tag).toBe("Success"); + expect(yield* memory.readRules(orgId)).toBe("Invented rules\n"); expect( (yield* writer.commitWakeup(orgId, "wake1", "completed").pipe(Effect.result))._tag, - ).toBe("Failure"); - expect(notices.at(-1)?.error._tag).toBe("MemoryCheckoutInvalid"); + ).toBe("Success"); return; } - if (scenario === "B1-shadow" || scenario === "B2-anchor") { + if (scenario === "B1-shadow" || scenario === "B2-anchor" || scenario === "P7-crash") { yield* fs.writeFileString(`${paths.read}/rules.md`, "Forged rules\n"); yield* git(paths.read, ["add", "rules.md"]); yield* commit(paths.read); @@ -220,7 +357,7 @@ it.live.each([ ); expect( (yield* writer.commitWakeup(orgId, "wake1", "completed").pipe(Effect.result))._tag, - ).toBe("Failure"); + ).toBe(scenario === "B2-anchor" || scenario === "P7-crash" ? "Success" : "Failure"); return; } if (scenario.startsWith("B6")) { @@ -355,6 +492,15 @@ it.live.each([ if (scenario === "B5-journal") { const journal = yield* git(paths.writer, ["show", `${branch}:journal/2026/11-08.md`]); expect(journal).toContain("Owner journal preserved."); + const chunks: Uint8Array[] = []; + yield* vcs.run({ + operation: "WriterTest.journalBytes", + command: "git", + cwd: paths.writer, + args: ["show", `${branch}:journal/2026/11-08.md`], + onStdoutChunk: (chunk) => chunks.push(chunk), + }); + expect(Buffer.concat(chunks).subarray(0, 2)).toEqual(Buffer.from([255, 10])); expect(journal).not.toContain("Forged journal"); } const again = yield* writer.commitWakeup( @@ -375,7 +521,7 @@ it.live.each([ `Outcome: ${scenario === "failed" ? "failed" : "completed"}`, ); } - }).pipe(Effect.provide(layer)); + }).pipe(Effect.provide(layer.pipe(Layer.provide(interceptedGit)))); }).pipe( Effect.provide( Layer.mergeAll( diff --git a/apps/server/src/kin/memory/MemoryWriter.ts b/apps/server/src/kin/memory/MemoryWriter.ts index e6f161e433ac7..921183a48aa3e 100644 --- a/apps/server/src/kin/memory/MemoryWriter.ts +++ b/apps/server/src/kin/memory/MemoryWriter.ts @@ -1,4 +1,7 @@ -import * as DateTime from "effect/DateTime"; +import * as Clock from "effect/Clock"; +import { readFiles, readCommit } from "./objects.ts"; +import { buildIndex } from "./buildIndex.ts"; +import * as MemoryCache from "./MemoryCache.ts"; import type { KinAgentId, KinOrgId, MemoryTrust } from "@t3tools/contracts"; import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; @@ -13,6 +16,7 @@ import { MemoryConfig } from "./MemoryConfig.ts"; import { Memory, MemoryCheckoutInvalid, + MemoryRecoveryHooks, MemoryError, MemorySyncConflict, MemoryConfigError, @@ -70,11 +74,6 @@ export class MemoryWriterHooks extends Context.Service< MemoryWriterHooks, { readonly notify: (notice: WriterNotice) => Effect.Effect; - readonly buildIndex: (input: { - orgId: KinOrgId; - writer: string; - recorded: string; - }) => Effect.Effect; } >()("t3/kin/memory/MemoryWriter/MemoryWriterHooks") {} export interface MemoryChange { @@ -122,6 +121,7 @@ const folders = ["objectives", "commitments", "deferrals", "decisions", "people" const render = (fields: object, body: string) => `---\n${stringify(fields)}---\n${body}`; const make = Effect.gen(function* () { const memory = yield* Memory; + const recovery = Option.getOrUndefined(yield* Effect.serviceOption(MemoryRecoveryHooks)); const authority = yield* MemoryWriterAuthority; const hooks = yield* MemoryWriterHooks; const scanner = yield* KinSecretScan; @@ -131,8 +131,20 @@ const make = Effect.gen(function* () { runMemoryGit(...args).pipe( Effect.provideService(FileSystem.FileSystem, fs), Effect.provideService(VcsProcess, vcs), + Effect.provideService(MemoryConfig, config), ); const config = yield* MemoryConfig; + const cache = yield* MemoryCache.MemoryCache; + const filesAt = (orgId: KinOrgId, writer: string, commit: string) => + readFiles(orgId, writer, commit).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(VcsProcess, vcs), + ); + const commitAt = (orgId: KinOrgId, writer: string, commit: string) => + readCommit(orgId, writer, commit).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(VcsProcess, vcs), + ); const locked = memory.withWriterLock; const sessions = new Map< KinOrgId, @@ -149,6 +161,65 @@ const make = Effect.gen(function* () { ) { return (yield* io(orgId, memoryGit(cwd, args, stdin, identity))).trim(); }); + const branchHead = Effect.fn("MemoryWriter.branchHead")(function* ( + orgId: KinOrgId, + writer: string, + branch: string, + ) { + const anchored = yield* memory.anchor(orgId, branch).pipe(Effect.result); + const local = yield* git(orgId, writer, ["rev-parse", "--verify", `refs/heads/${branch}`]).pipe( + Effect.result, + ); + if (anchored._tag === "Success") { + if (local._tag === "Failure") + return yield* new MemoryCheckoutInvalid({ + orgId, + operation: "branch", + reason: "Missing recorded branch", + }); + if (local.success !== anchored.success) { + yield* commitAt(orgId, writer, local.success); + yield* git(orgId, writer, ["merge-base", "--is-ancestor", anchored.success, local.success]); + yield* git(orgId, writer, [ + "update-ref", + `refs/heads/${branch}`, + anchored.success, + local.success, + ]); + const notice = { orgId, branch, reason: "Rolled back unrecorded branch tip" }; + yield* recovery ? recovery.notify(notice) : Effect.logWarning("Memory recovery", notice); + } + return anchored.success; + } + if (local._tag === "Success") + return yield* new MemoryCheckoutInvalid({ + orgId, + operation: "branch", + reason: "Unanchored local branch", + }); + const repo = config.repositories.find((repo) => repo.orgId === orgId); + if (!repo) + return yield* new MemoryConfigError({ orgId, operation: "branch", reason: "Unknown org" }); + const main = yield* memory.anchor(orgId, config.defaultBranch ?? "main"); + const advertised = yield* git(orgId, writer, [ + "ls-remote", + "--", + repo.repoUrl, + `refs/heads/${branch}`, + ]); + const remote = advertised + ? yield* git(orgId, writer, ["fetch", "--", repo.repoUrl, `refs/heads/${branch}`]) + : main; + yield* filesAt(orgId, writer, remote); + const recorded = yield* memory.anchor(orgId, branch, remote); + yield* git(orgId, writer, [ + "update-ref", + `refs/heads/${branch}`, + recorded, + "0".repeat(recorded.length), + ]); + return recorded; + }); const session = Effect.fn("MemoryWriter.session")(function* (orgId: KinOrgId, wakeupId: string) { const ctx = yield* authority.resolve(orgId, wakeupId); if ( @@ -242,7 +313,7 @@ const make = Effect.gen(function* () { return yield* new MemoryWriteRejected({ reason: "Use appendJournal for journal additions", }); - const parsed = yield* parseFrontMatter(orgId, change.path, change.content); + const parsed = yield* parseFrontMatter(orgId, change.path, change.content, true); const source = change.sourceEventId === undefined ? { saidBy: ctx.agentId, trust: "agent", source: `wakeup:${wakeupId}` } @@ -257,44 +328,25 @@ const make = Effect.gen(function* () { return yield* new MemoryWriteRejected({ reason: "Kind does not match path" }); const branch = ctx.level === "L0" ? `kin/shadow/${ctx.agentId}` : (config.defaultBranch ?? "main"); - const anchored = yield* memory.anchor(orgId, branch).pipe(Effect.result); - const local = yield* git(orgId, writer, [ - "rev-parse", - "--verify", - `refs/heads/${branch}`, - ]).pipe(Effect.result); - if ( - (anchored._tag === "Success" && - (local._tag === "Failure" || local.success !== anchored.success)) || - (anchored._tag === "Failure" && local._tag === "Success") - ) - return yield* new MemoryCheckoutInvalid({ - orgId, - operation: "stage", - reason: "Unrecorded branch tip", - }); - const recorded = - anchored._tag === "Success" - ? anchored.success - : yield* memory.anchor(orgId, config.defaultBranch ?? "main"); - const tree = yield* git(orgId, writer, [ - "ls-tree", - "-r", - "--name-only", - "-z", - recorded, - "--", - ...folders, - ]); + const recorded = yield* branchHead(orgId, writer, branch); + const recordedFiles = yield* filesAt(orgId, writer, recorded); const items = []; const bodies = new Map(); - for (const file of tree.split("\0").filter(Boolean)) { - const content = yield* io( - orgId, - memoryGit(writer, ["cat-file", "blob", `${recorded}:${file}`]), - ); + for (const [file, bytes] of recordedFiles) { + if (!folders.some((folder) => file.startsWith(`${folder}/`))) continue; + const content = bytes.toString("utf8"); const parsedItem = yield* parseFrontMatter(orgId, file, content).pipe(Effect.result); - if (parsedItem._tag === "Failure") continue; + if (parsedItem._tag === "Failure") { + if (file === change.path) + return yield* new MemoryWriteRejected({ + reason: "Cannot overwrite malformed recorded item", + }); + continue; + } + if (file === change.path && parsedItem.success.frontMatter.kind !== fm.kind) + return yield* new MemoryWriteRejected({ + reason: "Cannot change recorded item kind", + }); items.push({ ...parsedItem.success.frontMatter, orgId, @@ -319,6 +371,7 @@ const make = Effect.gen(function* () { if ( change.supersedes !== undefined && (!superseded || + superseded.status !== "open" || superseded.kind !== fm.kind || superseded.source !== source.source || Date.parse(fm.recorded_at) <= Date.parse(superseded.recorded_at) || @@ -376,18 +429,6 @@ const make = Effect.gen(function* () { reason: "Cannot replace a different item", }); } - const review = - fm.review_by !== undefined - ? fm.review_by - : fm.kind === "decision" - ? null - : fm.kind === "commitment" || fm.kind === "deferral" - ? fm.due - : DateTime.formatIso( - DateTime.makeUnsafe( - Date.parse(ctx.startedAt) + (fm.kind === "person" ? 90 : 30) * 86400000, - ), - ); const fields = { ...fm, said_by: source.saidBy, @@ -398,7 +439,7 @@ const make = Effect.gen(function* () { (fm.trust === "member" && source.trust === "owner") ? fm.trust : source.trust, - review_by: review, + review_by: fm.review_by, }; const existing = items.find((item) => item.id === fm.id); const text = render( @@ -471,48 +512,7 @@ const make = Effect.gen(function* () { }); const branch = ctx.level === "L0" ? `kin/shadow/${ctx.agentId}` : (config.defaultBranch ?? "main"); - const main = yield* memory.anchor(orgId, config.defaultBranch ?? "main"); - const anchored = yield* memory.anchor(orgId, branch).pipe(Effect.result); - let recorded: string; - if (anchored._tag === "Failure") { - const exists = yield* git(orgId, writer, [ - "rev-parse", - "--verify", - `refs/heads/${branch}`, - ]).pipe(Effect.result); - if (exists._tag === "Success") - return yield* new MemoryCheckoutInvalid({ - orgId, - operation: "branch", - reason: "Unanchored local branch", - }); - const fresh = yield* git(orgId, writer, [ - "fetch", - "--", - repo.repoUrl, - `refs/heads/${config.defaultBranch ?? "main"}`, - ]); - if (fresh !== main) - return yield* new MemoryCheckoutInvalid({ - orgId, - operation: "bootstrap", - reason: "Prepare required before branch creation", - }); - recorded = yield* memory.anchor(orgId, branch, fresh); - yield* git(orgId, writer, [ - "update-ref", - `refs/heads/${branch}`, - recorded, - "0000000000000000000000000000000000000000", - ]); - } else recorded = anchored.success; - const tip = yield* git(orgId, writer, ["rev-parse", `refs/heads/${branch}`]); - if (tip !== recorded) - return yield* new MemoryCheckoutInvalid({ - orgId, - operation: "branch", - reason: "Unrecorded branch tip", - }); + const recorded = yield* branchHead(orgId, writer, branch); const push = (sha: string) => git(orgId, writer, ["push", "--", repo.repoUrl, `${sha}:refs/heads/${branch}`]).pipe( Effect.mapError( @@ -538,12 +538,7 @@ const make = Effect.gen(function* () { lines: state.lines, }), ); - const body = yield* git(orgId, writer, [ - "show", - "--format=%B", - "--no-patch", - recorded, - ]); + const { message: body } = yield* commitAt(orgId, writer, recorded); if (body.split("\n")[0] === subject) { if ( !body.includes(`Kin-Outcome: ${outcome}`) || @@ -557,27 +552,42 @@ const make = Effect.gen(function* () { if (outcome === "failed") state.files.clear(); const entry = yield* journalEntry(ctx, outcome, state.lines); yield* safePath(orgId, writer, entry.path); - const oldResult = yield* git(orgId, writer, [ - "cat-file", - "blob", - `${recorded}:${entry.path}`, - ]).pipe(Effect.result); - // Preserve journal bytes; the general git wrapper trims command output. - const old = - oldResult._tag === "Success" - ? yield* io( - orgId, - memoryGit(writer, ["cat-file", "blob", `${recorded}:${entry.path}`]), - ) - : ""; - state.files.set(entry.path, old + entry.text); - const index = yield* hooks.buildIndex({ orgId, writer, recorded }); - if (index !== null) state.files.set("INDEX.md", index); + const recordedFiles = yield* filesAt(orgId, writer, recorded); + const journal = Buffer.concat([ + recordedFiles.get(entry.path) ?? Buffer.alloc(0), + Buffer.from(entry.text), + ]); + state.files.set(entry.path, journal.toString("utf8")); + const combined = new Map( + [...recordedFiles].map(([path, bytes]) => [path, bytes.toString("utf8")]), + ); + for (const [path, text] of state.files) combined.set(path, text); + const heads = []; + for (const [path, text] of combined) { + if (!folders.some((folder) => path.startsWith(`${folder}/`))) continue; + const parsed = yield* parseFrontMatter(orgId, path, text).pipe(Effect.result); + if (parsed._tag === "Success") + heads.push({ ...parsed.success.frontMatter, orgId, path }); + } + const reads = yield* cache + .lastReads(orgId) + .pipe( + Effect.mapError((cause) => new MemoryError({ orgId, operation: "index", cause })), + ); + const zone = yield* cache + .ownerTimeZone(orgId) + .pipe( + Effect.mapError((cause) => new MemoryError({ orgId, operation: "index", cause })), + ); + state.files.set( + "INDEX.md", + buildIndex(heads, reads, yield* Clock.currentTimeMillis, zone), + ); const findings = yield* scanner.scan({ orgId, files: [...state.files].map(([path, content]) => ({ path, - content: new TextEncoder().encode(content), + content: path === entry.path ? journal : new TextEncoder().encode(content), })), }); if (findings.length) return yield* new MemorySecretBlocked({ orgId, findings }); @@ -591,15 +601,19 @@ const make = Effect.gen(function* () { GIT_COMMITTER_NAME: config.botName ?? "Kin Memory Bot", GIT_COMMITTER_EMAIL: config.botEmail ?? "kin-memory@localhost", }; - yield* git(orgId, writer, ["read-tree", recorded], env); - for (const [file, text] of state.files) { - yield* safePath(orgId, writer, file, true); + yield* git(orgId, writer, ["read-tree", "--empty"], env); + const finalFiles = new Map(recordedFiles); + for (const [file, text] of state.files) + finalFiles.set(file, file === entry.path ? journal : Buffer.from(text)); + for (const [file, bytes] of finalFiles) { + if (state.files.has(file)) yield* safePath(orgId, writer, file, true); + const input = `${temporary}/blob`; + yield* io(orgId, fs.writeFile(input, bytes)); const blob = yield* git( orgId, writer, - ["hash-object", "-w", "--no-filters", "--stdin"], + ["hash-object", "-w", "--no-filters", input], env, - text, ); yield* git( orgId, @@ -635,4 +649,4 @@ const make = Effect.gen(function* () { ), }); }); -export const layer = Layer.effect(MemoryWriter, make); +export const layer = Layer.effect(MemoryWriter, make).pipe(Layer.provide(MemoryCache.layer)); diff --git a/apps/server/src/kin/memory/frontMatter.ts b/apps/server/src/kin/memory/frontMatter.ts index 737e7d8d60416..692502849fca7 100644 --- a/apps/server/src/kin/memory/frontMatter.ts +++ b/apps/server/src/kin/memory/frontMatter.ts @@ -2,6 +2,7 @@ import { MemoryItemFrontMatter, type KinOrgId } from "@t3tools/contracts"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Schema from "effect/Schema"; +import { defaultReviewBy } from "./freshness.ts"; import { isSeq, parseDocument } from "yaml"; export class MemoryItemError extends Schema.TaggedError()("MemoryItemError", { @@ -34,12 +35,19 @@ export const memoryDateMillis = (value: string | null) => { }; const decodeFrontMatter = Schema.decodeUnknownEffect(MemoryItemFrontMatter); +const decodeStaged = Schema.decodeUnknownEffect( + Schema.Struct({ + ...MemoryItemFrontMatter.fields, + review_by: Schema.optional(MemoryItemFrontMatter.fields.review_by), + }), +); // Parsing runs per blob under Memory.load; avoid creating a span for every item. export const parseFrontMatter = Effect.fnUntraced(function* ( orgId: KinOrgId, path: string, text: string, + staged = false, ) { const parts = /^(?:\uFEFF)?---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)([\s\S]*)$/.exec(text); if (!parts) { @@ -85,14 +93,14 @@ export const parseFrontMatter = Effect.fnUntraced(function* ( cause, }), }); - const frontMatter = yield* decodeFrontMatter(value).pipe( + const decoded = yield* (staged ? decodeStaged(value) : decodeFrontMatter(value)).pipe( Effect.mapError( (cause) => new MemoryItemError({ orgId, path, reason: "Invalid front matter schema", cause }), ), ); for (const field of ["recorded_at", "valid_from", "invalid_at", "review_by", "due"] as const) { - const value = frontMatter[field]; - if (value !== null && !Number.isFinite(memoryDateMillis(value))) { + const value = decoded[field]; + if (value !== null && value !== undefined && !Number.isFinite(memoryDateMillis(value))) { return yield* new MemoryItemError({ orgId, path, @@ -100,5 +108,16 @@ export const parseFrontMatter = Effect.fnUntraced(function* ( }); } } + const frontMatter = { + ...decoded, + review_by: defaultReviewBy( + { + kind: decoded.kind, + due: decoded.due, + ...(decoded.review_by === undefined ? {} : { review_by: decoded.review_by }), + }, + Date.parse(decoded.recorded_at), + ), + }; return { frontMatter, body: parts[2] ?? "" }; }); diff --git a/apps/server/src/kin/memory/git.ts b/apps/server/src/kin/memory/git.ts index 07befb3f9f9a7..cab52f6b95b45 100644 --- a/apps/server/src/kin/memory/git.ts +++ b/apps/server/src/kin/memory/git.ts @@ -1,8 +1,12 @@ +import * as Option from "effect/Option"; +import { MemoryConfig } from "./MemoryConfig.ts"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import { VcsProcess } from "../../vcs/VcsProcess.ts"; -// A private git directory excludes lead-controlled config, attributes, grafts and hooks. +const formats = new Map(); + +// A private bare git directory excludes lead-controlled config, grafts and hooks. export const memoryGit = Effect.fn("Memory.gitHardened")(function* ( cwd: string, args: ReadonlyArray, @@ -11,15 +15,26 @@ export const memoryGit = Effect.fn("Memory.gitHardened")(function* ( onStdoutChunk?: (chunk: Uint8Array) => void, ) { const fs = yield* FileSystem.FileSystem; + const config = Option.getOrUndefined(yield* Effect.serviceOption(MemoryConfig)); const vcs = yield* VcsProcess; return yield* Effect.scoped( Effect.gen(function* () { const directory = yield* fs.makeTempDirectoryScoped(); const repo = yield* fs.exists(`${cwd}/.git`); + let format = formats.get(cwd); + if (!format && repo) { + const text = yield* fs + .readFileString(`${cwd}/.git/config`) + .pipe(Effect.orElseSucceed(() => "")); + format = /objectformat\s*=\s*sha256/i.test(text) ? "sha256" : "sha1"; + formats.set(cwd, format); + } if (repo) { yield* fs.writeFileString( `${directory}/config`, - "[core]\nrepositoryformatversion = 0\nbare = false\n", + format === "sha256" + ? "[core]\nrepositoryformatversion = 1\nbare = true\n[extensions]\nobjectformat = sha256\n" + : "[core]\nrepositoryformatversion = 0\nbare = true\n", ); yield* fs.symlink(`${cwd}/.git/objects`, `${directory}/objects`); if (args[0] === "fsck") { @@ -33,13 +48,28 @@ export const memoryGit = Effect.fn("Memory.gitHardened")(function* ( } } } + let attributes: string[] = []; + if (repo && args[0] === "merge-tree") { + const empty = yield* vcs.run({ + operation: "Memory.emptyAttributes", + command: "git", + cwd, + args: [`--git-dir=${directory}`, "hash-object", "-w", "-t", "tree", "--stdin"], + stdin: "", + env: { GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: "/dev/null" }, + }); + attributes = [`--attr-source=${empty.stdout.trim()}`]; + } const result = yield* vcs.run({ operation: "Memory.gitHardened", command: "git", cwd, args: [ "--no-replace-objects", - ...(repo ? [`--git-dir=${directory}`, `--work-tree=${cwd}`] : []), + ...attributes, + ...(repo + ? [`--git-dir=${directory}`, ...(args[0] === "checkout" ? [`--work-tree=${cwd}`] : [])] + : []), "-c", "core.hooksPath=/dev/null", "-c", @@ -48,6 +78,9 @@ export const memoryGit = Effect.fn("Memory.gitHardened")(function* ( "core.attributesFile=/dev/null", "-c", "protocol.file.allow=user", + ...(config?.credentialHelper + ? ["-c", "credential.helper=", "-c", `credential.helper=${config.credentialHelper}`] + : []), ...args, ], env: { @@ -59,6 +92,7 @@ export const memoryGit = Effect.fn("Memory.gitHardened")(function* ( GIT_ASKPASS: "", GCM_INTERACTIVE: "never", ...env, + ...(config?.sshCommand ? { GIT_SSH_COMMAND: config.sshCommand } : {}), }, ...(stdin === undefined ? {} : { stdin }), ...(onStdoutChunk === undefined ? {} : { onStdoutChunk }), diff --git a/apps/server/src/kin/memory/objects.ts b/apps/server/src/kin/memory/objects.ts new file mode 100644 index 0000000000000..1e59847865277 --- /dev/null +++ b/apps/server/src/kin/memory/objects.ts @@ -0,0 +1,140 @@ +// @effect-diagnostics-next-line nodeBuiltinImport:off -- Git object identity uses the native byte-oriented hash API. +import * as NodeCrypto from "node:crypto"; +import type { KinOrgId } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import { memoryGit } from "./git.ts"; +import { MemoryCheckoutInvalid } from "./Memory.ts"; + +export const readObjects = Effect.fn("Memory.readObjects")(function* ( + orgId: KinOrgId, + cwd: string, + requests: ReadonlyArray<{ sha: string; type: "blob" | "tree" | "commit" }>, +) { + const invalid = () => + new MemoryCheckoutInvalid({ + orgId, + operation: "object", + reason: "Object hash or framing mismatch", + }); + if (requests.some(({ sha }) => !/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(sha))) + return yield* invalid(); + if (!requests.length) return []; + const chunks: Uint8Array[] = []; + yield* memoryGit( + cwd, + ["cat-file", "--batch"], + requests.map(({ sha }) => sha).join("\n") + "\n", + {}, + (chunk) => chunks.push(chunk), + ).pipe(Effect.mapError(invalid)); + const buffer = Buffer.concat(chunks); + const result: Buffer[] = []; + let offset = 0; + for (const { sha, type } of requests) { + const end = buffer.indexOf(10, offset); + const header = buffer.subarray(offset, end).toString("ascii"); + const match = /^(\w+) (blob|tree|commit) (\d+)$/.exec(header); + if (end < 0 || !match || match[1] !== sha || match[2] !== type) return yield* invalid(); + const length = Number(match[3]); + offset = end + 1; + const bytes = buffer.subarray(offset, offset + length); + if ( + bytes.length !== length || + buffer[offset + length] !== 10 || + NodeCrypto.createHash(sha.length === 40 ? "sha1" : "sha256") + .update(`${type} ${length}\0`) + .update(bytes) + .digest("hex") !== sha + ) + return yield* invalid(); + result.push(bytes); + offset += length + 1; + } + if (offset !== buffer.length) return yield* invalid(); + return result; +}); + +export const readCommit = Effect.fn("Memory.readCommit")(function* ( + orgId: KinOrgId, + cwd: string, + sha: string, +) { + const objects = yield* readObjects(orgId, cwd, [{ sha, type: "commit" }]); + const text = objects[0]?.toString("utf8") ?? ""; + const tree = /^tree ([a-f0-9]+)\n/.exec(text)?.[1]; + if (!tree) + return yield* new MemoryCheckoutInvalid({ + orgId, + operation: "commit", + reason: "Missing commit tree", + }); + return { + tree, + message: text.slice(text.indexOf("\n\n") + 2), + parent: /^parent ([a-f0-9]+)$/m.exec(text)?.[1], + }; +}); + +export const readTree = Effect.fn("Memory.readTree")(function* ( + orgId: KinOrgId, + cwd: string, + commit: string, +) { + const { tree } = yield* readCommit(orgId, cwd, commit); + const entries: Array<{ sha: string; relative: string }> = []; + const pending = [{ sha: tree, prefix: "" }]; + while (pending.length) { + const batch = pending.splice(0, 32); + const objects = yield* readObjects( + orgId, + cwd, + batch.map(({ sha }) => ({ sha, type: "tree" })), + ); + for (const [index, object] of objects.entries()) { + const request = batch[index]; + if (!request) continue; + let offset = 0; + while (offset < object.length) { + const space = object.indexOf(32, offset); + const nul = object.indexOf(0, space); + const width = request.sha.length / 2; + if (space < offset || nul < space || nul + 1 + width > object.length) + return yield* new MemoryCheckoutInvalid({ + orgId, + operation: "tree", + reason: "Invalid tree framing", + }); + const mode = object.subarray(offset, space).toString("ascii"); + const name = object.subarray(space + 1, nul).toString("utf8"); + const sha = object.subarray(nul + 1, nul + 1 + width).toString("hex"); + const relative = request.prefix + name; + if (mode === "40000") pending.push({ sha, prefix: `${relative}/` }); + else if (mode === "100644" || mode === "100755") entries.push({ sha, relative }); + offset = nul + 1 + width; + } + } + } + return entries; +}); + +export const readFiles = Effect.fn("Memory.readFiles")(function* ( + orgId: KinOrgId, + cwd: string, + commit: string, +) { + const entries = yield* readTree(orgId, cwd, commit); + const files = new Map(); + for (let index = 0; index < entries.length; index += 32) { + const batch = entries.slice(index, index + 32); + const objects = yield* readObjects( + orgId, + cwd, + batch.map(({ sha }) => ({ sha, type: "blob" })), + ); + for (const [index, entry] of batch.entries()) { + const bytes = objects[index]; + if (bytes) files.set(entry.relative, bytes); + } + } + return files; +}); diff --git a/docs/kin/specs/lead.md b/docs/kin/specs/lead.md index 875fffce64184..6e16c1eef9968 100644 --- a/docs/kin/specs/lead.md +++ b/docs/kin/specs/lead.md @@ -369,7 +369,7 @@ export class Memory extends Context.Service< Writer rules, enforced in code on `stage`: -- Paths: only `objectives/`, `commitments/`, `deferrals/`, `decisions/`, `people/`, `claims/` and `journal/`. `rules.md` is refused. `INDEX.md` is written only by `rebuildIndex`. `README.md` is never touched. +- Paths: only `objectives/`, `commitments/`, `deferrals/`, `decisions/`, `people/`, `claims/` and `journal/`. `rules.md` is refused. `INDEX.md` is generated from validated memory heads, never staged by a lead. `README.md` is never touched. - No deletes. `invalidate` sets `invalid_at`; `supersede` sets the old item's `superseded_by` and `status: superseded` and creates the new one. - Provenance comes from the event, not the model. The writer looks up `sourceEventId` in the current batch and sets `said_by`, `source` and `trust` from the event's `provenance`. Owner → `owner`, member → `member`, webhook or outside text → `external`, worker suggestion → `agent`. The lead may lower trust but never raise it. A member's statement that reads like a rule is stored under `claims/`. - `review_by` defaults by kind: person 90 days, commitment and deferral at `due`, decision `null`, claim 30 days; objective at the next milestone date (falling back to `due`, then +30 days from `recorded_at`). Explicit `review_by: null` is preserved. @@ -386,9 +386,9 @@ Front matter matches the kin-memory README, including snake_case keys. Date-only LEAD-2: the front matter parser must use YAML’s core or JSON schema; YAML 1.1 turns `review_by: 2026-11-13` into a Date. -The production `MemoryConfig` layer derives its server-owned base from `ServerConfig.stateDir`, using `${stateDir}/kin//memory` and `memory-read` (organization ids are path-encoded). Tests may supply an isolated base and relative paths. Paths must not overlap or escape that base. `ensureCheckouts` bootstraps a missing writer from a fresh fetch of the configured URL and configured default branch (`main` by default). K005 stores `(org_id, branch, head)` in `kin_memory_heads`; no writable file or local remote ref can bootstrap an existing checkout. Before committing or pushing, each local branch tip must match its anchor. Compare-and-swap branch updates and SQLite recording happen together in the writer operation; a crash between them fails closed and requires recovery. Prepare and commit share a per-org lock. Read clones are replaced from scratch on prepare: git is never run in an existing lead read clone. Every git invocation uses a private git directory excluding repository config, hooks, attributes and grafts, with system/global config disabled and hooks/fsmonitor disabled. +The production `MemoryConfig` layer derives its server-owned base from `ServerConfig.stateDir`, using `${stateDir}/kin//memory` and `memory-read` (organization ids are path-encoded). Tests may supply an isolated base and relative paths. Paths must not overlap or escape that base. `ensureCheckouts` bootstraps a missing writer from a fresh fetch of the configured URL and configured default branch (`main` by default). K005 stores `(org_id, branch, head)` in `kin_memory_heads`; no writable file or local remote ref can bootstrap an existing checkout. Before committing or pushing, each local branch tip must match its anchor. Compare-and-swap branch updates and SQLite recording happen together in the writer operation; a crash between them is rolled back to the recorded anchor with a compare-and-swap and an operator notice. Prepare and commit share a per-org lock. Read clones are replaced from scratch on prepare: git is never run in an existing lead read clone. Every git invocation uses a private git directory excluding repository config, hooks and grafts; merge plumbing uses an empty attribute source and no writer work tree, with system/global config disabled and hooks/fsmonitor disabled. -**Threat model.** Leads currently run full-access as the same OS user (`kin`) as the server. Assume leads can modify everything under `${stateDir}/kin/*/memory*`, including `.git` and file anchors. Code trusts only SQLite and server configuration, checks recorded branch tips, and builds commits from server-validated bytes rather than ambient working-tree/index content. These are not an OS sandbox: a same-user lead can still attack server processes or SQLite itself. Separating leads into an OS user unable to write `${stateDir}/kin` remains Nishchay's operations decision and is recommended before multi-org use. +**Threat model.** Leads currently run full-access as the same OS user (`kin`) as the server. Assume leads can modify everything under `${stateDir}/kin/*/memory*`, including `.git` and file anchors. Code trusts only SQLite and server configuration, checks recorded branch tips, and builds commits from server-validated bytes rather than ambient working-tree/index content. These are not an OS sandbox: a same-user lead can still attack server processes or SQLite itself. Separating leads into an OS user unable to write `${stateDir}/kin` remains Nishchay's operations decision and is required before production use. The read side pins the server-owned writer repository’s HEAD at wake-up (or first read), never the lead’s read-checkout HEAD or working-tree edits. Snapshots reload only on a new wake-up pin or explicit `rebuildCache`; blob contents and parse failures are cached by SHA. Symlinks are skipped; oversized blobs (over 256 KiB), malformed items and kind/folder mismatches are logged and skipped. Queries return duplicate ids with `duplicate: true` on each copy and mark them in the index; ambiguous `readItem` calls fail with `MemoryDuplicateId`. Queries include invalid items unless `excludeInvalid` is set; `invalid_at` remains on each returned `MemoryItemHead`. Date bounds are inclusive and compare instants; date-only strings mean midnight UTC. Missing or unparsable dates do not match a bounded date query. Overdue review means strictly before the current clock. @@ -628,6 +628,18 @@ LEAD-14. **End-to-end shadow week dry run.** Script that replays a day of record `MemoryWriterAuthority.resolve` is runner-only: resolve organization, wake-up, active lead, level, timestamps, timezone and event provenance from same-organization rows inside the runner's transaction. Never expose this context as tool input. Owner-confirmed sources must be verified there; absent a source, provenance is the lead's agent id. `MemoryWriterHooks.notify` consumes checkout, conflict and secret-block notices without chat transport coupling. -`KinSecretScan` is deliberately supplied separately. `stubLayer` returns no findings until TRIG-16 replaces it; it is not a production secret detector. Any finding or scanner failure blocks the whole commit. TRIG-16 must preserve this fail-closed interface: `scan({ orgId, files: ReadonlyArray<{ path: string; content: Uint8Array }> })` returns `{ path, ruleId, line }[]`, with typed `KinSecretScanError`. +`KinSecretScan` is deliberately supplied separately. `noFindingsStubLayer` returns no findings until TRIG-16 replaces it; it is not a production secret detector. Any finding or scanner failure blocks the whole commit. TRIG-16 must preserve this fail-closed interface: `scan({ orgId, files: ReadonlyArray<{ path: string; content: Uint8Array }> })` returns `{ path, ruleId, line }[]`, with typed `KinSecretScanError`. -TRIG-12's sectioned `buildIndex` has not landed on this base. `MemoryWriterHooks.buildIndex` receives the writer tree and returns generated text or null until wired; no read-checkout text is trusted. No migration is needed. The runner serializes prepare/stage/commit and dreaming per organization. Identical commit retries find the existing wake-up commit; failed pushes retain local commits, and failures discard uncommitted files. Journal additions use `appendJournal`, never replacement through `stage`. +`commitWakeup` builds TRIG-12's sectioned `INDEX.md` from the hash-verified recorded tree plus staged files, read history, the server clock and the owner's time zone, in the same wake-up commit. The read-side rebuild is display-only; no read-checkout text is trusted. No migration is needed. The runner serializes prepare/stage/commit and dreaming per organization. Identical commit retries find the existing wake-up commit; failed pushes retain local commits, and failures discard uncommitted files. Journal additions use `appendJournal`, never replacement through `stage`. + +### Memory writer operations + +Before production use, run leads under a separate OS user with no access to server state or other organizations. Only OS-user separation closes direct edits to `kin_memory_heads` and other SQLite state, other object/ref tampering, deleting the writer, changing `~/.ssh/config` for SSH pushes, racing read-clone replacement, and reading other organizations' directories. Hash-verified object reads and anchors are defense in depth, not a same-user sandbox. + +`prepareWakeup` checks git integrity once. Every application object read independently checks its hash. Journal-only append conflicts retain remote bytes followed by local entries in order. Other conflicts preserve the unpushed tip under `kin/conflict//` (or a server-generated recovery id when no wake-up id is supplied), then compare-and-swap the writer and anchor to the remote head and notify via `MemoryRecoveryHooks`. The next wake-up proceeds; the owner can inspect and cherry-pick the conflict branch. Existing conflict branches are never intentionally overwritten. + +Operators or owners can call server-only `Memory.recover(orgId)` after losing a writer or to reconcile its anchors. It fetches remote main and shadow heads, preserves surviving unpushed anchored work on a conflict branch, recreates a missing writer, records remote anchors and notifies. This method must never be registered as an agent tool. For writers predating K005 anchors, stop the runner and delete only that organization's writer directory, then bootstrap again. Do not delete anchored writers to recover unpushed work; call `recover` first. Recovery cannot restore commits lost with a deleted writer before they reached any remote. + +Each wake-up must start a fresh provider process with cwd set to the newly replaced read clone; LEAD-5 creates a fresh provider session for every wake-up. + +**Running on the Kin host.** Git pushes use the server's credentials. Since system/global/repository git config is excluded, supply a server-owned `MemoryConfig.credentialHelper` command for HTTPS or `MemoryConfig.sshCommand` for SSH. These are passed through command config/environment only; repository credential helpers cannot override them. Configure the Kin host's dedicated server user and credentials before enabling writes; never put credentials in memory, repository config, or lead instructions. See also “Running on the Kin host” in `triggers.md`. From c7d9dbf38d91242a07dde07529b7feef78c90860 Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Sun, 8 Nov 2026 03:14:17 +0530 Subject: [PATCH 05/11] fix(kin): keep snapshot reads outside ref reconciliation --- apps/server/src/kin/memory/Memory.ts | 7 ++-- .../src/kin/memory/MemoryWriter.test.ts | 33 +++++++++++++++++-- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/apps/server/src/kin/memory/Memory.ts b/apps/server/src/kin/memory/Memory.ts index eada248c87100..dd8465af9c633 100644 --- a/apps/server/src/kin/memory/Memory.ts +++ b/apps/server/src/kin/memory/Memory.ts @@ -377,6 +377,7 @@ const make = Effect.gen(function* () { const validateWriter = Effect.fn("Memory.validateWriter")(function* ( orgId: KinOrgId, paths: { writer: string; repo: { repoUrl: string } }, + checkTip = true, ) { yield* Effect.gen(function* () { const entry = yield* fs.lstat(path.join(paths.writer, ".git")); @@ -397,6 +398,7 @@ const make = Effect.gen(function* () { }); } const head = (yield* git(orgId, paths.writer, ["rev-parse", `refs/heads/${branch}`])).trim(); + if (!checkTip) return; if (head !== expected) { yield* readCommit(orgId, paths.writer, head).pipe( Effect.provideService(FileSystem.FileSystem, fs), @@ -424,7 +426,7 @@ const make = Effect.gen(function* () { const paths = yield* pathsFor(orgId); yield* io(orgId, "mkdir", fs.makeDirectory(paths.root, { recursive: true })); if (yield* io(orgId, "exists", fs.exists(paths.writer))) { - yield* validateWriter(orgId, paths); + yield* validateWriter(orgId, paths, false); } else { for (const entry of yield* io(orgId, "clones", fs.readDirectory(paths.root))) { if (/^memory\.clone-[A-Za-z0-9]+$/.test(entry)) @@ -473,7 +475,7 @@ const make = Effect.gen(function* () { const load = Effect.fn("Memory.load")(function* (orgId: KinOrgId, force = false) { const paths = yield* pathsFor(orgId); - yield* validateWriter(orgId, paths); + yield* validateWriter(orgId, paths, false); const head = pinnedHeads.get(orgId) ?? (yield* anchor(orgId, config.defaultBranch ?? "main")); pinnedHeads.set(orgId, head); const previous = snapshots.get(orgId); @@ -877,6 +879,7 @@ const make = Effect.gen(function* () { const prepare = Effect.fn("Memory.prepare")(function* (orgId: KinOrgId, wakeupId?: string) { const paths = yield* ensureCheckouts(orgId); + yield* validateWriter(orgId, yield* pathsFor(orgId)); yield* git(orgId, paths.writer, [ "fsck", "--full", diff --git a/apps/server/src/kin/memory/MemoryWriter.test.ts b/apps/server/src/kin/memory/MemoryWriter.test.ts index c1cb0db2270b2..4d9469b3ced95 100644 --- a/apps/server/src/kin/memory/MemoryWriter.test.ts +++ b/apps/server/src/kin/memory/MemoryWriter.test.ts @@ -44,6 +44,7 @@ it.live.each([ "review-default", "review-null", "credentials", + "read-during-record", ] as const)("writer real git: %s", (scenario) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -105,6 +106,16 @@ it.live.each([ yield* git(root, ["clone", "--bare", "--", source, remote]); let attack: { objectPath: string; sha: string; expression: string; bytes: Buffer } | undefined; let swapped = false; + let onRefChange: + | (() => Effect.Effect< + void, + | Memory.MemoryError + | Memory.MemoryCheckoutInvalid + | Memory.MemoryConfigError + | Memory.MemorySyncConflict + | Memory.MemoryQueryDecodeError + >) + | undefined; const interceptedGit = Layer.succeed( Vcs.VcsProcess, Vcs.VcsProcess.of({ @@ -119,7 +130,13 @@ it.live.each([ attack = undefined; swapped = true; } - return yield* vcs.run(input); + const result = yield* vcs.run(input); + if (onRefChange && input.args.includes("update-ref")) { + const callback = onRefChange; + onRefChange = undefined; + yield* callback().pipe(Effect.orDie); + } + return result; }), }), ); @@ -149,7 +166,12 @@ it.live.each([ orgId, wakeupId, agentId, - level: scenario === "conflict" || scenario === "B5-journal" ? "L1" : "L0", + level: + scenario === "conflict" || + scenario === "B5-journal" || + scenario === "read-during-record" + ? "L1" + : "L0", timeZone: "Asia/Kolkata", startedAt: "2026-11-08T00:00:00.000Z", eventIds: [], @@ -189,6 +211,13 @@ it.live.each([ yield* git(paths.writer, ["config", "user.name", "Fixture Bot"]); yield* git(paths.writer, ["config", "user.email", "fixture@example.invalid"]); const original = yield* git(paths.writer, ["rev-parse", "HEAD"]); + if (scenario === "read-during-record") { + onRefChange = () => memory.readRules(orgId).pipe(Effect.asVoid); + const { commit } = yield* writer.commitWakeup(orgId, "wake1", "completed"); + expect(yield* git(paths.writer, ["rev-parse", "refs/heads/main"])).toBe(commit); + expect(yield* memory.anchor(orgId, "main")).toBe(commit); + return; + } if (scenario === "credentials") { yield* git(paths.writer, ["config", "credential.helper", "fixture-repo-helper"]); const helper = yield* memoryGit(paths.writer, [ From dbe0bcc3da86233e296b0aee64b0c32424489e5f Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Sun, 8 Nov 2026 04:34:49 +0530 Subject: [PATCH 06/11] fix(kin): harden memory merges and verified tree writes --- apps/server/src/kin/memory/Memory.test.ts | 113 +++++++- apps/server/src/kin/memory/Memory.ts | 255 ++++++++---------- .../src/kin/memory/MemoryWriter.test.ts | 20 +- apps/server/src/kin/memory/MemoryWriter.ts | 29 +- apps/server/src/kin/memory/git.ts | 6 +- apps/server/src/kin/memory/merge.test.ts | 60 +++++ apps/server/src/kin/memory/merge.ts | 53 ++++ apps/server/src/kin/memory/objects.test.ts | 81 ++++++ apps/server/src/kin/memory/objects.ts | 70 ++++- docs/kin/specs/lead.md | 4 +- 10 files changed, 520 insertions(+), 171 deletions(-) create mode 100644 apps/server/src/kin/memory/merge.test.ts create mode 100644 apps/server/src/kin/memory/merge.ts create mode 100644 apps/server/src/kin/memory/objects.test.ts diff --git a/apps/server/src/kin/memory/Memory.test.ts b/apps/server/src/kin/memory/Memory.test.ts index 72f0cfc65ac3b..6e6d9264decb0 100644 --- a/apps/server/src/kin/memory/Memory.test.ts +++ b/apps/server/src/kin/memory/Memory.test.ts @@ -23,6 +23,13 @@ import * as MemoryConfig from "./MemoryConfig.ts"; import * as ServerConfig from "../../config.ts"; import { fixtureItem } from "./__fixtures__/items.ts"; +const memoryTestLayer = Memory.layer.pipe( + Layer.provide( + Layer.succeed(Memory.MemoryRecoveryHooks, { + notify: (notice) => Effect.logWarning("Memory recovery", notice), + }), + ), +); const orgA = KinOrgId.make("demo-a"); const orgB = KinOrgId.make("demo-b"); const itemId = KinMemoryItemId.make("def-001"); @@ -137,7 +144,7 @@ const fakeRepository = (fs: FileSystem.FileSystem) => { } onStdoutChunk?.(Buffer.from(stdout)); return Effect.gen(function* () { - if (args[0] === "clone") { + if (args[0] === "clone" || args[0] === "clone-snapshot") { yield* fs .makeDirectory(`${args.at(-1)}/.git/objects`, { recursive: true }) .pipe(Effect.orDie); @@ -176,7 +183,7 @@ const configLayer = (baseDirectory: string, repoUrl: string) => ], }); const memoryLayer = (base: string, repo: string) => - Memory.layer.pipe( + memoryTestLayer.pipe( Layer.provide(configLayer(base, repo)), Layer.provide(gitLayer), Layer.provide(NodeServices.layer), @@ -460,7 +467,7 @@ it.effect( expect(yield* memory.readIndex(orgA)).toContain("Invented service maintenance"); expect(yield* memory.readIndex(orgA)).not.toContain("Never opened fixture"); }); - const layer = Memory.layer.pipe( + const layer = memoryTestLayer.pipe( Layer.provide(configLayer(root, "unused")), Layer.provide(repository.layer), Layer.provide(NodeServices.layer), @@ -607,7 +614,7 @@ it.effect.each([ expect(yield* fs.readFileString(`${outside}/keep`)).toBe("untouched"); }).pipe( Effect.provide( - Memory.layer.pipe( + memoryTestLayer.pipe( Layer.provide( Layer.succeed(MemoryConfig.MemoryConfig, { baseDirectory: root, @@ -746,7 +753,7 @@ it.live.each([ expect(yield* fs.readDirectory(base)).toEqual([first]); }).pipe( Effect.provide( - Memory.layer.pipe( + memoryTestLayer.pipe( Layer.provide( Layer.succeed(MemoryConfig.MemoryConfig, { baseDirectory: base, @@ -827,6 +834,9 @@ it.live.each([ "rebase in progress", "journal conflict", "H2-merge-swap", + "unrelated", + "criss-cross", + "push retry", ])("syncs the writer with $0", (scenario) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -884,7 +894,7 @@ it.live.each([ const { writer } = yield* memory.ensureCheckouts(orgA); yield* git(writer, ["config", "user.name", "Fixture"]); yield* git(writer, ["config", "user.email", "fixture@example.invalid"]); - if (scenario === "journal conflict") { + if (scenario === "journal conflict" || scenario === "push retry") { yield* fs.makeDirectory(`${writer}/journal`, { recursive: true }); yield* fs.makeDirectory(`${source}/journal`, { recursive: true }); yield* fs.writeFileString(`${writer}/journal/day.md`, "local entry\n"); @@ -894,13 +904,28 @@ it.live.each([ `${writer}/rules.md`, scenario === "H2-merge-swap" ? h2Local : "Local invented rules\n", ); - const local = yield* commit(writer); + let local = yield* commit(writer); + if (scenario === "push retry") { + yield* git(writer, [ + "commit", + "--allow-empty", + "-m", + "Earlier journal outcome\n\nEvents: fixture-earlier", + ]); + yield* git(writer, [ + "commit", + "--allow-empty", + "-m", + "Later journal outcome\n\nEvents: fixture-later", + ]); + local = yield* git(writer, ["rev-parse", "HEAD"]); + } yield* memory.record(orgA, "main", initial.head, local); yield* fs.writeFileString( - `${source}/${scenario === "diverged" || scenario === "network down" || scenario === "dirty writer" || scenario === "H2-merge-swap" ? "remote-only" : "rules.md"}`, + `${source}/${scenario === "push retry" || scenario === "diverged" || scenario === "network down" || scenario === "dirty writer" || scenario === "H2-merge-swap" ? "remote-only" : "rules.md"}`, "Remote invented edit\n", ); - if (scenario === "journal conflict") { + if (scenario === "journal conflict" || scenario === "push retry") { yield* fs.writeFileString(`${source}/journal/day.md`, "remote entry\n"); yield* fs.writeFileString(`${source}/INDEX.md`, "remote generated index\n"); } @@ -920,8 +945,72 @@ it.live.each([ const sha = yield* git(writer, ["rev-parse", `${local}:rules.md`]); attack = { sha, path: `${writer}/.git/objects/${sha.slice(0, 2)}/${sha.slice(2)}` }; } - const result = yield* memory.prepareWakeup(orgA).pipe(Effect.result); - if (scenario === "H2-merge-swap") { + if (scenario === "unrelated" || scenario === "criss-cross") { + yield* git(writer, ["fetch", "--", remote, "main"]); + const remoteHead = yield* git(source, ["rev-parse", "HEAD"]); + const tree = yield* git(writer, ["rev-parse", `${local}^{tree}`]); + const parents = scenario === "criss-cross" ? ["-p", local, "-p", remoteHead] : []; + const replacement = yield* git(writer, [ + "commit-tree", + tree, + ...parents, + "-m", + "invented local merge", + ]); + if (scenario === "criss-cross") { + const other = yield* git(writer, [ + "commit-tree", + tree, + "-p", + remoteHead, + "-p", + local, + "-m", + "invented remote merge", + ]); + yield* git(writer, ["push", "--", remote, `${other}:refs/heads/main`]); + } + yield* git(writer, ["update-ref", "refs/heads/main", replacement, local]); + yield* memory.record(orgA, "main", local, replacement); + local = replacement; + } + const result = yield* memory.prepareWakeup(orgA, "fixture-wakeup").pipe(Effect.result); + if (scenario === "unrelated" || scenario === "criss-cross") { + expect(result._tag).toBe("Failure"); + if (result._tag === "Failure") expect(result.failure._tag).toBe("MemorySyncConflict"); + expect(yield* git(writer, ["rev-parse", "refs/heads/main"])).toBe(local); + expect( + yield* git(root, [ + `--git-dir=${remote}`, + "for-each-ref", + "--format=%(objectname)", + `refs/heads/kin/conflict/${orgA}/fixture-wakeup-*`, + ]), + ).toBe(local); + expect( + messages.some( + (message) => + message.includes("No unique verified merge base") && + message.includes("fixture-wakeup"), + ), + ).toBe(true); + } else if (scenario === "push retry") { + expect(result._tag).toBe("Success"); + if (result._tag === "Success") { + const merged = result.success.head; + expect((yield* git(writer, ["show", "-s", "--format=%P", merged])).split(" ")).toContain( + local, + ); + const messages = yield* git(writer, ["log", "--format=%B", merged]); + expect(messages).toContain("Events: fixture-earlier"); + expect(messages).toContain("Events: fixture-later"); + const journalBefore = yield* git(writer, ["show", `${merged}:journal/day.md`]); + yield* git(writer, ["update-ref", "refs/heads/main", local, merged]); + yield* memory.record(orgA, "main", merged, local); + expect((yield* memory.prepareWakeup(orgA)).head).toBe(merged); + expect(yield* git(writer, ["show", `${merged}:journal/day.md`])).toBe(journalBefore); + } + } else if (scenario === "H2-merge-swap") { expect(swapped).toBe(true); expect(result._tag).toBe("Failure"); expect(yield* git(root, [`--git-dir=${remote}`, "show", "main:rules.md"])).not.toContain( @@ -943,7 +1032,7 @@ it.live.each([ ); } } - } else if (scenario === "journal conflict") { + } else if (scenario === "journal conflict" || scenario === "push retry") { expect(result._tag).toBe("Success"); if (result._tag === "Success") expect(yield* git(writer, ["show", `${result.success.head}:journal/day.md`])).toBe( diff --git a/apps/server/src/kin/memory/Memory.ts b/apps/server/src/kin/memory/Memory.ts index 2330853e823bc..ac6a58c26807a 100644 --- a/apps/server/src/kin/memory/Memory.ts +++ b/apps/server/src/kin/memory/Memory.ts @@ -1,5 +1,12 @@ -import * as Option from "effect/Option"; -import { readTree, readObjects, readCommit, readFiles } from "./objects.ts"; +import { + readTree, + readObjects, + ancestors, + readCommit, + readFiles, + objectHash, + verifyTree, +} from "./objects.ts"; import { type KinMemoryItemId, type KinOrgId, @@ -22,6 +29,7 @@ import { toPersistenceSqlError } from "../../persistence/Errors.ts"; import { memoryGit as runMemoryGit } from "./git.ts"; import { VcsProcess } from "../../vcs/VcsProcess.ts"; import * as MemoryCache from "./MemoryCache.ts"; +import { mergeMemoryFiles, uniqueMergeBase } from "./merge.ts"; import { buildIndex } from "./buildIndex.ts"; import { MemoryConfig } from "./MemoryConfig.ts"; import { @@ -192,14 +200,7 @@ interface BlobCache { const make = Effect.gen(function* () { const config = yield* MemoryConfig; const fs = yield* FileSystem.FileSystem; - const recoveryHooks = Option.getOrUndefined(yield* Effect.serviceOption(MemoryRecoveryHooks)); - const notify = (notice: { - orgId: KinOrgId; - branch: string; - reason: string; - conflictBranch?: string; - }) => - recoveryHooks ? recoveryHooks.notify(notice) : Effect.logWarning("Memory recovery", notice); + const { notify } = yield* MemoryRecoveryHooks; const vcs = yield* VcsProcess; const memoryGit = (...args: Parameters) => runMemoryGit(...args).pipe( @@ -404,25 +405,11 @@ const make = Effect.gen(function* () { return { repo, root, writer, read }; }); - const ancestors = Effect.fn("Memory.ancestors")(function* ( - orgId: KinOrgId, - writer: string, - tip: string, - ) { - const seen = new Map>(); - const pending = [tip]; - while (pending.length) { - const sha = pending.pop(); - if (!sha || seen.has(sha)) continue; - const commit = yield* readCommit(orgId, writer, sha).pipe( - Effect.provideService(FileSystem.FileSystem, fs), - Effect.provideService(VcsProcess, vcs), - ); - seen.set(sha, commit.parents); - pending.push(...commit.parents); - } - return seen; - }); + const walkAncestors = (...args: Parameters) => + ancestors(...args).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(VcsProcess, vcs), + ); const validateWriter = Effect.fn("Memory.validateWriter")(function* ( orgId: KinOrgId, @@ -921,7 +908,7 @@ const make = Effect.gen(function* () { Effect.provideService(VcsProcess, vcs), Effect.result, ); - const present = yield* ancestors(orgId, paths.writer, head); + const present = yield* walkAncestors(orgId, paths.writer, head); if (local._tag === "Success" && !present.has(old) && !preserved.has(old)) { const conflictBranch = yield* preserve(orgId, paths.writer, paths.repo.repoUrl, old); preserved.add(old); @@ -981,116 +968,80 @@ const make = Effect.gen(function* () { if (fetched._tag === "Success") { const remote = fetched.success.trim(); if (remote !== head) { - const remoteAncestors = yield* ancestors(orgId, paths.writer, remote); - const localAncestors = yield* ancestors(orgId, paths.writer, head); + const remoteAncestors = yield* walkAncestors(orgId, paths.writer, remote, head); let next = remote; if (!remoteAncestors.has(head)) { + const localAncestors = yield* walkAncestors(orgId, paths.writer, head); if (localAncestors.has(remote)) next = head; else { - const common = new Set( - [...localAncestors.keys()].filter((sha) => remoteAncestors.has(sha)), - ); - const candidates = new Set(common); - for (const sha of common) - for (const parent of localAncestors.get(sha) ?? []) candidates.delete(parent); - const base = candidates.size === 1 ? [...candidates][0] : undefined; - if (!base) + const mergeBase = uniqueMergeBase(localAncestors, remoteAncestors); + if (!mergeBase) { + const conflictBranch = yield* preserve( + orgId, + paths.writer, + repo.repoUrl, + head, + wakeupId, + ); + yield* notify({ + orgId, + branch, + reason: "No unique verified merge base", + conflictBranch, + }); return yield* new MemorySyncConflict({ orgId, operation: "merge", reason: "No unique verified merge base", }); - const commits = [head]; - for (const commit of commits) { - const parent = base; - if (!parent) - return yield* new MemoryCheckoutInvalid({ - orgId, - operation: "rebase", - reason: "Missing parent", - }); - let tree: string | undefined; - { - const read = (sha: string) => - readFiles(orgId, paths.writer, sha).pipe( - Effect.provideService(FileSystem.FileSystem, fs), - Effect.provideService(VcsProcess, vcs), - ); - const base = yield* read(parent); - const local = yield* read(commit); - const remoteFiles = yield* read(next); - const metadata = new Map>(); - for (const sha of [parent, commit, next]) { - const entries = yield* readTree(orgId, paths.writer, sha).pipe( - Effect.provideService(FileSystem.FileSystem, fs), - Effect.provideService(VcsProcess, vcs), - ); - metadata.set(sha, new Map(entries.map((entry) => [entry.relative, entry]))); - } - const changed = [...new Set([...base.keys(), ...local.keys()])].filter( - (file) => - metadata.get(parent)?.get(file)?.sha !== metadata.get(commit)?.get(file)?.sha || - metadata.get(parent)?.get(file)?.mode !== metadata.get(commit)?.get(file)?.mode, + } + const commit = head; + const parent = mergeBase; + let tree: string | undefined; + { + const read = (sha: string) => + readFiles(orgId, paths.writer, sha).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(VcsProcess, vcs), ); - const modes = new Map( - [...(metadata.get(next) ?? [])].map(([file, entry]) => [file, entry.mode]), + const base = yield* read(parent); + const local = yield* read(commit); + const remoteFiles = yield* read(next); + const metadata = new Map>(); + for (const sha of [parent, commit, next]) { + const entries = yield* readTree(orgId, paths.writer, sha).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(VcsProcess, vcs), ); - - const same = (left: Buffer | undefined, right: Buffer | undefined) => - left === undefined - ? right === undefined - : right !== undefined && left.equals(right); - const journalOnly = changed.every( - (file) => - file === "INDEX.md" || - (same(remoteFiles.get(file), base.get(file)) && - metadata.get(parent)?.get(file)?.mode === modes.get(file)) || - (file.startsWith("journal/") && - ["100644", "100755"].includes(metadata.get(commit)?.get(file)?.mode ?? "") && - metadata.get(commit)?.get(file)?.mode === modes.get(file) && - local - .get(file) - ?.subarray(0, base.get(file)?.length ?? 0) - .equals(base.get(file) ?? Buffer.alloc(0)) && - remoteFiles - .get(file) - ?.subarray(0, base.get(file)?.length ?? 0) - .equals(base.get(file) ?? Buffer.alloc(0))), + metadata.set(sha, new Map(entries.map((entry) => [entry.relative, entry]))); + } + const merged = mergeMemoryFiles( + base, + local, + remoteFiles, + metadata.get(parent)!, + metadata.get(commit)!, + metadata.get(next)!, + ); + if (!merged) { + const conflictBranch = yield* preserve( + orgId, + paths.writer, + repo.repoUrl, + head, + wakeupId, ); - if (!journalOnly) { - const conflictBranch = yield* preserve( - orgId, - paths.writer, - repo.repoUrl, - head, - wakeupId, - ); - next = remote; - yield* notify({ - orgId, - branch, - reason: "Content conflict preserved", - conflictBranch, - }); - break; - } - for (const file of changed) { - if (file === "INDEX.md") continue; - const bytes = local.get(file); - if (same(remoteFiles.get(file), base.get(file))) { - if (bytes) { - remoteFiles.set(file, bytes); - modes.set(file, metadata.get(commit)?.get(file)?.mode ?? "100644"); - } else remoteFiles.delete(file); - } else - remoteFiles.set( - file, - Buffer.concat([ - remoteFiles.get(file) ?? Buffer.alloc(0), - bytes?.subarray(base.get(file)?.length ?? 0) ?? Buffer.alloc(0), - ]), - ); - } + next = remote; + yield* notify({ + orgId, + branch, + reason: "Content conflict preserved", + conflictBranch, + }); + } else { + const { files: mergedFiles, modes } = merged; + remoteFiles.clear(); + for (const [file, bytes] of mergedFiles) remoteFiles.set(file, bytes); const heads = []; for (const [file, bytes] of remoteFiles) { if (!folders.some((folder) => file.startsWith(`${folder}/`))) continue; @@ -1116,6 +1067,7 @@ const make = Effect.gen(function* () { (cause) => new MemoryError({ orgId, operation: "merge", cause }), ), ); + const expected = new Map(); for (const [file, bytes] of remoteFiles) { yield* io(orgId, "mergeBlob", fs.writeFile(`${temp}/blob`, bytes)); const mode = modes.get(file) ?? "100644"; @@ -1128,6 +1080,13 @@ const make = Effect.gen(function* () { "--no-filters", `${temp}/blob`, ])).trim(); + expected.set(file, { + mode, + sha: + mode === "160000" + ? bytes.toString("ascii") + : objectHash(bytes, head.length), + }); yield* memoryGit( paths.writer, ["update-index", "--add", "--cacheinfo", `${mode},${blob},${file}`], @@ -1139,14 +1098,26 @@ const make = Effect.gen(function* () { ), ); } - return (yield* memoryGit(paths.writer, ["write-tree"], undefined, env).pipe( + const written = (yield* memoryGit( + paths.writer, + ["write-tree"], + undefined, + env, + ).pipe( Effect.mapError( (cause) => new MemoryError({ orgId, operation: "merge", cause }), ), )).trim(); + yield* verifyTree(orgId, paths.writer, written, expected).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(VcsProcess, vcs), + ); + return written; }), ); } + } + if (next !== remote || tree) { if (!tree || !/^[0-9a-f]{40,64}$/.test(tree)) return yield* new MemorySyncConflict({ orgId, @@ -1157,12 +1128,17 @@ const make = Effect.gen(function* () { Effect.provideService(FileSystem.FileSystem, fs), Effect.provideService(VcsProcess, vcs), ); - next = (yield* memoryGit(paths.writer, ["commit-tree", tree, "-p", next], message, { - GIT_AUTHOR_NAME: config.botName ?? "Kin Memory Bot", - GIT_AUTHOR_EMAIL: config.botEmail ?? "kin-memory@localhost", - GIT_COMMITTER_NAME: config.botName ?? "Kin Memory Bot", - GIT_COMMITTER_EMAIL: config.botEmail ?? "kin-memory@localhost", - }).pipe( + next = (yield* memoryGit( + paths.writer, + ["commit-tree", tree, "-p", remote, "-p", head], + message, + { + GIT_AUTHOR_NAME: config.botName ?? "Kin Memory Bot", + GIT_AUTHOR_EMAIL: config.botEmail ?? "kin-memory@localhost", + GIT_COMMITTER_NAME: config.botName ?? "Kin Memory Bot", + GIT_COMMITTER_EMAIL: config.botEmail ?? "kin-memory@localhost", + }, + ).pipe( Effect.mapError((cause) => new MemoryError({ orgId, operation: "rebase", cause })), )).trim(); } @@ -1188,14 +1164,7 @@ const make = Effect.gen(function* () { Effect.scoped( Effect.gen(function* () { const temporary = yield* fs.makeTempDirectoryScoped(); - yield* git(orgId, temporary, [ - "clone", - "--no-hardlinks", - "--no-checkout", - "--", - paths.writer, - `${temporary}/read`, - ]); + yield* git(orgId, paths.writer, ["clone-snapshot", `${temporary}/read`]); yield* git(orgId, `${temporary}/read`, ["checkout", "--detach", head]); yield* fs.remove(paths.read, { recursive: true, force: true }); yield* fs.rename(`${temporary}/read`, paths.read); diff --git a/apps/server/src/kin/memory/MemoryWriter.test.ts b/apps/server/src/kin/memory/MemoryWriter.test.ts index 6aecd9b917843..ec28c34bbf048 100644 --- a/apps/server/src/kin/memory/MemoryWriter.test.ts +++ b/apps/server/src/kin/memory/MemoryWriter.test.ts @@ -49,6 +49,7 @@ it.live.each([ "M1-unanchored", "M2-nondescendant", "M4-modes", + "tree-substitution", ] as const)("writer real git: %s", (scenario) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -107,6 +108,8 @@ it.live.each([ } if (scenario === "M4-modes") { yield* fs.symlink("rules.md", `${source}/owner-link`); + yield* fs.makeDirectory(`${source}/claims`, { recursive: true }); + yield* fs.symlink(fixtureItem({ kind: "claim" }), `${source}/claims/link.md`); yield* fs.writeFileString(`${source}/owner-executable`, "fixture executable\n"); yield* fs.chmod(`${source}/owner-executable`, 0o755); } @@ -127,6 +130,7 @@ it.live.each([ yield* git(root, ["clone", "--bare", "--", source, remote]); let attack: { objectPath: string; sha: string; expression: string; bytes: Buffer } | undefined; let swapped = false; + let substituteTree: string | undefined; let onRefChange: | (() => Effect.Effect< void, @@ -152,6 +156,8 @@ it.live.each([ swapped = true; } const result = yield* vcs.run(input); + if (substituteTree && input.args.includes("write-tree")) + return { ...result, stdout: substituteTree }; if (onRefChange && input.args.includes("update-ref")) { const callback = onRefChange; onRefChange = undefined; @@ -167,7 +173,10 @@ it.live.each([ credentialHelper: "fixture-server-helper", repositories: [{ orgId, repoUrl: remote, path: "state" }], }); - const memoryLayer = Memory.layer.pipe(Layer.provide(config)); + const memoryLayer = Memory.layer.pipe( + Layer.provide(config), + Layer.provide(Layer.succeed(Memory.MemoryRecoveryHooks, { notify: () => Effect.void })), + ); const hooks = Layer.succeed(Writer.MemoryWriterHooks, { notify: Effect.fn("Test.notice")((notice) => Effect.sync(() => { @@ -219,6 +228,7 @@ it.live.each([ : Scan.noFindingsStubLayer; const layer = Writer.layer.pipe( Layer.provideMerge(memoryLayer), + Layer.provide(Layer.succeed(Memory.MemoryRecoveryHooks, { notify: () => Effect.void })), Layer.provide(config), Layer.provide(hooks), Layer.provide(auth), @@ -572,10 +582,18 @@ it.live.each([ yield* commit(source); yield* git(source, ["push", "--", remote, "main"]); } + if (scenario === "tree-substitution") + substituteTree = yield* git(paths.writer, ["rev-parse", `${original}^{tree}`]); const result = yield* writer .commitWakeup(orgId, "wake1", scenario === "failed" ? "failed" : "completed") .pipe(Effect.result); if (result._tag === "Failure") yield* Effect.logError(result.failure); + if (scenario === "tree-substitution") { + expect(result._tag).toBe("Failure"); + if (result._tag === "Failure") expect(result.failure._tag).toBe("MemoryCheckoutInvalid"); + expect(yield* git(paths.writer, ["rev-parse", "main"])).toBe(original); + return; + } if (scenario === "blocked" || scenario === "conflict") { expect(result._tag).toBe("Failure"); if (result._tag === "Failure") diff --git a/apps/server/src/kin/memory/MemoryWriter.ts b/apps/server/src/kin/memory/MemoryWriter.ts index dc8989e84c71f..a3927008d77ec 100644 --- a/apps/server/src/kin/memory/MemoryWriter.ts +++ b/apps/server/src/kin/memory/MemoryWriter.ts @@ -1,5 +1,5 @@ import * as Clock from "effect/Clock"; -import { readFiles, readCommit, readTree } from "./objects.ts"; +import { readFiles, readCommit, readTree, objectHash, verifyTree } from "./objects.ts"; import { buildIndex } from "./buildIndex.ts"; import * as MemoryCache from "./MemoryCache.ts"; import type { KinAgentId, KinOrgId, MemoryTrust } from "@t3tools/contracts"; @@ -121,7 +121,7 @@ const folders = ["objectives", "commitments", "deferrals", "decisions", "people" const render = (fields: object, body: string) => `---\n${stringify(fields)}---\n${body}`; const make = Effect.gen(function* () { const memory = yield* Memory; - const recovery = Option.getOrUndefined(yield* Effect.serviceOption(MemoryRecoveryHooks)); + const recovery = yield* MemoryRecoveryHooks; const authority = yield* MemoryWriterAuthority; const hooks = yield* MemoryWriterHooks; const scanner = yield* KinSecretScan; @@ -189,14 +189,14 @@ const make = Effect.gen(function* () { branch, reason: "Reset tampered branch to anchor; unpushed work at the replaced tip is lost", }; - yield* recovery ? recovery.notify(notice) : Effect.logWarning("Memory recovery", notice); + yield* recovery.notify(notice); } return anchored.success; } if (local._tag === "Success") { yield* git(orgId, writer, ["update-ref", "-d", `refs/heads/${branch}`, local.success]); const notice = { orgId, branch, reason: "Deleted unanchored local branch" }; - yield* recovery ? recovery.notify(notice) : Effect.logWarning("Memory recovery", notice); + yield* recovery.notify(notice); } const repo = config.repositories.find((repo) => repo.orgId === orgId); if (!repo) @@ -342,7 +342,16 @@ const make = Effect.gen(function* () { const recordedFiles = yield* filesAt(orgId, writer, recorded); const items = []; const bodies = new Map(); + const regular = new Set( + (yield* readTree(orgId, writer, recorded).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(VcsProcess, vcs), + )) + .filter((entry) => ["100644", "100755"].includes(entry.mode)) + .map((entry) => entry.relative), + ); for (const [file, bytes] of recordedFiles) { + if (!regular.has(file)) continue; if (!folders.some((folder) => file.startsWith(`${folder}/`))) continue; const content = bytes.toString("utf8"); const parsedItem = yield* parseFrontMatter(orgId, file, content).pipe(Effect.result); @@ -620,6 +629,7 @@ const make = Effect.gen(function* () { const finalFiles = new Map(recordedFiles); for (const [file, text] of state.files) finalFiles.set(file, file === entry.path ? journal : Buffer.from(text)); + const expected = new Map(); for (const [file, bytes] of finalFiles) { if (state.files.has(file)) yield* safePath(orgId, writer, file, true); const input = `${temporary}/blob`; @@ -636,6 +646,13 @@ const make = Effect.gen(function* () { ["hash-object", "-w", "--no-filters", input], env, ); + expected.set(file, { + mode, + sha: + mode === "160000" + ? bytes.toString("ascii") + : objectHash(bytes, recorded.length), + }); yield* git( orgId, writer, @@ -644,6 +661,10 @@ const make = Effect.gen(function* () { ); } const tree = yield* git(orgId, writer, ["write-tree"], env); + yield* verifyTree(orgId, writer, tree, expected).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(VcsProcess, vcs), + ); const message = `${subject}\n\nEvents: ${ctx.eventIds.join(", ")}\nKin-Outcome: ${outcome}\nKin-Request: ${fingerprint}\n`; const commit = yield* git( orgId, diff --git a/apps/server/src/kin/memory/git.ts b/apps/server/src/kin/memory/git.ts index bc4ca67d86ec3..e803680185786 100644 --- a/apps/server/src/kin/memory/git.ts +++ b/apps/server/src/kin/memory/git.ts @@ -63,7 +63,7 @@ export const memoryGit = Effect.fn("Memory.gitHardened")(function* ( cwd, args: [ "--no-replace-objects", - ...(repo + ...(repo && args[0] !== "clone-snapshot" ? [`--git-dir=${directory}`, ...(args[0] === "checkout" ? [`--work-tree=${cwd}`] : [])] : []), "-c", @@ -77,7 +77,9 @@ export const memoryGit = Effect.fn("Memory.gitHardened")(function* ( ...(config?.credentialHelper ? ["-c", "credential.helper=", "-c", `credential.helper=${config.credentialHelper}`] : []), - ...args, + ...(args[0] === "clone-snapshot" + ? ["clone", "--no-local", "--no-checkout", "--", directory, args[1]!] + : args), ], env: { GIT_CONFIG_NOSYSTEM: "1", diff --git a/apps/server/src/kin/memory/merge.test.ts b/apps/server/src/kin/memory/merge.test.ts new file mode 100644 index 0000000000000..a829b9e7f32a3 --- /dev/null +++ b/apps/server/src/kin/memory/merge.test.ts @@ -0,0 +1,60 @@ +import { expect, it } from "@effect/vitest"; +import { mergeMemoryFiles, uniqueMergeBase } from "./merge.ts"; + +it.each(["append", "rewrite", "disjoint", "remote rewrite", "rewrite disjoint"])( + "merges verified snapshots: %s", + (scenario) => { + const base = new Map([["journal/day.md", Buffer.from("original\n")]]); + const local = new Map(base); + const remote = new Map(base); + local.set( + "journal/day.md", + Buffer.from(scenario.startsWith("rewrite") ? "rewritten\n" : "original\nlocal\n"), + ); + if (!scenario.includes("disjoint")) + remote.set( + "journal/day.md", + Buffer.from(scenario === "remote rewrite" ? "rewritten remotely\n" : "original\nremote\n"), + ); + const metadata = (files: Map) => + new Map([...files].map(([file, bytes]) => [file, { mode: "100644", sha: bytes.toString() }])); + const result = mergeMemoryFiles( + base, + local, + remote, + metadata(base), + metadata(local), + metadata(remote), + ); + if (scenario.startsWith("rewrite") || scenario === "remote rewrite") + expect(result).toBeUndefined(); + else + expect(result?.files.get("journal/day.md")?.toString()).toBe( + scenario === "disjoint" ? "original\nlocal\n" : "original\nremote\nlocal\n", + ); + }, +); + +it("fails closed for unrelated histories and criss-cross merge bases", () => { + expect(uniqueMergeBase(new Map([["a", []]]), new Map([["b", []]]))).toBeUndefined(); + const common = [ + ["a", ["root"]], + ["b", ["root"]], + ["root", []], + ] as const; + expect( + uniqueMergeBase( + new Map>([...common, ["local", ["a", "b"]]]), + new Map>([...common, ["remote", ["b", "a"]]]), + ), + ).toBeUndefined(); + expect( + uniqueMergeBase( + new Map([ + ["a", []], + ["b", ["a"]], + ]), + new Map([["a", []]]), + ), + ).toBe("a"); +}); diff --git a/apps/server/src/kin/memory/merge.ts b/apps/server/src/kin/memory/merge.ts new file mode 100644 index 0000000000000..c7af0c991ddd7 --- /dev/null +++ b/apps/server/src/kin/memory/merge.ts @@ -0,0 +1,53 @@ +type Entry = { mode: string; sha: string }; + +export const uniqueMergeBase = ( + local: ReadonlyMap>, + remote: ReadonlyMap>, +) => { + const common = new Set([...local.keys()].filter((sha) => remote.has(sha))); + const candidates = new Set(common); + for (const sha of common) for (const parent of local.get(sha) ?? []) candidates.delete(parent); + return candidates.size === 1 ? [...candidates][0] : undefined; +}; + +export const mergeMemoryFiles = ( + base: ReadonlyMap, + local: ReadonlyMap, + remote: ReadonlyMap, + baseEntries: ReadonlyMap, + localEntries: ReadonlyMap, + remoteEntries: ReadonlyMap, +) => { + const files = new Map(remote); + const modes = new Map([...remoteEntries].map(([file, entry]) => [file, entry.mode])); + const same = (a: Buffer | undefined, b: Buffer | undefined) => + a === undefined ? b === undefined : b !== undefined && a.equals(b); + for (const file of new Set([...base.keys(), ...local.keys()])) { + const before = baseEntries.get(file); + const after = localEntries.get(file); + if (file === "INDEX.md" || (before?.sha === after?.sha && before?.mode === after?.mode)) + continue; + const original = base.get(file) ?? Buffer.alloc(0); + const bytes = local.get(file); + const journal = file.startsWith("journal/"); + if (journal && (!bytes || !bytes.subarray(0, original.length).equals(original))) + return undefined; + if (same(remote.get(file), base.get(file)) && before?.mode === modes.get(file)) { + if (bytes) { + files.set(file, bytes); + modes.set(file, after?.mode ?? "100644"); + } else { + files.delete(file); + modes.delete(file); + } + } else if ( + journal && + ["100644", "100755"].includes(after?.mode ?? "") && + after?.mode === modes.get(file) && + remote.get(file)?.subarray(0, original.length).equals(original) + ) { + files.set(file, Buffer.concat([remote.get(file)!, bytes!.subarray(original.length)])); + } else return undefined; + } + return { files, modes }; +}; diff --git a/apps/server/src/kin/memory/objects.test.ts b/apps/server/src/kin/memory/objects.test.ts new file mode 100644 index 0000000000000..009c6eb18df1d --- /dev/null +++ b/apps/server/src/kin/memory/objects.test.ts @@ -0,0 +1,81 @@ +// @effect-diagnostics-next-line nodeBuiltinImport:off -- Fixtures need authentic Git object identities. +import * as NodeCrypto from "node:crypto"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { expect, it } from "@effect/vitest"; +import { KinOrgId } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import { ChildProcessSpawner } from "effect/process"; +import { VcsProcess } from "../../vcs/VcsProcess.ts"; +import { ancestors, verifyTree } from "./objects.ts"; + +it.effect( + "batches a 200-commit frontier and stops at a verified ancestor; rejects a substituted tree", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const cwd = yield* fs.makeTempDirectoryScoped(); + const objects = new Map(); + const object = (type: string, text: string) => { + const bytes = Buffer.from(text); + const sha = NodeCrypto.createHash("sha1") + .update(`${type} ${bytes.length}\0`) + .update(bytes) + .digest("hex"); + objects.set(sha, { type, bytes }); + return sha; + }; + const tree = object("tree", ""); + const root = object("commit", `tree ${tree}\n\nroot\n`); + const frontier = Array.from({ length: 200 }, (_, n) => + object("commit", `tree ${tree}\nparent ${root}\n\ncommit ${n}\n`), + ); + const tip = object( + "commit", + `tree ${tree}\n${frontier.map((sha) => `parent ${sha}\n`).join("")}\nmerge\n`, + ); + let calls = 0; + const vcs = VcsProcess.of({ + run: Effect.fn("Test.batch")((input) => + Effect.sync(() => { + calls++; + const chunks = input + .stdin!.trim() + .split("\n") + .map((sha) => { + const entry = objects.get(sha)!; + return Buffer.concat([ + Buffer.from(`${sha} ${entry.type} ${entry.bytes.length}\n`), + entry.bytes, + Buffer.from("\n"), + ]); + }); + input.onStdoutChunk?.(Buffer.concat(chunks)); + return { + exitCode: ChildProcessSpawner.ExitCode(0), + stdout: "", + stderr: "", + stdoutTruncated: false, + stderrTruncated: false, + }; + }), + ), + }); + const org = KinOrgId.make("fixture"); + yield* Effect.gen(function* () { + expect((yield* ancestors(org, cwd, tip)).size).toBe(202); + expect(calls).toBe(3); + calls = 0; + expect((yield* ancestors(org, cwd, tip, frontier[0])).size).toBe(201); + expect(calls).toBe(2); + yield* verifyTree(org, cwd, tree, new Map()); + const rejected = yield* verifyTree( + org, + cwd, + tree, + new Map([["rules.md", { mode: "100644", sha: "0".repeat(40) }]]), + ).pipe(Effect.result); + expect(rejected._tag).toBe("Failure"); + }).pipe(Effect.provideService(VcsProcess, vcs)); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); diff --git a/apps/server/src/kin/memory/objects.ts b/apps/server/src/kin/memory/objects.ts index b935fdf50c83c..396d3cc14adc6 100644 --- a/apps/server/src/kin/memory/objects.ts +++ b/apps/server/src/kin/memory/objects.ts @@ -5,7 +5,8 @@ import * as Effect from "effect/Effect"; import { memoryGit } from "./git.ts"; import { MemoryCheckoutInvalid } from "./Memory.ts"; -export const readObjects = Effect.fn("Memory.readObjects")(function* ( +// Object batches share the enclosing operation span, including per-commit ancestry reads. +export const readObjects = Effect.fnUntraced(function* ( orgId: KinOrgId, cwd: string, requests: ReadonlyArray<{ sha: string; type: "blob" | "tree" | "commit" }>, @@ -54,11 +55,8 @@ export const readObjects = Effect.fn("Memory.readObjects")(function* ( return result; }); -export const readCommit = Effect.fn("Memory.readCommit")(function* ( - orgId: KinOrgId, - cwd: string, - sha: string, -) { +// Per-commit reads share the enclosing operation span. +export const readCommit = Effect.fnUntraced(function* (orgId: KinOrgId, cwd: string, sha: string) { const objects = yield* readObjects(orgId, cwd, [{ sha, type: "commit" }]); const text = objects[0]?.toString("utf8") ?? ""; const tree = /^tree ([a-f0-9]+)\n/.exec(text)?.[1]; @@ -82,8 +80,9 @@ export const readTree = Effect.fn("Memory.readTree")(function* ( orgId: KinOrgId, cwd: string, commit: string, + isTree = false, ) { - const { tree } = yield* readCommit(orgId, cwd, commit); + const tree = isTree ? commit : (yield* readCommit(orgId, cwd, commit)).tree; const entries: Array<{ sha: string; relative: string; mode: string }> = []; const pending = [{ sha: tree, prefix: "" }]; while (pending.length) { @@ -145,3 +144,60 @@ export const readFiles = Effect.fn("Memory.readFiles")(function* ( if (entry.mode === "160000") files.set(entry.relative, Buffer.from(entry.sha, "ascii")); return files; }); + +export const objectHash = (bytes: Buffer, width: number) => + NodeCrypto.createHash(width === 40 ? "sha1" : "sha256") + .update(`blob ${bytes.length}\0`) + .update(bytes) + .digest("hex"); + +export const verifyTree = Effect.fn("Memory.verifyTree")(function* ( + orgId: KinOrgId, + cwd: string, + tree: string, + expected: ReadonlyMap, +) { + const entries = yield* readTree(orgId, cwd, tree, true); + if ( + entries.length !== expected.size || + entries.some((entry) => { + const wanted = expected.get(entry.relative); + return wanted?.mode !== entry.mode || wanted.sha !== entry.sha; + }) + ) + return yield* new MemoryCheckoutInvalid({ + orgId, + operation: "writeTree", + reason: "Written tree differs from validated bytes", + }); +}); + +export const ancestors = Effect.fn("Memory.ancestors")(function* ( + orgId: KinOrgId, + writer: string, + tip: string, + stop?: string, +) { + const seen = new Map>(); + let pending = [tip]; + while (pending.length) { + const batch = [...new Set(pending)].filter((sha) => !seen.has(sha)); + if (!batch.length) break; + const objects = yield* readObjects( + orgId, + writer, + batch.map((sha) => ({ sha, type: "commit" as const })), + ); + pending = []; + for (const [index, bytes] of objects.entries()) { + const sha = batch[index]!; + const parents = [ + ...(bytes.toString("utf8").split("\n\n")[0] ?? "").matchAll(/^parent ([a-f0-9]+)$/gm), + ].map((m) => m[1]!); + seen.set(sha, parents); + pending.push(...parents); + } + if (stop && seen.has(stop)) break; + } + return seen; +}); diff --git a/docs/kin/specs/lead.md b/docs/kin/specs/lead.md index 263e432ed930d..e4b5b6c8e596b 100644 --- a/docs/kin/specs/lead.md +++ b/docs/kin/specs/lead.md @@ -384,7 +384,7 @@ Staged bytes live only in server memory. Discard requires no checkout validation Branches: at L1 and above the writer commits to the default branch. At L0 it commits to `kin/shadow/` (see Levels). Dreaming (triggers spec) commits to `dream/` branches and takes the same per-organization semaphore while it writes, so it never runs during a wake-up. -Pushes and fetches use the configured URL explicitly, never a local remote name or pushurl. A retry of an already recorded wake-up pushes again before returning success. Synchronization uses plumbing-only fast-forwards and replays divergent commits using `merge-tree` and `commit-tree`; conflicts leave the branch unchanged and fail with `MemorySyncConflict`. Failed fetches retain the SQLite anchor. Existing writers with mismatched or missing anchors fail closed as `MemoryCheckoutInvalid`. +Pushes and fetches use the configured URL explicitly, never a local remote name or pushurl. A retry of an already recorded wake-up pushes again before returning success. Synchronization uses verified ancestry and plumbing-only fast-forwards. Divergent trees merge disjoint edits and append-only journals in process, then use `commit-tree` with both parents. Content conflicts preserve the local head on a conflict branch, reset to the remote anchor and notify the owner. Missing or ambiguous merge bases preserve and notify, then fail closed without resetting. Failed fetches retain the SQLite anchor. Existing writers with mismatched or missing anchors fail closed as `MemoryCheckoutInvalid`. Front matter matches the kin-memory README, including snake_case keys. Date-only values mean UTC midnight; date-times require an explicit UTC offset, and invalid dates make an item malformed. `links` values containing `#` must be quoted. `owner` and `said_by` hold handles; `orgId` and `path` belong only to the server-side `MemoryItemHead`. @@ -634,7 +634,7 @@ LEAD-14. **End-to-end shadow week dry run.** Script that replays a day of record `KinSecretScan` is deliberately supplied separately. `noFindingsStubLayer` returns no findings until TRIG-16 replaces it; it is not a production secret detector. Any finding or scanner failure blocks the whole commit. TRIG-16 must preserve this fail-closed interface: `scan({ orgId, files: ReadonlyArray<{ path: string; content: Uint8Array }> })` returns `{ path, ruleId, line }[]`, with typed `KinSecretScanError`. -`commitWakeup` builds TRIG-12's sectioned `INDEX.md` from the hash-verified recorded tree plus staged files, read history, the server clock and the owner's time zone, in the same wake-up commit. The read-side rebuild is display-only; no read-checkout text is trusted. No migration is needed. The runner serializes prepare/stage/commit and dreaming per organization. Identical commit retries find the existing wake-up commit; failed pushes retain local commits, and failures discard uncommitted files. Journal additions use `appendJournal`, never replacement through `stage`. +`commitWakeup` builds TRIG-12's sectioned `INDEX.md` from the hash-verified recorded tree plus staged files, read history, the server clock and the owner's time zone, in the same wake-up commit. The read-side rebuild is display-only; no read-checkout text is trusted. K005 adds the SQLite branch anchors required by the writer. The runner serializes prepare/stage/commit and dreaming per organization. Identical commit retries find the existing wake-up commit; failed pushes retain local commits, and failures discard uncommitted files. Journal additions use `appendJournal`, never replacement through `stage`. ### Memory writer operations From fc30b004976225dbb4ed8c734bb9b3e06e401a5b Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Sun, 8 Nov 2026 04:37:29 +0530 Subject: [PATCH 07/11] test(kin): cover private upload-pack and ordered migration replay --- apps/server/src/kin/memory/Memory.test.ts | 7 ++++++- apps/server/src/kin/memory/MemoryWriter.test.ts | 1 + apps/server/src/kin/persistence/KinMigrations.test.ts | 8 ++++++-- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/apps/server/src/kin/memory/Memory.test.ts b/apps/server/src/kin/memory/Memory.test.ts index 3128cad25103f..1432dd7b10523 100644 --- a/apps/server/src/kin/memory/Memory.test.ts +++ b/apps/server/src/kin/memory/Memory.test.ts @@ -582,7 +582,12 @@ it.effect( yield* sql`UPDATE kin_trigger_settings SET time_zone = ${row.zone} WHERE org_id = ${orgA}`; expect(yield* cache.readOwnerTimeZone(orgA)).toEqual(row.expected); expect(yield* cache.readOwnerTimeZone(orgB)).toEqual({ _tag: "missing" }); - repository.commit(); + yield* memory.record( + orgA, + "main", + yield* memory.anchor(orgA, "main"), + repository.commit(), + ); yield* memory.prepareWakeup(orgA); if (row.expected._tag !== "zone") expect(yield* memory.readIndex(orgA)).toBe(utcIndex); } diff --git a/apps/server/src/kin/memory/MemoryWriter.test.ts b/apps/server/src/kin/memory/MemoryWriter.test.ts index ec28c34bbf048..194e65512ca73 100644 --- a/apps/server/src/kin/memory/MemoryWriter.test.ts +++ b/apps/server/src/kin/memory/MemoryWriter.test.ts @@ -528,6 +528,7 @@ it.live.each([ yield* git(paths.writer, ["config", "filter.attack.clean", script]); yield* fs.writeFileString(`${paths.writer}/.git/info/attributes`, "*.md filter=attack\n"); yield* git(paths.writer, ["config", "remote.origin.receivepack", script]); + yield* git(paths.writer, ["config", "uploadpack.packObjectsHook", script]); yield* memory.prepareWakeup(orgId); expect(yield* fs.exists(marker)).toBe(false); } diff --git a/apps/server/src/kin/persistence/KinMigrations.test.ts b/apps/server/src/kin/persistence/KinMigrations.test.ts index bc8e888d06b04..8e85b28bfa45d 100644 --- a/apps/server/src/kin/persistence/KinMigrations.test.ts +++ b/apps/server/src/kin/persistence/KinMigrations.test.ts @@ -214,8 +214,12 @@ describe("Kin migrations", () => { yield* sql`INSERT INTO kin_chat_members (member_id, org_id, chat_id, principal_kind, person_id, role, joined_at) VALUES ('existing-owner', 'org', 'existing-chat', 'person', 'owner', 'owner', ${timestamp})`; yield* sql`DROP INDEX kin_chat_members_one_owner`; - yield* sql`DELETE FROM kin_sql_migrations WHERE migration_id = 5`; - assert.deepStrictEqual(yield* runKinMigrations(), [[5, "OneChatOwner"]]); + yield* sql`DROP TABLE kin_memory_heads`; + yield* sql`DELETE FROM kin_sql_migrations WHERE migration_id >= 5`; + assert.deepStrictEqual(yield* runKinMigrations(), [ + [5, "OneChatOwner"], + [6, "MemoryHeads"], + ]); assert.deepStrictEqual( yield* sql`SELECT member_id FROM kin_chat_members WHERE org_id = 'org' ORDER BY member_id`, [{ member_id: "existing-owner" }], From 6c45cf3c72a1236d48a0d4bbe2670568303f2d90 Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Sun, 8 Nov 2026 04:55:38 +0530 Subject: [PATCH 08/11] fix(kin): runnable recovery runbook, reuse conflict branches, strict tree modes - Operator recover script provides the now-required MemoryRecoveryHooks. - A fail-closed org reuses the conflict branch already holding its head instead of pushing a new one per wake-up; recover unblocks it (tested). - readTree rejects unknown entry modes so tree read-back can't miss extras. - INDEX.md builders ignore non-regular entries, matching load and stage. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/kin/memory/Memory.test.ts | 14 ++++++++++++++ apps/server/src/kin/memory/Memory.ts | 17 +++++++++++++++++ apps/server/src/kin/memory/MemoryWriter.ts | 12 +++++++++++- apps/server/src/kin/memory/objects.ts | 6 ++++++ .../server/src/kin/persistence/KinMigrations.ts | 1 - docs/kin/specs/lead.md | 13 +++++++++++-- 6 files changed, 59 insertions(+), 4 deletions(-) diff --git a/apps/server/src/kin/memory/Memory.test.ts b/apps/server/src/kin/memory/Memory.test.ts index 1432dd7b10523..234f0407ef5a6 100644 --- a/apps/server/src/kin/memory/Memory.test.ts +++ b/apps/server/src/kin/memory/Memory.test.ts @@ -1016,6 +1016,20 @@ it.live.each([ message.includes("fixture-wakeup"), ), ).toBe(true); + const again = yield* memory.prepareWakeup(orgA, "fixture-wakeup-2").pipe(Effect.result); + expect(again._tag).toBe("Failure"); + expect( + yield* git(root, [ + `--git-dir=${remote}`, + "for-each-ref", + "--format=%(objectname)", + `refs/heads/kin/conflict/${orgA}/`, + ]), + ).toBe(local); + yield* memory.recover(orgA); + expect( + (yield* memory.prepareWakeup(orgA, "fixture-wakeup-3").pipe(Effect.result))._tag, + ).toBe("Success"); } else if (scenario === "push retry") { expect(result._tag).toBe("Success"); if (result._tag === "Success") { diff --git a/apps/server/src/kin/memory/Memory.ts b/apps/server/src/kin/memory/Memory.ts index f77d55990b382..4c24ce571802d 100644 --- a/apps/server/src/kin/memory/Memory.ts +++ b/apps/server/src/kin/memory/Memory.ts @@ -140,6 +140,7 @@ export class Memory extends Context.Service< branch: string, initial?: string, ) => Effect.Effect; + /** Server-only; never register as an agent tool. */ readonly record: ( orgId: KinOrgId, branch: string, @@ -796,6 +797,21 @@ const make = Effect.gen(function* () { operation: "preserve", reason: "Invalid wake-up id", }); + const existing = (yield* git(orgId, writer, [ + "ls-remote", + "--", + url, + `refs/heads/kin/conflict/${orgId}/*`, + ])) + .split("\n") + .flatMap((line) => { + const [sha, ref] = line.split("\t"); + return sha === head && ref?.startsWith("refs/heads/") + ? [ref.slice("refs/heads/".length)] + : []; + }) + .toSorted()[0]; + if (existing) return existing; const branch = `kin/conflict/${orgId}/${id}-${now.toString(36)}-${++conflictSequence}`; if ((yield* git(orgId, writer, ["ls-remote", "--", url, `refs/heads/${branch}`])).trim()) return yield* new MemorySyncConflict({ @@ -1037,6 +1053,7 @@ const make = Effect.gen(function* () { for (const [file, bytes] of mergedFiles) remoteFiles.set(file, bytes); const heads = []; for (const [file, bytes] of remoteFiles) { + if (!["100644", "100755"].includes(modes.get(file) ?? "")) continue; if (!folders.some((folder) => file.startsWith(`${folder}/`))) continue; const parsed = yield* parseFrontMatter(orgId, file, bytes.toString("utf8")).pipe( Effect.result, diff --git a/apps/server/src/kin/memory/MemoryWriter.ts b/apps/server/src/kin/memory/MemoryWriter.ts index a3927008d77ec..780b6114f5b47 100644 --- a/apps/server/src/kin/memory/MemoryWriter.ts +++ b/apps/server/src/kin/memory/MemoryWriter.ts @@ -577,8 +577,18 @@ const make = Effect.gen(function* () { Buffer.from(entry.text), ]); state.files.set(entry.path, journal.toString("utf8")); + const regular = new Set( + (yield* readTree(orgId, writer, recorded).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(VcsProcess, vcs), + )) + .filter((entry) => ["100644", "100755"].includes(entry.mode)) + .map((entry) => entry.relative), + ); const combined = new Map( - [...recordedFiles].map(([path, bytes]) => [path, bytes.toString("utf8")]), + [...recordedFiles] + .filter(([path]) => regular.has(path)) + .map(([path, bytes]) => [path, bytes.toString("utf8")]), ); for (const [path, text] of state.files) combined.set(path, text); const heads = []; diff --git a/apps/server/src/kin/memory/objects.ts b/apps/server/src/kin/memory/objects.ts index 396d3cc14adc6..e7c4d33155168 100644 --- a/apps/server/src/kin/memory/objects.ts +++ b/apps/server/src/kin/memory/objects.ts @@ -113,6 +113,12 @@ export const readTree = Effect.fn("Memory.readTree")(function* ( if (mode === "40000") pending.push({ sha, prefix: `${relative}/` }); else if (["100644", "100755", "120000", "160000"].includes(mode)) entries.push({ sha, relative, mode }); + else + return yield* new MemoryCheckoutInvalid({ + orgId, + operation: "tree", + reason: "Unknown tree entry mode", + }); offset = nul + 1 + width; } } diff --git a/apps/server/src/kin/persistence/KinMigrations.ts b/apps/server/src/kin/persistence/KinMigrations.ts index bab166cb32ce6..791ec54528d12 100644 --- a/apps/server/src/kin/persistence/KinMigrations.ts +++ b/apps/server/src/kin/persistence/KinMigrations.ts @@ -6,7 +6,6 @@ import Lead from "./migrations/K002_Lead.ts"; import Triggers from "./migrations/K003_Triggers.ts"; import MemoryReads from "./migrations/K004_MemoryReads.ts"; import OneChatOwner from "./migrations/K005_OneChatOwner.ts"; - import MemoryHeads from "./migrations/K006_MemoryHeads.ts"; const run = Migrator.make({}); diff --git a/docs/kin/specs/lead.md b/docs/kin/specs/lead.md index 9bce3219e86d8..5adb5bcdb7d42 100644 --- a/docs/kin/specs/lead.md +++ b/docs/kin/specs/lead.md @@ -384,7 +384,7 @@ Staged bytes live only in server memory. Discard requires no checkout validation Branches: at L1 and above the writer commits to the default branch. At L0 it commits to `kin/shadow/` (see Levels). Dreaming (triggers spec) commits to `dream/` branches and takes the same per-organization semaphore while it writes, so it never runs during a wake-up. -Pushes and fetches use the configured URL explicitly, never a local remote name or pushurl. A retry of an already recorded wake-up pushes again before returning success. Synchronization uses verified ancestry and plumbing-only fast-forwards. Divergent trees merge disjoint edits and append-only journals in process, then use `commit-tree` with both parents. Content conflicts preserve the local head on a conflict branch, reset to the remote anchor and notify the owner. Missing or ambiguous merge bases preserve and notify, then fail closed without resetting. Failed fetches retain the SQLite anchor. Existing writers with mismatched or missing anchors fail closed as `MemoryCheckoutInvalid`. +Pushes and fetches use the configured URL explicitly, never a local remote name or pushurl. A retry of an already recorded wake-up pushes again before returning success. Synchronization uses verified ancestry and plumbing-only fast-forwards. Divergent trees merge disjoint edits and append-only journals in process, then use `commit-tree` with both parents. Content conflicts preserve the local head on a conflict branch, reset to the remote anchor and notify the owner. Missing or ambiguous merge bases preserve and notify, then fail closed without resetting; the organization stays blocked until an operator runs `Memory.recover`, and repeated attempts reuse the conflict branch already holding that head. Failed fetches retain the SQLite anchor. Existing writers with mismatched or missing anchors fail closed as `MemoryCheckoutInvalid`. Front matter matches the kin-memory README, including snake_case keys. Date-only values mean UTC midnight; date-times require an explicit UTC offset, and invalid dates make an item malformed. `links` values containing `#` must be quoted. `owner` and `said_by` hold handles; `orgId` and `path` belong only to the server-side `MemoryItemHead`. @@ -671,7 +671,16 @@ const config = Layer.succeed(MemoryConfig, { await Effect.runPromise( Effect.scoped( Effect.flatMap(Memory.Memory, (memory) => memory.recover(orgId)).pipe( - Effect.provide(Memory.layer.pipe(Layer.provide(config))), + Effect.provide( + Memory.layer.pipe( + Layer.provide(config), + Layer.provide( + Layer.succeed(Memory.MemoryRecoveryHooks, { + notify: (notice) => Effect.logWarning("Memory recovery", notice), + }), + ), + ), + ), Effect.provide(database), Effect.provide(Vcs.layer), Effect.provide(NodeServices.layer), From 8e87b4aa7840d6fc9e66ad06062932d5b4d35498 Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:19:50 +0530 Subject: [PATCH 09/11] fix(kin): validate the writer on pinned writer operations; quiet supersession reads Writer ops reused the pinned checkout without checking .git, so a lead deleting it let git discover a parent repo. Reads validate only when they set a pin again. Wake-ups pass their id to prepareWakeup for conflict branch names. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/kin/lead/WakeupRunner.ts | 2 +- apps/server/src/kin/memory/Memory.test.ts | 5 +---- apps/server/src/kin/memory/Memory.ts | 9 ++++++--- apps/server/src/kin/memory/MemoryWriter.test.ts | 15 +++++++++++++++ apps/server/src/kin/memory/MemoryWriter.ts | 6 ++++-- docs/kin/specs/lead.md | 4 ++-- 6 files changed, 29 insertions(+), 12 deletions(-) diff --git a/apps/server/src/kin/lead/WakeupRunner.ts b/apps/server/src/kin/lead/WakeupRunner.ts index 1a551a23f4176..cac394b78d9c1 100644 --- a/apps/server/src/kin/lead/WakeupRunner.ts +++ b/apps/server/src/kin/lead/WakeupRunner.ts @@ -351,7 +351,7 @@ const make = Effect.gen(function* () { } if (events.length === 0) return; const budgetContext = yield* budget.promptContext(orgId); - const pinned = yield* memory.prepareWakeup(orgId).pipe( + const pinned = yield* memory.prepareWakeup(orgId, wakeupId).pipe( Effect.tapError((error) => Effect.logError("Kin memory preparation failed", { orgId, diff --git a/apps/server/src/kin/memory/Memory.test.ts b/apps/server/src/kin/memory/Memory.test.ts index 10f4c07c8815b..402e17e08e711 100644 --- a/apps/server/src/kin/memory/Memory.test.ts +++ b/apps/server/src/kin/memory/Memory.test.ts @@ -462,10 +462,7 @@ it.effect( expect( yield* sql`SELECT * FROM kin_memory_reads WHERE org_id = ${orgA} ORDER BY item_id`, ).toHaveLength(1); - expect(repository.counts()).toEqual({ - ...counts, - validations: counts.validations + 3, - }); + expect(repository.counts()).toEqual(counts); yield* TestClock.adjust("7 weeks"); yield* memory.readItem(orgA, itemId); const moved = repository.files.get("deferrals/def-001.md"); diff --git a/apps/server/src/kin/memory/Memory.ts b/apps/server/src/kin/memory/Memory.ts index b670a28d1daca..966f0bd38e82c 100644 --- a/apps/server/src/kin/memory/Memory.ts +++ b/apps/server/src/kin/memory/Memory.ts @@ -469,8 +469,10 @@ const make = Effect.gen(function* () { reusePin = false, ) { const paths = yield* pathsFor(orgId); - if (reusePin && initializedCheckouts.has(orgId) && pinnedHeads.has(orgId)) + if (reusePin && initializedCheckouts.has(orgId) && pinnedHeads.has(orgId)) { + yield* validateWriter(orgId, paths, false); return { writer: paths.writer, read: paths.read }; + } yield* io(orgId, "mkdir", fs.makeDirectory(paths.root, { recursive: true })); if (yield* io(orgId, "exists", fs.exists(paths.writer))) { yield* validateWriter(orgId, paths, false); @@ -518,8 +520,9 @@ const make = Effect.gen(function* () { const load = Effect.fn("Memory.load")(function* (orgId: KinOrgId, force = false) { const paths = yield* pathsFor(orgId); - yield* validateWriter(orgId, paths, false); - const head = pinnedHeads.get(orgId) ?? (yield* anchor(orgId, config.defaultBranch ?? "main")); + const pinned = pinnedHeads.get(orgId); + if (pinned === undefined) yield* validateWriter(orgId, paths, false); + const head = pinned ?? (yield* anchor(orgId, config.defaultBranch ?? "main")); pinnedHeads.set(orgId, head); const previous = snapshots.get(orgId); if (!force && previous?.head === head) return previous; diff --git a/apps/server/src/kin/memory/MemoryWriter.test.ts b/apps/server/src/kin/memory/MemoryWriter.test.ts index 194e65512ca73..5dc139d5b196c 100644 --- a/apps/server/src/kin/memory/MemoryWriter.test.ts +++ b/apps/server/src/kin/memory/MemoryWriter.test.ts @@ -40,6 +40,7 @@ it.live.each([ "P7-crash", "P8-lost-writer", "P9-shadow", + "P10-lost-git-dir", "sha256", "review-default", "review-null", @@ -356,6 +357,20 @@ it.live.each([ ); return; } + if (scenario === "P10-lost-git-dir") { + yield* git(root, ["init", "--quiet"]); + yield* fs.remove(`${paths.writer}/.git`, { recursive: true }); + const result = yield* writer + .stage(orgId, "wake1", { path: "claims/new.md", content: fixtureItem({ kind: "claim" }) }) + .pipe(Effect.result); + expect(result._tag).toBe("Failure"); + if (result._tag === "Failure") + expect(result.failure).toMatchObject({ + _tag: "MemoryCheckoutInvalid", + operation: "validate", + }); + return; + } if (scenario === "P8-lost-writer" || scenario === "P9-shadow") { if (scenario === "P9-shadow") yield* writer.commitWakeup(orgId, "wake0", "completed"); yield* fs.remove(paths.writer, { recursive: true }); diff --git a/apps/server/src/kin/memory/MemoryWriter.ts b/apps/server/src/kin/memory/MemoryWriter.ts index 780b6114f5b47..b5fd6ed3c4886 100644 --- a/apps/server/src/kin/memory/MemoryWriter.ts +++ b/apps/server/src/kin/memory/MemoryWriter.ts @@ -406,7 +406,8 @@ const make = Effect.gen(function* () { (item.id === fm.id || item.links.some((link) => fm.links.includes(link))) && item.id !== change.supersedes ) { - const body = bodies.get(item.path) ?? (yield* memory.readItem(orgId, item.id)).body; + const body = + bodies.get(item.path) ?? (yield* memory.readItemQuietly(orgId, item.id)).body; const rank = (trust: string) => ["external", "agent", "member", "owner"].indexOf(trust); if ( @@ -475,7 +476,8 @@ const make = Effect.gen(function* () { yield* parseFrontMatter(orgId, change.path, text); if (superseded) { const oldBody = - bodies.get(superseded.path) ?? (yield* memory.readItem(orgId, superseded.id)).body; + bodies.get(superseded.path) ?? + (yield* memory.readItemQuietly(orgId, superseded.id)).body; const { orgId: _org, path: oldPath, diff --git a/docs/kin/specs/lead.md b/docs/kin/specs/lead.md index 867a2cd312369..b6d23aaafa0b2 100644 --- a/docs/kin/specs/lead.md +++ b/docs/kin/specs/lead.md @@ -402,11 +402,11 @@ Front matter matches the kin-memory README, including snake_case keys. Date-only LEAD-2: the front matter parser must use YAML’s core or JSON schema; YAML 1.1 turns `review_by: 2026-11-13` into a Date. -The production `MemoryConfig` layer derives its server-owned base from `ServerConfig.stateDir`, using `${stateDir}/kin//memory` and `memory-read` (organization ids are path-encoded). Tests may supply an isolated base and relative paths. Paths must not overlap or escape that base. `ensureCheckouts` bootstraps a missing writer from a fresh fetch of the configured URL and configured default branch (`main` by default). K011 stores `(org_id, branch, head)` in `kin_memory_heads`; no writable file or local remote ref can bootstrap an existing checkout. Before committing or pushing, each local branch tip must match its anchor. Compare-and-swap branch updates and SQLite recording happen together in the writer operation; a crash between them is rolled back to the recorded anchor with a compare-and-swap and an operator notice. Prepare and commit share a per-org lock. Pinned reads reuse initialized checkout paths, but validate the writer structure on every load and read only the SQLite-anchored default-branch head. Read clones are replaced from scratch on prepare: git is never run in an existing lead read clone. Every git invocation uses a private git directory excluding repository config, hooks and grafts; merge plumbing uses verified bytes and a private index, never git merge-tree or a writer work tree, with system/global config disabled and hooks/fsmonitor disabled. +The production `MemoryConfig` layer derives its server-owned base from `ServerConfig.stateDir`, using `${stateDir}/kin//memory` and `memory-read` (organization ids are path-encoded). Tests may supply an isolated base and relative paths. Paths must not overlap or escape that base. `ensureCheckouts` bootstraps a missing writer from a fresh fetch of the configured URL and configured default branch (`main` by default). K011 stores `(org_id, branch, head)` in `kin_memory_heads`; no writable file or local remote ref can bootstrap an existing checkout. Before committing or pushing, each local branch tip must match its anchor. Compare-and-swap branch updates and SQLite recording happen together in the writer operation; a crash between them is rolled back to the recorded anchor with a compare-and-swap and an operator notice. Prepare and commit share a per-org lock. Pinned reads and writer operations skip re-bootstrap; writer operations still validate the writer checkout every time, and reads validate when they set a new pin. Read clones are replaced from scratch on prepare: git is never run in an existing lead read clone. Every git invocation uses a private git directory excluding repository config, hooks and grafts; merge plumbing uses verified bytes and a private index, never git merge-tree or a writer work tree, with system/global config disabled and hooks/fsmonitor disabled. **Threat model.** Leads currently run full-access as the same OS user (`kin`) as the server. Assume leads can modify everything under `${stateDir}/kin/*/memory*`, including `.git` and file anchors. Code trusts only SQLite and server configuration, checks recorded branch tips, and builds commits from server-validated bytes rather than ambient working-tree/index content. These are not an OS sandbox: a same-user lead can still attack server processes or SQLite itself. Separating leads into an OS user unable to write `${stateDir}/kin` remains Nishchay's operations decision and is required before production use. -The read side pins the server-owned writer repository’s HEAD at wake-up (or first read), never the lead’s read-checkout HEAD or working-tree edits. Snapshots reload only on a new wake-up pin or explicit `rebuildCache`; blob contents and parse failures are cached by SHA. Symlinks are skipped; oversized blobs (over 256 KiB), malformed items and kind/folder mismatches are logged and skipped. Queries return duplicate ids with `duplicate: true` on each copy and mark them in the index; ambiguous `readItem` calls fail with `MemoryDuplicateId`. Queries include invalid items unless `excludeInvalid` is set; `invalid_at` remains on each returned `MemoryItemHead`. Date bounds are inclusive and compare instants; date-only strings mean midnight UTC. Missing or unparsable dates do not match a bounded date query. Overdue review means strictly before the current clock. +The read side pins the SQLite-anchored default-branch head at wake-up (or first read), never the lead’s read-checkout HEAD or working-tree edits. Snapshots reload only on a new wake-up pin or explicit `rebuildCache`; blob contents and parse failures are cached by SHA. Symlinks are skipped; oversized blobs (over 256 KiB), malformed items and kind/folder mismatches are logged and skipped. Queries return duplicate ids with `duplicate: true` on each copy and mark them in the index; ambiguous `readItem` calls fail with `MemoryDuplicateId`. Queries include invalid items unless `excludeInvalid` is set; `invalid_at` remains on each returned `MemoryItemHead`. Date bounds are inclusive and compare instants; date-only strings mean midnight UTC. Missing or unparsable dates do not match a bounded date query. Overdue review means strictly before the current clock. `readItem` best-effort records integer-millisecond last-read times in Kin’s `kin_memory_reads` table, keyed by organization and item id so moves preserve history. Failed history writes are logged without failing the read. Never-read items use `recorded_at` as the baseline; index generation and queries do not count as reading an item. At eight weeks unread, items leave only the index, not search or git. Rebuilds use original heads, cap the index at 200 physical lines including the `As of ` first line, and flag invalid and overdue items. TRIG-12 extends this builder with sections, ordering and freshness defaults; no writer methods or automatic defaults are part of LEAD-2. From ca96d8c47fb1e52a82ca477d6be75b661bdeba56 Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:41:16 +0530 Subject: [PATCH 10/11] fix(kin): refuse repository-dependent memory git without .git Without --git-dir, git would discover a parent repository and its config. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/kin/memory/git.ts | 9 +++++++++ apps/server/src/kin/memory/objects.test.ts | 7 +++++++ 2 files changed, 16 insertions(+) diff --git a/apps/server/src/kin/memory/git.ts b/apps/server/src/kin/memory/git.ts index e803680185786..5d0e63e027849 100644 --- a/apps/server/src/kin/memory/git.ts +++ b/apps/server/src/kin/memory/git.ts @@ -6,6 +6,12 @@ import * as FileSystem from "effect/FileSystem"; import { VcsProcess } from "../../vcs/VcsProcess.ts"; class MemoryGitRefRace extends Schema.TaggedError()("MemoryGitRefRace", {}) {} +class MemoryGitMissingRepository extends Schema.TaggedError()( + "MemoryGitMissingRepository", + { command: Schema.String }, +) {} + +const repositoryFree = new Set(["clone", "clone-snapshot", "init", "ls-remote"]); const formats = new Map(); @@ -24,6 +30,9 @@ export const memoryGit = Effect.fn("Memory.gitHardened")(function* ( Effect.gen(function* () { const directory = yield* fs.makeTempDirectoryScoped(); const repo = yield* fs.exists(`${cwd}/.git`); + // Without --git-dir, git would discover a parent repository and its config. + if (!repo && !repositoryFree.has(args[0] ?? "")) + return yield* new MemoryGitMissingRepository({ command: args[0] ?? "" }); let packedBefore: string | undefined; let format = formats.get(cwd); if (!format && repo) { diff --git a/apps/server/src/kin/memory/objects.test.ts b/apps/server/src/kin/memory/objects.test.ts index 009c6eb18df1d..16f6fc8ce47bd 100644 --- a/apps/server/src/kin/memory/objects.test.ts +++ b/apps/server/src/kin/memory/objects.test.ts @@ -15,6 +15,8 @@ it.effect( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const cwd = yield* fs.makeTempDirectoryScoped(); + yield* fs.makeDirectory(`${cwd}/.git/objects`, { recursive: true }); + yield* fs.makeDirectory(`${cwd}/.git/refs`, { recursive: true }); const objects = new Map(); const object = (type: string, text: string) => { const bytes = Buffer.from(text); @@ -76,6 +78,11 @@ it.effect( new Map([["rules.md", { mode: "100644", sha: "0".repeat(40) }]]), ).pipe(Effect.result); expect(rejected._tag).toBe("Failure"); + yield* fs.remove(`${cwd}/.git`, { recursive: true }); + calls = 0; + const missing = yield* ancestors(org, cwd, tip).pipe(Effect.result); + expect(missing._tag).toBe("Failure"); + expect(calls).toBe(0); }).pipe(Effect.provideService(VcsProcess, vcs)); }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); From 05c2c37ffa156f8d147cac4fde778d822effebd4 Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Sun, 8 Nov 2026 14:04:09 +0530 Subject: [PATCH 11/11] fix(kin): memory git requires a real .git directory; report it as checkout invalid Co-Authored-By: Claude Opus 5.5 --- apps/server/src/kin/memory/Memory.ts | 12 +++++++++++- apps/server/src/kin/memory/MemoryWriter.ts | 14 +++++++++++++- apps/server/src/kin/memory/git.ts | 7 ++++--- apps/server/src/kin/memory/objects.test.ts | 21 ++++++++++++++++----- 4 files changed, 44 insertions(+), 10 deletions(-) diff --git a/apps/server/src/kin/memory/Memory.ts b/apps/server/src/kin/memory/Memory.ts index 966f0bd38e82c..4ecc9ded5fd74 100644 --- a/apps/server/src/kin/memory/Memory.ts +++ b/apps/server/src/kin/memory/Memory.ts @@ -321,7 +321,17 @@ const make = Effect.gen(function* () { onStdoutChunk?: (chunk: Uint8Array) => void, ) { return yield* memoryGit(cwd, args, stdin, {}, onStdoutChunk).pipe( - Effect.mapError((cause) => new MemoryError({ orgId, operation: "git", cause })), + Effect.mapError((cause) => + cause._tag === "MemoryGitMissingRepository" + ? new MemoryCheckoutInvalid({ + orgId, + operation: "git", + reason: "Writer .git is missing or not a directory", + path: cwd, + cause, + }) + : new MemoryError({ orgId, operation: "git", cause }), + ), ); }); const advertisedHead = Effect.fn("Memory.advertisedHead")(function* ( diff --git a/apps/server/src/kin/memory/MemoryWriter.ts b/apps/server/src/kin/memory/MemoryWriter.ts index b5fd6ed3c4886..113280e8906c5 100644 --- a/apps/server/src/kin/memory/MemoryWriter.ts +++ b/apps/server/src/kin/memory/MemoryWriter.ts @@ -159,7 +159,19 @@ const make = Effect.gen(function* () { identity: NodeJS.ProcessEnv = {}, stdin?: string, ) { - return (yield* io(orgId, memoryGit(cwd, args, stdin, identity))).trim(); + return (yield* memoryGit(cwd, args, stdin, identity).pipe( + Effect.mapError((cause) => + cause._tag === "MemoryGitMissingRepository" + ? new MemoryCheckoutInvalid({ + orgId, + operation: "git", + reason: "Writer .git is missing or not a directory", + path: cwd, + cause, + }) + : new MemoryError({ orgId, operation: "writer", cause }), + ), + )).trim(); }); const branchHead = Effect.fn("MemoryWriter.branchHead")(function* ( orgId: KinOrgId, diff --git a/apps/server/src/kin/memory/git.ts b/apps/server/src/kin/memory/git.ts index 5d0e63e027849..b17bf463e29c3 100644 --- a/apps/server/src/kin/memory/git.ts +++ b/apps/server/src/kin/memory/git.ts @@ -6,12 +6,12 @@ import * as FileSystem from "effect/FileSystem"; import { VcsProcess } from "../../vcs/VcsProcess.ts"; class MemoryGitRefRace extends Schema.TaggedError()("MemoryGitRefRace", {}) {} -class MemoryGitMissingRepository extends Schema.TaggedError()( +export class MemoryGitMissingRepository extends Schema.TaggedError()( "MemoryGitMissingRepository", { command: Schema.String }, ) {} -const repositoryFree = new Set(["clone", "clone-snapshot", "init", "ls-remote"]); +const repositoryFree = new Set(["clone", "ls-remote"]); const formats = new Map(); @@ -29,7 +29,8 @@ export const memoryGit = Effect.fn("Memory.gitHardened")(function* ( return yield* Effect.scoped( Effect.gen(function* () { const directory = yield* fs.makeTempDirectoryScoped(); - const repo = yield* fs.exists(`${cwd}/.git`); + const entry = yield* fs.lstat(`${cwd}/.git`).pipe(Effect.option); + const repo = Option.isSome(entry) && entry.value.type === "Directory"; // Without --git-dir, git would discover a parent repository and its config. if (!repo && !repositoryFree.has(args[0] ?? "")) return yield* new MemoryGitMissingRepository({ command: args[0] ?? "" }); diff --git a/apps/server/src/kin/memory/objects.test.ts b/apps/server/src/kin/memory/objects.test.ts index 16f6fc8ce47bd..12de5de57c533 100644 --- a/apps/server/src/kin/memory/objects.test.ts +++ b/apps/server/src/kin/memory/objects.test.ts @@ -78,11 +78,22 @@ it.effect( new Map([["rules.md", { mode: "100644", sha: "0".repeat(40) }]]), ).pipe(Effect.result); expect(rejected._tag).toBe("Failure"); - yield* fs.remove(`${cwd}/.git`, { recursive: true }); - calls = 0; - const missing = yield* ancestors(org, cwd, tip).pipe(Effect.result); - expect(missing._tag).toBe("Failure"); - expect(calls).toBe(0); + const elsewhere = yield* fs.makeTempDirectoryScoped(); + yield* fs.rename(`${cwd}/.git`, `${elsewhere}/.git`); + for (const shape of ["missing", "file", "symlink"] as const) { + if (shape === "file") + yield* fs.writeFileString(`${cwd}/.git`, `gitdir: ${elsewhere}/.git\n`); + if (shape === "symlink") { + yield* fs.remove(`${cwd}/.git`); + yield* fs.symlink(`${elsewhere}/.git`, `${cwd}/.git`); + } + calls = 0; + const refused = yield* ancestors(org, cwd, tip).pipe(Effect.result); + expect(refused._tag).toBe("Failure"); + if (refused._tag === "Failure") + expect(refused.failure._tag).toBe("MemoryCheckoutInvalid"); + expect(calls).toBe(0); + } }).pipe(Effect.provideService(VcsProcess, vcs)); }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), );