diff --git a/apps/server/src/kin/dreaming/checkDream.test.ts b/apps/server/src/kin/dreaming/checkDream.test.ts index e3bd6bc7e1686..e85a0b7a2e5d4 100644 --- a/apps/server/src/kin/dreaming/checkDream.test.ts +++ b/apps/server/src/kin/dreaming/checkDream.test.ts @@ -92,6 +92,20 @@ it.effect.each([ ".github/workflows/memory-check.yml is outside item files and journal/", ], }, + { + name: "rejects item and journal paths memory-check would refuse", + tree: new Map([ + ...goodTree, + ["claims/acme.renewal.md", "x\n"], + ["commitments/q4/ship.md", "x\n"], + ["journal/2026/week-45.md", "x\n"], + ]), + failures: [ + "claims/acme.renewal.md is outside item files and journal/", + "commitments/q4/ship.md is outside item files and journal/", + "journal/2026/week-45.md is outside item files and journal/", + ], + }, { name: "rejects a change dream.json does not list", dream: { ...goodDream, changes: goodDream.changes.slice(1) }, diff --git a/apps/server/src/kin/dreaming/checkDream.ts b/apps/server/src/kin/dreaming/checkDream.ts index 0b22b7879b905..4ae2606837405 100644 --- a/apps/server/src/kin/dreaming/checkDream.ts +++ b/apps/server/src/kin/dreaming/checkDream.ts @@ -3,6 +3,7 @@ import * as Effect from "effect/Effect"; import * as Schema from "effect/Schema"; import { parseFrontMatter } from "../memory/frontMatter.ts"; +import { kinShapeFailure } from "../memory/lint.ts"; import { itemFolders, type MaterialEntry } from "./material.ts"; const Sources = Schema.Array(Schema.String).check(Schema.isMinLength(1)); @@ -85,7 +86,7 @@ export const checkDream = Effect.fn("DreamRunner.check")(function* (input: { failures.push("INDEX.md is built by code"); continue; } - if (!isItemPath(path) && !isJournalPath(path)) { + if ((!isItemPath(path) && !isJournalPath(path)) || kinShapeFailure(path) !== undefined) { failures.push(`${path} is outside item files and journal/`); continue; } diff --git a/apps/server/src/kin/memory/Memory.ts b/apps/server/src/kin/memory/Memory.ts index c2968bbce9c99..f2e42bbfd82d6 100644 --- a/apps/server/src/kin/memory/Memory.ts +++ b/apps/server/src/kin/memory/Memory.ts @@ -210,9 +210,9 @@ export class Memory extends Context.Service< >()("t3/kin/memory/Memory") {} const decodeHead = Schema.decodeEffect(MemoryItemHead); -const folders = ["objectives", "commitments", "deferrals", "decisions", "people", "claims"]; +export const folders = ["objectives", "commitments", "deferrals", "decisions", "people", "claims"]; -const maxBlobBytes = 256 * 1024; +export const maxBlobBytes = 256 * 1024; interface Snapshot extends MemorySyncSnapshot { readonly files: ReadonlyMap; diff --git a/apps/server/src/kin/memory/MemoryWriter.test.ts b/apps/server/src/kin/memory/MemoryWriter.test.ts index 3970678e0aff4..4691c757fd1c1 100644 --- a/apps/server/src/kin/memory/MemoryWriter.test.ts +++ b/apps/server/src/kin/memory/MemoryWriter.test.ts @@ -582,6 +582,10 @@ it.live.each([ "/claims/a.md", "claims/.git/a.md", "claims/../../a.md", + // Paths memory-check refuses in Kin's unowned folders. + "deferrals/a.b.md", + "deferrals/nested/a.md", + "deferrals/CLAUDE.md", ]) { expect( (yield* writer diff --git a/apps/server/src/kin/memory/MemoryWriter.ts b/apps/server/src/kin/memory/MemoryWriter.ts index 1b69e0424e4ec..8a932232ba373 100644 --- a/apps/server/src/kin/memory/MemoryWriter.ts +++ b/apps/server/src/kin/memory/MemoryWriter.ts @@ -29,6 +29,7 @@ import { } from "./Memory.ts"; import { parseFrontMatter, MemoryItemError } from "./frontMatter.ts"; import { journalEntry, MemoryJournalError, type JournalContext } from "./journal.ts"; +import { kinShapeFailure } from "./lint.ts"; import { KinSecretScan, KinSecretScanError } from "./secretScan.ts"; export const LEAD_ENDED_WITHOUT_KIN_END = "Lead ended without calling kin_end."; @@ -363,6 +364,8 @@ const make = Effect.gen(function* () { (!(generated && relative === "INDEX.md") && ![...folders, "journal"].includes(parts[0] ?? "")) ) return yield* new MemoryWriteRejected({ reason: "Disallowed memory path" }); + const shape = kinShapeFailure(relative); + if (shape !== undefined) return yield* new MemoryWriteRejected({ reason: shape }); let current = writer; for (const part of parts) { current += `/${part}`; diff --git a/apps/server/src/kin/memory/gitleaksConfig.ts b/apps/server/src/kin/memory/gitleaksConfig.ts new file mode 100644 index 0000000000000..bb5788f786c60 --- /dev/null +++ b/apps/server/src/kin/memory/gitleaksConfig.ts @@ -0,0 +1,69 @@ +import { secretExceptions, secretRules } from "./secretScan.ts"; + +const space = String.raw`\s\v\p{Z}\x{feff}`; + +// RE2 caps counted repeats at 1000, has no lookarounds and its \s misses \v and Unicode spaces. +// Boundaries are dropped rather than emulated: a consuming boundary would hide an adjacent match. +const re2 = (pattern: RegExp) => { + let source = ""; + let inClass = false; + for (let index = 0; index < pattern.source.length; index++) { + const char = pattern.source[index]!; + if (char === "\\") { + const next = pattern.source[++index]!; + if (next === "S") throw new Error(`\\S has no RE2 translation: ${pattern.source}`); + source += next === "s" ? (inClass ? space : `[${space}]`) : `\\${next}`; + continue; + } + if (char === "[") inClass = true; + else if (char === "]") inClass = false; + source += char; + } + source = source.replace(/\{(\d+),(\d+)\}/g, (all, min: string, max: string) => + Number(max) > 1000 ? `{${min},}` : all, + ); + return pattern.flags.includes("i") ? `(?i:${source})` : source; +}; + +const literal = (value: string) => { + if (value.includes("'''")) throw new Error(`Pattern cannot be a TOML literal: ${value}`); + return `'''${value}'''`; +}; + +const rule = ([id, pattern]: (typeof secretRules)[number]) => { + const groups = (new RegExp(`${pattern.source}|`).exec("")?.length ?? 1) - 1; + const exception = secretExceptions[id]; + return [ + "[[rules]]", + `id = "${id}"`, + `description = "KinSecretScan ${id}"`, + `regex = ${literal(re2(pattern))}`, + ...(groups > 0 ? ["secretGroup = 1"] : []), + ...(exception?.minEntropy === undefined ? [] : [`entropy = ${exception.minEntropy}`]), + ...(["secret", "match"] as const).flatMap((target) => { + const patterns = exception?.[target]?.map((p) => literal(re2(p))); + if (patterns === undefined) return []; + // Wrapped the way the repo formatter wraps TOML arrays past 100 columns. + const inline = `regexes = [${patterns.join(", ")}]`; + return [ + "", + "[[rules.allowlists]]", + `regexTarget = "${target}"`, + ...(inline.length <= 100 + ? [inline] + : ["regexes = [", ...patterns.map((p) => ` ${p},`), "]"]), + ]; + }), + ].join("\n"); +}; + +export const gitleaksToml = [ + [ + "# Generated from apps/server/src/kin/memory/secretScan.ts; do not edit.", + "# Regenerate: node apps/server/src/kin/memory/gitleaksConfig.ts > ops/kin/memory-repo/gitleaks.toml", + 'title = "kin-memory"', + ].join("\n"), + ...secretRules.map(rule), +].join("\n\n"); + +if (import.meta.main) process.stdout.write(`${gitleaksToml}\n`); diff --git a/apps/server/src/kin/memory/lint.test.ts b/apps/server/src/kin/memory/lint.test.ts new file mode 100644 index 0000000000000..84d1d7d6cee56 --- /dev/null +++ b/apps/server/src/kin/memory/lint.test.ts @@ -0,0 +1,318 @@ +// @effect-diagnostics-next-line nodeBuiltinImport:off -- Builds a synthetic git index for the reader. +import * as NodeChildProcess from "node:child_process"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { expect, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; + +import { fixtureItem } from "./__fixtures__/items.ts"; +import { formatLintFailure, lintMemoryFiles } from "./lint.ts"; +import { readMemoryDirectory } from "./lintCli.ts"; + +const encode = (text: string) => new TextEncoder().encode(text); +const symlink = { mode: "120000" }; +const gitlink = { mode: "160000" }; +const tree = (files: Record) => + Object.entries(files).map(([path, content]) => + typeof content === "string" + ? { path, mode: "100644", content: encode(content) } + : content instanceof Uint8Array + ? { path, mode: "100644", content } + : { path, ...content }, + ); +const owners = [ + "* @invented-owner", + "/commitments/", + "/deferrals/", + "/claims/", + "/journal/", + "/INDEX.md", +]; +const codeowners = (...lines: ReadonlyArray) => ({ + ".github/CODEOWNERS": `${lines.join("\n")}\n`, +}); +const rules = { "rules.md": "Invented rules\n", ...codeowners(...owners) }; +const planted = "Invented-private-detail"; + +it.effect.each([ + { name: "a good item", files: { ...rules, "deferrals/def-001.md": fixtureItem() }, failures: [] }, + { + name: "files the server does not load", + files: { + ...rules, + "journal/2026/11-01.md": "no front matter", + "decisions/notes.txt": "x", + ".claude/settings.json": "{}", + "decisions/.hidden/notes.txt": "owner reviewed", + }, + failures: [], + }, + { + name: "a good tree", + files: { + ...rules, + "INDEX.md": "Index\n", + "commitments/com-1.md": fixtureItem({ id: "com-1", kind: "commitment" }), + "deferrals/def-001.md": fixtureItem(), + "claims/Claim_2.md": fixtureItem({ id: "claim-2", kind: "claim" }), + "journal/2026/12-31.md": "journal\n", + ".github/workflows/memory-check.yml": "name: x\n", + "people/demo.md": fixtureItem({ id: "demo", kind: "person" }), + }, + failures: [], + }, + { + name: "agent config in Kin's unowned folders", + files: { + ...rules, + "deferrals/CLAUDE.md": fixtureItem({ id: "x1" }), + "deferrals/AGENTS.md": fixtureItem({ id: "x2" }), + "deferrals/.claude/skills/y/SKILL.md": "x", + "journal/CLAUDE.md": "x", + "journal/.claude/skills/x/SKILL.md": "x", + "claims/.claude/settings.json": "{}", + "commitments/.cursor/rules/x.mdc": "x", + "INDEX.md/CLAUDE.md": "x", + "commitments/link.md": symlink, + }, + failures: [ + "claims/.claude/settings.json:1 Hidden paths are not allowed in Kin's unowned folders", + "commitments/.cursor/rules/x.mdc:1 Hidden paths are not allowed in Kin's unowned folders", + "commitments/link.md:1 Symlinks are not allowed", + "deferrals/.claude/skills/y/SKILL.md:1 Hidden paths are not allowed in Kin's unowned folders", + "deferrals/AGENTS.md:1 Kin's unowned folders only hold item and journal files", + "deferrals/CLAUDE.md:1 Kin's unowned folders only hold item and journal files", + "INDEX.md/CLAUDE.md:1 INDEX.md must be a file", + "journal/.claude/skills/x/SKILL.md:1 Hidden paths are not allowed in Kin's unowned folders", + "journal/CLAUDE.md:1 Kin's unowned folders only hold item and journal files", + ], + }, + { + name: "other shapes in Kin's unowned folders", + files: { + ...rules, + "deferrals/nested/def-001.md": fixtureItem(), + "claims/notes.txt": "x", + "journal/2026-11-01.md": "x", + "journal/2026/13-01.md": "x", + "commitments/gemini.md": "x", + "claims/GROK.md": "x", + "commitments/skill.md": "x", + commitments: "x", + }, + failures: [ + "claims/GROK.md:1 Kin's unowned folders only hold item and journal files", + "claims/notes.txt:1 Kin's unowned folders only hold item and journal files", + "commitments:1 Kin's unowned folders only hold item and journal files", + "commitments/gemini.md:1 Kin's unowned folders only hold item and journal files", + "commitments/skill.md:1 Kin's unowned folders only hold item and journal files", + "deferrals/nested/def-001.md:1 Kin's unowned folders only hold item and journal files", + "journal/2026-11-01.md:1 Kin's unowned folders only hold item and journal files", + "journal/2026/13-01.md:1 Kin's unowned folders only hold item and journal files", + ], + }, + { + name: "links and submodules anywhere", + files: { + ...rules, + "decisions/link.md": symlink, + "vendor/sub": gitlink, + "people/fifo": { mode: "unknown" }, + }, + failures: [ + "decisions/link.md:1 Symlinks are not allowed", + "people/fifo:1 Not a regular file", + "vendor/sub:1 Submodules are not allowed", + ], + }, + { + name: "a missing rules.md", + files: { ...codeowners(...owners), "deferrals/def-001.md": fixtureItem() }, + failures: ["rules.md:1 Missing rules.md"], + }, + { + name: "a missing CODEOWNERS", + files: { "rules.md": "Invented rules\n" }, + failures: [".github/CODEOWNERS:1 Missing .github/CODEOWNERS"], + }, + { + name: "CODEOWNERS with comments, co-owners and CRLF", + files: { + ...rules, + ".github/CODEOWNERS": `# owner\r\n* @invented-owner @co-owner # all\r\n${owners.slice(1).join("\r\n")}\r\n`, + }, + failures: [], + }, + { + name: "a CODEOWNERS placeholder", + files: { ...rules, ...codeowners("* @OWNER", ...owners.slice(1)) }, + failures: [".github/CODEOWNERS:1 CODEOWNERS still names the @OWNER placeholder"], + }, + { + name: "a CODEOWNERS without the * line first", + files: { ...rules, ...codeowners(...owners.slice(1), owners[0]!) }, + failures: [ + ".github/CODEOWNERS:1 CODEOWNERS must start with * and the owner", + ".github/CODEOWNERS:6 CODEOWNERS may only leave Kin's own paths unowned after the * line", + ".github/CODEOWNERS:1 CODEOWNERS lacks the /commitments/ line", + ], + }, + { + name: "a CODEOWNERS * line without an owner", + files: { ...rules, ...codeowners("*", ...owners.slice(1)) }, + failures: [".github/CODEOWNERS:1 CODEOWNERS must start with * and the owner"], + }, + { + name: "CODEOWNERS freeing a path the lead would load", + files: { ...rules, ...codeowners(...owners, "/CLAUDE.md", `/${planted}/ @kin-bot`) }, + failures: [ + ".github/CODEOWNERS:7 CODEOWNERS may only leave Kin's own paths unowned after the * line", + ".github/CODEOWNERS:8 CODEOWNERS may only leave Kin's own paths unowned after the * line", + ], + }, + { + name: "a CODEOWNERS missing a Kin path", + files: { ...rules, ...codeowners(...owners.filter((line) => line !== "/journal/")) }, + failures: [".github/CODEOWNERS:1 CODEOWNERS lacks the /journal/ line"], + }, + { + name: "an unresolved alias", + files: { ...rules, "decisions/dec-001.md": "---\nid: *PLANTED_alias\n---\n" }, + failures: ["decisions/dec-001.md:1 Malformed memory item"], + }, + { + name: "a control character in a path", + files: { ...rules, "claims/a\n::error::\u202e.md": "x".repeat(256 * 1024 + 1) }, + failures: [ + "claims/a\\u{a}::error::\\u{202e}.md:1 Kin's unowned folders only hold item and journal files", + ], + }, + { + name: "a missing required key", + files: { ...rules, "deferrals/def-001.md": fixtureItem().replace(/^said_by: .*\n/m, "") }, + failures: ["deferrals/def-001.md:1 Missing required key said_by"], + }, + { + name: "an invalid value", + files: { ...rules, "deferrals/def-001.md": fixtureItem({ trust: planted }) }, + failures: ["deferrals/def-001.md:16 Invalid value for key trust"], + }, + { + name: "an offsetless date-time", + files: { ...rules, "deferrals/def-001.md": fixtureItem({ due: "2026-11-13T09:00" }) }, + failures: ["deferrals/def-001.md:6 Invalid due; date-times require an explicit offset"], + }, + { + name: "malformed YAML", + files: { ...rules, "decisions/dec-001.md": `---\nid: dec-001\ntitle: [${planted}\n---\n` }, + failures: ["decisions/dec-001.md:3 Malformed YAML"], + }, + { + name: "a kind in the wrong folder", + files: { ...rules, "decisions/def-001.md": fixtureItem() }, + failures: ["decisions/def-001.md:3 Memory kind does not match folder"], + }, + { + name: "a duplicate id", + files: { + ...rules, + "deferrals/a.md": fixtureItem(), + "deferrals/b.md": fixtureItem(), + }, + failures: ["deferrals/b.md:2 Duplicate id, also in deferrals/a.md"], + }, + { + name: "invalid UTF-8 rules", + files: { ...rules, "rules.md": new Uint8Array([0xff, 0xfe]) }, + failures: ["rules.md:1 Not valid UTF-8"], + }, + { + name: "an oversized item", + files: { ...rules, "claims/c.md": "x".repeat(256 * 1024 + 1) }, + failures: ["claims/c.md:1 Larger than 262144 bytes"], + }, +])("lint reports $name", ({ files, failures }) => + Effect.gen(function* () { + const result = yield* lintMemoryFiles(tree(files)); + const lines = result.map(formatLintFailure); + expect(lines).toEqual(failures); + expect(lines.join("\n")).not.toMatch(new RegExp(`${planted}|PLANTED`)); + }), +); + +it.layer(NodeServices.layer)((it) => { + it.effect("the CODEOWNERS template passes only once the owner is filled in", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const here = yield* path.fromFileUrl(new URL(".", import.meta.url)); + const template = yield* fs.readFileString( + path.join(here, "../../../../../ops/kin/memory-repo/CODEOWNERS.template"), + ); + const lint = (text: string) => + lintMemoryFiles(tree({ ...rules, ".github/CODEOWNERS": text })).pipe( + Effect.map((failures) => failures.map(formatLintFailure)), + ); + expect(yield* lint(template)).toEqual([ + ".github/CODEOWNERS:5 CODEOWNERS still names the @OWNER placeholder", + ]); + expect(yield* lint(template.replace("* @OWNER", "* @invented-owner"))).toEqual([]); + }), + ); + it.effect("reads the whole tree with modes and never follows symlinks", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped(); + const outside = yield* fs.makeTempDirectoryScoped(); + yield* fs.writeFileString(`${outside}/secret.md`, fixtureItem()); + yield* fs.makeDirectory(`${root}/deferrals/.claude`, { recursive: true }); + yield* fs.makeDirectory(`${root}/INDEX.md`); + yield* fs.makeDirectory(`${root}/.github/workflows`, { recursive: true }); + yield* fs.writeFileString(`${root}/.github/CODEOWNERS`, "* @invented-owner\n"); + yield* fs.writeFileString(`${root}/.github/workflows/memory-check.yml`, "name: x\n"); + yield* fs.writeFileString(`${root}/rules.md`, "Invented rules\n"); + yield* fs.writeFileString(`${root}/deferrals/.claude/settings.json`, "{}"); + yield* fs.writeFileString(`${root}/INDEX.md/CLAUDE.md`, "x"); + yield* fs.symlink(`${outside}/secret.md`, `${root}/deferrals/link.md`); + yield* fs.symlink(outside, `${root}/people`); + const files = yield* readMemoryDirectory(root); + expect( + files + .map(({ path, mode, content }) => [path, mode, content !== undefined]) + .sort(([a], [b]) => String(a).localeCompare(String(b))), + ).toEqual([ + [".github/CODEOWNERS", "100644", true], + [".github/workflows/memory-check.yml", "100644", false], + ["deferrals/.claude/settings.json", "100644", false], + ["deferrals/link.md", "120000", false], + ["INDEX.md/CLAUDE.md", "100644", false], + ["people", "120000", false], + ["rules.md", "100644", true], + ]); + }).pipe(Effect.scoped), + ); + it.effect("reads git's index in a checkout, so a gitlink or symlink cannot hide", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped(); + const git = (...args: ReadonlyArray) => + expect(NodeChildProcess.spawnSync("git", ["-C", root, ...args]).status).toBe(0); + git("init", "-q"); + yield* fs.makeDirectory(`${root}/commitments`); + yield* fs.writeFileString(`${root}/rules.md`, "Invented rules\n"); + yield* fs.writeFileString(`${root}/commitments/x.md`, "x"); + yield* fs.symlink("../rules.md", `${root}/commitments/link.md`); + git("add", "rules.md", "commitments"); + git("update-index", "--add", "--cacheinfo", `160000,${"a".repeat(40)},claims/sub`); + yield* fs.writeFileString(`${root}/untracked.md`, "x"); + const failures = yield* lintMemoryFiles(yield* readMemoryDirectory(root)); + expect(failures.map(formatLintFailure)).toEqual([ + ".github/CODEOWNERS:1 Missing .github/CODEOWNERS", + "claims/sub:1 Submodules are not allowed", + "commitments/link.md:1 Symlinks are not allowed", + "commitments/x.md:1 Missing front matter delimiters", + ]); + }).pipe(Effect.scoped), + ); +}); diff --git a/apps/server/src/kin/memory/lint.ts b/apps/server/src/kin/memory/lint.ts new file mode 100644 index 0000000000000..599806d7af164 --- /dev/null +++ b/apps/server/src/kin/memory/lint.ts @@ -0,0 +1,197 @@ +import { KinOrgId } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; +import * as SchemaIssue from "effect/SchemaIssue"; +import { YAMLError } from "yaml"; + +import { parseFrontMatter, type MemoryItemError } from "./frontMatter.ts"; +import { folders, maxBlobBytes } from "./Memory.ts"; + +export interface LintFailure { + readonly path: string; + readonly line: number; + readonly reason: string; +} + +const orgId = KinOrgId.make("memory-lint"); +const formatIssue = SchemaIssue.makeFormatterStandardSchemaV1(); + +export interface MemoryTreeEntry { + readonly path: string; + readonly mode: string; + readonly content?: Uint8Array; +} + +const codeowners = ".github/CODEOWNERS"; +const kinOnlyPaths = ["/commitments/", "/deferrals/", "/claims/", "/journal/", "/INDEX.md"]; +const kinOnlyRoots = kinOnlyPaths.map((pattern) => pattern.slice(1).replace(/\/$/, "")); +const regularModes = ["100644", "100755"]; + +export const isLintedPath = (path: string) => + path === "rules.md" || + path === codeowners || + path === "INDEX.md" || + (path.endsWith(".md") && folders.some((folder) => path.startsWith(`${folder}/`))); + +// CODEOWNERS cannot negate a pattern, so nothing but Kin's own shapes may live under its paths. +// Agent instruction files would be loaded by any agent that reads the folder; keep this in step +// with the providers' nested context-file names. User-configured names are out of reach. +const agentFiles = /^(claude|agents?|gemini|grok|crush|qwen|warp|skill)$/i; +const itemPath = /^(?:commitments|deferrals|claims)\/([a-zA-Z0-9][a-zA-Z0-9_-]{0,199})\.md$/; +const journalPath = /^journal\/\d{4}\/(?:0[1-9]|1[0-2])-(?:0[1-9]|[12]\d|3[01])\.md$/; + +/** The shape check alone, for Kin's writers: a path they write must also pass `memory-check`. */ +export const kinShapeFailure = (path: string) => { + const [root, ...rest] = path.split("/"); + if (!kinOnlyRoots.includes(root ?? "")) return undefined; + if (root === "INDEX.md") return rest.length ? "INDEX.md must be a file" : undefined; + if (rest.some((segment) => segment.startsWith("."))) + return "Hidden paths are not allowed in Kin's unowned folders"; + const item = itemPath.exec(path)?.[1]; + if (journalPath.test(path) || (item !== undefined && !agentFiles.test(item))) return undefined; + return "Kin's unowned folders only hold item and journal files"; +}; + +const entryFailure = ({ path, mode }: MemoryTreeEntry) => { + if (mode === "120000") return "Symlinks are not allowed"; + if (mode === "160000") return "Submodules are not allowed"; + if (!regularModes.includes(mode)) return "Not a regular file"; + return kinShapeFailure(path); +}; + +const keyLine = (text: string, key: string) => { + const index = text.split("\n").findIndex((line) => line.startsWith(`${key}:`)); + return index === -1 ? 1 : index + 1; +}; + +// Deny by default: the owner owns everything but Kin's own paths, which is exactly the template. +const codeownersFailures = (text: string) => { + const failures: Array<{ line: number; reason: string }> = []; + const rules = text.split(/\r?\n/).flatMap((raw, index) => { + const tokens = raw.trim().split(/\s+/); + const end = tokens.findIndex((token) => token.startsWith("#")); + const [pattern, ...owners] = end === -1 ? tokens : tokens.slice(0, end); + return pattern ? [{ line: index + 1, pattern, owners }] : []; + }); + for (const rule of rules) + if (rule.owners.some((owner) => owner.toLowerCase() === "@owner")) + failures.push({ line: rule.line, reason: "CODEOWNERS still names the @OWNER placeholder" }); + const [first, ...rest] = rules; + if (first?.pattern !== "*" || first.owners.length === 0) + failures.push({ line: first?.line ?? 1, reason: "CODEOWNERS must start with * and the owner" }); + for (const rule of rest) + if (!kinOnlyPaths.includes(rule.pattern) || rule.owners.length > 0) + failures.push({ + line: rule.line, + reason: "CODEOWNERS may only leave Kin's own paths unowned after the * line", + }); + for (const path of kinOnlyPaths) + if (!rest.some((rule) => rule.pattern === path)) + failures.push({ line: 1, reason: `CODEOWNERS lacks the ${path} line` }); + return failures; +}; + +const dateField = + /^Invalid (recorded_at|valid_from|invalid_at|review_by|due); date-times require an explicit offset$/; + +// Reasons never quote the file: YAML and schema messages embed the offending source. +const describe = (error: MemoryItemError, text: string): { line: number; reason: string } => { + const field = dateField.exec(error.reason)?.[1]; + if (field) return { line: keyLine(text, field), reason: error.reason }; + if (error.reason === "Unknown field in on condition") + return { line: keyLine(text, "on"), reason: error.reason }; + if (error.reason === "Invalid front matter schema" && Schema.isSchemaError(error.cause)) { + const issue = formatIssue(error.cause.issue).issues[0]; + const key = issue?.path?.map(String).join(".") ?? ""; + const top = key.split(".")[0] ?? ""; + if (!top) return { line: 1, reason: error.reason }; + return issue?.message === "Missing key" + ? { line: 1, reason: `Missing required key ${key}` } + : { line: keyLine(text, top), reason: `Invalid value for key ${key}` }; + } + if (error.reason.startsWith("Malformed YAML")) { + const problem = error.cause instanceof Error ? error.cause.cause : undefined; + const line = problem instanceof YAMLError ? (problem.linePos?.[0].line ?? 0) + 1 : 1; + return { + line, + reason: + error.reason === "Malformed YAML; links containing # must be quoted" + ? error.reason + : "Malformed YAML", + }; + } + return { + line: 1, + reason: + error.reason === "Missing front matter delimiters" ? error.reason : "Malformed memory item", + }; +}; + +// Paths come from the checkout: a newline could forge a log line or a workflow command. +export const formatLintFailure = (failure: LintFailure) => + `${failure.path.replace(/[\p{Cc}\p{Cf}\u2028\u2029\\]/gu, (char) => `\\u{${char.codePointAt(0)!.toString(16)}}`)}:${failure.line} ${failure.reason}`; + +/** Checks the whole tree: Kin's unowned paths against an allowlist, memory files the way `Memory.load` reads them. */ +export const lintMemoryFiles = Effect.fn("MemoryLint.lint")(function* ( + files: ReadonlyArray, +) { + const failures: Array = []; + const ids = new Map(); + if (!files.some((file) => file.path === "rules.md")) + failures.push({ path: "rules.md", line: 1, reason: "Missing rules.md" }); + if (!files.some((file) => file.path === codeowners)) + failures.push({ path: codeowners, line: 1, reason: `Missing ${codeowners}` }); + for (const file of [...files].sort((a, b) => a.path.localeCompare(b.path))) { + const shape = entryFailure(file); + if (shape !== undefined) { + failures.push({ path: file.path, line: 1, reason: shape }); + continue; + } + if (!isLintedPath(file.path)) continue; + if (file.content === undefined) { + failures.push({ path: file.path, line: 1, reason: "Unreadable file" }); + continue; + } + if (file.content.byteLength > maxBlobBytes) { + failures.push({ path: file.path, line: 1, reason: `Larger than ${maxBlobBytes} bytes` }); + continue; + } + const text = yield* Effect.try({ + try: () => new TextDecoder("utf-8", { fatal: true }).decode(file.content), + catch: () => "Not valid UTF-8", + }).pipe(Effect.result); + if (text._tag === "Failure") { + failures.push({ path: file.path, line: 1, reason: text.failure }); + continue; + } + if (file.path === codeowners) { + failures.push(...codeownersFailures(text.success).map((f) => ({ path: file.path, ...f }))); + continue; + } + if (!folders.some((folder) => file.path.startsWith(`${folder}/`))) continue; + const parsed = yield* parseFrontMatter(orgId, file.path, text.success).pipe(Effect.result); + if (parsed._tag === "Failure") { + failures.push({ path: file.path, ...describe(parsed.failure, text.success) }); + continue; + } + const { kind, id } = parsed.success.frontMatter; + const folder = kind === "person" ? "people" : `${kind}s`; + if (!file.path.startsWith(`${folder}/`)) { + failures.push({ + path: file.path, + line: keyLine(text.success, "kind"), + reason: "Memory kind does not match folder", + }); + continue; + } + const first = ids.get(id); + if (first !== undefined) + failures.push({ + path: file.path, + line: keyLine(text.success, "id"), + reason: `Duplicate id, also in ${first}`, + }); + else ids.set(id, file.path); + } + return failures; +}); diff --git a/apps/server/src/kin/memory/lintCli.ts b/apps/server/src/kin/memory/lintCli.ts new file mode 100644 index 0000000000000..44921d6595625 --- /dev/null +++ b/apps/server/src/kin/memory/lintCli.ts @@ -0,0 +1,93 @@ +import * as NodeRuntime from "@effect/platform-node/NodeRuntime"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as Console from "effect/Console"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; +import type { PlatformError } from "effect/PlatformError"; +import * as Stream from "effect/Stream"; +import { Argument, Command } from "effect/cli"; +import { ChildProcess, ChildProcessSpawner } from "effect/process"; + +import { formatLintFailure, isLintedPath, lintMemoryFiles, type MemoryTreeEntry } from "./lint.ts"; + +const walkModes: Partial> = { + File: "100644", + SymbolicLink: "120000", +}; + +// Git's index carries modes, so a gitlink or symlink cannot pass for a directory or file. +const gitEntries = Effect.fn("MemoryLint.gitEntries")(function* (root: string) { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const { listing, exitCode } = yield* Effect.scoped( + Effect.gen(function* () { + const handle = yield* spawner.spawn( + ChildProcess.make("git", ["ls-files", "--stage", "-z"], { + cwd: root, + stdin: "ignore", + stderr: "ignore", + }), + ); + return yield* Effect.all( + { listing: Stream.mkString(Stream.decodeText(handle.stdout)), exitCode: handle.exitCode }, + { concurrency: 2 }, + ); + }), + ); + if (exitCode !== 0) return yield* Effect.fail("git ls-files failed" as const); + return listing.split("\0").flatMap((record) => { + const match = /^(\d+) [0-9a-f]+ \d\t(.+)$/s.exec(record); + return match?.[1] && match[2] ? [{ path: match[2], mode: match[1] }] : []; + }); +}); + +const walkEntries = Effect.fn("MemoryLint.walkEntries")(function* (root: string) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const entries: Array<{ path: string; mode: string }> = []; + const visit: (relative: string) => Effect.Effect = Effect.fnUntraced( + function* (relative) { + for (const entry of yield* fs.readDirectoryEntries(path.join(root, relative))) { + const child = relative ? `${relative}/${entry.name}` : entry.name; + if (entry.type === "Directory") yield* visit(child); + else entries.push({ path: child, mode: walkModes[entry.type] ?? "unknown" }); + } + }, + ); + yield* visit(""); + return entries; +}); + +/** Lists every entry in the tree; only regular memory files are read, so a symlink is never followed. */ +export const readMemoryDirectory = Effect.fn("MemoryLint.readDirectory")(function* (root: string) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const listed = (yield* fs.exists(path.join(root, ".git"))) + ? yield* gitEntries(root) + : yield* walkEntries(root); + const files: Array = []; + for (const entry of listed) + files.push( + ["100644", "100755"].includes(entry.mode) && isLintedPath(entry.path) + ? { ...entry, content: yield* fs.readFile(path.join(root, entry.path)) } + : entry, + ); + return files; +}); + +const lintCommand = Command.make( + "memory-lint", + { directory: Argument.String("directory") }, + Effect.fn("MemoryLint.cli")(function* ({ directory }) { + const failures = yield* lintMemoryFiles(yield* readMemoryDirectory(directory)); + for (const failure of failures) yield* Console.log(formatLintFailure(failure)); + if (failures.length > 0) process.exitCode = 1; + }), +).pipe(Command.withDescription("Check a Kin memory checkout against the server's item schema.")); + +if (import.meta.main) { + Command.run(lintCommand, { version: "0.0.0" }).pipe( + Effect.provide(NodeServices.layer), + NodeRuntime.runMain, + ); +} diff --git a/apps/server/src/kin/memory/memoryCheck.test.ts b/apps/server/src/kin/memory/memoryCheck.test.ts new file mode 100644 index 0000000000000..6ed8478e4dc90 --- /dev/null +++ b/apps/server/src/kin/memory/memoryCheck.test.ts @@ -0,0 +1,229 @@ +// @effect-diagnostics-next-line nodeBuiltinImport:off -- Runs the real gitleaks binary when it is installed. +import * as NodeChildProcess from "node:child_process"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { expect, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; +import * as Schema from "effect/Schema"; +import { parse } from "yaml"; + +import { gitleaksToml } from "./gitleaksConfig.ts"; +import { secretRules } from "./secretScan.ts"; + +const body = "aB3dE6gH9jK2mN5pQ8sT1vW4yZ7cF0iL3oR6"; +// Split so this public file never holds a literal token. +const plants: Record<(typeof secretRules)[number][0], string> = { + "github-token": ["gh", "p_"].join("") + body, + "anthropic-key": ["sk-", "ant-"].join("") + body + body, + "openai-key": ["sk-", "proj-"].join("") + body, + "aws-access-key": ["AK", "IA"].join("") + "QWERTYUIOPASDFGH", + "slack-token": ["xo", "xb-"].join("") + body, + "stripe-key": ["sk_", "live_"].join("") + body, + "google-key": ["AI", "za"].join("") + "X".repeat(35), + "pem-private-key": ["-----BEGIN ", "RSA PRIVATE KEY-----"].join(""), + jwt: ["ey", "J"].join("") + "a".repeat(40) + "." + ["ey", "J"].join("") + "b".repeat(40), + "url-password": "postgres://demo:Hunter2Hunter2xyz@db.example.invalid", + "secret-assignment": "api_key = " + body, +}; +const decoy = ["sk-", "proj-"].join("") + "a".repeat(40); +// Each hides a real token behind an allowed match or a space RE2's \s misses. +const hidden = [ + ["decoy-space", `${decoy} ${plants["openai-key"]}`, "openai-key"], + ["decoy-newline", `${decoy}\n${plants["openai-key"]}`, "openai-key"], + ["url-decoy", "postgres://a:${X}@postgres://u:Hunter2Hunter2xyz@db.invalid", "url-password"], + ["nbsp-assignment", `api_key\u00a0=\u00a0${body}`, "secret-assignment"], + ["vtab-assignment", `api_key\v=\v${body}`, "secret-assignment"], +] as const; +const nearMisses = [ + ["gh", "p_short"].join(""), + ["sk-", "proj-"].join("") + "lowercase".repeat(5), + "password = " + "a".repeat(24), + "postgres://demo:${DB_PASS}@db.example.invalid", + "token_scopes: " + body, +]; +const Step = Schema.Struct({ + name: Schema.String, + uses: Schema.optional(Schema.String), + run: Schema.optional(Schema.String), + with: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)), +}); +const Workflow = Schema.Struct({ + on: Schema.Record( + Schema.String, + Schema.NullOr(Schema.Struct({ branches: Schema.optional(Schema.Array(Schema.String)) })), + ), + permissions: Schema.Record(Schema.String, Schema.String), + jobs: Schema.Record( + Schema.String, + Schema.Struct({ + name: Schema.String, + permissions: Schema.optional(Schema.Unknown), + env: Schema.Record(Schema.String, Schema.String), + steps: Schema.Array(Step), + }), + ), +}); +const decodeWorkflow = Schema.decodeUnknownEffect(Workflow); +const decodeReport = Schema.decodeUnknownEffect( + Schema.fromJsonString( + Schema.Array( + Schema.Struct({ File: Schema.String, RuleID: Schema.String, StartLine: Schema.Number }), + ), + ), +); +const gitleaks = NodeChildProcess.spawnSync("gitleaks", ["version"]).status === 0; +const needsGitleaks = (name: string) => + gitleaks + ? name + : `SKIPPED, gitleaks is not on PATH, so this proof did not run (CI never runs it): ${name}`; +const scanFlags = (config: string, root: string) => [ + "--config", + config, + "--gitleaks-ignore-path", + `${root}/no-ignores`, + "--ignore-gitleaks-allow", + "--redact", + "--no-banner", + "--report-format", + "json", + "--report-path", + `${root}/report.json`, +]; + +const opsFile = (name: string) => + Effect.gen(function* () { + const path = yield* Path.Path; + const here = yield* path.fromFileUrl(new URL(".", import.meta.url)); + return path.join(here, "../../../../../ops/kin/memory-repo", name); + }); + +it.effect("the committed gitleaks config is generated from the scanner rules", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + expect(yield* fs.readFileString(yield* opsFile("gitleaks.toml"))).toBe(`${gitleaksToml}\n`); + expect(Object.keys(plants).sort()).toEqual(secretRules.map(([id]) => id).sort()); + }).pipe(Effect.provide(NodeServices.layer)), +); + +it.live.skipIf(!gitleaks)(needsGitleaks("gitleaks flags each planted token and no near miss"), () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped(); + yield* fs.makeDirectory(`${root}/memory/decisions`, { recursive: true }); + for (const [id, token] of Object.entries(plants)) + yield* fs.writeFileString(`${root}/memory/decisions/${id}.md`, `note ${token} end\n`); + for (const [name, text] of hidden) + yield* fs.writeFileString(`${root}/memory/decisions/${name}.md`, `note ${text} end\n`); + yield* fs.writeFileString(`${root}/memory/decisions/near.md`, `${nearMisses.join("\n")}\n`); + yield* fs.writeFileString(`${root}/no-ignores`, ""); + const result = NodeChildProcess.spawnSync("gitleaks", [ + "dir", + ...scanFlags(yield* opsFile("gitleaks.toml"), root), + `${root}/memory`, + ]); + expect(result.status).toBe(1); + const report = yield* decodeReport(yield* fs.readFileString(`${root}/report.json`)); + expect(report.map((f) => `${f.File.split("/").at(-1)} ${f.RuleID}`).sort()).toEqual( + [ + ...Object.keys(plants).map((id) => `${id}.md ${id}`), + ...hidden.map(([name, , id]) => `${name}.md ${id}`), + ].sort(), + ); + const newline = report.find((f) => f.File.endsWith("decoy-newline.md")); + expect(newline?.StartLine).toBe(2); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); + +it.live.skipIf(!gitleaks)( + needsGitleaks("memory-check's history scans see a token that only a merge resolution held"), + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped(); + const repo = `${root}/memory`; + yield* fs.makeDirectory(`${repo}/decisions`, { recursive: true }); + yield* fs.writeFileString(`${root}/no-ignores`, ""); + const env = { ...process.env, GIT_CONFIG_GLOBAL: "/dev/null", GIT_CONFIG_NOSYSTEM: "1" }; + const git = (...args: ReadonlyArray) => { + const run = NodeChildProcess.spawnSync( + "git", + ["-C", repo, "-c", "user.name=Fixture", "-c", "user.email=fixture@invalid", ...args], + { env, encoding: "utf8" }, + ); + expect(run.status, run.stderr).toBe(0); + return run.stdout.trim(); + }; + git("init", "-q", "-b", "main"); + yield* fs.writeFileString(`${repo}/decisions/base.md`, "base\n"); + git("add", "-A"); + git("commit", "-q", "-m", "base"); + const base = git("rev-parse", "HEAD"); + git("checkout", "-q", "-b", "side"); + yield* fs.writeFileString(`${repo}/decisions/side.md`, "side\n"); + git("add", "-A"); + git("commit", "-q", "-m", "side"); + git("checkout", "-q", "main"); + yield* fs.writeFileString(`${repo}/decisions/main.md`, "main\n"); + git("add", "-A"); + git("commit", "-q", "-m", "main"); + git("merge", "-q", "--no-ff", "--no-commit", "side"); + yield* fs.writeFileString(`${repo}/decisions/merge.md`, `note ${plants["github-token"]}\n`); + git("add", "-A"); + git("commit", "-q", "-m", "merge"); + yield* fs.remove(`${repo}/decisions/merge.md`); + git("add", "-A"); + git("commit", "-q", "-m", "remove"); + const head = git("rev-parse", "HEAD"); + const text = yield* fs.readFileString(yield* opsFile("memory-check.yml")); + const scans = text.split("\n").filter((line) => line.includes(`"$bin/gitleaks" git`)); + expect(scans).toHaveLength(2); + for (const scan of scans) { + const logOpts = /--log-opts="([^"]+)"/.exec(scan)?.[1]; + const result = NodeChildProcess.spawnSync("gitleaks", [ + "git", + ...scanFlags(yield* opsFile("gitleaks.toml"), root), + ...(logOpts === undefined + ? [] + : [`--log-opts=${logOpts.replace("$BASE", base).replace("$HEAD", head)}`]), + repo, + ]); + expect(result.status, scan).toBe(1); + const report = yield* decodeReport(yield* fs.readFileString(`${root}/report.json`)); + expect(report.map((f) => `${f.File} ${f.RuleID}`)).toContain( + "decisions/merge.md github-token", + ); + } + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); + +it.effect("memory-check only reads the memory repo and pins what it runs", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const text = yield* fs.readFileString(yield* opsFile("memory-check.yml")); + const workflow = yield* decodeWorkflow(parse(text)); + expect(Object.keys(workflow.on).sort()).toEqual(["pull_request", "push"]); + expect(workflow.on.push?.branches).toEqual(["main"]); + expect(text).not.toContain("pull_request_target"); + expect(workflow.permissions).toEqual({ contents: "read" }); + expect(Object.keys(workflow.jobs)).toEqual(["memory-check"]); + const job = workflow.jobs["memory-check"]; + expect(job?.name).toBe("memory-check"); + expect(job?.permissions).toBeUndefined(); + const steps = job?.steps ?? []; + for (const step of steps.filter((s) => s.uses !== undefined)) + expect(step.uses).toMatch(/^[\w.-]+\/[\w.-]+@[0-9a-f]{40}$/); + for (const step of steps.filter((s) => s.uses?.startsWith("actions/checkout@"))) + expect(step.with?.["persist-credentials"]).toBe(false); + const runs = steps.flatMap((s) => (s.run === undefined ? [] : [s.run])).join("\n"); + expect(runs).not.toContain("${{"); + expect(runs).toMatch(/echo "\$\{GITLEAKS_SHA256\} {2}\S+" \| sha256sum --check --strict/); + expect(job?.env.GITLEAKS_SHA256).toMatch(/^[0-9a-f]{64}$/); + expect(runs).toContain("--ignore-gitleaks-allow"); + expect(runs).toContain("--config t3code/ops/kin/memory-repo/gitleaks.toml"); + expect(runs).toContain("node t3code/apps/server/src/kin/memory/lintCli.ts memory"); + expect(runs).toMatch(/\^\[0-9a-f\]\{40\}\$/); + const t3code = steps.find((s) => s.with?.repository === "GLDRoger/t3code"); + expect(t3code?.with?.ref).toBe("${{ env.T3CODE_REF }}"); + }).pipe(Effect.provide(NodeServices.layer)), +); diff --git a/apps/server/src/kin/memory/secretScan.ts b/apps/server/src/kin/memory/secretScan.ts index 7e39bf38ff76f..d765d717e01b6 100644 --- a/apps/server/src/kin/memory/secretScan.ts +++ b/apps/server/src/kin/memory/secretScan.ts @@ -32,7 +32,7 @@ export class KinSecretScan extends Context.Service< } >()("t3/kin/memory/secretScan/KinSecretScan") {} -const rules = [ +export const secretRules = [ ["github-token", /(?:gh[pousr]_[A-Za-z0-9]{36}|github_pat_[A-Za-z0-9_]{82})/], ["anthropic-key", /sk-ant-[A-Za-z0-9_-]{40,200}/], [ @@ -51,19 +51,45 @@ const rules = [ /\b[A-Za-z0-9_]{0,64}(?:password|secret|token|api[_-]?key|access[_-]?key)[A-Za-z0-9_]{0,64}["']?\s{0,32}[:=]\s{0,32}["']?([^\s"';#]{20,512})/i, ], ] as const; -const compiledRules = rules.map(([id, pattern]) => { - const alphabet = - id === "github-token" || id === "aws-access-key" || id === "stripe-key" || id === "google-key" +const leftBoundary = "A-Za-z0-9"; +/** Matches each rule drops, as `memory-check`'s gitleaks allowlists do. Patterns must stay RE2-compatible. */ +export const secretExceptions: Partial< + Record< + (typeof secretRules)[number][0], + { + readonly minEntropy?: number; + readonly secret?: ReadonlyArray; + readonly match?: ReadonlyArray; + } + > +> = { + "openai-key": { secret: [/^[^A-Z]*$/] }, + "url-password": { secret: [/^\$\{[^}]+\}$/] }, + "secret-assignment": { + minEntropy: 3.5, + secret: [ + /^[a-f0-9]{40}(?:[a-f0-9]{24})?$/i, + /^https?:\/\//i, + /^projects\/[^/]+\/(?:locations\/[^/]+\/)?secrets\//, + ], + match: [/^[^:=]*token_scopes\s*[:=]/i], + }, +}; +/** The characters a match may not be followed by, or undefined when the rule has no right boundary. */ +const rightBoundary = (id: (typeof secretRules)[number][0]) => + id === "url-password" || id === "secret-assignment" || id === "jwt" + ? undefined + : id === "github-token" || id === "aws-access-key" || id === "stripe-key" || id === "google-key" ? "A-Za-z0-9" : id === "slack-token" ? "A-Za-z0-9-" : "A-Za-z0-9_-"; - const right = ["url-password", "secret-assignment", "jwt"].includes(id) - ? "" - : `(?![${alphabet}])`; +const compiledRules = secretRules.map(([id, pattern]) => { + const alphabet = rightBoundary(id); + const right = alphabet === undefined ? "" : `(?![${alphabet}])`; return [ id, - new RegExp(`(? pattern.test(secret)) || + exception.match?.some((pattern) => pattern.test(matched))) ) continue; if (allow.has(JSON.stringify([file.path, ruleId, fingerprint(ruleId, secret)]))) diff --git a/docs/kin/specs/triggers.md b/docs/kin/specs/triggers.md index 862a81305949e..99f86f2936e20 100644 --- a/docs/kin/specs/triggers.md +++ b/docs/kin/specs/triggers.md @@ -390,13 +390,15 @@ The builder takes item heads (optionally marked `misleading` by the caller), las **Consumer (TRIG-13).** `DreamRunner` registers on the Kin scheduler: it creates the `routine:dream` and `routine:dream_full` triggers and drains their fires. A fire is skipped (consumed) past 08:00 owner time or when suppressed by `fullRebuildWeekday`; it waits while the org is paused, `KinBudget.admit(orgId, "P4")` refuses, or a dream is already running. The dream turn is an ordinary thread in a per-dream scratch repo (its own `git init`) and gets no MCP credential on `/mcp` or `/kin/mcp`, because its material holds untrusted text. The checkout holds only item files, `journal/`, `rules.md` and `INDEX.md`, so a `.mcp.json`, `.claude/` or `CLAUDE.md` committed to the memory repo never reaches the provider. Dreams run only on Claude: a turn on any other driver, or on a Claude instance whose launch arguments set a permission or skip-permissions flag, fails before it starts. For `kin-dream:` threads, [`DreamIsolation.ts`](../../../apps/server/src/kin/dreaming/DreamIsolation.ts) wraps the Claude query runner so the query loads no settings sources or flag settings beyond model selection, uses a strict empty MCP config, adds no directories, forces `acceptEdits`, and offers only Read, Edit, Write, Glob and Grep. `DreamSession` declines every approval, and a request it cannot decline ends the run. Codex is refused: its shell sandbox still reads files outside the checkout, and no per-thread option removes the shell or the user's MCP servers. Material is bounded by INTEGER ms in `(materialFrom, materialUntil]`. Dream provider usage is not yet charged to the budget (charges key on wake-ups). The rejections list joins the material with `kin_dream_rejections` (TRIG-14). Step 5's evidence recheck and steps 6 and 7 are TRIG-14 or later; TRIG-13 rebuilds `INDEX.md` in the dream commit. -**Memory repo guard rails** (in `kin-memory`, outside this fork): +**Memory repo guard rails** (in `kin-memory`, outside this fork; templates in [`ops/kin/memory-repo/`](../../../ops/kin/memory-repo/README.md)): -- A `CODEOWNERS` file with `/rules.md /decisions/ /people/ /objectives/ @`. -- A ruleset on `main` that requires code-owner review. -- A required check `.github/workflows/memory-check.yml` that runs schema lint and gitleaks. +- A `CODEOWNERS` file that denies by default: `* @` first, then owner-less lines for only the paths Kin writes alone (`/commitments/`, `/deferrals/`, `/claims/`, `/journal/`, `/INDEX.md`). GitHub applies the last matching line, so `rules.md`, `decisions/`, `people/`, `objectives/`, `.github/` and any root file a lead turn would load from the read checkout (`CLAUDE.md`, `AGENTS.md`, `.claude/`, `.mcp.json`, `.codex/`) stay with the owner. `.github/` stops a PR from dropping the owner rule or the check itself. The lint fails when the file is missing, still names `@OWNER`, or differs from that shape. CODEOWNERS can't carve a path back out of an owner-less line, so the lint walks the whole tree from git's index and allows only Kin's own shapes there: `/.md` one level deep in the three item folders (no `CLAUDE.md`, `AGENTS.md` or `GEMINI.md`), `journal/YYYY/MM-DD.md` and a root `INDEX.md` file. A dot segment such as `.claude/` under those paths, a symlink or a submodule anywhere fails it. +- A ruleset on `main` that requires PRs and code-owner review, with no bypass for Kin's bot. +- A required check `.github/workflows/memory-check.yml` that runs schema lint and gitleaks. The lint is `apps/server/src/kin/memory/lintCli.ts`, which uses the server's own front-matter parser; the gitleaks config is generated from `KinSecretScan`'s rules. Both come from a pinned commit of this repo, never from the PR. `MemoryWriter` and the dream check use the lint's path shapes, so Kin never writes a path `memory-check` refuses. Before making the check required, run the lint against the memory repo's `main`, which may already hold older paths. Gitleaks parity is best effort: it misses a value swallowed by an allowed `secret-assignment` match (`secret=https://x/?db_password=…`), tokens split across lines or hidden with zero-width or fullwidth characters, and file paths. -GitHub then blocks a bad auto-merge even if Kin's classifier is wrong. This forces daytime lead writes onto PRs as well; see Open questions. +GitHub then blocks a bad auto-merge even if Kin's classifier is wrong. + +**Daytime lead writes (decided).** Because `main` requires PRs, LEAD's daytime memory writes land as PRs that Kin auto-merges once `memory-check` passes; a write touching an owner path waits for the owner's review like any other PR. APPR-8 (#44) and APPR-11 (#46) have merged; the TRIG-15 follow-up is the `MemoryWriter` PR path and running APPR-11's rulesets in a memory-repo mode, since its CODEOWNERS check expects every line to name the owner (lane A and lane C coordinate on #12). ### Secret scan before memory commits (gap 5) @@ -618,7 +620,6 @@ Kin runs on one always-on machine, the **Kin host**. The laptop and phone are cl ## Open questions -- **Daytime lead writes.** With the memory repo's `main` requiring PRs (needed for the CODEOWNERS guard), LEAD's daytime writes must land as auto-merged PRs too. Otherwise drop the ruleset and rely on Kin's checks only. - **Fencing T3's scheduling tools.** Can `McpToolAccess` remove `schedule_task` for specific threads, or does Kin need a check on `creationSource: "mcp"` in Kin projects? Decide with LEAD. - **Owner-check day.** Should the weekly owner check move to the owner's quietest weekday, learned from activity? - **Dream schedule per org.** Should one Kin host serving several orgs space their dreams, for example 15 minutes apart, so they don't compete for plan headroom? diff --git a/ops/kin/memory-repo/CODEOWNERS.template b/ops/kin/memory-repo/CODEOWNERS.template new file mode 100644 index 0000000000000..706df20a6f6da --- /dev/null +++ b/ops/kin/memory-repo/CODEOWNERS.template @@ -0,0 +1,10 @@ +# Install as .github/CODEOWNERS in the memory repo and replace @OWNER with the owner's GitHub login. +# Deny by default: the owner reviews every path, including a root CLAUDE.md, AGENTS.md, .claude/, +# .mcp.json or .codex/ that the lead's read checkout would load. The last matching line wins, so the +# owner-less lines below free only the paths Kin writes alone. memory-check's lint enforces this shape. +* @OWNER +/commitments/ +/deferrals/ +/claims/ +/journal/ +/INDEX.md diff --git a/ops/kin/memory-repo/README.md b/ops/kin/memory-repo/README.md new file mode 100644 index 0000000000000..6b5f431fe23d2 --- /dev/null +++ b/ops/kin/memory-repo/README.md @@ -0,0 +1,34 @@ +# Memory repo guard rails + +Templates for the private memory repo (`kin-memory`). This public repo holds only the templates and the code they run; never copy memory into it. The design is in [Memory repo guard rails](../../../docs/kin/specs/triggers.md#memory-repo-guard-rails). + +| File | Goes to | What it does | +| --------------------- | ------------------------------------------- | ------------------------------------------------------------------------------------------------------ | +| `CODEOWNERS.template` | `.github/CODEOWNERS` | Gives everything to the owner except the paths Kin writes alone. | +| `memory-check.yml` | `.github/workflows/memory-check.yml` | The `memory-check` job: schema lint, then gitleaks, on every PR and every push to `main`. | +| `gitleaks.toml` | stays here; the workflow reads it from here | `KinSecretScan`'s rules in gitleaks form, generated by `apps/server/src/kin/memory/gitleaksConfig.ts`. | + +## Install + +1. Copy `CODEOWNERS.template` to `.github/CODEOWNERS` and replace `@OWNER` with the owner's GitHub login. +2. Copy `memory-check.yml` to `.github/workflows/` and set `T3CODE_REF` to a full `GLDRoger/t3code` commit SHA that contains this folder. The job fails until it is set. Confirm the commit is on `kin` or `main` (`git branch -r --contains `): GitHub serves commits from any fork PR under the repo's name, so an unmerged SHA could carry someone else's lint. +3. Open both as one PR and check that `memory-check` passes. It lints the whole merged tree, so files already on `main` count too: memory written before these guard rails may hold paths the lint now refuses (nested items, names with dots). Move or rename those in the same PR. + +To pick up new lint or scanner rules, bump `T3CODE_REF` in a reviewed PR. The lint code and the gitleaks config always come from that commit, never from the PR's own files, which the job reads only as data. + +The ruleset on `main` (PRs required, code-owner review, `memory-check` required, no bypass for Kin's bot) comes in the TRIG-15 follow-up with the `MemoryWriter` PR path. APPR-8 and APPR-11 have merged, but APPR-11's ruleset setup needs a memory-repo mode first: its CODEOWNERS check expects every line to name the owner, and this file's Kin lines name none. + +## Cost + +Each run checks out `t3code` at the pinned commit without `.repos/`, installs the server's dependencies with `vp install --frozen-lockfile --ignore-scripts --filter=t3...` and runs the lint with Node. The install dominates: 881 packages, which took 19 s from an empty store on a fast connection, so budget about a minute per run on a hosted runner, plus Vite+ and Node setup. Nothing is cached between runs, so a PR can't seed a cache the check later trusts. Gitleaks is a 7 MB release binary, checked against its sha256 before it runs, and scans the tree plus the new commits in seconds. + +## Things to know + +- GitHub runs a PR's own copy of this workflow, so a PR could rewrite the check. The `*` line in `CODEOWNERS`, which also covers `.github/`, is what stops that: such a PR needs the owner's review. An org ruleset that requires this workflow from a pinned ref would remove the gap; consider it with the ruleset. +- A `.gitleaksignore` anywhere in the memory repo fails the check, because gitleaks would honour it. `gitleaks:allow` comments are ignored. +- A false positive the owner allowed in `kin_secret_allow` still fails gitleaks, which has no allowlist. Reword the text; whether the owner may merge past the check is settled with the ruleset. +- `CODEOWNERS` denies by default, so a new root file such as `CLAUDE.md`, `AGENTS.md`, `.claude/`, `.mcp.json` or `.codex/` needs the owner's review before the lead's read checkout loads it. The lint fails if the file still names `@OWNER` or frees any path beyond Kin's own. +- CODEOWNERS can't carve a path back out of Kin's owner-less lines, so the lint does it: under those paths it allows only `commitments/`, `deferrals/` and `claims/` items one level deep (`.md`, never an agent instruction file name such as `CLAUDE.md` or `AGENTS.md`), `journal/YYYY/MM-DD.md` and a root `INDEX.md` file. It reads git's index, so a `.claude/` folder, a symlink or a submodule fails it rather than being skipped; paths the owner reviews stay free-form. +- Gitleaks matches the scanner's patterns but not its Unicode folding, wrapped-token joining or word boundaries, so it can flag both more and less than the scanner. It adds its own decoding and covers every commit in the PR, merge resolutions included. +- Gitleaks scans file contents, not file paths. A token in a file or folder name passes this check; only `KinSecretScan` catches it, on Kin's own writes. +- Other known gitleaks misses, left as they are: a value an allowed `secret-assignment` match swallows (`secret=https://x/?db_password=…`, `token_scopes=read&api_key=…`), and a token split across lines or hidden with zero-width or fullwidth characters. diff --git a/ops/kin/memory-repo/gitleaks.toml b/ops/kin/memory-repo/gitleaks.toml new file mode 100644 index 0000000000000..d1f972e105b84 --- /dev/null +++ b/ops/kin/memory-repo/gitleaks.toml @@ -0,0 +1,81 @@ +# Generated from apps/server/src/kin/memory/secretScan.ts; do not edit. +# Regenerate: node apps/server/src/kin/memory/gitleaksConfig.ts > ops/kin/memory-repo/gitleaks.toml +title = "kin-memory" + +[[rules]] +id = "github-token" +description = "KinSecretScan github-token" +regex = '''(?:gh[pousr]_[A-Za-z0-9]{36}|github_pat_[A-Za-z0-9_]{82})''' + +[[rules]] +id = "anthropic-key" +description = "KinSecretScan anthropic-key" +regex = '''sk-ant-[A-Za-z0-9_-]{40,200}''' + +[[rules]] +id = "openai-key" +description = "KinSecretScan openai-key" +regex = '''sk-(?:(?:proj|svcacct|admin)-[A-Za-z0-9_-]{32,200}|[A-Za-z0-9]{20}T3BlbkFJ[A-Za-z0-9]{20})''' + +[[rules.allowlists]] +regexTarget = "secret" +regexes = ['''^[^A-Z]*$'''] + +[[rules]] +id = "aws-access-key" +description = "KinSecretScan aws-access-key" +regex = '''(?:AKIA|ASIA)[A-Z0-9]{16}''' + +[[rules]] +id = "slack-token" +description = "KinSecretScan slack-token" +regex = '''xox[abposr]-[A-Za-z0-9-]{20,200}''' + +[[rules]] +id = "stripe-key" +description = "KinSecretScan stripe-key" +regex = '''[rs]k_live_[A-Za-z0-9]{24,200}''' + +[[rules]] +id = "google-key" +description = "KinSecretScan google-key" +regex = '''AIza[A-Za-z0-9_-]{35}''' + +[[rules]] +id = "pem-private-key" +description = "KinSecretScan pem-private-key" +regex = '''-----BEGIN (?:[A-Z0-9]+ ){0,8}PRIVATE KEY(?: BLOCK)?-----''' + +[[rules]] +id = "jwt" +description = "KinSecretScan jwt" +regex = '''eyJ[A-Za-z0-9_-]{5,}\.eyJ[A-Za-z0-9_-]{8,}''' + +[[rules]] +id = "url-password" +description = "KinSecretScan url-password" +regex = '''(?i:\b[a-z][a-z0-9+.-]{0,31}:\/\/[^\s\v\p{Z}\x{feff}/:@]{1,512}:([^\s\v\p{Z}\x{feff}/@]{1,512})@)''' +secretGroup = 1 + +[[rules.allowlists]] +regexTarget = "secret" +regexes = ['''^\$\{[^}]+\}$'''] + +[[rules]] +id = "secret-assignment" +description = "KinSecretScan secret-assignment" +regex = '''(?i:\b[A-Za-z0-9_]{0,64}(?:password|secret|token|api[_-]?key|access[_-]?key)[A-Za-z0-9_]{0,64}["']?[\s\v\p{Z}\x{feff}]{0,32}[:=][\s\v\p{Z}\x{feff}]{0,32}["']?([^\s\v\p{Z}\x{feff}"';#]{20,512}))''' +secretGroup = 1 +entropy = 3.5 + +[[rules.allowlists]] +regexTarget = "secret" +regexes = [ + '''(?i:^[a-f0-9]{40}(?:[a-f0-9]{24})?$)''', + '''(?i:^https?:\/\/)''', + '''^projects\/[^/]+\/(?:locations\/[^/]+\/)?secrets\/''', +] + +[[rules.allowlists]] +regexTarget = "match" +regexes = ['''(?i:^[^:=]*token_scopes[\s\v\p{Z}\x{feff}]*[:=])'''] diff --git a/ops/kin/memory-repo/memory-check.yml b/ops/kin/memory-repo/memory-check.yml new file mode 100644 index 0000000000000..27798312f0b3c --- /dev/null +++ b/ops/kin/memory-repo/memory-check.yml @@ -0,0 +1,95 @@ +# Install as .github/workflows/memory-check.yml in the memory repo; see README.md beside this file. +# Memory files are only ever read as data. The lint code and gitleaks config come from a pinned +# GLDRoger/t3code commit, so a pull request cannot change what checks it. +name: memory-check + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +concurrency: + group: memory-check-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: true + +jobs: + memory-check: + name: memory-check + runs-on: ubuntu-24.04 + timeout-minutes: 15 + env: + # A GLDRoger/t3code commit on kin or main that contains ops/kin/memory-repo. Bump it deliberately. + T3CODE_REF: "" + GITLEAKS_VERSION: "8.30.1" + GITLEAKS_SHA256: "551f6fc83ea457d62a0d98237cbad105af8d557003051f41f3e7ca7b3f2470eb" + steps: + - name: Check pins + run: | + [[ "$T3CODE_REF" =~ ^[0-9a-f]{40}$ ]] || { echo "T3CODE_REF must be a full commit SHA"; exit 1; } + + - name: Checkout memory + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + path: memory + fetch-depth: 0 + persist-credentials: false + + - name: Checkout t3code + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + repository: GLDRoger/t3code + ref: ${{ env.T3CODE_REF }} + path: t3code + persist-credentials: false + sparse-checkout: | + /* + !/.repos/ + sparse-checkout-cone-mode: false + + - name: Setup Vite+ + uses: voidzero-dev/setup-vp@250f29ce396baf5e8f24498e17c0dfdebabc26eb # v1.15.0 + with: + working-directory: t3code + version-file: pnpm-workspace.yaml + node-version-file: package.json + cache: false + run-install: false + + - name: Install lint dependencies + working-directory: t3code + run: vp install --frozen-lockfile --ignore-scripts --filter=t3... + + - name: Schema lint + run: node t3code/apps/server/src/kin/memory/lintCli.ts memory + + - name: Gitleaks + env: + BASE: ${{ github.event.pull_request.base.sha || github.event.before }} + HEAD: ${{ github.event.pull_request.head.sha || github.sha }} + run: | + set -euo pipefail + # gitleaks honours a .gitleaksignore in the scanned tree whatever --gitleaks-ignore-path says. + if [[ -n "$(find memory -name .gitleaksignore -not -path 'memory/.git/*')" ]]; then + echo ".gitleaksignore is not allowed in the memory repo"; exit 1 + fi + bin="$RUNNER_TEMP/gitleaks" + mkdir -p "$bin" + archive="gitleaks_${GITLEAKS_VERSION}_linux_x64.tar.gz" + curl -fsSL --retry 3 -o "$bin/$archive" \ + "https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}/${archive}" + echo "${GITLEAKS_SHA256} $bin/$archive" | sha256sum --check --strict + tar -xzf "$bin/$archive" -C "$bin" gitleaks + : > "$bin/no-ignores" + flags=(--config t3code/ops/kin/memory-repo/gitleaks.toml --gitleaks-ignore-path "$bin/no-ignores" + --ignore-gitleaks-allow --redact --no-banner --exit-code 1) + "$bin/gitleaks" dir "${flags[@]}" memory + # New commits only; a missing or zero base (first push, rewritten history) scans everything. + # -m diffs merges against each parent, so a token only a merge resolution held is seen too. + if [[ "$BASE" =~ ^[0-9a-f]{40}$ && ! "$BASE" =~ ^0+$ ]] && git -C memory cat-file -e "$BASE^{commit}" 2>/dev/null; then + "$bin/gitleaks" git "${flags[@]}" --log-opts="-m $BASE..$HEAD" memory + else + "$bin/gitleaks" git "${flags[@]}" --log-opts="-m --all" memory + fi