diff --git a/apps/server/src/kin/memory/Memory.test.ts b/apps/server/src/kin/memory/Memory.test.ts index 402e17e08e711..942b89ffbba43 100644 --- a/apps/server/src/kin/memory/Memory.test.ts +++ b/apps/server/src/kin/memory/Memory.test.ts @@ -858,6 +858,10 @@ it.live("uses filesystem identity for a case-mismatched base without recloning", it.live.each([ "diverged", + "secret merge", + "scan failure merge", + "remote secret merge", + "large remote merge", "conflict", "network down", "dirty writer", @@ -932,7 +936,11 @@ it.live.each([ } else yield* fs.writeFileString( `${writer}/rules.md`, - scenario === "H2-merge-swap" ? h2Local : "Local invented rules\n", + scenario === "H2-merge-swap" + ? h2Local + : scenario === "secret merge" + ? ["gh", "p_", "a".repeat(36)].join("") + : "Local invented rules\n", ); let local = yield* commit(writer); if (scenario === "push retry") { @@ -952,13 +960,20 @@ it.live.each([ } yield* memory.record(orgA, "main", initial.head, local); yield* fs.writeFileString( - `${source}/${scenario === "push retry" || scenario === "diverged" || scenario === "network down" || scenario === "dirty writer" || scenario === "H2-merge-swap" ? "remote-only" : "rules.md"}`, + `${source}/${scenario === "push retry" || scenario === "diverged" || scenario === "scan failure merge" || scenario === "secret merge" || scenario === "remote secret merge" || scenario === "large remote merge" || scenario === "network down" || scenario === "dirty writer" || scenario === "H2-merge-swap" ? "remote-only" : "rules.md"}`, "Remote invented edit\n", ); if (scenario === "journal conflict" || scenario === "push retry") { yield* fs.writeFileString(`${source}/journal/day.md`, "remote entry\n"); yield* fs.writeFileString(`${source}/INDEX.md`, "remote generated index\n"); } + if (scenario === "remote secret merge") + yield* fs.writeFileString(`${source}/remote-only`, ["gh", "p_", "a".repeat(36)].join("")); + if (scenario === "large remote merge") { + yield* fs.makeDirectory(`${source}/archive`); + for (let i = 0; i < 1030; i++) + yield* fs.writeFileString(`${source}/archive/${i}.txt`, "clean"); + } if (scenario === "H2-merge-swap") yield* fs.writeFileString(`${source}/rules.md`, h2Base.replace("base tail", "remote tail")); yield* commit(source); @@ -1004,6 +1019,10 @@ it.live.each([ yield* memory.record(orgA, "main", local, replacement); local = replacement; } + if (scenario === "scan failure merge") { + const sql = yield* SqlClient.SqlClient; + yield* sql`DROP TABLE kin_secret_allow`; + } const result = yield* memory.prepareWakeup(orgA, "fixture-wakeup").pipe(Effect.result); if (scenario === "unrelated" || scenario === "criss-cross") { expect(result._tag).toBe("Failure"); @@ -1038,6 +1057,22 @@ it.live.each([ expect( (yield* memory.prepareWakeup(orgA, "fixture-wakeup-3").pipe(Effect.result))._tag, ).toBe("Success"); + } else if (scenario === "scan failure merge") { + expect(result._tag).toBe("Failure"); + expect( + messages.some((message) => message.includes("Secret scan failed: Allowlist unavailable")), + ).toBe(true); + expect(yield* memory.anchor(orgA, "main")).toBe(local); + } else if (scenario === "secret merge") { + expect(result._tag).toBe("Failure"); + if (result._tag === "Failure") { + expect(result.failure._tag).toBe("MemoryError"); + expect(result.failure.operation).toBe("secretScan"); + } + expect(yield* memory.anchor(orgA, "main")).toBe(local); + expect(yield* git(root, [`--git-dir=${remote}`, "rev-parse", "main"])).toBe( + yield* git(source, ["rev-parse", "HEAD"]), + ); } else if (scenario === "push retry") { expect(result._tag).toBe("Success"); if (result._tag === "Success") { @@ -1063,6 +1098,8 @@ it.live.each([ } else if ( scenario === "network down" || scenario === "diverged" || + scenario === "remote secret merge" || + scenario === "large remote merge" || scenario === "dirty writer" ) { expect(result._tag).toBe("Success"); @@ -1072,7 +1109,9 @@ it.live.each([ expect(result.success.head).not.toBe(local); expect(yield* memory.readRules(orgA)).toBe("Local invented rules\n"); expect(yield* git(writer, ["show", `${result.success.head}:remote-only`])).toBe( - "Remote invented edit", + scenario === "remote secret merge" + ? ["gh", "p_", "a".repeat(36)].join("") + : "Remote invented edit", ); } } diff --git a/apps/server/src/kin/memory/Memory.ts b/apps/server/src/kin/memory/Memory.ts index 4ecc9ded5fd74..d8aafb19a722a 100644 --- a/apps/server/src/kin/memory/Memory.ts +++ b/apps/server/src/kin/memory/Memory.ts @@ -1,3 +1,4 @@ +import * as SecretScan from "./secretScan.ts"; import { readTree, readObjects, @@ -203,6 +204,7 @@ interface BlobCache { } const make = Effect.gen(function* () { + const scanner = yield* SecretScan.KinSecretScan; const config = yield* MemoryConfig; const fs = yield* FileSystem.FileSystem; const { notify } = yield* MemoryRecoveryHooks; @@ -1079,6 +1081,7 @@ const make = Effect.gen(function* () { }); } else { const { files: mergedFiles, modes } = merged; + const remoteOriginal = new Map(remoteFiles); remoteFiles.clear(); for (const [file, bytes] of mergedFiles) remoteFiles.set(file, bytes); const heads = []; @@ -1098,6 +1101,36 @@ const make = Effect.gen(function* () { "INDEX.md", Buffer.from(buildIndex(heads, reads, yield* Clock.currentTimeMillis, zone)), ); + const findings = yield* scanner + .scan({ + orgId, + files: [...remoteFiles] + .filter( + ([path, bytes]) => + path === "INDEX.md" || !remoteOriginal.get(path)?.equals(bytes), + ) + .map(([path, content]) => ({ path, content })), + }) + .pipe( + Effect.tapError((cause) => + notify({ orgId, branch, reason: `Secret scan failed: ${cause.reason}` }), + ), + Effect.mapError( + (cause) => new MemoryError({ orgId, operation: "secretScan", cause }), + ), + ); + if (findings.length) { + yield* notify({ + orgId, + branch, + reason: `Secret scan blocked merge: ${findings[0]!.ruleId} in ${findings[0]!.pathHit ? "[redacted]" : (findings[0]!.itemPath ?? findings[0]!.path)}`, + }); + return yield* new MemoryError({ + orgId, + operation: "secretScan", + reason: `Secret scan blocked merge: ${findings[0]!.ruleId} in ${findings[0]!.pathHit ? "[redacted]" : (findings[0]!.itemPath ?? findings[0]!.path)}`, + }); + } tree = yield* Effect.scoped( Effect.gen(function* () { const temp = yield* io(orgId, "journalMerge", fs.makeTempDirectoryScoped()); @@ -1286,4 +1319,7 @@ const make = Effect.gen(function* () { }); }); -export const layer = Layer.effect(Memory, make).pipe(Layer.provide(MemoryCache.layer)); +export const layer = Layer.effect(Memory, make).pipe( + Layer.provide(MemoryCache.layer), + Layer.provide(SecretScan.layer), +); diff --git a/apps/server/src/kin/memory/MemoryWriter.test.ts b/apps/server/src/kin/memory/MemoryWriter.test.ts index 5dc139d5b196c..9b9faefc29e13 100644 --- a/apps/server/src/kin/memory/MemoryWriter.test.ts +++ b/apps/server/src/kin/memory/MemoryWriter.test.ts @@ -1,3 +1,4 @@ +import * as SqlClient from "effect/sql/SqlClient"; import { memoryGit } from "./git.ts"; import * as NodeZlib from "node:zlib"; import * as NodeServices from "@effect/platform-node/NodeServices"; @@ -23,6 +24,8 @@ it.live.each([ "completed", "failed", "blocked", + "scan unavailable", + "scan limit", "forged", "conflict", "B1-shadow", @@ -223,10 +226,7 @@ it.live.each([ }), ), }); - const scan = - scenario === "blocked" - ? Scan.testLayer([{ path: "claims/new.md", ruleId: "fixture", line: 1 }]) - : Scan.noFindingsStubLayer; + const scan = Scan.layer; const layer = Writer.layer.pipe( Layer.provideMerge(memoryLayer), Layer.provide(Layer.succeed(Memory.MemoryRecoveryHooks, { notify: () => Effect.void })), @@ -600,10 +600,52 @@ it.live.each([ } if (scenario === "tree-substitution") substituteTree = yield* git(paths.writer, ["rev-parse", `${original}^{tree}`]); + if (scenario === "blocked") { + const tokenPath = "claims/" + ["gh", "p_", "a".repeat(36)].join("") + ".md"; + const malformed = yield* writer + .stage(orgId, "wake1", { + path: tokenPath, + content: "---\nid: [\n" + tokenPath + ": [\n---\nbody", + }) + .pipe(Effect.result); + expect(malformed._tag).toBe("Failure"); + if (malformed._tag === "Failure") { + expect(malformed.failure._tag).toBe("MemoryItemError"); + expect(JSON.stringify(malformed.failure).includes(tokenPath.slice(7, -3))).toBe(false); + if (malformed.failure._tag === "MemoryItemError") + expect(malformed.failure.path).toBe("[redacted]"); + } + const inaccessible = yield* writer + .stage(orgId, "wake1", { + path: tokenPath.slice(0, -3) + "_".repeat(260) + ".md", + content: "missing front matter", + }) + .pipe(Effect.result); + expect(inaccessible._tag).toBe("Failure"); + if (inaccessible._tag === "Failure") + expect(JSON.stringify(inaccessible.failure).includes(tokenPath.slice(7, -3))).toBe(false); + } + if (scenario === "blocked") + yield* writer.appendJournal(orgId, "wake1", [ + ["gh", "p_"].join("") + "aB3dE6gH9jK2mN5pQ8sT1vW4yZ7cF0iL3oR6", + ]); + if (scenario === "scan unavailable") { + const sql = yield* SqlClient.SqlClient; + yield* sql`DROP TABLE kin_secret_allow`; + } + if (scenario === "scan limit") + yield* writer.appendJournal(orgId, "wake1", ["a".repeat(1024 * 1024)]); const result = yield* writer .commitWakeup(orgId, "wake1", scenario === "failed" ? "failed" : "completed") .pipe(Effect.result); if (result._tag === "Failure") yield* Effect.logError(result.failure); + if (scenario === "scan unavailable" || scenario === "scan limit") { + expect(result._tag).toBe("Failure"); + expect(notices).toHaveLength(1); + expect(notices[0]!.error._tag).toBe("KinSecretScanError"); + expect(yield* git(paths.writer, ["rev-parse", "HEAD"])).toBe(original); + return; + } if (scenario === "tree-substitution") { expect(result._tag).toBe("Failure"); if (result._tag === "Failure") expect(result.failure._tag).toBe("MemoryCheckoutInvalid"); @@ -618,6 +660,19 @@ it.live.each([ ); expect(notices).toHaveLength(1); + if (scenario === "blocked") { + expect(JSON.stringify(notices)).not.toContain("aB3dE6gH9jK2mN5pQ8sT1vW4yZ7cF0iL3oR6"); + expect(yield* git(paths.writer, ["rev-parse", "HEAD"])).toBe(original); + expect( + (yield* git(root, [ + `--git-dir=${remote}`, + "show-ref", + "--verify", + `refs/heads/kin/shadow/${agentId}`, + ]).pipe(Effect.result))._tag, + ).toBe("Failure"); + expect(yield* fs.exists(`${paths.writer}/claims/new.md`)).toBe(false); + } expect(yield* fs.exists(`${paths.writer}/.git/rebase-merge`)).toBe(false); } else { expect(result._tag).toBe("Success"); diff --git a/apps/server/src/kin/memory/MemoryWriter.ts b/apps/server/src/kin/memory/MemoryWriter.ts index 113280e8906c5..dd5fbd7224057 100644 --- a/apps/server/src/kin/memory/MemoryWriter.ts +++ b/apps/server/src/kin/memory/MemoryWriter.ts @@ -61,14 +61,24 @@ export class MemorySecretBlocked extends Schema.TaggedError { orgId: Schema.String, findings: Schema.Array( - Schema.Struct({ path: Schema.String, ruleId: Schema.String, line: Schema.Number }), + Schema.Struct({ + path: Schema.String, + ruleId: Schema.String, + line: Schema.Number, + pathHit: Schema.optional(Schema.Boolean), + itemPath: Schema.optional(Schema.String), + }), ), }, ) {} export type WriterNotice = { readonly orgId: KinOrgId; readonly wakeupId: string; - readonly error: MemoryCheckoutInvalid | MemorySyncConflict | MemorySecretBlocked; + readonly error: + | MemoryCheckoutInvalid + | MemorySyncConflict + | MemorySecretBlocked + | KinSecretScanError; }; export class MemoryWriterHooks extends Context.Service< MemoryWriterHooks, @@ -288,7 +298,11 @@ const make = Effect.gen(function* () { const result = yield* fs.lstat(current).pipe(Effect.result); if (result._tag === "Failure") { if (result.failure.reason._tag !== "NotFound") - return yield* new MemoryError({ orgId, operation: "path", cause: result.failure }); + return yield* new MemoryError({ + orgId, + operation: "path", + cause: "Unable to inspect memory path", + }); } else if ( result.success.type === "SymbolicLink" || (result.success.type === "File" && @@ -306,10 +320,25 @@ const make = Effect.gen(function* () { }); const report = (orgId: KinOrgId, wakeupId: string, effect: Effect.Effect) => effect.pipe( + Effect.catchTags({ + MemoryItemError: (error) => + scanner.scan({ orgId, files: [{ path: error.path, content: new Uint8Array() }] }).pipe( + Effect.flatMap((findings) => + Effect.fail( + new MemoryItemError({ + orgId: error.orgId, + path: findings.length ? "[redacted]" : error.path, + reason: error.reason, + }), + ), + ), + ), + }), Effect.tapError((error) => error._tag === "MemoryCheckoutInvalid" || error._tag === "MemorySyncConflict" || - error._tag === "MemorySecretBlocked" + error._tag === "MemorySecretBlocked" || + error._tag === "KinSecretScanError" ? hooks .notify({ orgId, wakeupId, error }) .pipe( diff --git a/apps/server/src/kin/memory/frontMatter.ts b/apps/server/src/kin/memory/frontMatter.ts index 692502849fca7..49b7bb49b70b7 100644 --- a/apps/server/src/kin/memory/frontMatter.ts +++ b/apps/server/src/kin/memory/frontMatter.ts @@ -89,8 +89,12 @@ export const parseFrontMatter = Effect.fnUntraced(function* ( new MemoryItemError({ orgId, path, - reason: cause instanceof Error ? cause.message : "Malformed YAML", - cause, + reason: staged + ? "Malformed YAML front matter" + : cause instanceof Error + ? cause.message + : "Malformed YAML", + ...(staged ? {} : { cause }), }), }); const decoded = yield* (staged ? decodeStaged(value) : decodeFrontMatter(value)).pipe( diff --git a/apps/server/src/kin/memory/secretScan.test.ts b/apps/server/src/kin/memory/secretScan.test.ts new file mode 100644 index 0000000000000..319b4fc2aff03 --- /dev/null +++ b/apps/server/src/kin/memory/secretScan.test.ts @@ -0,0 +1,387 @@ +// @effect-diagnostics-next-line nodeBuiltinImport:off -- Bounded synchronous fingerprint hashing shares the scan loop. +import * as NodeCrypto from "node:crypto"; +import { vi } from "vite-plus/test"; +import { expect, it } from "@effect/vitest"; +import { KinOrgId } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as SqlClient from "effect/sql/SqlClient"; +import * as Sqlite from "../../persistence/Sqlite.ts"; +import * as Persistence from "../persistence/KinPersistence.ts"; +import * as Scan from "./secretScan.ts"; + +const orgId = KinOrgId.make("fixture-org"); +const database = Layer.provideMerge(Persistence.layer, Sqlite.layerMemory); +const layer = Scan.layer.pipe(Layer.provideMerge(database)); +const body = "aB3dE6gH9jK2mN5pQ8sT1vW4yZ7cF0iL3oR6"; +const github = ["gh", "p_"].join("") + body; +const fixtures = [ + ...[ + ["stripe-key", ["sk_", "live_"].join("") + body + "_old"], + ["slack-token", ["xo", "xb-"].join("") + body + "_old"], + ["google-key", ["AI", "za"].join("") + "X".repeat(35) + "_old"], + ["google-key", "key_" + ["AI", "za"].join("") + "X".repeat(35)], + ["anthropic-key", "prod-" + ["sk-", "ant-"].join("") + body.repeat(2)], + ...[ + Buffer.from(JSON.stringify({ alg: "RS256", x5c: ["a".repeat(499)] })).toString("base64url"), + ["ey", "J"].join("") + "a".repeat(2048), + ].map((header) => [ + "jwt", + header + "." + ["ey", "J"].join("") + "b".repeat(2600) + ".signature", + ]), + ].map(([rule, hit]) => [rule, hit, "task-follow-up-with-design-team"]), + + ...["X-Api-Key: ", "client-secret: ", '{"password": "', '"api_key":"'].map((prefix) => [ + "secret-assignment", + prefix + body + '"', + prefix + "short", + ]), + ...["_", "token_"].map((prefix) => [ + "github-token", + prefix + github + "_rotated", + "task-follow-up-with-design-team", + ]), + ["aws-access-key", ["AK", "IA"].join("") + "Q".repeat(16) + "_old", "ASIAPACIFICREGION2026"], + [ + "jwt", + ["ey", "J"].join("") + "a".repeat(100) + ".eyJ" + "b".repeat(1800) + "." + "c".repeat(100), + "eyJshort.two.parts", + ], + + ...["p", "o", "u", "s", "r"].map((kind) => [ + "github-token", + "gh" + kind + "_" + body, + "gh" + kind + "_short", + ]), + [ + "github-token", + ["github_", "pat_"].join("") + "a".repeat(82), + ["github_", "pat_short"].join(""), + ], + ["anthropic-key", ["sk-", "ant-"].join("") + body.repeat(2), ["sk-", "ant-short"].join("")], + ["openai-key", "sk-" + body.slice(0, 20) + "T3BlbkFJ" + body.slice(0, 20), "sk-short"], + ...["proj", "svcacct", "admin"].map((kind) => [ + "openai-key", + ["sk-", kind, "-", body.repeat(3)].join(""), + ["sk-", kind, "-", "a".repeat(40)].join(""), + ]), + ...["AK", "AS"].map((kind) => [ + "aws-access-key", + kind + "IA" + "X".repeat(16), + kind + "IA" + "X".repeat(15), + ]), + ["slack-token", ["xo", "xb-"].join("") + body, ["xo", "xb-short"].join("")], + ["stripe-key", ["sk_", "live_"].join("") + body, "sk_live_short"], + ["google-key", ["AI", "za"].join("") + "X".repeat(35), ["AI", "za_short"].join("")], + ["pem-private-key", "-----BEGIN RSA PRIVATE KEY-----", "-----BEGIN PUBLIC KEY-----"], + [ + "jwt", + ["ey", "J"].join("") + "a".repeat(10) + ".eyJ" + "b".repeat(12) + "." + "c".repeat(20), + "eyJshort.two.parts", + ], + [ + "url-password", + "https://fixture:fake-password@example.invalid", + "https://fixture@example.invalid", + ], + ["secret-assignment", "password=" + body, "password=" + "a".repeat(40)], + ["secret-assignment", "export API_KEY='" + body + "'", "export API_KEY='short'"], + ["secret-assignment", "AWS_SECRET_ACCESS_KEY=" + body + "aB3d", "AWS_SECRET_ACCESS_KEY=example"], +]; +it.effect.each(fixtures.map(([rule, hit, miss], i) => ({ rule, hit, miss, id: i })))( + "rule fixture $id: $rule", + ({ rule, hit, miss }) => + Effect.gen(function* () { + const scanner = yield* Scan.KinSecretScan; + const run = (content: string) => + scanner.scan({ orgId, files: [{ path: "claims/test.md", content: Buffer.from(content) }] }); + // Assert only metadata so a failed assertion never prints the synthetic matched token. + expect((yield* run(hit!)).some((finding) => finding.ruleId === rule)).toBe(true); + expect((yield* run(miss!)).length).toBe(0); + }).pipe(Effect.provide(layer)), +); +it.effect("blocks encoded, wrapped and confusable tokens with safe findings", () => + Effect.gen(function* () { + const scanner = yield* Scan.KinSecretScan; + const values = [ + github, + "The deploy key is " + github.slice(0, 12) + "\n" + github.slice(12), + github.slice(0, 12) + "\n" + github.slice(12) + " and more", + github.slice(0, 12) + "\n" + github.slice(12) + "\nThanks", + "The deploy key is " + + Buffer.from(github).toString("base64").slice(0, 16) + + "\n" + + Buffer.from(github).toString("base64").slice(16) + + " and more", + "%67" + github.slice(1), + ...["/", "-", "_", "+", "word"].map( + (prefix) => prefix + Buffer.from(github).toString("base64"), + ), + Buffer.from("é" + github + "é").toString("base64"), + [...Buffer.from(github.slice(0, 5) + "\u034f" + github.slice(5))] + .map((byte) => `%${byte.toString(16).padStart(2, "0")}`) + .join(""), + "%FF" + [...github].map((c) => `%${c.charCodeAt(0).toString(16)}`).join(""), + ...["\u00ad", "\u034f", "\ufe0f", "\u{e0061}"].map( + (separator) => github.slice(0, 5) + separator + github.slice(5), + ), + github.replace("g", "ɡ"), + github.replace("h", "һ"), + ["gh", "ѕ_", body].join(""), + ["gh", "ο_", body].join(""), + " ".repeat(8190) + github, + Buffer.from(github).toString("base64"), + Buffer.from(Buffer.from(github).toString("base64")).toString("base64"), + [...github].map((c) => `%${c.charCodeAt(0).toString(16)}`).join(""), + github.slice(0, 12) + "\n" + github.slice(12), + ...["next word", " next word"].map( + (suffix) => github.slice(0, 12) + "\n" + github.slice(12) + "\n" + suffix, + ), + github.slice(0, 12) + "\\\n " + github.slice(12), + github.replace("p", "р"), + [...github].map((c) => String.fromCharCode(c.charCodeAt(0) + 0xfee0)).join(""), + github.slice(0, 5) + "\u200b" + github.slice(5), + ]; + for (const content of values) { + const result = yield* scanner.scan({ + orgId, + files: [{ path: "journal/test.md", content: Buffer.from(content) }], + }); + expect(result.some((f) => f.ruleId === "github-token")).toBe(true); + expect(JSON.stringify(result).includes(body)).toBe(false); + expect(Object.keys(result[0]!).sort()).toEqual(["line", "path", "ruleId"]); + } + }).pipe(Effect.provide(layer)), +); +it.effect("reports source lines and blocks AWS credential pairs without leaking values", () => + Effect.gen(function* () { + const scanner = yield* Scan.KinSecretScan; + for (const content of [ + "safe\n\n" + github, + "safe\n\n" + github.slice(0, 12) + "\n" + github.slice(12), + ]) { + const result = yield* scanner.scan({ + orgId, + files: [{ path: "claims/test.md", content: Buffer.from(content) }], + }); + expect(result.every((f) => f.line === 3)).toBe(true); + expect(result.length).toBeGreaterThan(0); + } + const pair = [ + "AWS_ACCESS_KEY_ID=" + ["AK", "IA", "X".repeat(16)].join(""), + "AWS_SECRET_ACCESS_KEY=" + body + "aB3d", + ].join("\n"); + const findings = yield* scanner.scan({ + orgId, + files: [{ path: ".env", content: Buffer.from(pair) }], + }); + expect(findings.some((f) => f.ruleId === "aws-access-key")).toBe(true); + expect(findings.some((f) => f.ruleId === "secret-assignment")).toBe(true); + }).pipe(Effect.provide(layer)), +); +it.effect("allowlist is exact, org scoped, and read only to scanner callers", () => + Effect.gen(function* () { + const scanner = yield* Scan.KinSecretScan; + const sql = yield* SqlClient.SqlClient; + const hash = NodeCrypto.createHash("sha256") + .update("github-token\0") + .update(github) + .digest("hex"); + // Fixture stands in for the future APPR owner-signed executor; scanner has no write method. + yield* sql`INSERT INTO kin_secret_allow VALUES (${orgId}, 'claims/test.md', 'github-token', ${hash}, 'signed-fixture', 0)`; + const run = (org: KinOrgId, path: string, value = github) => + scanner.scan({ orgId: org, files: [{ path, content: Buffer.from(value) }] }); + expect((yield* run(orgId, "claims/test.md")).length).toBe(0); + expect((yield* run(KinOrgId.make("other"), "claims/test.md")).length).toBeGreaterThan(0); + expect((yield* run(orgId, "claims/other.md")).length).toBeGreaterThan(0); + expect((yield* run(orgId, "claims/test.md", github.replace("a", "z"))).length).toBeGreaterThan( + 0, + ); + expect(Object.keys(scanner)).toEqual(["scan"]); + yield* sql`DROP TABLE kin_secret_allow`; + expect((yield* run(orgId, "claims/test.md").pipe(Effect.result))._tag).toBe("Failure"); + }).pipe(Effect.provide(layer)), +); +it.effect("fails closed when decoded findings or the allowlist exceed their bounds", () => + Effect.gen(function* () { + const scanner = yield* Scan.KinSecretScan; + const content = Buffer.from( + Array.from({ length: 1100 }, () => Buffer.from(github).toString("base64")).join("\n"), + ); + expect( + (yield* scanner + .scan({ orgId, files: [{ path: "claims/many.md", content }] }) + .pipe(Effect.result))._tag, + ).toBe("Failure"); + const exhausted = yield* scanner + .scan({ + orgId, + files: Array.from({ length: 8 }, (_, i) => ({ + path: `claims/${i}.md`, + content: Buffer.from("a".repeat(1024 * 1024 - 100)), + })), + }) + .pipe(Effect.result); + expect(exhausted._tag).toBe("Failure"); + if (exhausted._tag === "Failure") + expect(exhausted.failure.reason).toBe("Scan work limit exceeded"); + const sql = yield* SqlClient.SqlClient; + yield* sql`WITH RECURSIVE entries(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM entries WHERE n < 4097) + INSERT INTO kin_secret_allow SELECT ${orgId}, 'claims/' || n, 'github-token', ${"0".repeat(64)}, 'signed-fixture', 0 FROM entries`; + expect((yield* scanner.scan({ orgId, files: [] }).pipe(Effect.result))._tag).toBe("Failure"); + }).pipe(Effect.provide(layer)), +); +it.effect("fails closed at file, batch and count limits", () => + Effect.gen(function* () { + const scanner = yield* Scan.KinSecretScan; + const scan = (files: { path: string; content: Uint8Array }[]) => scanner.scan({ orgId, files }); + for (const files of [ + [{ path: "huge", content: Buffer.alloc(1024 * 1024 + 1) }], + Array.from({ length: 9 }, () => ({ path: "batch", content: Buffer.alloc(1024 * 1024) })), + Array.from({ length: 1025 }, () => ({ path: "count", content: Buffer.alloc(0) })), + ]) + expect((yield* scan(files).pipe(Effect.result))._tag).toBe("Failure"); + const content = Buffer.from( + "a".repeat(1024 * 1024 - github.length - 1 - "large".length) + "\n" + github, + ); + expect( + (yield* scan([{ path: "large", content }])).some((f) => f.ruleId === "github-token"), + ).toBe(true); + }).pipe(Effect.provide(layer)), +); + +it.effect("near misses and private key headers", () => + Effect.gen(function* () { + const scanner = yield* Scan.KinSecretScan; + for (const content of [ + "task-follow-up-with-design-team-on-onboarding", + "risk-follow-up-with-design-team-on-onboarding", + "desk-follow-up-with-design-team-on-onboarding", + "token: 0123456789abcdef0123456789abcdef01234567", + "Password: https://vault.example.invalid/credentials/team", + "secret: projects/example-project/secrets/database/versions/latest", + "github_token_scopes: repo,workflow,admin:org,read:user", + "postgres://user:${DB_PASSWORD}@example.invalid/db", + "ASIAPACIFICREGION2026", + ["sk-", "proj-2026-11-follow-up-with-design-team-on-onboarding"].join(""), + ["sk-", "admin-rotation-plan-for-2026-q3-and-next-year"].join(""), + "z".repeat(4096) + ["AK", "IA"].join("") + "Q".repeat(16), + " ".repeat(8173) + ["AK", "IA"].join("") + "Q".repeat(16) + "z", + ]) + expect( + (yield* scanner.scan({ + orgId, + files: [{ path: "claims/test.md", content: Buffer.from(content) }], + })).length, + ).toBe(0); + for (const content of [ + "-----BEGIN PGP PRIVATE KEY BLOCK-----", + "-----BEGIN CUSTOM PRIVATE KEY-----", + ]) + expect( + (yield* scanner.scan({ + orgId, + files: [{ path: "claims/test.md", content: Buffer.from(content) }], + })).some((f) => f.ruleId === "pem-private-key"), + ).toBe(true); + }).pipe(Effect.provide(layer)), +); +it.effect("scans paths without exposing them", () => + Effect.gen(function* () { + const scanner = yield* Scan.KinSecretScan; + const findings = yield* scanner.scan({ + orgId, + files: [{ path: `claims/${github}.md`, content: Buffer.from("clean") }], + }); + expect(findings.some((f) => f.ruleId === "github-token")).toBe(true); + expect(JSON.stringify(findings).includes(github)).toBe(false); + const indexFindings = yield* scanner.scan({ + orgId, + files: [ + { path: "claims/item.md", content: Buffer.from("clean") }, + { + path: "INDEX.md", + content: Buffer.from("- [claim (claims/wrong.md)] " + github + " (claims/item.md)"), + }, + ], + }); + expect(indexFindings.some((f) => f.itemPath === "claims/item.md")).toBe(true); + }).pipe(Effect.provide(layer)), +); +it.effect("bounds adversarial rule inputs structurally", () => + Effect.gen(function* () { + const scanner = yield* Scan.KinSecretScan; + for (const prefix of ["eyJ", "sk-", "AKIA"]) { + const content = prefix.repeat(Math.floor((1024 * 1024 - 100) / prefix.length)); + let examined = 0; + for (const window of Scan.ruleWindows(content)) { + expect(window.text.length).toBeLessThanOrEqual(8194); + examined += window.text.length; + } + expect(examined).toBeLessThanOrEqual( + content.length * 2 + Math.ceil(content.length / 4096) * 2, + ); + expect( + (yield* scanner.scan({ + orgId, + files: [{ path: "claims/test.md", content: Buffer.from(content) }], + })).length, + ).toBe(0); + } + }).pipe(Effect.provide(layer)), +); + +it.effect.each([ + { name: "256 KiB journal", count: 1, bytes: 256 * 1024 }, + { name: "30 x 60 KiB batch", count: 30, bytes: 60 * 1024 }, +])("scans realistic $name without exhausting work", ({ count, bytes }) => + Effect.gen(function* () { + const scanner = yield* Scan.KinSecretScan; + const journal = + [ + "Discussed the next steps with the team. Follow up tomorrow.", + "Commit " + "0123456789abcdef".repeat(2) + "01234567 reviewed.", + "Updated apps/server/src/kin/memory/secretScan.test.ts and docs/operations/development.md", + "Trace 12345678-1234-5678-9abc-123456789abc completed successfully.", + ].join("\n") + "\n"; + const files = Array.from({ length: count }, (_, i) => ({ + path: `journal/${i}.md`, + content: Buffer.from(journal.repeat(Math.ceil(bytes / journal.length)).slice(0, bytes)), + })); + expect((yield* scanner.scan({ orgId, files })).length).toBe(0); + }).pipe(Effect.provide(layer)), +); + +it.effect("bounds rule executions for random runs, word lines and multi-file inputs", () => + Effect.gen(function* () { + const scanner = yield* Scan.KinSecretScan; + let seed = 42; + const random = (length: number) => + Array.from({ length }, () => { + seed = (Math.imul(seed, 1664525) + 1013904223) >>> 0; + return "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"[seed % 62]; + }).join(""); + const runs = Array.from({ length: Math.floor((1024 * 1024 - 100) / 25) }, () => + random(24), + ).join(" "); + const words = (Array.from({ length: 10 }, () => random(7)).join(" ") + "\n").repeat(13000); + for (const files of [ + [{ path: "runs", content: Buffer.from(runs) }], + [{ path: "words", content: Buffer.from(words) }], + Array.from({ length: 8 }, (_, i) => ({ path: `runs-${i}`, content: Buffer.from(runs) })), + ]) { + let executions = 0; + const exec = RegExp.prototype.exec; + using _spy = vi.spyOn(RegExp.prototype, "exec").mockImplementation(function ( + this: RegExp, + value: string, + ) { + if (this.source.startsWith("(?()( "KinSecretScanError", { reason: Schema.String, + cause: Schema.optional(Schema.Defect()), }, ) {} export interface SecretFinding { readonly path: string; readonly ruleId: string; readonly line: number; + readonly pathHit?: boolean; + readonly itemPath?: string; } export class KinSecretScan extends Context.Service< KinSecretScan, @@ -25,14 +32,263 @@ export class KinSecretScan extends Context.Service< } >()("t3/kin/memory/secretScan/KinSecretScan") {} -export const noFindingsStubLayer = Layer.effect( - KinSecretScan, - Effect.gen(function* () { - yield* Effect.logWarning("SECURITY: Kin secret scanning is disabled (no-findings stub)"); - return KinSecretScan.of({ scan: Effect.fn("KinSecretScan.stub")(() => Effect.succeed([])) }); - }), -); -export const testLayer = (findings: ReadonlyArray) => - Layer.succeed(KinSecretScan, { - scan: Effect.fn("KinSecretScan.fixture")(() => Effect.succeed(findings)), +const rules = [ + ["github-token", /(?:gh[pousr]_[A-Za-z0-9]{36}|github_pat_[A-Za-z0-9_]{82})/], + ["anthropic-key", /sk-ant-[A-Za-z0-9_-]{40,200}/], + [ + "openai-key", + /sk-(?:(?:proj|svcacct|admin)-[A-Za-z0-9_-]{32,200}|[A-Za-z0-9]{20}T3BlbkFJ[A-Za-z0-9]{20})/, + ], + ["aws-access-key", /(?:AKIA|ASIA)[A-Z0-9]{16}/], + ["slack-token", /xox[abposr]-[A-Za-z0-9-]{20,200}/], + ["stripe-key", /[rs]k_live_[A-Za-z0-9]{24,200}/], + ["google-key", /AIza[A-Za-z0-9_-]{35}/], + ["pem-private-key", /-----BEGIN (?:[A-Z0-9]+ ){0,8}PRIVATE KEY(?: BLOCK)?-----/], + ["jwt", /eyJ[A-Za-z0-9_-]{5,2048}\.eyJ[A-Za-z0-9_-]{8,1024}/], + ["url-password", /\b[a-z][a-z0-9+.-]{0,31}:\/\/[^\s/:@]{1,512}:([^\s/@]{1,512})@/i], + [ + "secret-assignment", + /\b[A-Za-z0-9_]{0,64}(?:password|secret|token|api[_-]?key|access[_-]?key)[A-Za-z0-9_]{0,64}["']?\s{0,32}[:=]\s{0,32}["']?([^\s"';#]{20,512})/i, + ], +] as const; +const compiledRules = rules.map(([id, pattern]) => { + const alphabet = + id === "github-token" || id === "aws-access-key" || id === "stripe-key" || id === "google-key" + ? "A-Za-z0-9" + : id === "slack-token" + ? "A-Za-z0-9-" + : "A-Za-z0-9_-"; + const right = ["url-password", "secret-assignment", "jwt"].includes(id) + ? "" + : `(?![${alphabet}])`; + return [ + id, + new RegExp(`(? 0 ? 1 : 0, + rightContext: offset + 8192 < value.length ? 1 : 0, + }; +}; +const entropy = (value: string) => { + const counts = new Map(); + for (const char of value) counts.set(char, (counts.get(char) ?? 0) + 1); + let bits = 0; + for (const count of counts.values()) { + const p = count / value.length; + bits -= p * Math.log2(p); + } + return bits; +}; +// NFKC covers fullwidth forms; common Cyrillic/Greek confusables are normalized as well. +const lookalikes: Readonly> = { + ѕ: "s", + һ: "h", + ο: "o", + ɡ: "g", + а: "a", + е: "e", + о: "o", + р: "p", + с: "c", + х: "x", + у: "y", + і: "i", + ј: "j", + А: "A", + В: "B", + Е: "E", + К: "K", + М: "M", + Н: "H", + О: "O", + Р: "P", + С: "C", + Т: "T", + Х: "X", + Α: "A", + Β: "B", + Ε: "E", + Ι: "I", + Κ: "K", + Μ: "M", + Ν: "N", + Ο: "O", + Ρ: "P", + Τ: "T", + Χ: "X", +}; +const normalize = (text: string) => + text + .normalize("NFKC") + .replace(/\p{Default_Ignorable_Code_Point}/gu, "") + .replace(/[ѕһοɡаеорсхуіјАВЕКМНОРСТХΑΒΕΙΚΜΝΟΡΤΧ]/g, (c) => lookalikes[c] ?? c); +const fingerprint = (ruleId: string, value: string) => + NodeCrypto.createHash("sha256").update(ruleId).update("\0").update(value).digest("hex"); +const make = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const scan = Effect.fn("KinSecretScan.scan")(function* (input: { + orgId: KinOrgId; + files: ReadonlyArray<{ path: string; content: Uint8Array }>; + }) { + if ( + input.files.length > 1024 || + input.files.some((f) => f.content.byteLength + Buffer.byteLength(f.path) > 1024 * 1024) || + input.files.reduce((sum, f) => sum + f.content.byteLength + Buffer.byteLength(f.path), 0) > + 8 * 1024 * 1024 + ) + return yield* new KinSecretScanError({ reason: "Scan size limit exceeded" }); + // No mutation API: the owner-signed action executor owns allowlist insertion. + const allowed = yield* sql<{ path: string; rule_id: string; fingerprint: string }>` + SELECT path, rule_id, fingerprint FROM kin_secret_allow WHERE org_id = ${input.orgId} LIMIT 4097 + `.pipe( + Effect.mapError(toPersistenceSqlError("KinSecretScan.allowlist")), + Effect.mapError( + (cause) => new KinSecretScanError({ reason: "Allowlist unavailable", cause }), + ), + ); + if (allowed.length > 4096) + return yield* new KinSecretScanError({ reason: "Allowlist size limit exceeded" }); + const allow = new Set(allowed.map((r) => JSON.stringify([r.path, r.rule_id, r.fingerprint]))); + const findings: SecretFinding[] = []; + const seen = new Set(); + let budget = 64 * 1024 * 1024; + for (const file of input.files) { + const text = normalize(Buffer.from(file.content).toString("utf8")); + const lines = text.split("\n"); + const pathFindingStart = findings.length; + let pathHit = true; + let redactPath = false; + const inspect = function* (value: string, baseLine: number, depth: number): Generator { + budget -= value.length; + if (budget < 0 || findings.length > 1024) return; + for (const window of ruleWindows(value)) { + budget -= window.text.length; + if (budget < 0) return; + yield; + if (!anchors.test(window.text)) continue; + for (const [ruleId, bounded] of compiledRules) { + bounded.lastIndex = 0; + let match: RegExpExecArray | null; + while ((match = bounded.exec(window.text)) !== null) { + budget -= 64; + if (budget < 0) return; + if ( + match.index < window.leftContext || + match.index + match[0].length > window.text.length - window.rightContext + ) + continue; + const index = window.offset + match.index; + const secret = match[1] ?? match[0]; + if (ruleId === "openai-key" && !/[A-Z]/.test(secret)) continue; + if (ruleId === "url-password" && /^\$\{[^}]+\}$/.test(secret)) continue; + if ( + ruleId === "secret-assignment" && + (entropy(secret) < 3.5 || + /^[a-f0-9]{40}(?:[a-f0-9]{24})?$/i.test(secret) || + /^https?:\/\//i.test(secret) || + /^projects\/[^/]+\/(?:locations\/[^/]+\/)?secrets\//.test(secret) || + /^[^:=]*token_scopes\s*[:=]/i.test(match[0])) + ) + continue; + if (allow.has(JSON.stringify([file.path, ruleId, fingerprint(ruleId, secret)]))) + continue; + const line = baseLine + (value.slice(0, index).match(/\n/g)?.length ?? 0); + const key = JSON.stringify([file.path, ruleId, line, pathHit]); + if (!seen.has(key)) { + const indexLine = file.path === "INDEX.md" ? lines[line - 1] : undefined; + const itemPath = Array.from(indexLine?.matchAll(/\(([^()]+\.md)\)/g) ?? []).at( + -1, + )?.[1]; + seen.add(key); + findings.push({ + path: pathHit || redactPath ? "[redacted]" : file.path, + ruleId, + line, + ...(pathHit ? { pathHit: true } : {}), + ...(itemPath && input.files.some((f) => f.path === itemPath) ? { itemPath } : {}), + }); + } + break; + } + } + } + if (depth === 2) return; + let cursor = 0; + let decodedLine = baseLine; + for (const match of value.matchAll(/(?:%[0-9a-fA-F]{2}|[A-Za-z0-9_+/-]){20,}={0,2}/g)) { + const encoded = match[0]; + // Mixed percent/plain encodings can represent 20 bytes in fewer than 60 characters. + if (!encoded.includes("%") && encoded.length < 27) continue; + decodedLine += value.slice(cursor, match.index).match(/\n/g)?.length ?? 0; + cursor = match.index; + yield; + budget -= encoded.length; + if (budget < 0) return; + const candidates = encoded.includes("%") + ? [ + Buffer.from( + encoded.replace(/%([0-9a-f]{2})/gi, (_, hex: string) => + String.fromCharCode(Number.parseInt(hex, 16)), + ), + "latin1", + ).toString("utf8"), + ] + : Array.from({ length: 4 }, (_, alignment) => + Buffer.from(encoded.slice(alignment), "base64url").toString("utf8"), + ); + for (const candidate of candidates) { + if (budget < 0 || findings.length > 1024) return; + const decoded = normalize(candidate).replace(/[^\x09\x0a\x0d\x20-\x7e]/g, " "); + if (decoded !== encoded) yield* inspect(decoded, decodedLine, depth + 1); + } + } + }; + for (const _ of inspect(normalize(file.path), 1, 0)) yield* Effect.yieldNow; + redactPath = findings.length > pathFindingStart; + pathHit = false; + for (const _ of inspect(text, 1, 0)) yield* Effect.yieldNow; + let run: { text: string; line: number }[] = []; + const append = function* (text: string, line: number): Generator { + run.push({ text, line }); + while (run.length > 8 || run.reduce((sum, part) => sum + part.text.length, 0) > 4096) + run.shift(); + if (run.length < 2) return; + const joined = run.map((part) => part.text).join(""); + // No rule matches without an anchor and no decoder starts below 20 characters. + if (joined.length >= 20 || anchors.test(joined)) yield* inspect(joined, run[0]!.line, 0); + }; + for (const [index, raw] of lines.entries()) { + if (index % 1024 === 1023) yield* Effect.yieldNow; + const words = raw + .replace(/\\?\r$/, "") + .replace(/\\$/, "") + .trim() + .split(/\s+/); + const first = words[0]!; + if (!first) { + run = []; + continue; + } + for (const _ of append(first, index + 1)) yield* Effect.yieldNow; + if (words.length > 1) run = [{ text: words.at(-1)!, line: index + 1 }]; + } + } + if (budget < 0 || findings.length > 1024) + return yield* new KinSecretScanError({ reason: "Scan work limit exceeded" }); + const unsafePaths = findings.some((f) => f.pathHit); + return findings.map(({ itemPath, ...finding }) => ({ + ...finding, + ...(itemPath && !unsafePaths ? { itemPath } : {}), + })); }); + return KinSecretScan.of({ scan }); +}); +export const layer = Layer.effect(KinSecretScan, make); diff --git a/apps/server/src/kin/persistence/KinMigrations.test.ts b/apps/server/src/kin/persistence/KinMigrations.test.ts index 4b855164bf64d..467d14d994cd8 100644 --- a/apps/server/src/kin/persistence/KinMigrations.test.ts +++ b/apps/server/src/kin/persistence/KinMigrations.test.ts @@ -36,6 +36,7 @@ const tableNames = [ "kin_ping_items", "kin_ping_ledger", "kin_read_state", + "kin_secret_allow", "kin_session_bindings", "kin_shadow_entries", "kin_trigger_fires", @@ -98,6 +99,7 @@ describe("Kin migrations", () => { { migration_id: 9, name: "JudgeLog" }, { migration_id: 10, name: "ChatChanges" }, { migration_id: 11, name: "MemoryHeads" }, + { migration_id: 12, name: "SecretAllow" }, ]); assert.deepStrictEqual( upstreamBefore.map((row) => [row.migration_id, row.name] as const), @@ -151,6 +153,7 @@ describe("Kin migrations", () => { { migration_id: 9, name: "JudgeLog" }, { migration_id: 10, name: "ChatChanges" }, { migration_id: 11, name: "MemoryHeads" }, + { migration_id: 12, name: "SecretAllow" }, ]); assert.deepStrictEqual(yield* upstreamHistory, upstreamBefore); assert.deepStrictEqual(yield* sql`SELECT * FROM kin_orgs`, [ @@ -229,6 +232,7 @@ describe("Kin migrations", () => { [9, "JudgeLog"], [10, "ChatChanges"], [11, "MemoryHeads"], + [12, "SecretAllow"], ]); const before = yield* kinSchema; assert.deepStrictEqual( @@ -266,6 +270,7 @@ describe("Kin migrations", () => { yield* sql`DROP TABLE kin_ping_items`; yield* sql`DROP INDEX kin_chat_members_one_owner`; yield* sql`DROP TABLE kin_memory_heads`; + yield* sql`DROP TABLE kin_secret_allow`; yield* sql`DROP TABLE kin_budget_charges`; yield* sql`ALTER TABLE kin_events DROP COLUMN budget_deferred`; yield* sql`UPDATE kin_chats SET last_seq = 1 WHERE org_id = 'org' AND chat_id = 'existing-chat'`; @@ -280,6 +285,7 @@ describe("Kin migrations", () => { [9, "JudgeLog"], [10, "ChatChanges"], [11, "MemoryHeads"], + [12, "SecretAllow"], ]); assert.deepStrictEqual( yield* sql`SELECT member_id FROM kin_chat_members WHERE org_id = 'org' ORDER BY member_id`, @@ -316,6 +322,7 @@ describe("Kin migrations", () => { { migration_id: 9, name: "JudgeLog" }, { migration_id: 10, name: "ChatChanges" }, { migration_id: 11, name: "MemoryHeads" }, + { migration_id: 12, name: "SecretAllow" }, ]); assert.deepStrictEqual(yield* runKinMigrations(), []); assert.deepStrictEqual(yield* runMigrations(), []); diff --git a/apps/server/src/kin/persistence/KinMigrations.ts b/apps/server/src/kin/persistence/KinMigrations.ts index b7bba32965691..2a407ca6d5f7b 100644 --- a/apps/server/src/kin/persistence/KinMigrations.ts +++ b/apps/server/src/kin/persistence/KinMigrations.ts @@ -15,6 +15,8 @@ import JudgeLog from "./migrations/K009_JudgeLog.ts"; import ChatChanges from "./migrations/K010_ChatChanges.ts"; import MemoryHeads from "./migrations/K011_MemoryHeads.ts"; +import SecretAllow from "./migrations/K012_SecretAllow.ts"; + const run = Migrator.make({}); const loader = Migrator.fromRecord({ "1_ChatCore": ChatCore, @@ -28,6 +30,7 @@ const loader = Migrator.fromRecord({ "9_JudgeLog": JudgeLog, "10_ChatChanges": ChatChanges, "11_MemoryHeads": MemoryHeads, + "12_SecretAllow": SecretAllow, }); export const runKinMigrations = Effect.fn("KinMigrations.run")(function* () { diff --git a/apps/server/src/kin/persistence/migrations/K012_SecretAllow.ts b/apps/server/src/kin/persistence/migrations/K012_SecretAllow.ts new file mode 100644 index 0000000000000..c5563c34c8717 --- /dev/null +++ b/apps/server/src/kin/persistence/migrations/K012_SecretAllow.ts @@ -0,0 +1,15 @@ +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/sql/SqlClient"; + +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql`CREATE TABLE kin_secret_allow ( + org_id TEXT NOT NULL, + path TEXT NOT NULL, + rule_id TEXT NOT NULL, + fingerprint TEXT NOT NULL CHECK(length(fingerprint) = 64), + signed_action_id TEXT NOT NULL, + created_at INTEGER NOT NULL, + PRIMARY KEY (org_id, path, rule_id, fingerprint) + )`; +}); diff --git a/docs/kin/specs/lead.md b/docs/kin/specs/lead.md index b6d23aaafa0b2..2a9f3237a5978 100644 --- a/docs/kin/specs/lead.md +++ b/docs/kin/specs/lead.md @@ -600,7 +600,7 @@ LEAD-1. **Contracts and tables.** `kinLead.ts` schemas; Kin migrations for all l LEAD-2. **Memory read side.** `ensureCheckouts`, `prepareWakeup`, front matter parser, cache, `query`, `readItem`, `rebuildIndex` (200-line cap, "As of" line, fading after 8 weeks unread). Files: `apps/server/src/kin/memory/{Memory,MemoryCache,frontMatter}.ts`. Depends on: LEAD-1. Done when: tests create a temporary git repo from invented fixtures matching the kin-memory README; kind/date/review queries, malformed-item isolation, organization isolation, cache invalidation, index cap/As-of and TestClock fading all pass. Never use real organization memory as a fixture. Owner: backend. -LEAD-3. **Memory git writer.** `stage` with every writer rule, `commitWakeup`, push and rebase, shadow branch, the secret scan hook (stub until the triggers spec lands). Files: `MemoryWriter.ts`, `journal.ts`. Depends on: LEAD-2, TRIG-16 (stub allowed). Done when: tests show provenance override, `rules.md` refusal, conflict error, one commit per wake-up, discard on failure. Owner: backend. +LEAD-3. **Memory git writer.** `stage` with every writer rule, `commitWakeup`, push and rebase, shadow branch, the secret scan hook (TRIG-16 supplies the scanner layer). Files: `MemoryWriter.ts`, `journal.ts`. Depends on: LEAD-2, TRIG-16 (stub allowed). Done when: tests show provenance override, `rules.md` refusal, conflict error, one commit per wake-up, discard on failure. Owner: backend. LEAD-4. **Event queue and coalescer.** `enqueue`, dedupe, priorities, quiet window, claim, restart requeue, per-org semaphore. Files: `KinEventQueue.ts`. Depends on: LEAD-1. Done when: coalescing and one-writer tests pass with `TestClock`. Owner: backend. @@ -647,7 +647,7 @@ LEAD-14. **End-to-end shadow week dry run.** Script that replays a day of record `MemoryWriterAuthority.resolve` is runner-only: resolve organization, wake-up, active lead, level, timestamps, timezone and event provenance from same-organization rows inside the runner's transaction. Never expose this context as tool input. Owner-confirmed sources must be verified there; absent a source, provenance is the lead's agent id. `MemoryWriterHooks.notify` consumes checkout, conflict and secret-block notices without chat transport coupling. -`KinSecretScan` is deliberately supplied separately. `noFindingsStubLayer` returns no findings until TRIG-16 replaces it; it is not a production secret detector. Any finding or scanner failure blocks the whole commit. TRIG-16 must preserve this fail-closed interface: `scan({ orgId, files: ReadonlyArray<{ path: string; content: Uint8Array }> })` returns `{ path, ruleId, line }[]`, with typed `KinSecretScanError`. +`KinSecretScan` is deliberately supplied separately. `secretScan.layer` supplies the TRIG-16 scanner and reads the owner-signed allowlist; there is no no-findings production stub. Any finding or scanner failure blocks the whole commit. TRIG-16 must preserve this fail-closed interface: `scan({ orgId, files: ReadonlyArray<{ path: string; content: Uint8Array }> })` returns `{ path, ruleId, line }[]`, with typed `KinSecretScanError`. `commitWakeup` builds TRIG-12's sectioned `INDEX.md` from the hash-verified recorded tree plus staged files, read history, the server clock and the owner's time zone, in the same wake-up commit. The read-side rebuild is display-only; no read-checkout text is trusted. K011 adds the SQLite branch anchors required by the writer. The runner serializes prepare/stage/commit and dreaming per organization. Identical commit retries find the existing wake-up commit; failed pushes retain local commits, and failures discard uncommitted files. Journal additions use `appendJournal`, never replacement through `stage`. diff --git a/docs/kin/specs/triggers.md b/docs/kin/specs/triggers.md index 1890e5a05a9df..f2f928008be37 100644 --- a/docs/kin/specs/triggers.md +++ b/docs/kin/specs/triggers.md @@ -324,12 +324,12 @@ GitHub then blocks a bad auto-merge even if Kin's classifier is wrong. This forc **Rules:** - GitHub tokens (`ghp_`, `gho_`, `ghu_`, `ghs_`, `ghr_`, `github_pat_`); -- Anthropic `sk-ant-`, OpenAI `sk-`/`sk-proj-`; +- Anthropic `sk-ant-`; OpenAI legacy `sk-` with the `T3BlbkFJ` marker and `sk-proj-`/`sk-svcacct-`/`sk-admin-` containing an uppercase letter; - AWS `AKIA…`/`ASIA…`; - Slack `xox[abposr]-`; - Stripe `sk_live_`/`rk_live_`; - Google `AIza`; -- PEM `-----BEGIN … PRIVATE KEY-----`; +- PEM `-----BEGIN … PRIVATE KEY-----`, including `PGP PRIVATE KEY BLOCK`; - JWTs (`eyJ…`, three base64url parts); - URLs with a password in them (`scheme://user:pass@`); - `password|secret|token|api_key = `. @@ -342,7 +342,11 @@ GitHub then blocks a bad auto-merge even if Kin's classifier is wrong. This forc **On a finding**, the writer blocks the entire commit, discards staged text and emits a typed owner-notification hook. The runner journals the refusal without secret content and tells the owner to review and rotate the affected credential. Automatic redaction is not part of the writer contract. -**False positives:** the owner adds the fingerprint to `kin_secret_allow` through a signed action. A model can never add to it. +**False positives:** the owner adds the fingerprint to `kin_secret_allow` through a signed action. A model can never add to it. Entries are scoped to organization, exact path, rule and SHA-256 of `ruleId + "\0" + normalized matched value`, with `created_at` in integer milliseconds and `signed_action_id` reserved for the future APPR executor id. The scanner is read-only; the APPR executor must verify the owner grant before inserting. Until that executor lands, there is no agent or MCP allowlist-write path. Migration K012 is provisional and will be renumbered at merge. + +The scanner checks paths as well as content, redacts paths that contain a finding, normalizes NFKC, common Cyrillic/Greek/Latin lookalikes and Default Ignorable code points, checks newline-wrapped tokens, and decodes URL/base64 content up to two levels. Base64 decoding tries all four alignments; malformed percent bytes and non-ASCII decoded bytes become separators. Prefix rules use alphanumeric left boundaries and body-specific right boundaries (fixed-width Google keys also tolerate underscore metadata suffixes); each precompiled regex sees at most 8192 characters plus one character of boundary context on each side, with 4096-character overlap. JWT detection matches a bounded prefix: an eyJ header followed by 5–2048 base64url characters, then an eyJ payload prefix followed by 8–1024 characters, without requiring the signature or the payload end. Its maximum match is 3080 characters, within the overlap. Wrapped tokens are inspected after each appended fragment once the joined run has 20 characters or an anchor, retaining up to eight fragments and 4 KiB. A prose line closes the prior run with its first word and opens a new run with its last word, so adjacent prose and trailing words do not hide a wrapped token. Merge scanning covers only bytes changed from the fetched remote plus the rebuilt `INDEX.md`, not pre-existing remote content. Scanner failures notify through the writer hook, just like findings. Staged YAML parser errors omit source-bearing diagnostics and filesystem path errors omit the underlying path-bearing cause. Merge blocks use the recovery hook; production owner delivery still waits on a notifier. It fails closed above 1 MiB per file, 8 MiB or 1024 files per call, 4096 allowlist entries per organization, or 64 MiB of weighted work (inspected characters, encoded candidate characters, window characters, and 64 units per regex match). A cheap anchor pre-filter skips rule execution on irrelevant windows, including decoded candidates; recursive decoding remains active to find double-encoded tokens. Scanning yields between windows and decoder candidates so multi-file batches do not monopolize the event loop. Base64 runs shorter than 27 characters and fully percent-encoded runs shorter than 60 characters are not decoded; mixed percent/plain runs retain a 20-byte minimum to prevent partial-encoding bypasses. Wrapped runs beyond eight fragments or 4 KiB, splits by a space, `\r`, U+2028, across journal bullets, markdown or HTML comments, `\u` escapes, and hex encoding remain out of scope. These are conservative detectors, not a guarantee against arbitrary encryption or steganography. + +Known gaps, accepted for phase 1: JWE (five-part) tokens and JWT headers longer than 2051 characters; multi-part secrets (JWT, PEM header, URL password, `KEY=value`) with only some bytes percent-encoded; lookalikes outside the folding table (for example Greek `ρ`, Cyrillic `І`/`Ј`/`к`, small caps, U+2800, combining marks inside an anchor). A file of about 1 MiB made only of long whitespace-free lines (such as one path per line) exhausts the work budget after about 15 s and fails closed, blocking that commit; realistic journals up to 1 MiB pass. ### Owner-signed purge (gap 5) @@ -475,7 +479,7 @@ Kin runs on one always-on machine, the **Kin host**. The laptop and phone are cl - **Sink:** a signed GitHub `pull_request` request through the real `ScheduledTaskService` gives exactly one inbox row and no thread launch. A replay gives `duplicate` and no new row. A bot-sender request gives `filtered`. A non-Kin task still launches normally. - **Deferral:** unchanged recheck, then heads-up, then silence, then `started` at due. A linked issue closed before due leads to `asked`, and nothing starts. With the stop switch on, it asks. [Monday] re-arms to next Monday 09:00. - **Dream:** a dream touching `rules.md` fails, and `main` is unchanged. A dream with one journal fold and one `decisions/` change opens two PRs, and only the auto one merges. -- **Secret scan:** a journal line holding `ghp_` plus 36 characters is committed as `[redacted:github-token]`, and the owner gets one message. +- **Secret scan:** a journal line holding `ghp_` plus 36 characters blocks the entire commit, and the owner gets one typed notification without the value. - **Purge:** on a fixture repo, the file is gone from `git log --all -p`, a tombstone exists, and an audit row exists. - **Contract tests:** `vp test run kin-contract` passes on today's upstream. Renaming `scheduledTaskId` in a scratch branch makes it fail.