From 768c397d635360dbc9b095573e38e38682801097 Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:15:01 +0530 Subject: [PATCH 1/7] feat(kin): block secrets in memory writes (TRIG-16) --- apps/server/src/kin/memory/Memory.test.ts | 16 +- apps/server/src/kin/memory/Memory.ts | 29 ++- .../src/kin/memory/MemoryWriter.test.ts | 22 ++- apps/server/src/kin/memory/secretScan.test.ts | 129 +++++++++++++ apps/server/src/kin/memory/secretScan.ts | 173 +++++++++++++++++- .../src/kin/persistence/KinMigrations.test.ts | 7 + .../src/kin/persistence/KinMigrations.ts | 3 + .../migrations/K012_SecretAllow.ts | 14 ++ docs/kin/specs/triggers.md | 6 +- 9 files changed, 381 insertions(+), 18 deletions(-) create mode 100644 apps/server/src/kin/memory/secretScan.test.ts create mode 100644 apps/server/src/kin/persistence/migrations/K012_SecretAllow.ts diff --git a/apps/server/src/kin/memory/Memory.test.ts b/apps/server/src/kin/memory/Memory.test.ts index 10f4c07c8815b..d3d23ba9258c1 100644 --- a/apps/server/src/kin/memory/Memory.test.ts +++ b/apps/server/src/kin/memory/Memory.test.ts @@ -861,6 +861,7 @@ it.live("uses filesystem identity for a case-mismatched base without recloning", it.live.each([ "diverged", + "secret merge", "conflict", "network down", "dirty writer", @@ -955,13 +956,15 @@ it.live.each([ } yield* memory.record(orgA, "main", initial.head, local); yield* fs.writeFileString( - `${source}/${scenario === "push retry" || scenario === "diverged" || scenario === "network down" || scenario === "dirty writer" || scenario === "H2-merge-swap" ? "remote-only" : "rules.md"}`, + `${source}/${scenario === "push retry" || scenario === "diverged" || scenario === "secret merge" || scenario === "network down" || scenario === "dirty writer" || scenario === "H2-merge-swap" ? "remote-only" : "rules.md"}`, "Remote invented edit\n", ); if (scenario === "journal conflict" || scenario === "push retry") { yield* fs.writeFileString(`${source}/journal/day.md`, "remote entry\n"); yield* fs.writeFileString(`${source}/INDEX.md`, "remote generated index\n"); } + if (scenario === "secret merge") + yield* fs.writeFileString(`${source}/remote-only`, ["gh", "p_", "a".repeat(36)].join("")); if (scenario === "H2-merge-swap") yield* fs.writeFileString(`${source}/rules.md`, h2Base.replace("base tail", "remote tail")); yield* commit(source); @@ -1041,6 +1044,16 @@ it.live.each([ expect( (yield* memory.prepareWakeup(orgA, "fixture-wakeup-3").pipe(Effect.result))._tag, ).toBe("Success"); + } else if (scenario === "secret merge") { + expect(result._tag).toBe("Failure"); + if (result._tag === "Failure") { + expect(result.failure._tag).toBe("MemoryError"); + expect(result.failure.operation).toBe("secretScan"); + } + expect(yield* memory.anchor(orgA, "main")).toBe(local); + expect(yield* git(root, [`--git-dir=${remote}`, "rev-parse", "main"])).toBe( + yield* git(source, ["rev-parse", "HEAD"]), + ); } else if (scenario === "push retry") { expect(result._tag).toBe("Success"); if (result._tag === "Success") { @@ -1066,6 +1079,7 @@ it.live.each([ } else if ( scenario === "network down" || scenario === "diverged" || + scenario === "secret merge" || scenario === "dirty writer" ) { expect(result._tag).toBe("Success"); diff --git a/apps/server/src/kin/memory/Memory.ts b/apps/server/src/kin/memory/Memory.ts index b670a28d1daca..1e091006c7925 100644 --- a/apps/server/src/kin/memory/Memory.ts +++ b/apps/server/src/kin/memory/Memory.ts @@ -1,3 +1,4 @@ +import * as SecretScan from "./secretScan.ts"; import { readTree, readObjects, @@ -203,6 +204,7 @@ interface BlobCache { } const make = Effect.gen(function* () { + const scanner = yield* SecretScan.KinSecretScan; const config = yield* MemoryConfig; const fs = yield* FileSystem.FileSystem; const { notify } = yield* MemoryRecoveryHooks; @@ -1085,6 +1087,28 @@ const make = Effect.gen(function* () { "INDEX.md", Buffer.from(buildIndex(heads, reads, yield* Clock.currentTimeMillis, zone)), ); + const findings = yield* scanner + .scan({ + orgId, + files: [...remoteFiles].map(([path, content]) => ({ path, content })), + }) + .pipe( + Effect.mapError( + (cause) => new MemoryError({ orgId, operation: "secretScan", cause }), + ), + ); + if (findings.length) { + yield* notify({ + orgId, + branch, + reason: `Secret scan blocked merge: ${findings[0]!.ruleId} in ${findings[0]!.path}`, + }); + return yield* new MemoryError({ + orgId, + operation: "secretScan", + reason: `Secret scan blocked merge: ${findings[0]!.ruleId} in ${findings[0]!.path}`, + }); + } tree = yield* Effect.scoped( Effect.gen(function* () { const temp = yield* io(orgId, "journalMerge", fs.makeTempDirectoryScoped()); @@ -1273,4 +1297,7 @@ const make = Effect.gen(function* () { }); }); -export const layer = Layer.effect(Memory, make).pipe(Layer.provide(MemoryCache.layer)); +export const layer = Layer.effect(Memory, make).pipe( + Layer.provide(MemoryCache.layer), + Layer.provide(SecretScan.layer), +); diff --git a/apps/server/src/kin/memory/MemoryWriter.test.ts b/apps/server/src/kin/memory/MemoryWriter.test.ts index 194e65512ca73..9db49d2440eb4 100644 --- a/apps/server/src/kin/memory/MemoryWriter.test.ts +++ b/apps/server/src/kin/memory/MemoryWriter.test.ts @@ -222,10 +222,7 @@ it.live.each([ }), ), }); - const scan = - scenario === "blocked" - ? Scan.testLayer([{ path: "claims/new.md", ruleId: "fixture", line: 1 }]) - : Scan.noFindingsStubLayer; + const scan = Scan.layer; const layer = Writer.layer.pipe( Layer.provideMerge(memoryLayer), Layer.provide(Layer.succeed(Memory.MemoryRecoveryHooks, { notify: () => Effect.void })), @@ -585,6 +582,10 @@ it.live.each([ } if (scenario === "tree-substitution") substituteTree = yield* git(paths.writer, ["rev-parse", `${original}^{tree}`]); + if (scenario === "blocked") + yield* writer.appendJournal(orgId, "wake1", [ + ["gh", "p_"].join("") + "aB3dE6gH9jK2mN5pQ8sT1vW4yZ7cF0iL3oR6", + ]); const result = yield* writer .commitWakeup(orgId, "wake1", scenario === "failed" ? "failed" : "completed") .pipe(Effect.result); @@ -603,6 +604,19 @@ it.live.each([ ); expect(notices).toHaveLength(1); + if (scenario === "blocked") { + expect(JSON.stringify(notices)).not.toContain("aB3dE6gH9jK2mN5pQ8sT1vW4yZ7cF0iL3oR6"); + expect(yield* git(paths.writer, ["rev-parse", "HEAD"])).toBe(original); + expect( + (yield* git(root, [ + `--git-dir=${remote}`, + "show-ref", + "--verify", + `refs/heads/kin/shadow/${agentId}`, + ]).pipe(Effect.result))._tag, + ).toBe("Failure"); + expect(yield* fs.exists(`${paths.writer}/claims/new.md`)).toBe(false); + } expect(yield* fs.exists(`${paths.writer}/.git/rebase-merge`)).toBe(false); } else { expect(result._tag).toBe("Success"); diff --git a/apps/server/src/kin/memory/secretScan.test.ts b/apps/server/src/kin/memory/secretScan.test.ts new file mode 100644 index 0000000000000..2a52acb8a3b77 --- /dev/null +++ b/apps/server/src/kin/memory/secretScan.test.ts @@ -0,0 +1,129 @@ +// @effect-diagnostics-next-line nodeBuiltinImport:off -- Bounded synchronous fingerprint hashing shares the scan loop. +import * as NodeCrypto from "node:crypto"; +import { expect, it } from "@effect/vitest"; +import { KinOrgId } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as SqlClient from "effect/sql/SqlClient"; +import * as Sqlite from "../../persistence/Sqlite.ts"; +import * as Persistence from "../persistence/KinPersistence.ts"; +import * as Scan from "./secretScan.ts"; + +const orgId = KinOrgId.make("fixture-org"); +const database = Layer.provideMerge(Persistence.layer, Sqlite.layerMemory); +const layer = Scan.layer.pipe(Layer.provideMerge(database)); +const body = "aB3dE6gH9jK2mN5pQ8sT1vW4yZ7cF0iL3oR6"; +const github = ["gh", "p_"].join("") + body; +const fixtures = [ + ...["p", "o", "u", "s", "r"].map((kind) => [ + "github-token", + "gh" + kind + "_" + body, + "gh" + kind + "_short", + ]), + [ + "github-token", + ["github_", "pat_"].join("") + "a".repeat(82), + ["github_", "pat_short"].join(""), + ], + ["anthropic-key", ["sk-", "ant-"].join("") + body.repeat(2), ["sk-", "ant-short"].join("")], + ["openai-key", "sk-" + body, "sk-short"], + ["openai-key", ["sk-", "proj-"].join("") + body.repeat(3), "sk-proj-short"], + ...["AK", "AS"].map((kind) => [ + "aws-access-key", + kind + "IA" + "X".repeat(16), + kind + "IA" + "X".repeat(15), + ]), + ["slack-token", ["xo", "xb-"].join("") + body, ["xo", "xb-short"].join("")], + ["stripe-key", ["sk_", "live_"].join("") + body, "sk_live_short"], + ["google-key", ["AI", "za"].join("") + "X".repeat(35), ["AI", "za_short"].join("")], + ["pem-private-key", "-----BEGIN RSA PRIVATE KEY-----", "-----BEGIN PUBLIC KEY-----"], + [ + "jwt", + ["ey", "J"].join("") + "a".repeat(10) + "." + "b".repeat(12) + "." + "c".repeat(20), + "eyJshort.two.parts", + ], + [ + "url-password", + "https://fixture:fake-password@example.invalid", + "https://fixture@example.invalid", + ], + ["secret-assignment", "password=" + body, "password=" + "a".repeat(40)], + ["secret-assignment", "export API_KEY='" + body + "'", "export API_KEY='short'"], + ["secret-assignment", "AWS_SECRET_ACCESS_KEY=" + body + "aB3d", "AWS_SECRET_ACCESS_KEY=example"], +]; +it.effect.each(fixtures.map(([rule, hit, miss], i) => ({ rule, hit, miss, id: i })))( + "rule fixture $id: $rule", + ({ rule, hit, miss }) => + Effect.gen(function* () { + const scanner = yield* Scan.KinSecretScan; + const run = (content: string) => + scanner.scan({ orgId, files: [{ path: "claims/test.md", content: Buffer.from(content) }] }); + // Assert only metadata so a failed assertion never prints the synthetic matched token. + expect((yield* run(hit!)).some((finding) => finding.ruleId === rule)).toBe(true); + expect((yield* run(miss!)).length).toBe(0); + }).pipe(Effect.provide(layer)), +); +it.effect("blocks encoded, wrapped and confusable tokens with safe findings", () => + Effect.gen(function* () { + const scanner = yield* Scan.KinSecretScan; + const values = [ + github, + Buffer.from(github).toString("base64"), + Buffer.from(Buffer.from(github).toString("base64")).toString("base64"), + [...github].map((c) => `%${c.charCodeAt(0).toString(16)}`).join(""), + github.slice(0, 12) + "\n" + github.slice(12), + github.slice(0, 12) + "\\\n " + github.slice(12), + github.replace("p", "р"), + [...github].map((c) => String.fromCharCode(c.charCodeAt(0) + 0xfee0)).join(""), + github.slice(0, 5) + "\u200b" + github.slice(5), + ]; + for (const content of values) { + const result = yield* scanner.scan({ + orgId, + files: [{ path: "journal/test.md", content: Buffer.from(content) }], + }); + expect(result.some((f) => f.ruleId === "github-token")).toBe(true); + expect(JSON.stringify(result).includes(body)).toBe(false); + expect(Object.keys(result[0]!).sort()).toEqual(["line", "path", "ruleId"]); + } + }).pipe(Effect.provide(layer)), +); +it.effect("allowlist is exact, org scoped, and read only to scanner callers", () => + Effect.gen(function* () { + const scanner = yield* Scan.KinSecretScan; + const sql = yield* SqlClient.SqlClient; + const hash = NodeCrypto.createHash("sha256") + .update("github-token\0") + .update(github) + .digest("hex"); + // Fixture stands in for the future APPR owner-signed executor; scanner has no write method. + yield* sql`INSERT INTO kin_secret_allow VALUES (${orgId}, 'claims/test.md', 'github-token', ${hash}, 'signed-fixture')`; + const run = (org: KinOrgId, path: string, value = github) => + scanner.scan({ orgId: org, files: [{ path, content: Buffer.from(value) }] }); + expect((yield* run(orgId, "claims/test.md")).length).toBe(0); + expect((yield* run(KinOrgId.make("other"), "claims/test.md")).length).toBeGreaterThan(0); + expect((yield* run(orgId, "claims/other.md")).length).toBeGreaterThan(0); + expect((yield* run(orgId, "claims/test.md", github.replace("a", "z"))).length).toBeGreaterThan( + 0, + ); + expect(Object.keys(scanner)).toEqual(["scan"]); + yield* sql`DROP TABLE kin_secret_allow`; + expect((yield* run(orgId, "claims/test.md").pipe(Effect.result))._tag).toBe("Failure"); + }).pipe(Effect.provide(layer)), +); +it.effect("fails closed at file, batch and count limits", () => + Effect.gen(function* () { + const scanner = yield* Scan.KinSecretScan; + const scan = (files: { path: string; content: Uint8Array }[]) => scanner.scan({ orgId, files }); + for (const files of [ + [{ path: "huge", content: Buffer.alloc(1024 * 1024 + 1) }], + Array.from({ length: 9 }, () => ({ path: "batch", content: Buffer.alloc(1024 * 1024) })), + Array.from({ length: 1025 }, () => ({ path: "count", content: Buffer.alloc(0) })), + ]) + expect((yield* scan(files).pipe(Effect.result))._tag).toBe("Failure"); + const content = Buffer.from("a".repeat(1024 * 1024 - github.length - 1) + "\n" + github); + expect( + (yield* scan([{ path: "large", content }])).some((f) => f.ruleId === "github-token"), + ).toBe(true); + }).pipe(Effect.provide(layer)), +); diff --git a/apps/server/src/kin/memory/secretScan.ts b/apps/server/src/kin/memory/secretScan.ts index f7ef2760efa17..ca0e84b88adba 100644 --- a/apps/server/src/kin/memory/secretScan.ts +++ b/apps/server/src/kin/memory/secretScan.ts @@ -1,3 +1,6 @@ +// @effect-diagnostics-next-line nodeBuiltinImport:off -- Bounded synchronous fingerprint hashing shares the scan loop. +import * as NodeCrypto from "node:crypto"; +import * as SqlClient from "effect/sql/SqlClient"; import type { KinOrgId } from "@t3tools/contracts"; import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; @@ -25,14 +28,164 @@ export class KinSecretScan extends Context.Service< } >()("t3/kin/memory/secretScan/KinSecretScan") {} -export const noFindingsStubLayer = Layer.effect( - KinSecretScan, - Effect.gen(function* () { - yield* Effect.logWarning("SECURITY: Kin secret scanning is disabled (no-findings stub)"); - return KinSecretScan.of({ scan: Effect.fn("KinSecretScan.stub")(() => Effect.succeed([])) }); - }), -); -export const testLayer = (findings: ReadonlyArray) => - Layer.succeed(KinSecretScan, { - scan: Effect.fn("KinSecretScan.fixture")(() => Effect.succeed(findings)), +const rules = [ + ["github-token", /\b(?:gh[pousr]_[A-Za-z0-9]{36}|github_pat_[A-Za-z0-9_]{82})\b/], + ["anthropic-key", /\bsk-ant-[A-Za-z0-9_-]{40,200}\b/], + ["openai-key", /\bsk-(?!ant-)(?:proj-)?[A-Za-z0-9_-]{32,200}\b/], + ["aws-access-key", /\b(?:AKIA|ASIA)[A-Z0-9]{16}\b/], + ["slack-token", /\bxox[abposr]-[A-Za-z0-9-]{20,200}\b/], + ["stripe-key", /\b[rs]k_live_[A-Za-z0-9]{24,200}\b/], + ["google-key", /\bAIza[A-Za-z0-9_-]{35}\b/], + ["pem-private-key", /-----BEGIN (?:(?:RSA|EC|DSA|OPENSSH|ENCRYPTED) )?PRIVATE KEY-----/], + ["jwt", /\beyJ[A-Za-z0-9_-]{5,8192}\.[A-Za-z0-9_-]{8,8192}\.[A-Za-z0-9_-]{16,8192}\b/], + ["url-password", /\b[a-z][a-z0-9+.-]{0,31}:\/\/[^\s/:@]{1,512}:([^\s/@]{1,512})@/i], + [ + "secret-assignment", + /\b[A-Za-z0-9_]{0,64}(?:password|secret|token|api[_-]?key|access[_-]?key)[A-Za-z0-9_]{0,64}\s*[:=]\s*["']?([^\s"';#]{20,512})/i, + ], +] as const; +const entropy = (value: string) => { + const counts = new Map(); + for (const char of value) counts.set(char, (counts.get(char) ?? 0) + 1); + let bits = 0; + for (const count of counts.values()) { + const p = count / value.length; + bits -= p * Math.log2(p); + } + return bits; +}; +// NFKC covers fullwidth forms; common Cyrillic/Greek confusables are normalized as well. +const lookalikes: Readonly> = { + а: "a", + е: "e", + о: "o", + р: "p", + с: "c", + х: "x", + у: "y", + і: "i", + ј: "j", + А: "A", + В: "B", + Е: "E", + К: "K", + М: "M", + Н: "H", + О: "O", + Р: "P", + С: "C", + Т: "T", + Х: "X", + Α: "A", + Β: "B", + Ε: "E", + Ι: "I", + Κ: "K", + Μ: "M", + Ν: "N", + Ο: "O", + Ρ: "P", + Τ: "T", + Χ: "X", +}; +const normalize = (text: string) => + text + .normalize("NFKC") + .replace(/[\u200B-\u200F\u202A-\u202E\u2060-\u206F\uFEFF\uFFFD]/g, "") + .replace(/[аеорсхуіјАВЕКМНОРСТХΑΒΕΙΚΜΝΟΡΤΧ]/g, (c) => lookalikes[c] ?? c); +const fingerprint = (ruleId: string, value: string) => + NodeCrypto.createHash("sha256").update(ruleId).update("\0").update(value).digest("hex"); +const make = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const scan = Effect.fn("KinSecretScan.scan")(function* (input: { + orgId: KinOrgId; + files: ReadonlyArray<{ path: string; content: Uint8Array }>; + }) { + if ( + input.files.length > 1024 || + input.files.some((f) => f.content.byteLength > 1024 * 1024) || + input.files.reduce((sum, f) => sum + f.content.byteLength, 0) > 8 * 1024 * 1024 + ) + return yield* new KinSecretScanError({ reason: "Scan size limit exceeded" }); + // No mutation API: the owner-signed action executor owns allowlist insertion. + const allowed = yield* sql<{ path: string; rule_id: string; fingerprint: string }>` + SELECT path, rule_id, fingerprint FROM kin_secret_allow WHERE org_id = ${input.orgId} LIMIT 4097 + `.pipe(Effect.mapError(() => new KinSecretScanError({ reason: "Allowlist unavailable" }))); + if (allowed.length > 4096) + return yield* new KinSecretScanError({ reason: "Allowlist size limit exceeded" }); + const allow = new Set(allowed.map((r) => JSON.stringify([r.path, r.rule_id, r.fingerprint]))); + const findings: SecretFinding[] = []; + const seen = new Set(); + let budget = 32 * 1024 * 1024; + for (const file of input.files) { + const text = normalize(Buffer.from(file.content).toString("utf8")); + const inspect = ( + value: string, + baseLine: number, + depth: number, + compact = false, + lineAt?: Uint32Array, + ): void => { + budget -= value.length; + if (budget < 0 || findings.length > 1024) return; + for (const [ruleId, pattern] of rules) { + for (const match of value.matchAll(new RegExp(pattern.source, `${pattern.flags}g`))) { + const secret = match[1] ?? match[0]; + if (ruleId === "secret-assignment" && entropy(secret) < 3.5) continue; + if (allow.has(JSON.stringify([file.path, ruleId, fingerprint(ruleId, secret)]))) + continue; + const line = + lineAt?.[match.index] ?? + baseLine + (value.slice(0, match.index).match(/\n/g)?.length ?? 0); + const key = JSON.stringify([file.path, ruleId, line]); + if (!seen.has(key)) { + seen.add(key); + findings.push({ path: file.path, ruleId, line }); + } + break; + } + } + if (depth === 2 || compact) return; + let cursor = 0; + let decodedLine = baseLine; + for (const match of value.matchAll(/(?:%[0-9a-fA-F]{2}|[A-Za-z0-9_+/-]){24,}={0,2}/g)) { + decodedLine += value.slice(cursor, match.index).match(/\n/g)?.length ?? 0; + cursor = match.index; + const encoded = match[0]; + let decoded: string; + if (encoded.includes("%")) { + try { + decoded = decodeURIComponent(encoded); + } catch { + continue; + } + } else { + decoded = Buffer.from(encoded, "base64url").toString("utf8"); + if (!/^[\x09\x0a\x0d\x20-\x7e]+$/.test(decoded)) continue; + } + if (decoded === encoded) continue; + inspect(normalize(decoded), decodedLine, depth + 1); + } + }; + inspect(text, 1, 0); + const lineAt = new Uint32Array(text.length); + let line = 1; + let offset = 0; + const compact = text.replace(/\\?\r?\n[\t ]*|[^\r\n\\]+|./g, (chunk) => { + if (chunk.includes("\n")) { + line++; + return ""; + } + lineAt.fill(line, offset, offset + chunk.length); + offset += chunk.length; + return chunk; + }); + inspect(compact, 1, 2, true, lineAt); + } + if (budget < 0 || findings.length > 1024) + return yield* new KinSecretScanError({ reason: "Scan work limit exceeded" }); + return findings; }); + return KinSecretScan.of({ scan }); +}); +export const layer = Layer.effect(KinSecretScan, make); diff --git a/apps/server/src/kin/persistence/KinMigrations.test.ts b/apps/server/src/kin/persistence/KinMigrations.test.ts index 4b855164bf64d..467d14d994cd8 100644 --- a/apps/server/src/kin/persistence/KinMigrations.test.ts +++ b/apps/server/src/kin/persistence/KinMigrations.test.ts @@ -36,6 +36,7 @@ const tableNames = [ "kin_ping_items", "kin_ping_ledger", "kin_read_state", + "kin_secret_allow", "kin_session_bindings", "kin_shadow_entries", "kin_trigger_fires", @@ -98,6 +99,7 @@ describe("Kin migrations", () => { { migration_id: 9, name: "JudgeLog" }, { migration_id: 10, name: "ChatChanges" }, { migration_id: 11, name: "MemoryHeads" }, + { migration_id: 12, name: "SecretAllow" }, ]); assert.deepStrictEqual( upstreamBefore.map((row) => [row.migration_id, row.name] as const), @@ -151,6 +153,7 @@ describe("Kin migrations", () => { { migration_id: 9, name: "JudgeLog" }, { migration_id: 10, name: "ChatChanges" }, { migration_id: 11, name: "MemoryHeads" }, + { migration_id: 12, name: "SecretAllow" }, ]); assert.deepStrictEqual(yield* upstreamHistory, upstreamBefore); assert.deepStrictEqual(yield* sql`SELECT * FROM kin_orgs`, [ @@ -229,6 +232,7 @@ describe("Kin migrations", () => { [9, "JudgeLog"], [10, "ChatChanges"], [11, "MemoryHeads"], + [12, "SecretAllow"], ]); const before = yield* kinSchema; assert.deepStrictEqual( @@ -266,6 +270,7 @@ describe("Kin migrations", () => { yield* sql`DROP TABLE kin_ping_items`; yield* sql`DROP INDEX kin_chat_members_one_owner`; yield* sql`DROP TABLE kin_memory_heads`; + yield* sql`DROP TABLE kin_secret_allow`; yield* sql`DROP TABLE kin_budget_charges`; yield* sql`ALTER TABLE kin_events DROP COLUMN budget_deferred`; yield* sql`UPDATE kin_chats SET last_seq = 1 WHERE org_id = 'org' AND chat_id = 'existing-chat'`; @@ -280,6 +285,7 @@ describe("Kin migrations", () => { [9, "JudgeLog"], [10, "ChatChanges"], [11, "MemoryHeads"], + [12, "SecretAllow"], ]); assert.deepStrictEqual( yield* sql`SELECT member_id FROM kin_chat_members WHERE org_id = 'org' ORDER BY member_id`, @@ -316,6 +322,7 @@ describe("Kin migrations", () => { { migration_id: 9, name: "JudgeLog" }, { migration_id: 10, name: "ChatChanges" }, { migration_id: 11, name: "MemoryHeads" }, + { migration_id: 12, name: "SecretAllow" }, ]); assert.deepStrictEqual(yield* runKinMigrations(), []); assert.deepStrictEqual(yield* runMigrations(), []); diff --git a/apps/server/src/kin/persistence/KinMigrations.ts b/apps/server/src/kin/persistence/KinMigrations.ts index b7bba32965691..2a407ca6d5f7b 100644 --- a/apps/server/src/kin/persistence/KinMigrations.ts +++ b/apps/server/src/kin/persistence/KinMigrations.ts @@ -15,6 +15,8 @@ import JudgeLog from "./migrations/K009_JudgeLog.ts"; import ChatChanges from "./migrations/K010_ChatChanges.ts"; import MemoryHeads from "./migrations/K011_MemoryHeads.ts"; +import SecretAllow from "./migrations/K012_SecretAllow.ts"; + const run = Migrator.make({}); const loader = Migrator.fromRecord({ "1_ChatCore": ChatCore, @@ -28,6 +30,7 @@ const loader = Migrator.fromRecord({ "9_JudgeLog": JudgeLog, "10_ChatChanges": ChatChanges, "11_MemoryHeads": MemoryHeads, + "12_SecretAllow": SecretAllow, }); export const runKinMigrations = Effect.fn("KinMigrations.run")(function* () { diff --git a/apps/server/src/kin/persistence/migrations/K012_SecretAllow.ts b/apps/server/src/kin/persistence/migrations/K012_SecretAllow.ts new file mode 100644 index 0000000000000..fec65afd99e4a --- /dev/null +++ b/apps/server/src/kin/persistence/migrations/K012_SecretAllow.ts @@ -0,0 +1,14 @@ +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/sql/SqlClient"; + +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql`CREATE TABLE kin_secret_allow ( + org_id TEXT NOT NULL, + path TEXT NOT NULL, + rule_id TEXT NOT NULL, + fingerprint TEXT NOT NULL CHECK(length(fingerprint) = 64), + signed_action_id TEXT NOT NULL, + PRIMARY KEY (org_id, path, rule_id, fingerprint) + )`; +}); diff --git a/docs/kin/specs/triggers.md b/docs/kin/specs/triggers.md index 1890e5a05a9df..a2373fcaf6b38 100644 --- a/docs/kin/specs/triggers.md +++ b/docs/kin/specs/triggers.md @@ -342,7 +342,9 @@ GitHub then blocks a bad auto-merge even if Kin's classifier is wrong. This forc **On a finding**, the writer blocks the entire commit, discards staged text and emits a typed owner-notification hook. The runner journals the refusal without secret content and tells the owner to review and rotate the affected credential. Automatic redaction is not part of the writer contract. -**False positives:** the owner adds the fingerprint to `kin_secret_allow` through a signed action. A model can never add to it. +**False positives:** the owner adds the fingerprint to `kin_secret_allow` through a signed action. A model can never add to it. Entries are scoped to organization, exact path, rule and SHA-256 of `ruleId + "\0" + normalized matched value`, with the signed action id retained. The scanner is read-only; the APPR executor must verify the owner grant before inserting. Until that executor lands, there is no agent or MCP allowlist-write path. Migration K012 is provisional and will be renumbered at merge. + +The scanner normalizes NFKC, common Cyrillic/Greek lookalikes and invisible format characters, checks wrapped tokens, and decodes URL/base64 content up to two levels. It fails closed above 1 MiB per file, 8 MiB or 1024 files per call, 4096 allowlist entries per organization, or 32 MiB of transformed scan text. These are conservative detectors, not a guarantee against arbitrary encryption or steganography. ### Owner-signed purge (gap 5) @@ -475,7 +477,7 @@ Kin runs on one always-on machine, the **Kin host**. The laptop and phone are cl - **Sink:** a signed GitHub `pull_request` request through the real `ScheduledTaskService` gives exactly one inbox row and no thread launch. A replay gives `duplicate` and no new row. A bot-sender request gives `filtered`. A non-Kin task still launches normally. - **Deferral:** unchanged recheck, then heads-up, then silence, then `started` at due. A linked issue closed before due leads to `asked`, and nothing starts. With the stop switch on, it asks. [Monday] re-arms to next Monday 09:00. - **Dream:** a dream touching `rules.md` fails, and `main` is unchanged. A dream with one journal fold and one `decisions/` change opens two PRs, and only the auto one merges. -- **Secret scan:** a journal line holding `ghp_` plus 36 characters is committed as `[redacted:github-token]`, and the owner gets one message. +- **Secret scan:** a journal line holding `ghp_` plus 36 characters blocks the entire commit, and the owner gets one typed notification without the value. - **Purge:** on a fixture repo, the file is gone from `git log --all -p`, a tombstone exists, and an audit row exists. - **Contract tests:** `vp test run kin-contract` passes on today's upstream. Renaming `scheduledTaskId` in a scratch branch makes it fail. From e7aba15d84f7ba0438e3458b8dd40d2ebe4b2c63 Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:19:11 +0530 Subject: [PATCH 2/7] test(kin): harden scanner bounds and wrapped-token coverage --- apps/server/src/kin/memory/secretScan.test.ts | 43 +++++++++++++++++++ apps/server/src/kin/memory/secretScan.ts | 23 +++++----- docs/kin/specs/lead.md | 4 +- 3 files changed, 56 insertions(+), 14 deletions(-) diff --git a/apps/server/src/kin/memory/secretScan.test.ts b/apps/server/src/kin/memory/secretScan.test.ts index 2a52acb8a3b77..db74dd37b5f6d 100644 --- a/apps/server/src/kin/memory/secretScan.test.ts +++ b/apps/server/src/kin/memory/secretScan.test.ts @@ -88,6 +88,32 @@ it.effect("blocks encoded, wrapped and confusable tokens with safe findings", () } }).pipe(Effect.provide(layer)), ); +it.effect("reports source lines and blocks AWS credential pairs without leaking values", () => + Effect.gen(function* () { + const scanner = yield* Scan.KinSecretScan; + for (const content of [ + "safe\n\n" + github, + "safe\n\n" + github.slice(0, 12) + "\n" + github.slice(12), + ]) { + const result = yield* scanner.scan({ + orgId, + files: [{ path: "claims/test.md", content: Buffer.from(content) }], + }); + expect(result.every((f) => f.line === 3)).toBe(true); + expect(result.length).toBeGreaterThan(0); + } + const pair = [ + "AWS_ACCESS_KEY_ID=" + ["AK", "IA", "X".repeat(16)].join(""), + "AWS_SECRET_ACCESS_KEY=" + body + "aB3d", + ].join("\n"); + const findings = yield* scanner.scan({ + orgId, + files: [{ path: ".env", content: Buffer.from(pair) }], + }); + expect(findings.some((f) => f.ruleId === "aws-access-key")).toBe(true); + expect(findings.some((f) => f.ruleId === "secret-assignment")).toBe(true); + }).pipe(Effect.provide(layer)), +); it.effect("allowlist is exact, org scoped, and read only to scanner callers", () => Effect.gen(function* () { const scanner = yield* Scan.KinSecretScan; @@ -111,6 +137,23 @@ it.effect("allowlist is exact, org scoped, and read only to scanner callers", () expect((yield* run(orgId, "claims/test.md").pipe(Effect.result))._tag).toBe("Failure"); }).pipe(Effect.provide(layer)), ); +it.effect("fails closed when decoded findings or the allowlist exceed their bounds", () => + Effect.gen(function* () { + const scanner = yield* Scan.KinSecretScan; + const content = Buffer.from( + Array.from({ length: 1100 }, () => Buffer.from(github).toString("base64")).join("\n"), + ); + expect( + (yield* scanner + .scan({ orgId, files: [{ path: "claims/many.md", content }] }) + .pipe(Effect.result))._tag, + ).toBe("Failure"); + const sql = yield* SqlClient.SqlClient; + yield* sql`WITH RECURSIVE entries(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM entries WHERE n < 4097) + INSERT INTO kin_secret_allow SELECT ${orgId}, 'claims/' || n, 'github-token', ${"0".repeat(64)}, 'signed-fixture' FROM entries`; + expect((yield* scanner.scan({ orgId, files: [] }).pipe(Effect.result))._tag).toBe("Failure"); + }).pipe(Effect.provide(layer)), +); it.effect("fails closed at file, batch and count limits", () => Effect.gen(function* () { const scanner = yield* Scan.KinSecretScan; diff --git a/apps/server/src/kin/memory/secretScan.ts b/apps/server/src/kin/memory/secretScan.ts index ca0e84b88adba..3f5e0a4430121 100644 --- a/apps/server/src/kin/memory/secretScan.ts +++ b/apps/server/src/kin/memory/secretScan.ts @@ -29,15 +29,15 @@ export class KinSecretScan extends Context.Service< >()("t3/kin/memory/secretScan/KinSecretScan") {} const rules = [ - ["github-token", /\b(?:gh[pousr]_[A-Za-z0-9]{36}|github_pat_[A-Za-z0-9_]{82})\b/], - ["anthropic-key", /\bsk-ant-[A-Za-z0-9_-]{40,200}\b/], - ["openai-key", /\bsk-(?!ant-)(?:proj-)?[A-Za-z0-9_-]{32,200}\b/], - ["aws-access-key", /\b(?:AKIA|ASIA)[A-Z0-9]{16}\b/], - ["slack-token", /\bxox[abposr]-[A-Za-z0-9-]{20,200}\b/], - ["stripe-key", /\b[rs]k_live_[A-Za-z0-9]{24,200}\b/], - ["google-key", /\bAIza[A-Za-z0-9_-]{35}\b/], + ["github-token", /(?:gh[pousr]_[A-Za-z0-9]{36}|github_pat_[A-Za-z0-9_]{82})/], + ["anthropic-key", /sk-ant-[A-Za-z0-9_-]{40,200}/], + ["openai-key", /sk-(?!ant-)(?:proj-)?[A-Za-z0-9_-]{32,200}/], + ["aws-access-key", /(?:AKIA|ASIA)[A-Z0-9]{16}/], + ["slack-token", /xox[abposr]-[A-Za-z0-9-]{20,200}/], + ["stripe-key", /[rs]k_live_[A-Za-z0-9]{24,200}/], + ["google-key", /AIza[A-Za-z0-9_-]{35}/], ["pem-private-key", /-----BEGIN (?:(?:RSA|EC|DSA|OPENSSH|ENCRYPTED) )?PRIVATE KEY-----/], - ["jwt", /\beyJ[A-Za-z0-9_-]{5,8192}\.[A-Za-z0-9_-]{8,8192}\.[A-Za-z0-9_-]{16,8192}\b/], + ["jwt", /eyJ[A-Za-z0-9_-]{5,8192}\.[A-Za-z0-9_-]{8,8192}\.[A-Za-z0-9_-]{16,8192}/], ["url-password", /\b[a-z][a-z0-9+.-]{0,31}:\/\/[^\s/:@]{1,512}:([^\s/@]{1,512})@/i], [ "secret-assignment", @@ -123,7 +123,6 @@ const make = Effect.gen(function* () { value: string, baseLine: number, depth: number, - compact = false, lineAt?: Uint32Array, ): void => { budget -= value.length; @@ -145,7 +144,7 @@ const make = Effect.gen(function* () { break; } } - if (depth === 2 || compact) return; + if (depth === 2) return; let cursor = 0; let decodedLine = baseLine; for (const match of value.matchAll(/(?:%[0-9a-fA-F]{2}|[A-Za-z0-9_+/-]){24,}={0,2}/g)) { @@ -164,7 +163,7 @@ const make = Effect.gen(function* () { if (!/^[\x09\x0a\x0d\x20-\x7e]+$/.test(decoded)) continue; } if (decoded === encoded) continue; - inspect(normalize(decoded), decodedLine, depth + 1); + inspect(normalize(decoded), lineAt?.[match.index] ?? decodedLine, depth + 1); } }; inspect(text, 1, 0); @@ -180,7 +179,7 @@ const make = Effect.gen(function* () { offset += chunk.length; return chunk; }); - inspect(compact, 1, 2, true, lineAt); + inspect(compact, 1, 0, lineAt); } if (budget < 0 || findings.length > 1024) return yield* new KinSecretScanError({ reason: "Scan work limit exceeded" }); diff --git a/docs/kin/specs/lead.md b/docs/kin/specs/lead.md index 867a2cd312369..134bc9b50aecb 100644 --- a/docs/kin/specs/lead.md +++ b/docs/kin/specs/lead.md @@ -600,7 +600,7 @@ LEAD-1. **Contracts and tables.** `kinLead.ts` schemas; Kin migrations for all l LEAD-2. **Memory read side.** `ensureCheckouts`, `prepareWakeup`, front matter parser, cache, `query`, `readItem`, `rebuildIndex` (200-line cap, "As of" line, fading after 8 weeks unread). Files: `apps/server/src/kin/memory/{Memory,MemoryCache,frontMatter}.ts`. Depends on: LEAD-1. Done when: tests create a temporary git repo from invented fixtures matching the kin-memory README; kind/date/review queries, malformed-item isolation, organization isolation, cache invalidation, index cap/As-of and TestClock fading all pass. Never use real organization memory as a fixture. Owner: backend. -LEAD-3. **Memory git writer.** `stage` with every writer rule, `commitWakeup`, push and rebase, shadow branch, the secret scan hook (stub until the triggers spec lands). Files: `MemoryWriter.ts`, `journal.ts`. Depends on: LEAD-2, TRIG-16 (stub allowed). Done when: tests show provenance override, `rules.md` refusal, conflict error, one commit per wake-up, discard on failure. Owner: backend. +LEAD-3. **Memory git writer.** `stage` with every writer rule, `commitWakeup`, push and rebase, shadow branch, the secret scan hook (TRIG-16 supplies the scanner layer). Files: `MemoryWriter.ts`, `journal.ts`. Depends on: LEAD-2, TRIG-16 (stub allowed). Done when: tests show provenance override, `rules.md` refusal, conflict error, one commit per wake-up, discard on failure. Owner: backend. LEAD-4. **Event queue and coalescer.** `enqueue`, dedupe, priorities, quiet window, claim, restart requeue, per-org semaphore. Files: `KinEventQueue.ts`. Depends on: LEAD-1. Done when: coalescing and one-writer tests pass with `TestClock`. Owner: backend. @@ -647,7 +647,7 @@ LEAD-14. **End-to-end shadow week dry run.** Script that replays a day of record `MemoryWriterAuthority.resolve` is runner-only: resolve organization, wake-up, active lead, level, timestamps, timezone and event provenance from same-organization rows inside the runner's transaction. Never expose this context as tool input. Owner-confirmed sources must be verified there; absent a source, provenance is the lead's agent id. `MemoryWriterHooks.notify` consumes checkout, conflict and secret-block notices without chat transport coupling. -`KinSecretScan` is deliberately supplied separately. `noFindingsStubLayer` returns no findings until TRIG-16 replaces it; it is not a production secret detector. Any finding or scanner failure blocks the whole commit. TRIG-16 must preserve this fail-closed interface: `scan({ orgId, files: ReadonlyArray<{ path: string; content: Uint8Array }> })` returns `{ path, ruleId, line }[]`, with typed `KinSecretScanError`. +`KinSecretScan` is deliberately supplied separately. `secretScan.layer` supplies the TRIG-16 scanner and reads the owner-signed allowlist; there is no no-findings production stub. Any finding or scanner failure blocks the whole commit. TRIG-16 must preserve this fail-closed interface: `scan({ orgId, files: ReadonlyArray<{ path: string; content: Uint8Array }> })` returns `{ path, ruleId, line }[]`, with typed `KinSecretScanError`. `commitWakeup` builds TRIG-12's sectioned `INDEX.md` from the hash-verified recorded tree plus staged files, read history, the server clock and the owner's time zone, in the same wake-up commit. The read-side rebuild is display-only; no read-checkout text is trusted. K011 adds the SQLite branch anchors required by the writer. The runner serializes prepare/stage/commit and dreaming per organization. Identical commit retries find the existing wake-up commit; failed pushes retain local commits, and failures discard uncommitted files. Journal additions use `appendJournal`, never replacement through `stage`. From cb8f2ff7e6a6ab1f1ce5502cbee3ca4672050e6d Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Sun, 8 Nov 2026 14:12:17 +0530 Subject: [PATCH 3/7] fix(kin): bound secret scanning and scan only merged changes --- apps/server/src/kin/memory/Memory.test.ts | 24 +++- apps/server/src/kin/memory/Memory.ts | 12 +- .../src/kin/memory/MemoryWriter.test.ts | 16 +++ apps/server/src/kin/memory/MemoryWriter.ts | 16 ++- apps/server/src/kin/memory/secretScan.test.ts | 96 ++++++++++++- apps/server/src/kin/memory/secretScan.ts | 132 +++++++++++++----- .../migrations/K012_SecretAllow.ts | 1 + docs/kin/specs/triggers.md | 8 +- 8 files changed, 251 insertions(+), 54 deletions(-) diff --git a/apps/server/src/kin/memory/Memory.test.ts b/apps/server/src/kin/memory/Memory.test.ts index 0188ef14accd0..674998fcc7b2f 100644 --- a/apps/server/src/kin/memory/Memory.test.ts +++ b/apps/server/src/kin/memory/Memory.test.ts @@ -859,6 +859,8 @@ it.live("uses filesystem identity for a case-mismatched base without recloning", it.live.each([ "diverged", "secret merge", + "remote secret merge", + "large remote merge", "conflict", "network down", "dirty writer", @@ -933,7 +935,11 @@ it.live.each([ } else yield* fs.writeFileString( `${writer}/rules.md`, - scenario === "H2-merge-swap" ? h2Local : "Local invented rules\n", + scenario === "H2-merge-swap" + ? h2Local + : scenario === "secret merge" + ? ["gh", "p_", "a".repeat(36)].join("") + : "Local invented rules\n", ); let local = yield* commit(writer); if (scenario === "push retry") { @@ -953,15 +959,20 @@ it.live.each([ } yield* memory.record(orgA, "main", initial.head, local); yield* fs.writeFileString( - `${source}/${scenario === "push retry" || scenario === "diverged" || scenario === "secret merge" || scenario === "network down" || scenario === "dirty writer" || scenario === "H2-merge-swap" ? "remote-only" : "rules.md"}`, + `${source}/${scenario === "push retry" || scenario === "diverged" || scenario === "secret merge" || scenario === "remote secret merge" || scenario === "large remote merge" || scenario === "network down" || scenario === "dirty writer" || scenario === "H2-merge-swap" ? "remote-only" : "rules.md"}`, "Remote invented edit\n", ); if (scenario === "journal conflict" || scenario === "push retry") { yield* fs.writeFileString(`${source}/journal/day.md`, "remote entry\n"); yield* fs.writeFileString(`${source}/INDEX.md`, "remote generated index\n"); } - if (scenario === "secret merge") + if (scenario === "remote secret merge") yield* fs.writeFileString(`${source}/remote-only`, ["gh", "p_", "a".repeat(36)].join("")); + if (scenario === "large remote merge") { + yield* fs.makeDirectory(`${source}/archive`); + for (let i = 0; i < 1030; i++) + yield* fs.writeFileString(`${source}/archive/${i}.txt`, "clean"); + } if (scenario === "H2-merge-swap") yield* fs.writeFileString(`${source}/rules.md`, h2Base.replace("base tail", "remote tail")); yield* commit(source); @@ -1076,7 +1087,8 @@ it.live.each([ } else if ( scenario === "network down" || scenario === "diverged" || - scenario === "secret merge" || + scenario === "remote secret merge" || + scenario === "large remote merge" || scenario === "dirty writer" ) { expect(result._tag).toBe("Success"); @@ -1086,7 +1098,9 @@ it.live.each([ expect(result.success.head).not.toBe(local); expect(yield* memory.readRules(orgA)).toBe("Local invented rules\n"); expect(yield* git(writer, ["show", `${result.success.head}:remote-only`])).toBe( - "Remote invented edit", + scenario === "remote secret merge" + ? ["gh", "p_", "a".repeat(36)].join("") + : "Remote invented edit", ); } } diff --git a/apps/server/src/kin/memory/Memory.ts b/apps/server/src/kin/memory/Memory.ts index f74f6557e43b2..d2d35e8ae1c84 100644 --- a/apps/server/src/kin/memory/Memory.ts +++ b/apps/server/src/kin/memory/Memory.ts @@ -1071,6 +1071,7 @@ const make = Effect.gen(function* () { }); } else { const { files: mergedFiles, modes } = merged; + const remoteOriginal = new Map(remoteFiles); remoteFiles.clear(); for (const [file, bytes] of mergedFiles) remoteFiles.set(file, bytes); const heads = []; @@ -1093,7 +1094,12 @@ const make = Effect.gen(function* () { const findings = yield* scanner .scan({ orgId, - files: [...remoteFiles].map(([path, content]) => ({ path, content })), + files: [...remoteFiles] + .filter( + ([path, bytes]) => + path === "INDEX.md" || !remoteOriginal.get(path)?.equals(bytes), + ) + .map(([path, content]) => ({ path, content })), }) .pipe( Effect.mapError( @@ -1104,12 +1110,12 @@ const make = Effect.gen(function* () { yield* notify({ orgId, branch, - reason: `Secret scan blocked merge: ${findings[0]!.ruleId} in ${findings[0]!.path}`, + reason: `Secret scan blocked merge: ${findings[0]!.ruleId} in ${findings[0]!.pathHit ? "[redacted]" : findings[0]!.path}`, }); return yield* new MemoryError({ orgId, operation: "secretScan", - reason: `Secret scan blocked merge: ${findings[0]!.ruleId} in ${findings[0]!.path}`, + reason: `Secret scan blocked merge: ${findings[0]!.ruleId} in ${findings[0]!.pathHit ? "[redacted]" : findings[0]!.path}`, }); } tree = yield* Effect.scoped( diff --git a/apps/server/src/kin/memory/MemoryWriter.test.ts b/apps/server/src/kin/memory/MemoryWriter.test.ts index 780093834215f..df01b3857d435 100644 --- a/apps/server/src/kin/memory/MemoryWriter.test.ts +++ b/apps/server/src/kin/memory/MemoryWriter.test.ts @@ -1,3 +1,4 @@ +import * as SqlClient from "effect/sql/SqlClient"; import { memoryGit } from "./git.ts"; import * as NodeZlib from "node:zlib"; import * as NodeServices from "@effect/platform-node/NodeServices"; @@ -23,6 +24,8 @@ it.live.each([ "completed", "failed", "blocked", + "scan unavailable", + "scan limit", "forged", "conflict", "B1-shadow", @@ -601,10 +604,23 @@ it.live.each([ yield* writer.appendJournal(orgId, "wake1", [ ["gh", "p_"].join("") + "aB3dE6gH9jK2mN5pQ8sT1vW4yZ7cF0iL3oR6", ]); + if (scenario === "scan unavailable") { + const sql = yield* SqlClient.SqlClient; + yield* sql`DROP TABLE kin_secret_allow`; + } + if (scenario === "scan limit") + yield* writer.appendJournal(orgId, "wake1", ["a".repeat(1024 * 1024)]); const result = yield* writer .commitWakeup(orgId, "wake1", scenario === "failed" ? "failed" : "completed") .pipe(Effect.result); if (result._tag === "Failure") yield* Effect.logError(result.failure); + if (scenario === "scan unavailable" || scenario === "scan limit") { + expect(result._tag).toBe("Failure"); + expect(notices).toHaveLength(1); + expect(notices[0]!.error._tag).toBe("KinSecretScanError"); + expect(yield* git(paths.writer, ["rev-parse", "HEAD"])).toBe(original); + return; + } if (scenario === "tree-substitution") { expect(result._tag).toBe("Failure"); if (result._tag === "Failure") expect(result.failure._tag).toBe("MemoryCheckoutInvalid"); diff --git a/apps/server/src/kin/memory/MemoryWriter.ts b/apps/server/src/kin/memory/MemoryWriter.ts index b5fd6ed3c4886..a45193e5275cc 100644 --- a/apps/server/src/kin/memory/MemoryWriter.ts +++ b/apps/server/src/kin/memory/MemoryWriter.ts @@ -61,14 +61,23 @@ export class MemorySecretBlocked extends Schema.TaggedError { orgId: Schema.String, findings: Schema.Array( - Schema.Struct({ path: Schema.String, ruleId: Schema.String, line: Schema.Number }), + Schema.Struct({ + path: Schema.String, + ruleId: Schema.String, + line: Schema.Number, + pathHit: Schema.optional(Schema.Boolean), + }), ), }, ) {} export type WriterNotice = { readonly orgId: KinOrgId; readonly wakeupId: string; - readonly error: MemoryCheckoutInvalid | MemorySyncConflict | MemorySecretBlocked; + readonly error: + | MemoryCheckoutInvalid + | MemorySyncConflict + | MemorySecretBlocked + | KinSecretScanError; }; export class MemoryWriterHooks extends Context.Service< MemoryWriterHooks, @@ -297,7 +306,8 @@ const make = Effect.gen(function* () { Effect.tapError((error) => error._tag === "MemoryCheckoutInvalid" || error._tag === "MemorySyncConflict" || - error._tag === "MemorySecretBlocked" + error._tag === "MemorySecretBlocked" || + error._tag === "KinSecretScanError" ? hooks .notify({ orgId, wakeupId, error }) .pipe( diff --git a/apps/server/src/kin/memory/secretScan.test.ts b/apps/server/src/kin/memory/secretScan.test.ts index db74dd37b5f6d..63ff116caed3c 100644 --- a/apps/server/src/kin/memory/secretScan.test.ts +++ b/apps/server/src/kin/memory/secretScan.test.ts @@ -26,8 +26,12 @@ const fixtures = [ ["github_", "pat_short"].join(""), ], ["anthropic-key", ["sk-", "ant-"].join("") + body.repeat(2), ["sk-", "ant-short"].join("")], - ["openai-key", "sk-" + body, "sk-short"], - ["openai-key", ["sk-", "proj-"].join("") + body.repeat(3), "sk-proj-short"], + ["openai-key", "sk-" + body.slice(0, 20) + "T3BlbkFJ" + body.slice(0, 20), "sk-short"], + ...["proj", "svcacct", "admin"].map((kind) => [ + "openai-key", + ["sk-", kind, "-", body.repeat(3)].join(""), + ["sk-", kind, "-", "a".repeat(40)].join(""), + ]), ...["AK", "AS"].map((kind) => [ "aws-access-key", kind + "IA" + "X".repeat(16), @@ -68,6 +72,22 @@ it.effect("blocks encoded, wrapped and confusable tokens with safe findings", () const scanner = yield* Scan.KinSecretScan; const values = [ github, + ...["/", "-", "_", "+", "word"].map( + (prefix) => prefix + Buffer.from(github).toString("base64"), + ), + Buffer.from("é" + github + "é").toString("base64"), + [...Buffer.from(github.slice(0, 5) + "\u034f" + github.slice(5))] + .map((byte) => `%${byte.toString(16).padStart(2, "0")}`) + .join(""), + "%FF" + [...github].map((c) => `%${c.charCodeAt(0).toString(16)}`).join(""), + ...["\u00ad", "\u034f", "\ufe0f", "\u{e0061}"].map( + (separator) => github.slice(0, 5) + separator + github.slice(5), + ), + github.replace("g", "ɡ"), + github.replace("h", "һ"), + ["gh", "ѕ_", body].join(""), + ["gh", "ο_", body].join(""), + " ".repeat(8190) + github, Buffer.from(github).toString("base64"), Buffer.from(Buffer.from(github).toString("base64")).toString("base64"), [...github].map((c) => `%${c.charCodeAt(0).toString(16)}`).join(""), @@ -123,7 +143,7 @@ it.effect("allowlist is exact, org scoped, and read only to scanner callers", () .update(github) .digest("hex"); // Fixture stands in for the future APPR owner-signed executor; scanner has no write method. - yield* sql`INSERT INTO kin_secret_allow VALUES (${orgId}, 'claims/test.md', 'github-token', ${hash}, 'signed-fixture')`; + yield* sql`INSERT INTO kin_secret_allow VALUES (${orgId}, 'claims/test.md', 'github-token', ${hash}, 'signed-fixture', 0)`; const run = (org: KinOrgId, path: string, value = github) => scanner.scan({ orgId: org, files: [{ path, content: Buffer.from(value) }] }); expect((yield* run(orgId, "claims/test.md")).length).toBe(0); @@ -150,7 +170,7 @@ it.effect("fails closed when decoded findings or the allowlist exceed their boun ).toBe("Failure"); const sql = yield* SqlClient.SqlClient; yield* sql`WITH RECURSIVE entries(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM entries WHERE n < 4097) - INSERT INTO kin_secret_allow SELECT ${orgId}, 'claims/' || n, 'github-token', ${"0".repeat(64)}, 'signed-fixture' FROM entries`; + INSERT INTO kin_secret_allow SELECT ${orgId}, 'claims/' || n, 'github-token', ${"0".repeat(64)}, 'signed-fixture', 0 FROM entries`; expect((yield* scanner.scan({ orgId, files: [] }).pipe(Effect.result))._tag).toBe("Failure"); }).pipe(Effect.provide(layer)), ); @@ -164,9 +184,75 @@ it.effect("fails closed at file, batch and count limits", () => Array.from({ length: 1025 }, () => ({ path: "count", content: Buffer.alloc(0) })), ]) expect((yield* scan(files).pipe(Effect.result))._tag).toBe("Failure"); - const content = Buffer.from("a".repeat(1024 * 1024 - github.length - 1) + "\n" + github); + const content = Buffer.from( + "a".repeat(1024 * 1024 - github.length - 1 - "large".length) + "\n" + github, + ); expect( (yield* scan([{ path: "large", content }])).some((f) => f.ruleId === "github-token"), ).toBe(true); }).pipe(Effect.provide(layer)), ); + +it.effect("round 2 near misses and private key headers", () => + Effect.gen(function* () { + const scanner = yield* Scan.KinSecretScan; + for (const content of [ + "task-follow-up-with-design-team-on-onboarding", + "risk-follow-up-with-design-team-on-onboarding", + "desk-follow-up-with-design-team-on-onboarding", + "token: 0123456789abcdef0123456789abcdef01234567", + "Password: https://vault.example.invalid/credentials/team", + "secret: projects/example-project/secrets/database/versions/latest", + "github_token_scopes: repo,workflow,admin:org,read:user", + "postgres://user:${DB_PASSWORD}@example.invalid/db", + "ASIAPACIFICREGION2026", + ]) + expect( + (yield* scanner.scan({ + orgId, + files: [{ path: "claims/test.md", content: Buffer.from(content) }], + })).length, + ).toBe(0); + for (const content of [ + "-----BEGIN PGP PRIVATE KEY BLOCK-----", + "-----BEGIN CUSTOM PRIVATE KEY-----", + ]) + expect( + (yield* scanner.scan({ + orgId, + files: [{ path: "claims/test.md", content: Buffer.from(content) }], + })).some((f) => f.ruleId === "pem-private-key"), + ).toBe(true); + }).pipe(Effect.provide(layer)), +); +it.effect("round 2 scans paths without exposing them", () => + Effect.gen(function* () { + const scanner = yield* Scan.KinSecretScan; + const findings = yield* scanner.scan({ + orgId, + files: [{ path: `claims/${github}.md`, content: Buffer.from("clean") }], + }); + expect(findings.some((f) => f.ruleId === "github-token")).toBe(true); + expect(JSON.stringify(findings).includes(github)).toBe(false); + }).pipe(Effect.provide(layer)), +); +it.effect("round 2 bounds adversarial rule inputs structurally", () => + Effect.gen(function* () { + const scanner = yield* Scan.KinSecretScan; + for (const prefix of ["eyJ", "sk-", "AKIA"]) { + const content = prefix.repeat(Math.floor((1024 * 1024 - 100) / prefix.length)); + let examined = 0; + for (const window of Scan.ruleWindows(content)) { + expect(window.text.length).toBeLessThanOrEqual(8192); + examined += window.text.length; + } + expect(examined).toBeLessThanOrEqual(content.length * 2); + expect( + (yield* scanner.scan({ + orgId, + files: [{ path: "claims/test.md", content: Buffer.from(content) }], + })).length, + ).toBe(0); + } + }).pipe(Effect.provide(layer)), +); diff --git a/apps/server/src/kin/memory/secretScan.ts b/apps/server/src/kin/memory/secretScan.ts index 3f5e0a4430121..cdf2ae08a9bd5 100644 --- a/apps/server/src/kin/memory/secretScan.ts +++ b/apps/server/src/kin/memory/secretScan.ts @@ -6,17 +6,20 @@ import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Schema from "effect/Schema"; +import { toPersistenceSqlError } from "../../persistence/Errors.ts"; export class KinSecretScanError extends Schema.TaggedError()( "KinSecretScanError", { reason: Schema.String, + cause: Schema.optional(Schema.Defect()), }, ) {} export interface SecretFinding { readonly path: string; readonly ruleId: string; readonly line: number; + readonly pathHit?: boolean; } export class KinSecretScan extends Context.Service< KinSecretScan, @@ -31,19 +34,27 @@ export class KinSecretScan extends Context.Service< const rules = [ ["github-token", /(?:gh[pousr]_[A-Za-z0-9]{36}|github_pat_[A-Za-z0-9_]{82})/], ["anthropic-key", /sk-ant-[A-Za-z0-9_-]{40,200}/], - ["openai-key", /sk-(?!ant-)(?:proj-)?[A-Za-z0-9_-]{32,200}/], + [ + "openai-key", + /sk-(?:(?:proj|svcacct|admin)-[A-Za-z0-9_-]{32,200}|[A-Za-z0-9]{20}T3BlbkFJ[A-Za-z0-9]{20})/, + ], ["aws-access-key", /(?:AKIA|ASIA)[A-Z0-9]{16}/], ["slack-token", /xox[abposr]-[A-Za-z0-9-]{20,200}/], ["stripe-key", /[rs]k_live_[A-Za-z0-9]{24,200}/], ["google-key", /AIza[A-Za-z0-9_-]{35}/], - ["pem-private-key", /-----BEGIN (?:(?:RSA|EC|DSA|OPENSSH|ENCRYPTED) )?PRIVATE KEY-----/], - ["jwt", /eyJ[A-Za-z0-9_-]{5,8192}\.[A-Za-z0-9_-]{8,8192}\.[A-Za-z0-9_-]{16,8192}/], + ["pem-private-key", /-----BEGIN (?:[A-Z0-9]+ ){0,8}PRIVATE KEY(?: BLOCK)?-----/], + ["jwt", /eyJ[A-Za-z0-9_-]{5,1024}\.[A-Za-z0-9_-]{8,1024}\.[A-Za-z0-9_-]{16,1024}/], ["url-password", /\b[a-z][a-z0-9+.-]{0,31}:\/\/[^\s/:@]{1,512}:([^\s/@]{1,512})@/i], [ "secret-assignment", - /\b[A-Za-z0-9_]{0,64}(?:password|secret|token|api[_-]?key|access[_-]?key)[A-Za-z0-9_]{0,64}\s*[:=]\s*["']?([^\s"';#]{20,512})/i, + /\b[A-Za-z0-9_]{0,64}(?:password|secret|token|api[_-]?key|access[_-]?key)[A-Za-z0-9_]{0,64}\s{0,32}[:=]\s{0,32}["']?([^\s"';#]{20,512})/i, ], ] as const; +// Every rule sees bounded windows; overlap exceeds the longest possible match (a JWT). +export const ruleWindows = function* (value: string) { + for (let offset = 0; offset < value.length; offset += 4096) + yield { text: value.slice(offset, offset + 8192), offset }; +}; const entropy = (value: string) => { const counts = new Map(); for (const char of value) counts.set(char, (counts.get(char) ?? 0) + 1); @@ -56,6 +67,10 @@ const entropy = (value: string) => { }; // NFKC covers fullwidth forms; common Cyrillic/Greek confusables are normalized as well. const lookalikes: Readonly> = { + ѕ: "s", + һ: "h", + ο: "o", + ɡ: "g", а: "a", е: "e", о: "o", @@ -91,8 +106,8 @@ const lookalikes: Readonly> = { const normalize = (text: string) => text .normalize("NFKC") - .replace(/[\u200B-\u200F\u202A-\u202E\u2060-\u206F\uFEFF\uFFFD]/g, "") - .replace(/[аеорсхуіјАВЕКМНОРСТХΑΒΕΙΚΜΝΟΡΤΧ]/g, (c) => lookalikes[c] ?? c); + .replace(/\p{Default_Ignorable_Code_Point}/gu, "") + .replace(/[ѕһοɡаеорсхуіјАВЕКМНОРСТХΑΒΕΙΚΜΝΟΡΤΧ]/g, (c) => lookalikes[c] ?? c); const fingerprint = (ruleId: string, value: string) => NodeCrypto.createHash("sha256").update(ruleId).update("\0").update(value).digest("hex"); const make = Effect.gen(function* () { @@ -103,14 +118,20 @@ const make = Effect.gen(function* () { }) { if ( input.files.length > 1024 || - input.files.some((f) => f.content.byteLength > 1024 * 1024) || - input.files.reduce((sum, f) => sum + f.content.byteLength, 0) > 8 * 1024 * 1024 + input.files.some((f) => f.content.byteLength + Buffer.byteLength(f.path) > 1024 * 1024) || + input.files.reduce((sum, f) => sum + f.content.byteLength + Buffer.byteLength(f.path), 0) > + 8 * 1024 * 1024 ) return yield* new KinSecretScanError({ reason: "Scan size limit exceeded" }); // No mutation API: the owner-signed action executor owns allowlist insertion. const allowed = yield* sql<{ path: string; rule_id: string; fingerprint: string }>` SELECT path, rule_id, fingerprint FROM kin_secret_allow WHERE org_id = ${input.orgId} LIMIT 4097 - `.pipe(Effect.mapError(() => new KinSecretScanError({ reason: "Allowlist unavailable" }))); + `.pipe( + Effect.mapError(toPersistenceSqlError("KinSecretScan.allowlist")), + Effect.mapError( + (cause) => new KinSecretScanError({ reason: "Allowlist unavailable", cause }), + ), + ); if (allowed.length > 4096) return yield* new KinSecretScanError({ reason: "Allowlist size limit exceeded" }); const allow = new Set(allowed.map((r) => JSON.stringify([r.path, r.rule_id, r.fingerprint]))); @@ -119,6 +140,9 @@ const make = Effect.gen(function* () { let budget = 32 * 1024 * 1024; for (const file of input.files) { const text = normalize(Buffer.from(file.content).toString("utf8")); + const pathFindingStart = findings.length; + let pathHit = true; + let redactPath = false; const inspect = ( value: string, baseLine: number, @@ -128,20 +152,45 @@ const make = Effect.gen(function* () { budget -= value.length; if (budget < 0 || findings.length > 1024) return; for (const [ruleId, pattern] of rules) { - for (const match of value.matchAll(new RegExp(pattern.source, `${pattern.flags}g`))) { - const secret = match[1] ?? match[0]; - if (ruleId === "secret-assignment" && entropy(secret) < 3.5) continue; - if (allow.has(JSON.stringify([file.path, ruleId, fingerprint(ruleId, secret)]))) - continue; - const line = - lineAt?.[match.index] ?? - baseLine + (value.slice(0, match.index).match(/\n/g)?.length ?? 0); - const key = JSON.stringify([file.path, ruleId, line]); - if (!seen.has(key)) { - seen.add(key); - findings.push({ path: file.path, ruleId, line }); + for (const window of ruleWindows(value)) { + const bounded = new RegExp( + `(? + String.fromCharCode(Number.parseInt(hex, 16)), + ), + "latin1", + ).toString("utf8"), + ] + : Array.from({ length: 4 }, (_, alignment) => + Buffer.from(encoded.slice(alignment), "base64url").toString("utf8"), + ); + for (const candidate of candidates) { + if (budget < 0 || findings.length > 1024) return; + const decoded = normalize(candidate).replace(/[^\x09\x0a\x0d\x20-\x7e]/g, " "); + if (decoded !== encoded) + inspect(decoded, lineAt?.[match.index] ?? decodedLine, depth + 1); } - if (decoded === encoded) continue; - inspect(normalize(decoded), lineAt?.[match.index] ?? decodedLine, depth + 1); } }; + inspect(normalize(file.path), 1, 0); + redactPath = findings.length > pathFindingStart; + pathHit = false; inspect(text, 1, 0); const lineAt = new Uint32Array(text.length); let line = 1; let offset = 0; + const starts = new Set(); const compact = text.replace(/\\?\r?\n[\t ]*|[^\r\n\\]+|./g, (chunk) => { if (chunk.includes("\n")) { line++; + starts.add(offset); return ""; } lineAt.fill(line, offset, offset + chunk.length); @@ -180,6 +239,11 @@ const make = Effect.gen(function* () { return chunk; }); inspect(compact, 1, 0, lineAt); + // A wrapped token may start after prose that compacting joined to its left boundary. + for (const start of starts) { + if (budget < 0 || findings.length > 1024) break; + inspect(compact.slice(start, start + 8192), lineAt[start] ?? 1, 0, lineAt.subarray(start)); + } } if (budget < 0 || findings.length > 1024) return yield* new KinSecretScanError({ reason: "Scan work limit exceeded" }); diff --git a/apps/server/src/kin/persistence/migrations/K012_SecretAllow.ts b/apps/server/src/kin/persistence/migrations/K012_SecretAllow.ts index fec65afd99e4a..c5563c34c8717 100644 --- a/apps/server/src/kin/persistence/migrations/K012_SecretAllow.ts +++ b/apps/server/src/kin/persistence/migrations/K012_SecretAllow.ts @@ -9,6 +9,7 @@ export default Effect.gen(function* () { rule_id TEXT NOT NULL, fingerprint TEXT NOT NULL CHECK(length(fingerprint) = 64), signed_action_id TEXT NOT NULL, + created_at INTEGER NOT NULL, PRIMARY KEY (org_id, path, rule_id, fingerprint) )`; }); diff --git a/docs/kin/specs/triggers.md b/docs/kin/specs/triggers.md index a2373fcaf6b38..26607c9744cd0 100644 --- a/docs/kin/specs/triggers.md +++ b/docs/kin/specs/triggers.md @@ -324,12 +324,12 @@ GitHub then blocks a bad auto-merge even if Kin's classifier is wrong. This forc **Rules:** - GitHub tokens (`ghp_`, `gho_`, `ghu_`, `ghs_`, `ghr_`, `github_pat_`); -- Anthropic `sk-ant-`, OpenAI `sk-`/`sk-proj-`; +- Anthropic `sk-ant-`; OpenAI legacy `sk-` with the `T3BlbkFJ` marker and `sk-proj-`/`sk-svcacct-`/`sk-admin-` with mixed case or digits; - AWS `AKIA…`/`ASIA…`; - Slack `xox[abposr]-`; - Stripe `sk_live_`/`rk_live_`; - Google `AIza`; -- PEM `-----BEGIN … PRIVATE KEY-----`; +- PEM `-----BEGIN … PRIVATE KEY-----`, including `PGP PRIVATE KEY BLOCK`; - JWTs (`eyJ…`, three base64url parts); - URLs with a password in them (`scheme://user:pass@`); - `password|secret|token|api_key = `. @@ -342,9 +342,9 @@ GitHub then blocks a bad auto-merge even if Kin's classifier is wrong. This forc **On a finding**, the writer blocks the entire commit, discards staged text and emits a typed owner-notification hook. The runner journals the refusal without secret content and tells the owner to review and rotate the affected credential. Automatic redaction is not part of the writer contract. -**False positives:** the owner adds the fingerprint to `kin_secret_allow` through a signed action. A model can never add to it. Entries are scoped to organization, exact path, rule and SHA-256 of `ruleId + "\0" + normalized matched value`, with the signed action id retained. The scanner is read-only; the APPR executor must verify the owner grant before inserting. Until that executor lands, there is no agent or MCP allowlist-write path. Migration K012 is provisional and will be renumbered at merge. +**False positives:** the owner adds the fingerprint to `kin_secret_allow` through a signed action. A model can never add to it. Entries are scoped to organization, exact path, rule and SHA-256 of `ruleId + "\0" + normalized matched value`, with `created_at` in integer milliseconds and `signed_action_id` reserved for the future APPR executor id. The scanner is read-only; the APPR executor must verify the owner grant before inserting. Until that executor lands, there is no agent or MCP allowlist-write path. Migration K012 is provisional and will be renumbered at merge. -The scanner normalizes NFKC, common Cyrillic/Greek lookalikes and invisible format characters, checks wrapped tokens, and decodes URL/base64 content up to two levels. It fails closed above 1 MiB per file, 8 MiB or 1024 files per call, 4096 allowlist entries per organization, or 32 MiB of transformed scan text. These are conservative detectors, not a guarantee against arbitrary encryption or steganography. +The scanner checks paths as well as content, redacts paths that contain a finding, normalizes NFKC, common Cyrillic/Greek/Latin lookalikes and Default Ignorable code points, checks newline-wrapped tokens, and decodes URL/base64 content up to two levels. Base64 decoding tries all four alignments; malformed percent bytes and non-ASCII decoded bytes become separators. Prefix rules have token boundaries and bounded lengths; each regex sees at most 8192 characters with 4096-character overlap. Merge scanning covers only bytes changed from the fetched remote plus the rebuilt `INDEX.md`, not pre-existing remote content. Scanner failures notify through the writer hook, just like findings. Merge blocks use the recovery hook; production owner delivery still waits on a notifier. It fails closed above 1 MiB per file, 8 MiB or 1024 files per call, 4096 allowlist entries per organization, or 32 MiB of transformed scan text. Splits by a space, `\r`, U+2028, across journal bullets, markdown or HTML comments, `\u` escapes, and hex encoding remain out of scope. These are conservative detectors, not a guarantee against arbitrary encryption or steganography. ### Owner-signed purge (gap 5) From 0a5f36dcd30648bdede2dddceb4b45b05344a04f Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Sun, 8 Nov 2026 14:40:05 +0530 Subject: [PATCH 4/7] fix(kin): bound secret scan work and preserve token boundaries --- apps/server/src/kin/memory/Memory.test.ts | 13 +- apps/server/src/kin/memory/Memory.ts | 7 +- .../src/kin/memory/MemoryWriter.test.ts | 15 +++ apps/server/src/kin/memory/MemoryWriter.ts | 15 +++ apps/server/src/kin/memory/secretScan.test.ts | 100 +++++++++++++- apps/server/src/kin/memory/secretScan.ts | 127 +++++++++++------- docs/kin/specs/triggers.md | 4 +- 7 files changed, 223 insertions(+), 58 deletions(-) diff --git a/apps/server/src/kin/memory/Memory.test.ts b/apps/server/src/kin/memory/Memory.test.ts index 674998fcc7b2f..942b89ffbba43 100644 --- a/apps/server/src/kin/memory/Memory.test.ts +++ b/apps/server/src/kin/memory/Memory.test.ts @@ -859,6 +859,7 @@ it.live("uses filesystem identity for a case-mismatched base without recloning", it.live.each([ "diverged", "secret merge", + "scan failure merge", "remote secret merge", "large remote merge", "conflict", @@ -959,7 +960,7 @@ it.live.each([ } yield* memory.record(orgA, "main", initial.head, local); yield* fs.writeFileString( - `${source}/${scenario === "push retry" || scenario === "diverged" || scenario === "secret merge" || scenario === "remote secret merge" || scenario === "large remote merge" || scenario === "network down" || scenario === "dirty writer" || scenario === "H2-merge-swap" ? "remote-only" : "rules.md"}`, + `${source}/${scenario === "push retry" || scenario === "diverged" || scenario === "scan failure merge" || scenario === "secret merge" || scenario === "remote secret merge" || scenario === "large remote merge" || scenario === "network down" || scenario === "dirty writer" || scenario === "H2-merge-swap" ? "remote-only" : "rules.md"}`, "Remote invented edit\n", ); if (scenario === "journal conflict" || scenario === "push retry") { @@ -1018,6 +1019,10 @@ it.live.each([ yield* memory.record(orgA, "main", local, replacement); local = replacement; } + if (scenario === "scan failure merge") { + const sql = yield* SqlClient.SqlClient; + yield* sql`DROP TABLE kin_secret_allow`; + } const result = yield* memory.prepareWakeup(orgA, "fixture-wakeup").pipe(Effect.result); if (scenario === "unrelated" || scenario === "criss-cross") { expect(result._tag).toBe("Failure"); @@ -1052,6 +1057,12 @@ it.live.each([ expect( (yield* memory.prepareWakeup(orgA, "fixture-wakeup-3").pipe(Effect.result))._tag, ).toBe("Success"); + } else if (scenario === "scan failure merge") { + expect(result._tag).toBe("Failure"); + expect( + messages.some((message) => message.includes("Secret scan failed: Allowlist unavailable")), + ).toBe(true); + expect(yield* memory.anchor(orgA, "main")).toBe(local); } else if (scenario === "secret merge") { expect(result._tag).toBe("Failure"); if (result._tag === "Failure") { diff --git a/apps/server/src/kin/memory/Memory.ts b/apps/server/src/kin/memory/Memory.ts index c38760e061e22..d8aafb19a722a 100644 --- a/apps/server/src/kin/memory/Memory.ts +++ b/apps/server/src/kin/memory/Memory.ts @@ -1112,6 +1112,9 @@ const make = Effect.gen(function* () { .map(([path, content]) => ({ path, content })), }) .pipe( + Effect.tapError((cause) => + notify({ orgId, branch, reason: `Secret scan failed: ${cause.reason}` }), + ), Effect.mapError( (cause) => new MemoryError({ orgId, operation: "secretScan", cause }), ), @@ -1120,12 +1123,12 @@ const make = Effect.gen(function* () { yield* notify({ orgId, branch, - reason: `Secret scan blocked merge: ${findings[0]!.ruleId} in ${findings[0]!.pathHit ? "[redacted]" : findings[0]!.path}`, + reason: `Secret scan blocked merge: ${findings[0]!.ruleId} in ${findings[0]!.pathHit ? "[redacted]" : (findings[0]!.itemPath ?? findings[0]!.path)}`, }); return yield* new MemoryError({ orgId, operation: "secretScan", - reason: `Secret scan blocked merge: ${findings[0]!.ruleId} in ${findings[0]!.pathHit ? "[redacted]" : findings[0]!.path}`, + reason: `Secret scan blocked merge: ${findings[0]!.ruleId} in ${findings[0]!.pathHit ? "[redacted]" : (findings[0]!.itemPath ?? findings[0]!.path)}`, }); } tree = yield* Effect.scoped( diff --git a/apps/server/src/kin/memory/MemoryWriter.test.ts b/apps/server/src/kin/memory/MemoryWriter.test.ts index df01b3857d435..3f0789c6874e0 100644 --- a/apps/server/src/kin/memory/MemoryWriter.test.ts +++ b/apps/server/src/kin/memory/MemoryWriter.test.ts @@ -600,6 +600,21 @@ it.live.each([ } if (scenario === "tree-substitution") substituteTree = yield* git(paths.writer, ["rev-parse", `${original}^{tree}`]); + if (scenario === "blocked") { + const tokenPath = "claims/" + ["gh", "p_", "a".repeat(36)].join("") + ".md"; + const malformed = yield* writer + .stage(orgId, "wake1", { + path: tokenPath, + content: "missing front matter", + }) + .pipe(Effect.result); + expect(malformed._tag).toBe("Failure"); + if (malformed._tag === "Failure") { + expect(malformed.failure._tag).toBe("MemoryItemError"); + if (malformed.failure._tag === "MemoryItemError") + expect(malformed.failure.path).toBe("[redacted]"); + } + } if (scenario === "blocked") yield* writer.appendJournal(orgId, "wake1", [ ["gh", "p_"].join("") + "aB3dE6gH9jK2mN5pQ8sT1vW4yZ7cF0iL3oR6", diff --git a/apps/server/src/kin/memory/MemoryWriter.ts b/apps/server/src/kin/memory/MemoryWriter.ts index 180af1d60d20f..39dfb13f92f2e 100644 --- a/apps/server/src/kin/memory/MemoryWriter.ts +++ b/apps/server/src/kin/memory/MemoryWriter.ts @@ -66,6 +66,7 @@ export class MemorySecretBlocked extends Schema.TaggedError ruleId: Schema.String, line: Schema.Number, pathHit: Schema.optional(Schema.Boolean), + itemPath: Schema.optional(Schema.String), }), ), }, @@ -315,6 +316,20 @@ const make = Effect.gen(function* () { }); const report = (orgId: KinOrgId, wakeupId: string, effect: Effect.Effect) => effect.pipe( + Effect.catchTags({ + MemoryItemError: (error) => + scanner.scan({ orgId, files: [{ path: error.path, content: new Uint8Array() }] }).pipe( + Effect.flatMap((findings) => + Effect.fail( + new MemoryItemError({ + orgId: error.orgId, + path: findings.length ? "[redacted]" : error.path, + reason: error.reason, + }), + ), + ), + ), + }), Effect.tapError((error) => error._tag === "MemoryCheckoutInvalid" || error._tag === "MemorySyncConflict" || diff --git a/apps/server/src/kin/memory/secretScan.test.ts b/apps/server/src/kin/memory/secretScan.test.ts index 63ff116caed3c..fe3010170cc20 100644 --- a/apps/server/src/kin/memory/secretScan.test.ts +++ b/apps/server/src/kin/memory/secretScan.test.ts @@ -1,5 +1,6 @@ // @effect-diagnostics-next-line nodeBuiltinImport:off -- Bounded synchronous fingerprint hashing shares the scan loop. import * as NodeCrypto from "node:crypto"; +import { vi } from "vite-plus/test"; import { expect, it } from "@effect/vitest"; import { KinOrgId } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; @@ -15,6 +16,23 @@ const layer = Scan.layer.pipe(Layer.provideMerge(database)); const body = "aB3dE6gH9jK2mN5pQ8sT1vW4yZ7cF0iL3oR6"; const github = ["gh", "p_"].join("") + body; const fixtures = [ + ...["X-Api-Key: ", "client-secret: ", '{"password": "', '"api_key":"'].map((prefix) => [ + "secret-assignment", + prefix + body + '"', + prefix + "short", + ]), + ...["_", "token_"].map((prefix) => [ + "github-token", + prefix + github + "_rotated", + "task-follow-up-with-design-team", + ]), + ["aws-access-key", ["AK", "IA"].join("") + "Q".repeat(16) + "_old", "ASIAPACIFICREGION2026"], + [ + "jwt", + ["ey", "J"].join("") + "a".repeat(100) + "." + "b".repeat(1800) + "." + "c".repeat(100), + "eyJshort.two.parts", + ], + ...["p", "o", "u", "s", "r"].map((kind) => [ "github-token", "gh" + kind + "_" + body, @@ -72,6 +90,7 @@ it.effect("blocks encoded, wrapped and confusable tokens with safe findings", () const scanner = yield* Scan.KinSecretScan; const values = [ github, + "%67" + github.slice(1), ...["/", "-", "_", "+", "word"].map( (prefix) => prefix + Buffer.from(github).toString("base64"), ), @@ -92,6 +111,9 @@ it.effect("blocks encoded, wrapped and confusable tokens with safe findings", () Buffer.from(Buffer.from(github).toString("base64")).toString("base64"), [...github].map((c) => `%${c.charCodeAt(0).toString(16)}`).join(""), github.slice(0, 12) + "\n" + github.slice(12), + ...["next word", " next word"].map( + (suffix) => github.slice(0, 12) + "\n" + github.slice(12) + "\n" + suffix, + ), github.slice(0, 12) + "\\\n " + github.slice(12), github.replace("p", "р"), [...github].map((c) => String.fromCharCode(c.charCodeAt(0) + 0xfee0)).join(""), @@ -193,7 +215,7 @@ it.effect("fails closed at file, batch and count limits", () => }).pipe(Effect.provide(layer)), ); -it.effect("round 2 near misses and private key headers", () => +it.effect("near misses and private key headers", () => Effect.gen(function* () { const scanner = yield* Scan.KinSecretScan; for (const content of [ @@ -206,6 +228,10 @@ it.effect("round 2 near misses and private key headers", () => "github_token_scopes: repo,workflow,admin:org,read:user", "postgres://user:${DB_PASSWORD}@example.invalid/db", "ASIAPACIFICREGION2026", + ["sk-", "proj-2026-11-follow-up-with-design-team-on-onboarding"].join(""), + ["sk-", "admin-rotation-plan-for-2026-q3-and-next-year"].join(""), + "z".repeat(4096) + ["AK", "IA"].join("") + "Q".repeat(16), + " ".repeat(8173) + ["AK", "IA"].join("") + "Q".repeat(16) + "z", ]) expect( (yield* scanner.scan({ @@ -225,7 +251,7 @@ it.effect("round 2 near misses and private key headers", () => ).toBe(true); }).pipe(Effect.provide(layer)), ); -it.effect("round 2 scans paths without exposing them", () => +it.effect("scans paths without exposing them", () => Effect.gen(function* () { const scanner = yield* Scan.KinSecretScan; const findings = yield* scanner.scan({ @@ -234,19 +260,29 @@ it.effect("round 2 scans paths without exposing them", () => }); expect(findings.some((f) => f.ruleId === "github-token")).toBe(true); expect(JSON.stringify(findings).includes(github)).toBe(false); + const indexFindings = yield* scanner.scan({ + orgId, + files: [ + { path: "claims/item.md", content: Buffer.from("clean") }, + { path: "INDEX.md", content: Buffer.from("- [claim] " + github + " (claims/item.md)") }, + ], + }); + expect(indexFindings.some((f) => f.itemPath === "claims/item.md")).toBe(true); }).pipe(Effect.provide(layer)), ); -it.effect("round 2 bounds adversarial rule inputs structurally", () => +it.effect("bounds adversarial rule inputs structurally", () => Effect.gen(function* () { const scanner = yield* Scan.KinSecretScan; for (const prefix of ["eyJ", "sk-", "AKIA"]) { const content = prefix.repeat(Math.floor((1024 * 1024 - 100) / prefix.length)); let examined = 0; for (const window of Scan.ruleWindows(content)) { - expect(window.text.length).toBeLessThanOrEqual(8192); + expect(window.text.length).toBeLessThanOrEqual(8194); examined += window.text.length; } - expect(examined).toBeLessThanOrEqual(content.length * 2); + expect(examined).toBeLessThanOrEqual( + content.length * 2 + Math.ceil(content.length / 4096) * 2, + ); expect( (yield* scanner.scan({ orgId, @@ -256,3 +292,57 @@ it.effect("round 2 bounds adversarial rule inputs structurally", () => } }).pipe(Effect.provide(layer)), ); + +it.effect("scans ordinary journals and multi-file batches without exhausting work", () => + Effect.gen(function* () { + const scanner = yield* Scan.KinSecretScan; + const journal = "Discussed the next steps with the team. Follow up tomorrow.\n"; + for (const files of [ + [ + { + path: "journal/test.md", + content: Buffer.from(journal.repeat(Math.ceil((256 * 1024) / journal.length))), + }, + ], + Array.from({ length: 30 }, (_, i) => ({ + path: `journal/${i}.md`, + content: Buffer.from(journal.repeat(Math.ceil((60 * 1024) / journal.length))), + })), + ]) + expect((yield* scanner.scan({ orgId, files })).length).toBe(0); + }).pipe(Effect.provide(layer)), +); + +it.effect("bounds rule executions for random runs, word lines and multi-file inputs", () => + Effect.gen(function* () { + const scanner = yield* Scan.KinSecretScan; + let seed = 42; + const random = (length: number) => + Array.from({ length }, () => { + seed = (Math.imul(seed, 1664525) + 1013904223) >>> 0; + return "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"[seed % 62]; + }).join(""); + const runs = Array.from({ length: Math.floor((1024 * 1024 - 100) / 25) }, () => + random(24), + ).join(" "); + const words = (Array.from({ length: 10 }, () => random(7)).join(" ") + "\n").repeat(13000); + for (const files of [ + [{ path: "runs", content: Buffer.from(runs) }], + [{ path: "words", content: Buffer.from(words) }], + Array.from({ length: 8 }, (_, i) => ({ path: `runs-${i}`, content: Buffer.from(runs) })), + ]) { + let executions = 0; + const exec = RegExp.prototype.exec; + using _spy = vi.spyOn(RegExp.prototype, "exec").mockImplementation(function ( + this: RegExp, + value: string, + ) { + if (this.source.startsWith("(? { + const alphabet = + id === "github-token" || id === "aws-access-key" + ? "A-Za-z0-9" + : id === "secret-assignment" + ? "A-Za-z0-9_" + : "A-Za-z0-9_-"; + const right = id === "url-password" || id === "secret-assignment" ? "" : `(?![${alphabet}])`; + return [ + id, + new RegExp(`(? 0 ? 1 : 0, + rightContext: offset + 8192 < value.length ? 1 : 0, + }; }; const entropy = (value: string) => { const counts = new Map(); @@ -137,34 +156,35 @@ const make = Effect.gen(function* () { const allow = new Set(allowed.map((r) => JSON.stringify([r.path, r.rule_id, r.fingerprint]))); const findings: SecretFinding[] = []; const seen = new Set(); - let budget = 32 * 1024 * 1024; + let budget = 64 * 1024 * 1024; for (const file of input.files) { const text = normalize(Buffer.from(file.content).toString("utf8")); + const lines = text.split("\n"); const pathFindingStart = findings.length; let pathHit = true; let redactPath = false; - const inspect = ( - value: string, - baseLine: number, - depth: number, - lineAt?: Uint32Array, - ): void => { - budget -= value.length; + const inspect = function* (value: string, baseLine: number, depth: number): Generator { + // Charge fixed work as well as bytes to bound decoder fan-out on short values. + budget -= value.length + 4096; if (budget < 0 || findings.length > 1024) return; - for (const [ruleId, pattern] of rules) { - for (const window of ruleWindows(value)) { - const bounded = new RegExp( - `(? window.text.length - window.rightContext ) continue; + const index = window.offset + match.index; + const secret = match[1] ?? match[0]; + if (ruleId === "openai-key" && !/[A-Z]/.test(secret)) continue; if (ruleId === "url-password" && /^\$\{[^}]+\}$/.test(secret)) continue; if ( ruleId === "secret-assignment" && @@ -177,16 +197,18 @@ const make = Effect.gen(function* () { continue; if (allow.has(JSON.stringify([file.path, ruleId, fingerprint(ruleId, secret)]))) continue; - const line = - lineAt?.[index] ?? baseLine + (value.slice(0, index).match(/\n/g)?.length ?? 0); + const line = baseLine + (value.slice(0, index).match(/\n/g)?.length ?? 0); const key = JSON.stringify([file.path, ruleId, line, pathHit]); if (!seen.has(key)) { + const indexLine = file.path === "INDEX.md" ? lines[line - 1] : undefined; + const itemPath = indexLine?.match(/\(([^()]+\.md)\)/)?.[1]; seen.add(key); findings.push({ path: pathHit || redactPath ? "[redacted]" : file.path, ruleId, line, ...(pathHit ? { pathHit: true } : {}), + ...(itemPath && input.files.some((f) => f.path === itemPath) ? { itemPath } : {}), }); } break; @@ -196,10 +218,15 @@ const make = Effect.gen(function* () { if (depth === 2) return; let cursor = 0; let decodedLine = baseLine; - for (const match of value.matchAll(/(?:%[0-9a-fA-F]{2}|[A-Za-z0-9_+/-]){24,}={0,2}/g)) { + for (const match of value.matchAll(/(?:%[0-9a-fA-F]{2}|[A-Za-z0-9_+/-]){20,}={0,2}/g)) { + const encoded = match[0]; + // Mixed percent/plain encodings can represent 20 bytes in fewer than 60 characters. + if (!encoded.includes("%") && encoded.length < 27) continue; decodedLine += value.slice(cursor, match.index).match(/\n/g)?.length ?? 0; cursor = match.index; - const encoded = match[0]; + yield; + budget -= 4096; + if (budget < 0) return; const candidates = encoded.includes("%") ? [ Buffer.from( @@ -215,39 +242,43 @@ const make = Effect.gen(function* () { for (const candidate of candidates) { if (budget < 0 || findings.length > 1024) return; const decoded = normalize(candidate).replace(/[^\x09\x0a\x0d\x20-\x7e]/g, " "); - if (decoded !== encoded) - inspect(decoded, lineAt?.[match.index] ?? decodedLine, depth + 1); + if (decoded !== encoded) yield* inspect(decoded, decodedLine, depth + 1); } } }; - inspect(normalize(file.path), 1, 0); + for (const _ of inspect(normalize(file.path), 1, 0)) yield* Effect.yieldNow; redactPath = findings.length > pathFindingStart; pathHit = false; - inspect(text, 1, 0); - const lineAt = new Uint32Array(text.length); + for (const _ of inspect(text, 1, 0)) yield* Effect.yieldNow; + let run = ""; + let runLine = 1; let line = 1; - let offset = 0; - const starts = new Set(); - const compact = text.replace(/\\?\r?\n[\t ]*|[^\r\n\\]+|./g, (chunk) => { - if (chunk.includes("\n")) { - line++; - starts.add(offset); - return ""; + let fragments = 0; + for (const raw of lines) { + const fragment = raw + .replace(/\\?\r$/, "") + .replace(/\\$/, "") + .trim(); + if (fragment && !/\s/.test(fragment)) { + if (!fragments) runLine = line; + run += fragment; + fragments++; + } else { + if (fragments > 1) for (const _ of inspect(run, runLine, 0)) yield* Effect.yieldNow; + run = ""; + fragments = 0; } - lineAt.fill(line, offset, offset + chunk.length); - offset += chunk.length; - return chunk; - }); - inspect(compact, 1, 0, lineAt); - // A wrapped token may start after prose that compacting joined to its left boundary. - for (const start of starts) { - if (budget < 0 || findings.length > 1024) break; - inspect(compact.slice(start, start + 8192), lineAt[start] ?? 1, 0, lineAt.subarray(start)); + line++; } + if (fragments > 1) for (const _ of inspect(run, runLine, 0)) yield* Effect.yieldNow; } if (budget < 0 || findings.length > 1024) return yield* new KinSecretScanError({ reason: "Scan work limit exceeded" }); - return findings; + const unsafePaths = findings.some((f) => f.pathHit); + return findings.map(({ itemPath, ...finding }) => ({ + ...finding, + ...(itemPath && !unsafePaths ? { itemPath } : {}), + })); }); return KinSecretScan.of({ scan }); }); diff --git a/docs/kin/specs/triggers.md b/docs/kin/specs/triggers.md index 26607c9744cd0..5606832973d83 100644 --- a/docs/kin/specs/triggers.md +++ b/docs/kin/specs/triggers.md @@ -324,7 +324,7 @@ GitHub then blocks a bad auto-merge even if Kin's classifier is wrong. This forc **Rules:** - GitHub tokens (`ghp_`, `gho_`, `ghu_`, `ghs_`, `ghr_`, `github_pat_`); -- Anthropic `sk-ant-`; OpenAI legacy `sk-` with the `T3BlbkFJ` marker and `sk-proj-`/`sk-svcacct-`/`sk-admin-` with mixed case or digits; +- Anthropic `sk-ant-`; OpenAI legacy `sk-` with the `T3BlbkFJ` marker and `sk-proj-`/`sk-svcacct-`/`sk-admin-` containing an uppercase letter; - AWS `AKIA…`/`ASIA…`; - Slack `xox[abposr]-`; - Stripe `sk_live_`/`rk_live_`; @@ -344,7 +344,7 @@ GitHub then blocks a bad auto-merge even if Kin's classifier is wrong. This forc **False positives:** the owner adds the fingerprint to `kin_secret_allow` through a signed action. A model can never add to it. Entries are scoped to organization, exact path, rule and SHA-256 of `ruleId + "\0" + normalized matched value`, with `created_at` in integer milliseconds and `signed_action_id` reserved for the future APPR executor id. The scanner is read-only; the APPR executor must verify the owner grant before inserting. Until that executor lands, there is no agent or MCP allowlist-write path. Migration K012 is provisional and will be renumbered at merge. -The scanner checks paths as well as content, redacts paths that contain a finding, normalizes NFKC, common Cyrillic/Greek/Latin lookalikes and Default Ignorable code points, checks newline-wrapped tokens, and decodes URL/base64 content up to two levels. Base64 decoding tries all four alignments; malformed percent bytes and non-ASCII decoded bytes become separators. Prefix rules have token boundaries and bounded lengths; each regex sees at most 8192 characters with 4096-character overlap. Merge scanning covers only bytes changed from the fetched remote plus the rebuilt `INDEX.md`, not pre-existing remote content. Scanner failures notify through the writer hook, just like findings. Merge blocks use the recovery hook; production owner delivery still waits on a notifier. It fails closed above 1 MiB per file, 8 MiB or 1024 files per call, 4096 allowlist entries per organization, or 32 MiB of transformed scan text. Splits by a space, `\r`, U+2028, across journal bullets, markdown or HTML comments, `\u` escapes, and hex encoding remain out of scope. These are conservative detectors, not a guarantee against arbitrary encryption or steganography. +The scanner checks paths as well as content, redacts paths that contain a finding, normalizes NFKC, common Cyrillic/Greek/Latin lookalikes and Default Ignorable code points, checks newline-wrapped tokens, and decodes URL/base64 content up to two levels. Base64 decoding tries all four alignments; malformed percent bytes and non-ASCII decoded bytes become separators. Prefix rules have token boundaries and bounded lengths; each precompiled regex sees at most 8192 characters plus one character of boundary context on each side, with 4096-character overlap. JWT headers, payloads and signatures are capped at 512, 2500 and 1024 characters. Wrapped tokens are scanned as consecutive whitespace-free line runs with independent boundaries, not joined to subsequent prose. Merge scanning covers only bytes changed from the fetched remote plus the rebuilt `INDEX.md`, not pre-existing remote content. Scanner failures notify through the writer hook, just like findings. Merge blocks use the recovery hook; production owner delivery still waits on a notifier. It fails closed above 1 MiB per file, 8 MiB or 1024 files per call, 4096 allowlist entries per organization, or 64 MiB of weighted work (transformed characters, 4096 units per inspect and decoder run, window characters, and 64 units per regex match). Scanning yields between windows and decoder candidates so multi-file batches do not monopolize the event loop. Base64 runs shorter than 27 characters and fully percent-encoded runs shorter than 60 characters are not decoded; mixed percent/plain runs retain a 20-byte minimum to prevent partial-encoding bypasses. Splits by a space, `\r`, U+2028, across journal bullets, markdown or HTML comments, `\u` escapes, and hex encoding remain out of scope. These are conservative detectors, not a guarantee against arbitrary encryption or steganography. ### Owner-signed purge (gap 5) From ec5bf6d4218b0f7b3ba06625bd432ea98b464151 Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Sun, 8 Nov 2026 15:12:17 +0530 Subject: [PATCH 5/7] fix(kin): scan realistic journals and wrapped secret prefixes --- .../src/kin/memory/MemoryWriter.test.ts | 12 ++- apps/server/src/kin/memory/MemoryWriter.ts | 6 +- apps/server/src/kin/memory/frontMatter.ts | 8 +- apps/server/src/kin/memory/secretScan.test.ts | 73 ++++++++++++++----- apps/server/src/kin/memory/secretScan.ts | 61 +++++++++------- docs/kin/specs/triggers.md | 2 +- 6 files changed, 112 insertions(+), 50 deletions(-) diff --git a/apps/server/src/kin/memory/MemoryWriter.test.ts b/apps/server/src/kin/memory/MemoryWriter.test.ts index 3f0789c6874e0..9b9faefc29e13 100644 --- a/apps/server/src/kin/memory/MemoryWriter.test.ts +++ b/apps/server/src/kin/memory/MemoryWriter.test.ts @@ -605,15 +605,25 @@ it.live.each([ const malformed = yield* writer .stage(orgId, "wake1", { path: tokenPath, - content: "missing front matter", + content: "---\nid: [\n" + tokenPath + ": [\n---\nbody", }) .pipe(Effect.result); expect(malformed._tag).toBe("Failure"); if (malformed._tag === "Failure") { expect(malformed.failure._tag).toBe("MemoryItemError"); + expect(JSON.stringify(malformed.failure).includes(tokenPath.slice(7, -3))).toBe(false); if (malformed.failure._tag === "MemoryItemError") expect(malformed.failure.path).toBe("[redacted]"); } + const inaccessible = yield* writer + .stage(orgId, "wake1", { + path: tokenPath.slice(0, -3) + "_".repeat(260) + ".md", + content: "missing front matter", + }) + .pipe(Effect.result); + expect(inaccessible._tag).toBe("Failure"); + if (inaccessible._tag === "Failure") + expect(JSON.stringify(inaccessible.failure).includes(tokenPath.slice(7, -3))).toBe(false); } if (scenario === "blocked") yield* writer.appendJournal(orgId, "wake1", [ diff --git a/apps/server/src/kin/memory/MemoryWriter.ts b/apps/server/src/kin/memory/MemoryWriter.ts index 39dfb13f92f2e..dd5fbd7224057 100644 --- a/apps/server/src/kin/memory/MemoryWriter.ts +++ b/apps/server/src/kin/memory/MemoryWriter.ts @@ -298,7 +298,11 @@ const make = Effect.gen(function* () { const result = yield* fs.lstat(current).pipe(Effect.result); if (result._tag === "Failure") { if (result.failure.reason._tag !== "NotFound") - return yield* new MemoryError({ orgId, operation: "path", cause: result.failure }); + return yield* new MemoryError({ + orgId, + operation: "path", + cause: "Unable to inspect memory path", + }); } else if ( result.success.type === "SymbolicLink" || (result.success.type === "File" && diff --git a/apps/server/src/kin/memory/frontMatter.ts b/apps/server/src/kin/memory/frontMatter.ts index 692502849fca7..49b7bb49b70b7 100644 --- a/apps/server/src/kin/memory/frontMatter.ts +++ b/apps/server/src/kin/memory/frontMatter.ts @@ -89,8 +89,12 @@ export const parseFrontMatter = Effect.fnUntraced(function* ( new MemoryItemError({ orgId, path, - reason: cause instanceof Error ? cause.message : "Malformed YAML", - cause, + reason: staged + ? "Malformed YAML front matter" + : cause instanceof Error + ? cause.message + : "Malformed YAML", + ...(staged ? {} : { cause }), }), }); const decoded = yield* (staged ? decodeStaged(value) : decodeFrontMatter(value)).pipe( diff --git a/apps/server/src/kin/memory/secretScan.test.ts b/apps/server/src/kin/memory/secretScan.test.ts index fe3010170cc20..319b4fc2aff03 100644 --- a/apps/server/src/kin/memory/secretScan.test.ts +++ b/apps/server/src/kin/memory/secretScan.test.ts @@ -16,6 +16,21 @@ const layer = Scan.layer.pipe(Layer.provideMerge(database)); const body = "aB3dE6gH9jK2mN5pQ8sT1vW4yZ7cF0iL3oR6"; const github = ["gh", "p_"].join("") + body; const fixtures = [ + ...[ + ["stripe-key", ["sk_", "live_"].join("") + body + "_old"], + ["slack-token", ["xo", "xb-"].join("") + body + "_old"], + ["google-key", ["AI", "za"].join("") + "X".repeat(35) + "_old"], + ["google-key", "key_" + ["AI", "za"].join("") + "X".repeat(35)], + ["anthropic-key", "prod-" + ["sk-", "ant-"].join("") + body.repeat(2)], + ...[ + Buffer.from(JSON.stringify({ alg: "RS256", x5c: ["a".repeat(499)] })).toString("base64url"), + ["ey", "J"].join("") + "a".repeat(2048), + ].map((header) => [ + "jwt", + header + "." + ["ey", "J"].join("") + "b".repeat(2600) + ".signature", + ]), + ].map(([rule, hit]) => [rule, hit, "task-follow-up-with-design-team"]), + ...["X-Api-Key: ", "client-secret: ", '{"password": "', '"api_key":"'].map((prefix) => [ "secret-assignment", prefix + body + '"', @@ -29,7 +44,7 @@ const fixtures = [ ["aws-access-key", ["AK", "IA"].join("") + "Q".repeat(16) + "_old", "ASIAPACIFICREGION2026"], [ "jwt", - ["ey", "J"].join("") + "a".repeat(100) + "." + "b".repeat(1800) + "." + "c".repeat(100), + ["ey", "J"].join("") + "a".repeat(100) + ".eyJ" + "b".repeat(1800) + "." + "c".repeat(100), "eyJshort.two.parts", ], @@ -61,7 +76,7 @@ const fixtures = [ ["pem-private-key", "-----BEGIN RSA PRIVATE KEY-----", "-----BEGIN PUBLIC KEY-----"], [ "jwt", - ["ey", "J"].join("") + "a".repeat(10) + "." + "b".repeat(12) + "." + "c".repeat(20), + ["ey", "J"].join("") + "a".repeat(10) + ".eyJ" + "b".repeat(12) + "." + "c".repeat(20), "eyJshort.two.parts", ], [ @@ -90,6 +105,14 @@ it.effect("blocks encoded, wrapped and confusable tokens with safe findings", () const scanner = yield* Scan.KinSecretScan; const values = [ github, + "The deploy key is " + github.slice(0, 12) + "\n" + github.slice(12), + github.slice(0, 12) + "\n" + github.slice(12) + " and more", + github.slice(0, 12) + "\n" + github.slice(12) + "\nThanks", + "The deploy key is " + + Buffer.from(github).toString("base64").slice(0, 16) + + "\n" + + Buffer.from(github).toString("base64").slice(16) + + " and more", "%67" + github.slice(1), ...["/", "-", "_", "+", "word"].map( (prefix) => prefix + Buffer.from(github).toString("base64"), @@ -190,6 +213,18 @@ it.effect("fails closed when decoded findings or the allowlist exceed their boun .scan({ orgId, files: [{ path: "claims/many.md", content }] }) .pipe(Effect.result))._tag, ).toBe("Failure"); + const exhausted = yield* scanner + .scan({ + orgId, + files: Array.from({ length: 8 }, (_, i) => ({ + path: `claims/${i}.md`, + content: Buffer.from("a".repeat(1024 * 1024 - 100)), + })), + }) + .pipe(Effect.result); + expect(exhausted._tag).toBe("Failure"); + if (exhausted._tag === "Failure") + expect(exhausted.failure.reason).toBe("Scan work limit exceeded"); const sql = yield* SqlClient.SqlClient; yield* sql`WITH RECURSIVE entries(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM entries WHERE n < 4097) INSERT INTO kin_secret_allow SELECT ${orgId}, 'claims/' || n, 'github-token', ${"0".repeat(64)}, 'signed-fixture', 0 FROM entries`; @@ -264,7 +299,10 @@ it.effect("scans paths without exposing them", () => orgId, files: [ { path: "claims/item.md", content: Buffer.from("clean") }, - { path: "INDEX.md", content: Buffer.from("- [claim] " + github + " (claims/item.md)") }, + { + path: "INDEX.md", + content: Buffer.from("- [claim (claims/wrong.md)] " + github + " (claims/item.md)"), + }, ], }); expect(indexFindings.some((f) => f.itemPath === "claims/item.md")).toBe(true); @@ -293,23 +331,24 @@ it.effect("bounds adversarial rule inputs structurally", () => }).pipe(Effect.provide(layer)), ); -it.effect("scans ordinary journals and multi-file batches without exhausting work", () => +it.effect.each([ + { name: "256 KiB journal", count: 1, bytes: 256 * 1024 }, + { name: "30 x 60 KiB batch", count: 30, bytes: 60 * 1024 }, +])("scans realistic $name without exhausting work", ({ count, bytes }) => Effect.gen(function* () { const scanner = yield* Scan.KinSecretScan; - const journal = "Discussed the next steps with the team. Follow up tomorrow.\n"; - for (const files of [ + const journal = [ - { - path: "journal/test.md", - content: Buffer.from(journal.repeat(Math.ceil((256 * 1024) / journal.length))), - }, - ], - Array.from({ length: 30 }, (_, i) => ({ - path: `journal/${i}.md`, - content: Buffer.from(journal.repeat(Math.ceil((60 * 1024) / journal.length))), - })), - ]) - expect((yield* scanner.scan({ orgId, files })).length).toBe(0); + "Discussed the next steps with the team. Follow up tomorrow.", + "Commit " + "0123456789abcdef".repeat(2) + "01234567 reviewed.", + "Updated apps/server/src/kin/memory/secretScan.test.ts and docs/operations/development.md", + "Trace 12345678-1234-5678-9abc-123456789abc completed successfully.", + ].join("\n") + "\n"; + const files = Array.from({ length: count }, (_, i) => ({ + path: `journal/${i}.md`, + content: Buffer.from(journal.repeat(Math.ceil(bytes / journal.length)).slice(0, bytes)), + })); + expect((yield* scanner.scan({ orgId, files })).length).toBe(0); }).pipe(Effect.provide(layer)), ); diff --git a/apps/server/src/kin/memory/secretScan.ts b/apps/server/src/kin/memory/secretScan.ts index 39e749d354e8e..9a232393079fa 100644 --- a/apps/server/src/kin/memory/secretScan.ts +++ b/apps/server/src/kin/memory/secretScan.ts @@ -44,7 +44,7 @@ const rules = [ ["stripe-key", /[rs]k_live_[A-Za-z0-9]{24,200}/], ["google-key", /AIza[A-Za-z0-9_-]{35}/], ["pem-private-key", /-----BEGIN (?:[A-Z0-9]+ ){0,8}PRIVATE KEY(?: BLOCK)?-----/], - ["jwt", /eyJ[A-Za-z0-9_-]{5,509}\.[A-Za-z0-9_-]{8,2500}\.[A-Za-z0-9_-]{16,1024}/], + ["jwt", /eyJ[A-Za-z0-9_-]{5,2048}\.eyJ[A-Za-z0-9_-]{8,1024}/], ["url-password", /\b[a-z][a-z0-9+.-]{0,31}:\/\/[^\s/:@]{1,512}:([^\s/@]{1,512})@/i], [ "secret-assignment", @@ -53,18 +53,21 @@ const rules = [ ] as const; const compiledRules = rules.map(([id, pattern]) => { const alphabet = - id === "github-token" || id === "aws-access-key" + id === "github-token" || id === "aws-access-key" || id === "stripe-key" || id === "google-key" ? "A-Za-z0-9" - : id === "secret-assignment" - ? "A-Za-z0-9_" + : id === "slack-token" + ? "A-Za-z0-9-" : "A-Za-z0-9_-"; - const right = id === "url-password" || id === "secret-assignment" ? "" : `(?![${alphabet}])`; + const right = ["url-password", "secret-assignment", "jwt"].includes(id) + ? "" + : `(?![${alphabet}])`; return [ id, - new RegExp(`(? { - // Charge fixed work as well as bytes to bound decoder fan-out on short values. - budget -= value.length + 4096; + budget -= value.length; if (budget < 0 || findings.length > 1024) return; for (const window of ruleWindows(value)) { budget -= window.text.length; if (budget < 0) return; yield; + if (!anchors.test(window.text)) continue; for (const [ruleId, bounded] of compiledRules) { bounded.lastIndex = 0; let match: RegExpExecArray | null; @@ -201,7 +204,9 @@ const make = Effect.gen(function* () { const key = JSON.stringify([file.path, ruleId, line, pathHit]); if (!seen.has(key)) { const indexLine = file.path === "INDEX.md" ? lines[line - 1] : undefined; - const itemPath = indexLine?.match(/\(([^()]+\.md)\)/)?.[1]; + const itemPath = Array.from(indexLine?.matchAll(/\(([^()]+\.md)\)/g) ?? []).at( + -1, + )?.[1]; seen.add(key); findings.push({ path: pathHit || redactPath ? "[redacted]" : file.path, @@ -225,7 +230,7 @@ const make = Effect.gen(function* () { decodedLine += value.slice(cursor, match.index).match(/\n/g)?.length ?? 0; cursor = match.index; yield; - budget -= 4096; + budget -= encoded.length; if (budget < 0) return; const candidates = encoded.includes("%") ? [ @@ -250,27 +255,27 @@ const make = Effect.gen(function* () { redactPath = findings.length > pathFindingStart; pathHit = false; for (const _ of inspect(text, 1, 0)) yield* Effect.yieldNow; - let run = ""; - let runLine = 1; - let line = 1; - let fragments = 0; - for (const raw of lines) { - const fragment = raw + let run: { text: string; line: number }[] = []; + const append = function* (text: string, line: number): Generator { + run.push({ text, line }); + while (run.length > 8 || run.reduce((sum, part) => sum + part.text.length, 0) > 4096) + run.shift(); + if (run.length > 1) yield* inspect(run.map((part) => part.text).join(""), run[0]!.line, 0); + }; + for (const [index, raw] of lines.entries()) { + const words = raw .replace(/\\?\r$/, "") .replace(/\\$/, "") - .trim(); - if (fragment && !/\s/.test(fragment)) { - if (!fragments) runLine = line; - run += fragment; - fragments++; - } else { - if (fragments > 1) for (const _ of inspect(run, runLine, 0)) yield* Effect.yieldNow; - run = ""; - fragments = 0; + .trim() + .split(/\s+/); + const first = words[0]!; + if (!first) { + run = []; + continue; } - line++; + for (const _ of append(first, index + 1)) yield* Effect.yieldNow; + if (words.length > 1) run = [{ text: words.at(-1)!, line: index + 1 }]; } - if (fragments > 1) for (const _ of inspect(run, runLine, 0)) yield* Effect.yieldNow; } if (budget < 0 || findings.length > 1024) return yield* new KinSecretScanError({ reason: "Scan work limit exceeded" }); diff --git a/docs/kin/specs/triggers.md b/docs/kin/specs/triggers.md index 5606832973d83..944aee345322f 100644 --- a/docs/kin/specs/triggers.md +++ b/docs/kin/specs/triggers.md @@ -344,7 +344,7 @@ GitHub then blocks a bad auto-merge even if Kin's classifier is wrong. This forc **False positives:** the owner adds the fingerprint to `kin_secret_allow` through a signed action. A model can never add to it. Entries are scoped to organization, exact path, rule and SHA-256 of `ruleId + "\0" + normalized matched value`, with `created_at` in integer milliseconds and `signed_action_id` reserved for the future APPR executor id. The scanner is read-only; the APPR executor must verify the owner grant before inserting. Until that executor lands, there is no agent or MCP allowlist-write path. Migration K012 is provisional and will be renumbered at merge. -The scanner checks paths as well as content, redacts paths that contain a finding, normalizes NFKC, common Cyrillic/Greek/Latin lookalikes and Default Ignorable code points, checks newline-wrapped tokens, and decodes URL/base64 content up to two levels. Base64 decoding tries all four alignments; malformed percent bytes and non-ASCII decoded bytes become separators. Prefix rules have token boundaries and bounded lengths; each precompiled regex sees at most 8192 characters plus one character of boundary context on each side, with 4096-character overlap. JWT headers, payloads and signatures are capped at 512, 2500 and 1024 characters. Wrapped tokens are scanned as consecutive whitespace-free line runs with independent boundaries, not joined to subsequent prose. Merge scanning covers only bytes changed from the fetched remote plus the rebuilt `INDEX.md`, not pre-existing remote content. Scanner failures notify through the writer hook, just like findings. Merge blocks use the recovery hook; production owner delivery still waits on a notifier. It fails closed above 1 MiB per file, 8 MiB or 1024 files per call, 4096 allowlist entries per organization, or 64 MiB of weighted work (transformed characters, 4096 units per inspect and decoder run, window characters, and 64 units per regex match). Scanning yields between windows and decoder candidates so multi-file batches do not monopolize the event loop. Base64 runs shorter than 27 characters and fully percent-encoded runs shorter than 60 characters are not decoded; mixed percent/plain runs retain a 20-byte minimum to prevent partial-encoding bypasses. Splits by a space, `\r`, U+2028, across journal bullets, markdown or HTML comments, `\u` escapes, and hex encoding remain out of scope. These are conservative detectors, not a guarantee against arbitrary encryption or steganography. +The scanner checks paths as well as content, redacts paths that contain a finding, normalizes NFKC, common Cyrillic/Greek/Latin lookalikes and Default Ignorable code points, checks newline-wrapped tokens, and decodes URL/base64 content up to two levels. Base64 decoding tries all four alignments; malformed percent bytes and non-ASCII decoded bytes become separators. Prefix rules use alphanumeric left boundaries and body-specific right boundaries (fixed-width Google keys also tolerate underscore metadata suffixes); each precompiled regex sees at most 8192 characters plus one character of boundary context on each side, with 4096-character overlap. JWT detection matches a bounded prefix: an eyJ header followed by 5–2048 base64url characters, then an eyJ payload prefix followed by 8–1024 characters, without requiring the signature or the payload end. Its maximum match is 3080 characters, within the overlap. Wrapped tokens are inspected after each appended fragment, retaining up to eight fragments and 4 KiB. A prose line closes the prior run with its first word and opens a new run with its last word, so adjacent prose and trailing words do not hide a wrapped token. Merge scanning covers only bytes changed from the fetched remote plus the rebuilt `INDEX.md`, not pre-existing remote content. Scanner failures notify through the writer hook, just like findings. Staged YAML parser errors omit source-bearing diagnostics and filesystem path errors omit the underlying path-bearing cause. Merge blocks use the recovery hook; production owner delivery still waits on a notifier. It fails closed above 1 MiB per file, 8 MiB or 1024 files per call, 4096 allowlist entries per organization, or 64 MiB of weighted work (inspected characters, encoded candidate characters, window characters, and 64 units per regex match). A cheap anchor pre-filter skips rule execution on irrelevant windows, including decoded candidates; recursive decoding remains active to find double-encoded tokens. Scanning yields between windows and decoder candidates so multi-file batches do not monopolize the event loop. Base64 runs shorter than 27 characters and fully percent-encoded runs shorter than 60 characters are not decoded; mixed percent/plain runs retain a 20-byte minimum to prevent partial-encoding bypasses. Wrapped runs beyond eight fragments or 4 KiB, splits by a space, `\r`, U+2028, across journal bullets, markdown or HTML comments, `\u` escapes, and hex encoding remain out of scope. These are conservative detectors, not a guarantee against arbitrary encryption or steganography. ### Owner-signed purge (gap 5) From 303938160d037520df85c14af99b664f228ffbdf Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Sun, 8 Nov 2026 15:53:02 +0530 Subject: [PATCH 6/7] fix(kin): skip short anchorless wrapped runs; yield in the line loop Every line re-inspected its joined run, so 8 MiB of short lines took ~30 s. No rule matches without an anchor and no decoder starts below 20 characters. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/kin/memory/secretScan.ts | 6 +++++- docs/kin/specs/triggers.md | 4 +++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/apps/server/src/kin/memory/secretScan.ts b/apps/server/src/kin/memory/secretScan.ts index 9a232393079fa..7e39bf38ff76f 100644 --- a/apps/server/src/kin/memory/secretScan.ts +++ b/apps/server/src/kin/memory/secretScan.ts @@ -260,9 +260,13 @@ const make = Effect.gen(function* () { run.push({ text, line }); while (run.length > 8 || run.reduce((sum, part) => sum + part.text.length, 0) > 4096) run.shift(); - if (run.length > 1) yield* inspect(run.map((part) => part.text).join(""), run[0]!.line, 0); + if (run.length < 2) return; + const joined = run.map((part) => part.text).join(""); + // No rule matches without an anchor and no decoder starts below 20 characters. + if (joined.length >= 20 || anchors.test(joined)) yield* inspect(joined, run[0]!.line, 0); }; for (const [index, raw] of lines.entries()) { + if (index % 1024 === 1023) yield* Effect.yieldNow; const words = raw .replace(/\\?\r$/, "") .replace(/\\$/, "") diff --git a/docs/kin/specs/triggers.md b/docs/kin/specs/triggers.md index 944aee345322f..1b1a3f7917edc 100644 --- a/docs/kin/specs/triggers.md +++ b/docs/kin/specs/triggers.md @@ -344,7 +344,9 @@ GitHub then blocks a bad auto-merge even if Kin's classifier is wrong. This forc **False positives:** the owner adds the fingerprint to `kin_secret_allow` through a signed action. A model can never add to it. Entries are scoped to organization, exact path, rule and SHA-256 of `ruleId + "\0" + normalized matched value`, with `created_at` in integer milliseconds and `signed_action_id` reserved for the future APPR executor id. The scanner is read-only; the APPR executor must verify the owner grant before inserting. Until that executor lands, there is no agent or MCP allowlist-write path. Migration K012 is provisional and will be renumbered at merge. -The scanner checks paths as well as content, redacts paths that contain a finding, normalizes NFKC, common Cyrillic/Greek/Latin lookalikes and Default Ignorable code points, checks newline-wrapped tokens, and decodes URL/base64 content up to two levels. Base64 decoding tries all four alignments; malformed percent bytes and non-ASCII decoded bytes become separators. Prefix rules use alphanumeric left boundaries and body-specific right boundaries (fixed-width Google keys also tolerate underscore metadata suffixes); each precompiled regex sees at most 8192 characters plus one character of boundary context on each side, with 4096-character overlap. JWT detection matches a bounded prefix: an eyJ header followed by 5–2048 base64url characters, then an eyJ payload prefix followed by 8–1024 characters, without requiring the signature or the payload end. Its maximum match is 3080 characters, within the overlap. Wrapped tokens are inspected after each appended fragment, retaining up to eight fragments and 4 KiB. A prose line closes the prior run with its first word and opens a new run with its last word, so adjacent prose and trailing words do not hide a wrapped token. Merge scanning covers only bytes changed from the fetched remote plus the rebuilt `INDEX.md`, not pre-existing remote content. Scanner failures notify through the writer hook, just like findings. Staged YAML parser errors omit source-bearing diagnostics and filesystem path errors omit the underlying path-bearing cause. Merge blocks use the recovery hook; production owner delivery still waits on a notifier. It fails closed above 1 MiB per file, 8 MiB or 1024 files per call, 4096 allowlist entries per organization, or 64 MiB of weighted work (inspected characters, encoded candidate characters, window characters, and 64 units per regex match). A cheap anchor pre-filter skips rule execution on irrelevant windows, including decoded candidates; recursive decoding remains active to find double-encoded tokens. Scanning yields between windows and decoder candidates so multi-file batches do not monopolize the event loop. Base64 runs shorter than 27 characters and fully percent-encoded runs shorter than 60 characters are not decoded; mixed percent/plain runs retain a 20-byte minimum to prevent partial-encoding bypasses. Wrapped runs beyond eight fragments or 4 KiB, splits by a space, `\r`, U+2028, across journal bullets, markdown or HTML comments, `\u` escapes, and hex encoding remain out of scope. These are conservative detectors, not a guarantee against arbitrary encryption or steganography. +The scanner checks paths as well as content, redacts paths that contain a finding, normalizes NFKC, common Cyrillic/Greek/Latin lookalikes and Default Ignorable code points, checks newline-wrapped tokens, and decodes URL/base64 content up to two levels. Base64 decoding tries all four alignments; malformed percent bytes and non-ASCII decoded bytes become separators. Prefix rules use alphanumeric left boundaries and body-specific right boundaries (fixed-width Google keys also tolerate underscore metadata suffixes); each precompiled regex sees at most 8192 characters plus one character of boundary context on each side, with 4096-character overlap. JWT detection matches a bounded prefix: an eyJ header followed by 5–2048 base64url characters, then an eyJ payload prefix followed by 8–1024 characters, without requiring the signature or the payload end. Its maximum match is 3080 characters, within the overlap. Wrapped tokens are inspected after each appended fragment once the joined run has 20 characters or an anchor, retaining up to eight fragments and 4 KiB. A prose line closes the prior run with its first word and opens a new run with its last word, so adjacent prose and trailing words do not hide a wrapped token. Merge scanning covers only bytes changed from the fetched remote plus the rebuilt `INDEX.md`, not pre-existing remote content. Scanner failures notify through the writer hook, just like findings. Staged YAML parser errors omit source-bearing diagnostics and filesystem path errors omit the underlying path-bearing cause. Merge blocks use the recovery hook; production owner delivery still waits on a notifier. It fails closed above 1 MiB per file, 8 MiB or 1024 files per call, 4096 allowlist entries per organization, or 64 MiB of weighted work (inspected characters, encoded candidate characters, window characters, and 64 units per regex match). A cheap anchor pre-filter skips rule execution on irrelevant windows, including decoded candidates; recursive decoding remains active to find double-encoded tokens. Scanning yields between windows and decoder candidates so multi-file batches do not monopolize the event loop. Base64 runs shorter than 27 characters and fully percent-encoded runs shorter than 60 characters are not decoded; mixed percent/plain runs retain a 20-byte minimum to prevent partial-encoding bypasses. Wrapped runs beyond eight fragments or 4 KiB, splits by a space, `\r`, U+2028, across journal bullets, markdown or HTML comments, `\u` escapes, and hex encoding remain out of scope. These are conservative detectors, not a guarantee against arbitrary encryption or steganography. + +Known gaps, accepted for phase 1: JWE (five-part) tokens and JWT headers over 2048 characters; multi-part secrets (JWT, PEM header, URL password, `KEY=value`) with only some bytes percent-encoded; lookalikes outside the folding table (for example Greek `ρ`, Cyrillic `І`/`Ј`/`к`, small caps, U+2800, combining marks inside an anchor). ### Owner-signed purge (gap 5) From 7df9d3f5a0bd7665fd8c7b7d8bba0b0d6839287e Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Sun, 8 Nov 2026 16:11:34 +0530 Subject: [PATCH 7/7] docs(kin): correct JWT header limit; note path-list work limit Co-Authored-By: Claude Opus 5.5 --- docs/kin/specs/triggers.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/kin/specs/triggers.md b/docs/kin/specs/triggers.md index 1b1a3f7917edc..f2f928008be37 100644 --- a/docs/kin/specs/triggers.md +++ b/docs/kin/specs/triggers.md @@ -346,7 +346,7 @@ GitHub then blocks a bad auto-merge even if Kin's classifier is wrong. This forc The scanner checks paths as well as content, redacts paths that contain a finding, normalizes NFKC, common Cyrillic/Greek/Latin lookalikes and Default Ignorable code points, checks newline-wrapped tokens, and decodes URL/base64 content up to two levels. Base64 decoding tries all four alignments; malformed percent bytes and non-ASCII decoded bytes become separators. Prefix rules use alphanumeric left boundaries and body-specific right boundaries (fixed-width Google keys also tolerate underscore metadata suffixes); each precompiled regex sees at most 8192 characters plus one character of boundary context on each side, with 4096-character overlap. JWT detection matches a bounded prefix: an eyJ header followed by 5–2048 base64url characters, then an eyJ payload prefix followed by 8–1024 characters, without requiring the signature or the payload end. Its maximum match is 3080 characters, within the overlap. Wrapped tokens are inspected after each appended fragment once the joined run has 20 characters or an anchor, retaining up to eight fragments and 4 KiB. A prose line closes the prior run with its first word and opens a new run with its last word, so adjacent prose and trailing words do not hide a wrapped token. Merge scanning covers only bytes changed from the fetched remote plus the rebuilt `INDEX.md`, not pre-existing remote content. Scanner failures notify through the writer hook, just like findings. Staged YAML parser errors omit source-bearing diagnostics and filesystem path errors omit the underlying path-bearing cause. Merge blocks use the recovery hook; production owner delivery still waits on a notifier. It fails closed above 1 MiB per file, 8 MiB or 1024 files per call, 4096 allowlist entries per organization, or 64 MiB of weighted work (inspected characters, encoded candidate characters, window characters, and 64 units per regex match). A cheap anchor pre-filter skips rule execution on irrelevant windows, including decoded candidates; recursive decoding remains active to find double-encoded tokens. Scanning yields between windows and decoder candidates so multi-file batches do not monopolize the event loop. Base64 runs shorter than 27 characters and fully percent-encoded runs shorter than 60 characters are not decoded; mixed percent/plain runs retain a 20-byte minimum to prevent partial-encoding bypasses. Wrapped runs beyond eight fragments or 4 KiB, splits by a space, `\r`, U+2028, across journal bullets, markdown or HTML comments, `\u` escapes, and hex encoding remain out of scope. These are conservative detectors, not a guarantee against arbitrary encryption or steganography. -Known gaps, accepted for phase 1: JWE (five-part) tokens and JWT headers over 2048 characters; multi-part secrets (JWT, PEM header, URL password, `KEY=value`) with only some bytes percent-encoded; lookalikes outside the folding table (for example Greek `ρ`, Cyrillic `І`/`Ј`/`к`, small caps, U+2800, combining marks inside an anchor). +Known gaps, accepted for phase 1: JWE (five-part) tokens and JWT headers longer than 2051 characters; multi-part secrets (JWT, PEM header, URL password, `KEY=value`) with only some bytes percent-encoded; lookalikes outside the folding table (for example Greek `ρ`, Cyrillic `І`/`Ј`/`к`, small caps, U+2800, combining marks inside an anchor). A file of about 1 MiB made only of long whitespace-free lines (such as one path per line) exhausts the work budget after about 15 s and fails closed, blocking that commit; realistic journals up to 1 MiB pass. ### Owner-signed purge (gap 5)