Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 42 additions & 3 deletions apps/server/src/kin/memory/Memory.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -858,6 +858,10 @@ it.live("uses filesystem identity for a case-mismatched base without recloning",

it.live.each([
"diverged",
"secret merge",
"scan failure merge",
"remote secret merge",
"large remote merge",
"conflict",
"network down",
"dirty writer",
Expand Down Expand Up @@ -932,7 +936,11 @@ it.live.each([
} else
yield* fs.writeFileString(
`${writer}/rules.md`,
scenario === "H2-merge-swap" ? h2Local : "Local invented rules\n",
scenario === "H2-merge-swap"
? h2Local
: scenario === "secret merge"
? ["gh", "p_", "a".repeat(36)].join("")
: "Local invented rules\n",
);
let local = yield* commit(writer);
if (scenario === "push retry") {
Expand All @@ -952,13 +960,20 @@ it.live.each([
}
yield* memory.record(orgA, "main", initial.head, local);
yield* fs.writeFileString(
`${source}/${scenario === "push retry" || scenario === "diverged" || scenario === "network down" || scenario === "dirty writer" || scenario === "H2-merge-swap" ? "remote-only" : "rules.md"}`,
`${source}/${scenario === "push retry" || scenario === "diverged" || scenario === "scan failure merge" || scenario === "secret merge" || scenario === "remote secret merge" || scenario === "large remote merge" || scenario === "network down" || scenario === "dirty writer" || scenario === "H2-merge-swap" ? "remote-only" : "rules.md"}`,
"Remote invented edit\n",
);
if (scenario === "journal conflict" || scenario === "push retry") {
yield* fs.writeFileString(`${source}/journal/day.md`, "remote entry\n");
yield* fs.writeFileString(`${source}/INDEX.md`, "remote generated index\n");
}
if (scenario === "remote secret merge")
yield* fs.writeFileString(`${source}/remote-only`, ["gh", "p_", "a".repeat(36)].join(""));
if (scenario === "large remote merge") {
yield* fs.makeDirectory(`${source}/archive`);
for (let i = 0; i < 1030; i++)
yield* fs.writeFileString(`${source}/archive/${i}.txt`, "clean");
}
if (scenario === "H2-merge-swap")
yield* fs.writeFileString(`${source}/rules.md`, h2Base.replace("base tail", "remote tail"));
yield* commit(source);
Expand Down Expand Up @@ -1004,6 +1019,10 @@ it.live.each([
yield* memory.record(orgA, "main", local, replacement);
local = replacement;
}
if (scenario === "scan failure merge") {
const sql = yield* SqlClient.SqlClient;
yield* sql`DROP TABLE kin_secret_allow`;
}
const result = yield* memory.prepareWakeup(orgA, "fixture-wakeup").pipe(Effect.result);
if (scenario === "unrelated" || scenario === "criss-cross") {
expect(result._tag).toBe("Failure");
Expand Down Expand Up @@ -1038,6 +1057,22 @@ it.live.each([
expect(
(yield* memory.prepareWakeup(orgA, "fixture-wakeup-3").pipe(Effect.result))._tag,
).toBe("Success");
} else if (scenario === "scan failure merge") {
expect(result._tag).toBe("Failure");
expect(
messages.some((message) => message.includes("Secret scan failed: Allowlist unavailable")),
).toBe(true);
expect(yield* memory.anchor(orgA, "main")).toBe(local);
} else if (scenario === "secret merge") {
expect(result._tag).toBe("Failure");
if (result._tag === "Failure") {
expect(result.failure._tag).toBe("MemoryError");
expect(result.failure.operation).toBe("secretScan");
}
expect(yield* memory.anchor(orgA, "main")).toBe(local);
expect(yield* git(root, [`--git-dir=${remote}`, "rev-parse", "main"])).toBe(
yield* git(source, ["rev-parse", "HEAD"]),
);
} else if (scenario === "push retry") {
expect(result._tag).toBe("Success");
if (result._tag === "Success") {
Expand All @@ -1063,6 +1098,8 @@ it.live.each([
} else if (
scenario === "network down" ||
scenario === "diverged" ||
scenario === "remote secret merge" ||
scenario === "large remote merge" ||
scenario === "dirty writer"
) {
expect(result._tag).toBe("Success");
Expand All @@ -1072,7 +1109,9 @@ it.live.each([
expect(result.success.head).not.toBe(local);
expect(yield* memory.readRules(orgA)).toBe("Local invented rules\n");
expect(yield* git(writer, ["show", `${result.success.head}:remote-only`])).toBe(
"Remote invented edit",
scenario === "remote secret merge"
? ["gh", "p_", "a".repeat(36)].join("")
: "Remote invented edit",
);
}
}
Expand Down
38 changes: 37 additions & 1 deletion apps/server/src/kin/memory/Memory.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import * as SecretScan from "./secretScan.ts";
import {
readTree,
readObjects,
Expand Down Expand Up @@ -203,6 +204,7 @@ interface BlobCache {
}

const make = Effect.gen(function* () {
const scanner = yield* SecretScan.KinSecretScan;
const config = yield* MemoryConfig;
const fs = yield* FileSystem.FileSystem;
const { notify } = yield* MemoryRecoveryHooks;
Expand Down Expand Up @@ -1079,6 +1081,7 @@ const make = Effect.gen(function* () {
});
} else {
const { files: mergedFiles, modes } = merged;
const remoteOriginal = new Map(remoteFiles);
remoteFiles.clear();
for (const [file, bytes] of mergedFiles) remoteFiles.set(file, bytes);
const heads = [];
Expand All @@ -1098,6 +1101,36 @@ const make = Effect.gen(function* () {
"INDEX.md",
Buffer.from(buildIndex(heads, reads, yield* Clock.currentTimeMillis, zone)),
);
const findings = yield* scanner
.scan({
orgId,
files: [...remoteFiles]
.filter(
([path, bytes]) =>
path === "INDEX.md" || !remoteOriginal.get(path)?.equals(bytes),
)
.map(([path, content]) => ({ path, content })),
})
.pipe(
Effect.tapError((cause) =>
notify({ orgId, branch, reason: `Secret scan failed: ${cause.reason}` }),
),
Effect.mapError(
(cause) => new MemoryError({ orgId, operation: "secretScan", cause }),
),
);
if (findings.length) {
yield* notify({
orgId,
branch,
reason: `Secret scan blocked merge: ${findings[0]!.ruleId} in ${findings[0]!.pathHit ? "[redacted]" : (findings[0]!.itemPath ?? findings[0]!.path)}`,
});
return yield* new MemoryError({
orgId,
operation: "secretScan",
reason: `Secret scan blocked merge: ${findings[0]!.ruleId} in ${findings[0]!.pathHit ? "[redacted]" : (findings[0]!.itemPath ?? findings[0]!.path)}`,
});
}
tree = yield* Effect.scoped(
Effect.gen(function* () {
const temp = yield* io(orgId, "journalMerge", fs.makeTempDirectoryScoped());
Expand Down Expand Up @@ -1286,4 +1319,7 @@ const make = Effect.gen(function* () {
});
});

export const layer = Layer.effect(Memory, make).pipe(Layer.provide(MemoryCache.layer));
export const layer = Layer.effect(Memory, make).pipe(
Layer.provide(MemoryCache.layer),
Layer.provide(SecretScan.layer),
);
63 changes: 59 additions & 4 deletions apps/server/src/kin/memory/MemoryWriter.test.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import * as SqlClient from "effect/sql/SqlClient";
import { memoryGit } from "./git.ts";
import * as NodeZlib from "node:zlib";
import * as NodeServices from "@effect/platform-node/NodeServices";
Expand All @@ -23,6 +24,8 @@ it.live.each([
"completed",
"failed",
"blocked",
"scan unavailable",
"scan limit",
"forged",
"conflict",
"B1-shadow",
Expand Down Expand Up @@ -223,10 +226,7 @@ it.live.each([
}),
),
});
const scan =
scenario === "blocked"
? Scan.testLayer([{ path: "claims/new.md", ruleId: "fixture", line: 1 }])
: Scan.noFindingsStubLayer;
const scan = Scan.layer;
const layer = Writer.layer.pipe(
Layer.provideMerge(memoryLayer),
Layer.provide(Layer.succeed(Memory.MemoryRecoveryHooks, { notify: () => Effect.void })),
Expand Down Expand Up @@ -600,10 +600,52 @@ it.live.each([
}
if (scenario === "tree-substitution")
substituteTree = yield* git(paths.writer, ["rev-parse", `${original}^{tree}`]);
if (scenario === "blocked") {
const tokenPath = "claims/" + ["gh", "p_", "a".repeat(36)].join("") + ".md";
const malformed = yield* writer
.stage(orgId, "wake1", {
path: tokenPath,
content: "---\nid: [\n" + tokenPath + ": [\n---\nbody",
})
.pipe(Effect.result);
expect(malformed._tag).toBe("Failure");
if (malformed._tag === "Failure") {
expect(malformed.failure._tag).toBe("MemoryItemError");
expect(JSON.stringify(malformed.failure).includes(tokenPath.slice(7, -3))).toBe(false);
if (malformed.failure._tag === "MemoryItemError")
expect(malformed.failure.path).toBe("[redacted]");
}
const inaccessible = yield* writer
.stage(orgId, "wake1", {
path: tokenPath.slice(0, -3) + "_".repeat(260) + ".md",
content: "missing front matter",
})
.pipe(Effect.result);
expect(inaccessible._tag).toBe("Failure");
if (inaccessible._tag === "Failure")
expect(JSON.stringify(inaccessible.failure).includes(tokenPath.slice(7, -3))).toBe(false);
}
if (scenario === "blocked")
yield* writer.appendJournal(orgId, "wake1", [
["gh", "p_"].join("") + "aB3dE6gH9jK2mN5pQ8sT1vW4yZ7cF0iL3oR6",
]);
if (scenario === "scan unavailable") {
const sql = yield* SqlClient.SqlClient;
yield* sql`DROP TABLE kin_secret_allow`;
}
if (scenario === "scan limit")
yield* writer.appendJournal(orgId, "wake1", ["a".repeat(1024 * 1024)]);
const result = yield* writer
.commitWakeup(orgId, "wake1", scenario === "failed" ? "failed" : "completed")
.pipe(Effect.result);
if (result._tag === "Failure") yield* Effect.logError(result.failure);
if (scenario === "scan unavailable" || scenario === "scan limit") {
expect(result._tag).toBe("Failure");
expect(notices).toHaveLength(1);
expect(notices[0]!.error._tag).toBe("KinSecretScanError");
expect(yield* git(paths.writer, ["rev-parse", "HEAD"])).toBe(original);
return;
}
if (scenario === "tree-substitution") {
expect(result._tag).toBe("Failure");
if (result._tag === "Failure") expect(result.failure._tag).toBe("MemoryCheckoutInvalid");
Expand All @@ -618,6 +660,19 @@ it.live.each([
);

expect(notices).toHaveLength(1);
if (scenario === "blocked") {
expect(JSON.stringify(notices)).not.toContain("aB3dE6gH9jK2mN5pQ8sT1vW4yZ7cF0iL3oR6");
expect(yield* git(paths.writer, ["rev-parse", "HEAD"])).toBe(original);
expect(
(yield* git(root, [
`--git-dir=${remote}`,
"show-ref",
"--verify",
`refs/heads/kin/shadow/${agentId}`,
]).pipe(Effect.result))._tag,
).toBe("Failure");
expect(yield* fs.exists(`${paths.writer}/claims/new.md`)).toBe(false);
}
expect(yield* fs.exists(`${paths.writer}/.git/rebase-merge`)).toBe(false);
} else {
expect(result._tag).toBe("Success");
Expand Down
37 changes: 33 additions & 4 deletions apps/server/src/kin/memory/MemoryWriter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -61,14 +61,24 @@ export class MemorySecretBlocked extends Schema.TaggedError<MemorySecretBlocked>
{
orgId: Schema.String,
findings: Schema.Array(
Schema.Struct({ path: Schema.String, ruleId: Schema.String, line: Schema.Number }),
Schema.Struct({
path: Schema.String,
ruleId: Schema.String,
line: Schema.Number,
pathHit: Schema.optional(Schema.Boolean),
itemPath: Schema.optional(Schema.String),
}),
),
},
) {}
export type WriterNotice = {
readonly orgId: KinOrgId;
readonly wakeupId: string;
readonly error: MemoryCheckoutInvalid | MemorySyncConflict | MemorySecretBlocked;
readonly error:
| MemoryCheckoutInvalid
| MemorySyncConflict
| MemorySecretBlocked
| KinSecretScanError;
};
export class MemoryWriterHooks extends Context.Service<
MemoryWriterHooks,
Expand Down Expand Up @@ -288,7 +298,11 @@ const make = Effect.gen(function* () {
const result = yield* fs.lstat(current).pipe(Effect.result);
if (result._tag === "Failure") {
if (result.failure.reason._tag !== "NotFound")
return yield* new MemoryError({ orgId, operation: "path", cause: result.failure });
return yield* new MemoryError({
orgId,
operation: "path",
cause: "Unable to inspect memory path",
});
} else if (
result.success.type === "SymbolicLink" ||
(result.success.type === "File" &&
Expand All @@ -306,10 +320,25 @@ const make = Effect.gen(function* () {
});
const report = <A>(orgId: KinOrgId, wakeupId: string, effect: Effect.Effect<A, Failure>) =>
effect.pipe(
Effect.catchTags({
MemoryItemError: (error) =>
scanner.scan({ orgId, files: [{ path: error.path, content: new Uint8Array() }] }).pipe(
Effect.flatMap((findings) =>
Effect.fail(
new MemoryItemError({
orgId: error.orgId,
path: findings.length ? "[redacted]" : error.path,
reason: error.reason,
}),
),
),
),
}),
Effect.tapError((error) =>
error._tag === "MemoryCheckoutInvalid" ||
error._tag === "MemorySyncConflict" ||
error._tag === "MemorySecretBlocked"
error._tag === "MemorySecretBlocked" ||
error._tag === "KinSecretScanError"
? hooks
.notify({ orgId, wakeupId, error })
.pipe(
Expand Down
8 changes: 6 additions & 2 deletions apps/server/src/kin/memory/frontMatter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -89,8 +89,12 @@ export const parseFrontMatter = Effect.fnUntraced(function* (
new MemoryItemError({
orgId,
path,
reason: cause instanceof Error ? cause.message : "Malformed YAML",
cause,
reason: staged
? "Malformed YAML front matter"
: cause instanceof Error
? cause.message
: "Malformed YAML",
...(staged ? {} : { cause }),
}),
});
const decoded = yield* (staged ? decodeStaged(value) : decodeFrontMatter(value)).pipe(
Expand Down
Loading