Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions apps/server/src/kin/lead/LeadPrompt.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,9 @@ const memory = Layer.succeed(
anchor: () => Effect.succeed("fixture"),
record: () => Effect.void,
recover: () => Effect.void,
withHistoryRewrite: () => Effect.die("Unused"),
purgeInFlight: () => Effect.die("Unused"),
historyGeneration: () => Effect.die("Unused"),
readPromptSnapshot: (org, head) =>
org !== orgId || !["head-one", "head-two"].includes(head)
? Effect.fail(new Memory.MemoryError({ orgId: org, operation: "promptSnapshot" }))
Expand Down
3 changes: 3 additions & 0 deletions apps/server/src/kin/lead/layer.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,9 @@ vi.mock("../memory/Memory.ts", async (importOriginal) => {
anchor: () => Effect.die("unused"),
record: () => Effect.die("unused"),
recover: () => Effect.die("unused"),
withHistoryRewrite: () => Effect.die("unused"),
purgeInFlight: () => Effect.die("unused"),
historyGeneration: () => Effect.die("unused"),
ensureCheckouts: () => Effect.succeed({ writer: "/private/tmp", read: "/private/tmp" }),
prepareWakeup: () =>
Effect.sync(() => {
Expand Down
121 changes: 111 additions & 10 deletions apps/server/src/kin/memory/Memory.ts
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,13 @@ export class MemorySyncHooks extends Context.Reference<{
defaultValue: () => ({ syncItems: () => Effect.void }),
}) {}

export interface MemoryHistoryRewrite {
/** Writes the durable marker, then deletes the writer and read checkouts that hold the old history. */
readonly quarantine: (marker: string) => Effect.Effect<void, MemoryFailure>;
/** Clones fresh, records the remote heads without preserving old anchors, rebuilds, then clears the marker. */
readonly reclone: () => Effect.Effect<void, MemoryFailure>;
}

export class Memory extends Context.Service<
Memory,
{
Expand All @@ -162,6 +169,15 @@ export class Memory extends Context.Service<
head: string,
) => Effect.Effect<void, MemoryFailure>;
readonly recover: (orgId: KinOrgId) => Effect.Effect<void, MemoryFailure>;
/** Server-only, for `KinPurge`: runs `body` under the writer lock with the checkout operations a history rewrite needs. */
readonly withHistoryRewrite: <A, E>(
orgId: KinOrgId,
body: (rewrite: MemoryHistoryRewrite) => Effect.Effect<A, E>,
) => Effect.Effect<A, E>;
/** The durable marker of an unfinished history rewrite; while present, nothing pushes old anchors. */
readonly purgeInFlight: (orgId: KinOrgId) => Effect.Effect<string | undefined, MemoryFailure>;
/** Bumped whenever a history rewrite drops the checkouts; work staged under an older value is stale. */
readonly historyGeneration: (orgId: KinOrgId) => Effect.Effect<number>;
readonly ensureCheckouts: (orgId: KinOrgId) => Effect.Effect<MemoryPaths, MemoryFailure>;
readonly prepareWakeup: (
orgId: KinOrgId,
Expand Down Expand Up @@ -314,6 +330,7 @@ const make = Effect.gen(function* () {
const cache = yield* MemoryCache.MemoryCache;
const locks = new Map<KinOrgId, Semaphore.Semaphore>();
const initializedCheckouts = new Set<KinOrgId>();
const historyGenerations = new Map<KinOrgId, number>();
const duplicateWarnings = new Map<KinOrgId, ReadonlySet<string>>();
const snapshots = new Map<KinOrgId, Snapshot>();
const blobCaches = new Map<KinOrgId, BlobCache>();
Expand Down Expand Up @@ -438,6 +455,37 @@ const make = Effect.gen(function* () {
return { repo, root, writer, read };
});

const markerPath = (root: string) => path.join(root, "purge-in-flight");
/** Half-finished clones from a crash; after a purge they can hold the old history. */
const removeStaleClones = Effect.fn("Memory.removeStaleClones")(function* (
orgId: KinOrgId,
root: string,
) {
for (const entry of yield* io(orgId, "clones", fs.readDirectory(root)))
if (/^memory\.clone-[A-Za-z0-9]+$/.test(entry))
yield* io(
orgId,
"recoverClone",
fs.remove(path.join(root, entry), { recursive: true, force: true }),
);
});
const purgeInFlight = Effect.fn("Memory.purgeInFlight")(function* (orgId: KinOrgId) {
const { root } = yield* pathsFor(orgId);
if (!(yield* io(orgId, "purgeMarker", fs.exists(markerPath(root))))) return undefined;
return yield* io(orgId, "purgeMarker", fs.readFileString(markerPath(root)));
});
const refuseDuringPurge = Effect.fn("Memory.refuseDuringPurge")(function* (
orgId: KinOrgId,
operation: string,
) {
if ((yield* purgeInFlight(orgId)) !== undefined)
return yield* new MemorySyncConflict({
orgId,
operation,
reason: "A memory purge is in flight; old history must not be pushed",
});
});

const walkAncestors = (...args: Parameters<typeof ancestors>) =>
ancestors(...args).pipe(
Effect.provideService(FileSystem.FileSystem, fs),
Expand Down Expand Up @@ -504,14 +552,9 @@ const make = Effect.gen(function* () {
if (yield* io(orgId, "exists", fs.exists(paths.writer))) {
yield* validateWriter(orgId, paths, false);
} else {
for (const entry of yield* io(orgId, "clones", fs.readDirectory(paths.root))) {
if (/^memory\.clone-[A-Za-z0-9]+$/.test(entry))
yield* io(
orgId,
"recoverClone",
fs.remove(path.join(paths.root, entry), { recursive: true, force: true }),
);
}
// The remote can still hold the old history until the purge's push lands.
yield* refuseDuringPurge(orgId, "clone");
yield* removeStaleClones(orgId, paths.root);
yield* io(
orgId,
"clone",
Expand Down Expand Up @@ -845,6 +888,7 @@ const make = Effect.gen(function* () {
head: string,
wakeupId?: string,
) {
yield* refuseDuringPurge(orgId, "preserve");
const now = yield* Clock.currentTimeMillis;
const id = wakeupId ?? `recovery-${now}-${head.slice(0, 12)}`;
if (!/^[a-zA-Z0-9_-]+$/.test(id) || !/^[a-zA-Z0-9_-]+$/.test(orgId))
Expand Down Expand Up @@ -883,10 +927,11 @@ const make = Effect.gen(function* () {
return branch;
});

const recover = Effect.fn("Memory.recover")(function* (orgId: KinOrgId) {
const recover = Effect.fn("Memory.recover")(function* (orgId: KinOrgId, reclone = false) {
const paths = yield* pathsFor(orgId);
yield* io(orgId, "mkdir", fs.makeDirectory(paths.root, { recursive: true }));
const exists = yield* io(orgId, "exists", fs.exists(paths.writer));
if (!exists && !reclone) yield* refuseDuringPurge(orgId, "clone");
if (!exists)
yield* git(orgId, paths.root, [
"clone",
Expand Down Expand Up @@ -1242,13 +1287,15 @@ const make = Effect.gen(function* () {
}
}
if (next !== head) {
if (next !== remote)
if (next !== remote) {
yield* refuseDuringPurge(orgId, "merge");
yield* git(orgId, paths.writer, [
"push",
"--",
repo.repoUrl,
`${next}:refs/heads/${branch}`,
]);
}
yield* git(orgId, paths.writer, ["update-ref", `refs/heads/${branch}`, next, head]);
yield* record(orgId, branch, head, next);
head = next;
Expand Down Expand Up @@ -1289,8 +1336,62 @@ const make = Effect.gen(function* () {
return { head, indexAsOf: DateTime.toUtc(indexAsOf) };
});

const dropCheckouts = Effect.fn("Memory.dropCheckouts")(function* (orgId: KinOrgId) {
const paths = yield* pathsFor(orgId);
historyGenerations.set(orgId, (historyGenerations.get(orgId) ?? 0) + 1);
// `dreams/` holds plain copies of memory files for a running dream.
for (const directory of [paths.writer, paths.read, path.join(paths.root, "dreams")])
yield* io(orgId, "quarantine", fs.remove(directory, { recursive: true, force: true }));
if (yield* io(orgId, "exists", fs.exists(paths.root)))
yield* removeStaleClones(orgId, paths.root);
initializedCheckouts.delete(orgId);
pinnedHeads.delete(orgId);
snapshots.delete(orgId);
blobCaches.delete(orgId);
return paths;
});
const historyRewrite = (orgId: KinOrgId): MemoryHistoryRewrite => ({
quarantine: Effect.fn("Memory.quarantine")((marker: string) =>
locked(
orgId,
Effect.gen(function* () {
const { root } = yield* pathsFor(orgId);
yield* io(orgId, "mkdir", fs.makeDirectory(root, { recursive: true }));
yield* io(orgId, "purgeMarker", fs.writeFileString(`${markerPath(root)}.tmp`, marker));
yield* io(orgId, "purgeMarker", fs.rename(`${markerPath(root)}.tmp`, markerPath(root)));
yield* dropCheckouts(orgId);
}),
),
),
reclone: Effect.fn("Memory.reclone")(() =>
locked(
orgId,
Effect.gen(function* () {
const { root } = yield* dropCheckouts(orgId);
// With no writer checkout, recover clones fresh and records the rewritten heads.
yield* recover(orgId, true);
yield* prepare(orgId);
yield* rebuildIndex(orgId);
yield* io(orgId, "purgeMarker", fs.remove(markerPath(root), { force: true }));
}),
),
),
});

const withHistoryRewrite = <A, E>(
orgId: KinOrgId,
body: (rewrite: MemoryHistoryRewrite) => Effect.Effect<A, E>,
) => withWriterLock(orgId, body(historyRewrite(orgId)));

return Memory.of({
withWriterLock,
withHistoryRewrite,
purgeInFlight: Effect.fn("Memory.purgeInFlightLocked")((orgId) =>
locked(orgId, purgeInFlight(orgId)),
),
historyGeneration: Effect.fn("Memory.historyGeneration")((orgId) =>
Effect.sync(() => historyGenerations.get(orgId) ?? 0),
),
recover: Effect.fn("Memory.recoverLocked")((orgId) =>
withWriterLock(orgId, locked(orgId, recover(orgId))),
),
Expand Down
38 changes: 38 additions & 0 deletions apps/server/src/kin/memory/MemoryWriter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,7 @@ const make = Effect.gen(function* () {
KinOrgId,
{
wakeupId: string;
generation: number;
files: Map<string, string>;
lines: string[];
supersessions: Map<string, string>;
Expand Down Expand Up @@ -307,8 +308,19 @@ const make = Effect.gen(function* () {
const previous = sessions.get(orgId);
if (previous && previous.wakeupId !== wakeupId)
return yield* new MemoryWriteRejected({ reason: "Another wake-up owns the writer" });
const generation = yield* memory.historyGeneration(orgId);
// A purge rewrote history after this wake-up staged; committing it could restore purged content.
if (previous && previous.generation !== generation) {
sessions.delete(orgId);
return yield* new MemorySyncConflict({
orgId,
operation: "push",
reason: "Memory history was rewritten during this wake-up; its staged changes were dropped",
});
}
const state = previous ?? {
wakeupId,
generation,
files: new Map<string, string>(),
lines: [],
supersessions: new Map<string, string>(),
Expand Down Expand Up @@ -377,6 +389,17 @@ const make = Effect.gen(function* () {
const paths = yield* memory.ensureCheckouts(orgId);
return paths;
});
// Every caller holds the writer lock, and the marker only changes under it.
const refuseDuringPurge = Effect.fn("MemoryWriter.refuseDuringPurge")(function* (
orgId: KinOrgId,
) {
if ((yield* memory.purgeInFlight(orgId)) !== undefined)
return yield* new MemorySyncConflict({
orgId,
operation: "push",
reason: "A memory purge is in flight; old history must not be pushed",
});
});
const report = <A>(orgId: KinOrgId, wakeupId: string, effect: Effect.Effect<A, Failure>) =>
effect.pipe(
Effect.catchTags({
Expand Down Expand Up @@ -770,6 +793,7 @@ const make = Effect.gen(function* () {
Effect.gen(function* () {
const { ctx, state } = yield* session(orgId, wakeupId);
const { writer } = yield* validate(orgId);
yield* refuseDuringPurge(orgId);
const repo = config.repositories.find((entry) => entry.orgId === orgId);
if (!repo)
return yield* new MemoryConfigError({
Expand Down Expand Up @@ -899,7 +923,20 @@ const make = Effect.gen(function* () {
if (!dreamBranch.test(input.branch))
return yield* new MemoryWriteRejected({ reason: "Invalid dream branch" });
const { writer } = yield* validate(orgId);
yield* refuseDuringPurge(orgId);
const repo = yield* repoFor(orgId);
// A purge's reclone drops the old history, so a dream started before it has no base here.
if (
!/^[a-f0-9]{40}([a-f0-9]{24})?$/.test(input.base) ||
(yield* git(orgId, writer, ["cat-file", "-e", `${input.base}^{commit}`]).pipe(
Effect.result,
))._tag === "Failure"
)
return yield* new MemorySyncConflict({
orgId,
operation: "push",
reason: "The dream's base is no longer in memory history",
});
const baseFiles = yield* filesAt(orgId, writer, input.base);
const regular = yield* regularPaths(orgId, writer, input.base);
const combined = new Map(
Expand Down Expand Up @@ -964,6 +1001,7 @@ const make = Effect.gen(function* () {
if (!dreamBranch.test(branch))
return yield* new MemoryWriteRejected({ reason: "Invalid dream branch" });
const { writer } = yield* validate(orgId);
yield* refuseDuringPurge(orgId);
const repo = yield* repoFor(orgId);
const advertised = yield* git(orgId, writer, [
"ls-remote",
Expand Down
4 changes: 2 additions & 2 deletions apps/server/src/kin/memory/objects.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import { MemoryCheckoutInvalid } from "./Memory.ts";
export const readObjects = Effect.fnUntraced(function* (
orgId: KinOrgId,
cwd: string,
requests: ReadonlyArray<{ sha: string; type: "blob" | "tree" | "commit" }>,
requests: ReadonlyArray<{ sha: string; type: "blob" | "tree" | "commit" | "tag" }>,
) {
const invalid = () =>
new MemoryCheckoutInvalid({
Expand All @@ -34,7 +34,7 @@ export const readObjects = Effect.fnUntraced(function* (
for (const { sha, type } of requests) {
const end = buffer.indexOf(10, offset);
const header = buffer.subarray(offset, end).toString("ascii");
const match = /^(\w+) (blob|tree|commit) (\d+)$/.exec(header);
const match = /^(\w+) (blob|tree|commit|tag) (\d+)$/.exec(header);
if (end < 0 || !match || match[1] !== sha || match[2] !== type) return yield* invalid();
const length = Number(match[3]);
offset = end + 1;
Expand Down
Loading