diff --git a/.gitignore b/.gitignore index 18353491f33..5b23066efec 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ projects/ state/ data/ -scratchpad/ +scratchpad* .no-mistakes/ .atelier/ .fm-secondmate-home diff --git a/.pi/extensions/fm-branch-supervision.ts b/.pi/extensions/fm-branch-supervision.ts new file mode 100644 index 00000000000..eaffba80e71 --- /dev/null +++ b/.pi/extensions/fm-branch-supervision.ts @@ -0,0 +1,738 @@ +// Firstmate supervision branch for Pi (docs/pi-supervision-branch.md). +// +// A persistent second AgentSession - the supervision BRANCH - inside the same +// pi process as the captain's MAIN session. The watcher extension offers each +// actionable wake here (lib/fm-branch-dispatch.ts); the branch handles it with +// real tools and reports through the fm_branch_report custom tool, which +// writes the durable outcome store FIRST (bin/fm-branch-outcome.sh) and then +// merges an append-only note to main's tail. Main's captain/assistant dialog +// is mirrored into the branch as read-only fm-main-mirror context at main's +// turn_end. Pi-only by construction: this file lives in .pi/extensions, so no +// other harness ever loads it. Supervision is default-on for every task once +// this Pi session owns the fleet lock: no captain grant file is required. +// Away mode (or a broken branch) keeps today's wake-to-main behavior +// untouched regardless. +// +// Prefix stability (the cache contract, owner: bin/fm-branch-prompt.sh +// header): the branch's system prompt is the generator's byte-stable output, +// the tool set is BRANCH_TOOL_NAMES in that fixed order on every spawn, and +// one shared per-home prompt_cache_key is set for branch requests in a +// before_provider_request hook - main keeps Pi's default per-session key. +// Wakes, mirrored dialog, and merge notes are all appends at a tail. +// +// Session-lock ownership: every branch side-effect boundary re-evaluates the +// current extension generation and lock ownership LAZILY, the same way the +// watcher extension evaluates ownership at arm time. A cold +// Pi start acquires the lock only when the session runs fm-session-start.sh, +// so latching ownership once at session_start would leave the branch inert +// for the whole process; and a secondary read-only Pi session that never owns +// the lock must never write markers, clean leases, or accept wakes. +// +// Failure direction: every path that cannot reach a working branch falls back +// to delivering the wake to MAIN exactly as before the branch existed - a +// broken branch degrades to today's behavior, never to a lost wake. The wake +// queue itself stays durable until the handler runs the drain's +// acknowledgement, so a branch that dies mid-handling re-presents its rows at +// the next drain exactly as a mid-handling main crash always has. +// +// Threat model (captain-decided): the branch's actor identity is +// CONFUSED-AGENT-GRADE - deterministic spawnHook env injection plus a +// readonly-variable shell prelude so an accidental override fails loudly +// inside the branch's own shell. bin/fm-lease-lib.sh documents the grade and +// its deliberate limits. +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { + createAgentSession, + createBashToolDefinition, + DefaultResourceLoader, + getAgentDir, + SessionManager, + type AgentSession, + type ExtensionAPI, + type ToolDefinition, +} from "@earendil-works/pi-coding-agent"; +import { Text } from "@earendil-works/pi-tui"; +import { Type } from "typebox"; +import { + FM_BRANCH_DISPATCH_EVENT, + scopeForUnreadWake, + type BranchDispatchOffer, +} from "./lib/fm-branch-dispatch.ts"; +import { encodeFirstmateOperationalInput } from "./lib/fm-operational-input.ts"; + +const extensionFile = fileURLToPath(import.meta.url); +const extensionDir = dirname(extensionFile); +const root = resolve(extensionDir, "../.."); +const fmHome = process.env.FM_HOME || process.env.FM_ROOT_OVERRIDE || root; +const fmRoot = process.env.FM_ROOT_OVERRIDE || root; +const state = process.env.FM_STATE_OVERRIDE || `${fmHome}/state`; +const config = process.env.FM_CONFIG_OVERRIDE || `${fmHome}/config`; +const afkFlag = join(state, ".afk"); +const sessionsDir = join(state, "branch-session"); +const sessionPointer = join(state, ".branch-session"); +const mirrorCursorFile = join(state, ".branch-mirror-cursor"); +const promptScript = join(fmRoot, "bin", "fm-branch-prompt.sh"); +const outcomeScript = join(fmRoot, "bin", "fm-branch-outcome.sh"); +const leaseScript = join(fmRoot, "bin", "fm-lease.sh"); +const loadedMarker = join(state, ".pi-branch-extension-loaded"); + +// Same tool set in the same order on every request (part of the cached +// prefix). "bash" resolves to the customTools override below, which injects +// the branch actor identity deterministically into every shell command. +const BRANCH_TOOL_NAMES = ["read", "bash", "fm_branch_report"] as const; + +// One shared prompt_cache_key per home for ALL branch sessions, derived only +// from the home path so it survives restarts; main keeps its own session key. +const branchCacheKey = `fm-branch-${createHash("sha256").update(fmHome).digest("hex").slice(0, 24)}`; + +const MIRROR_MESSAGE_CAP = 4000; +const MERGE_NOTE_BOAT = "⛵"; +type MirrorItem = { tag: "captain" | "main"; text: string }; +type MirrorCursor = { file: string; index: number }; +type Verdict = "routine" | "captain"; +type LockOwnership = "owned" | "other" | "missing"; + +const scriptEnv = { + ...process.env, + FM_HOME: fmHome, + FM_ROOT_OVERRIDE: fmRoot, + FM_STATE_OVERRIDE: state, + FM_CONFIG_OVERRIDE: config, +}; + +function offerEligible(offer: BranchDispatchOffer): boolean { + return offer.eligible === true; +} + +function afkActive(): boolean { + return existsSync(afkFlag); +} + +function parentPid(pid: string): string { + const result = spawnSync("ps", ["-o", "ppid=", "-p", pid], { encoding: "utf8" }); + if (result.status !== 0) return ""; + return result.stdout.trim(); +} + +function pidAlive(pid: string): boolean { + try { + process.kill(Number(pid), 0); + return true; + } catch { + return false; + } +} + +let ownedLockPid = ""; + +// Same ownership read as the watcher extension's lockOwnership(): the lock +// names the harness pid, and this process owns it when that pid appears in +// its own ancestry. +function lockOwnership(): LockOwnership { + ownedLockPid = ""; + let lockPid = ""; + try { + lockPid = readFileSync(`${state}/.lock`, "utf8").trim(); + } catch { + return "missing"; + } + if (!/^[0-9]+$/.test(lockPid) || lockPid === "1") return "other"; + let pid = String(process.pid); + for (let i = 0; i < 8; i += 1) { + if (pid === lockPid) { + ownedLockPid = lockPid; + return "owned"; + } + pid = parentPid(pid); + if (!pid || pid === "1") break; + } + return pidAlive(lockPid) ? "other" : "missing"; +} + +function textOfContent(content: unknown): string { + if (typeof content === "string") return content; + if (Array.isArray(content)) { + return content + .map((part) => { + const p = part as { type?: string; text?: string }; + return p && p.type === "text" && typeof p.text === "string" ? p.text : ""; + }) + .filter((piece) => piece.length > 0) + .join("\n"); + } + return ""; +} + +// Operational injections (watcher wakes, away-supervisor escalations, launch +// briefs) are fleet machinery, not captain dialog; the report's volume +// analysis counts them apart from dialog, and mirroring them would feed the +// branch its own supervision traffic back. Current injections start with the +// U+2063 operational prefix; the plain legacy form starts with FIRSTMATE. +function isOperationalUserText(text: string): boolean { + return text.startsWith("⁣") || /^FIRSTMATE[ _]/.test(text); +} + +function capMirrorText(text: string): string { + if (text.length <= MIRROR_MESSAGE_CAP) return text; + return `${text.slice(0, MIRROR_MESSAGE_CAP)}\n[mirror truncated at ${MIRROR_MESSAGE_CAP} characters]`; +} + +function readMirrorCursor(): MirrorCursor { + try { + const parsed = JSON.parse(readFileSync(mirrorCursorFile, "utf8")) as Partial; + if (typeof parsed.file === "string" && typeof parsed.index === "number" && parsed.index >= 0) { + return { file: parsed.file, index: Math.floor(parsed.index) }; + } + } catch { + // Absent or torn cursor: re-mirror the current main session from its + // start. Idempotent context, so over-mirroring is safe; dropping is not. + } + return { file: "", index: 0 }; +} + +function writeMirrorCursor(cursor: MirrorCursor): void { + mkdirSync(state, { recursive: true }); + writeFileSync(mirrorCursorFile, `${JSON.stringify(cursor)}\n`); +} + +type ReadonlyEntries = { + getSessionFile(): string | undefined; + getEntries(): Array<{ type: string }>; +}; + +// Volatile mirror-collection state. Instance-scoped and cleared at the +// session replacement boundary, so a replacement extension instance +// reconstructs EXCLUSIVELY from the durable cursor: dialog collected but not +// yet delivered re-mirrors rather than dropping (the durable cursor advances +// only in flushMirror after delivery). +type MirrorCollectionState = { + collectAnchor: MirrorCursor | null; + pendingCursor: MirrorCursor | null; +}; + +function collectMainDialog(sessionManager: ReadonlyEntries, collection: MirrorCollectionState): MirrorItem[] { + const file = sessionManager.getSessionFile() ?? ""; + const entries = sessionManager.getEntries(); + const anchor = collection.collectAnchor ?? readMirrorCursor(); + const start = anchor.file === file ? Math.min(anchor.index, entries.length) : 0; + const items: MirrorItem[] = []; + for (const entry of entries.slice(start)) { + if (entry.type !== "message") continue; + const message = (entry as { message?: { role?: string; content?: unknown } }).message; + if (!message) continue; + if (message.role !== "user" && message.role !== "assistant") continue; + const text = textOfContent(message.content).trim(); + if (!text) continue; + if (message.role === "user" && isOperationalUserText(text)) continue; + items.push({ tag: message.role === "user" ? "captain" : "main", text: capMirrorText(text) }); + } + collection.collectAnchor = { file, index: entries.length }; + collection.pendingCursor = collection.collectAnchor; + return items; +} + +export default function (pi: ExtensionAPI) { + let branch: AgentSession | null = null; + let branchBroken = ""; + let mainStreaming = false; + let shuttingDown = false; + // Bumps at every session replacement so a stale chain continuation from the + // prior generation cannot act into the new one. + let generation = 0; + // One-time per-generation activation work (marker write + stray branch + // lease cleanup); ownership itself is re-read lazily at every boundary. + let activatedGeneration = -1; + // Serializes branch work: mirror appends and wake turns run strictly in + // dispatch order, one at a time (the branch runs drain -> handle -> ack + // serially by design). + let branchChain: Promise = Promise.resolve(); + const pendingMirror: MirrorItem[] = []; + const mirrorCollection: MirrorCollectionState = { collectAnchor: null, pendingCursor: null }; + + function generationOwnsLock(expectedGeneration: number): boolean { + return !shuttingDown && expectedGeneration === generation && lockOwnership() === "owned"; + } + + function markLoaded(): void { + try { + mkdirSync(state, { recursive: true }); + writeFileSync(loadedMarker, `${process.pid}\n`); + } catch { + // Diagnostic marker only; never block activation on it. + } + } + + // A replaced branch conversation must not leave its per-task leases behind + // (the session-lock holder pid is still alive, so the sweep alone would + // keep them). One bulk release per generation, at activation. + function releaseBranchLeases(expectedGeneration: number): boolean { + if (!generationOwnsLock(expectedGeneration)) return false; + try { + const result = spawnSync("bash", [leaseScript, "release-actor", "--actor", "branch"], { + cwd: fmRoot, + encoding: "utf8", + env: { ...scriptEnv, FM_SUPERVISION_ACTOR: "branch" }, + }); + return result.status === 0; + } catch { + return false; + } + } + + // Lazy, per-action ownership evaluation (see the header). Returns true only + // when this session owns the fleet lock right now; the first true evaluation + // of a generation also writes the diagnostic marker and clears stray branch + // leases from a prior generation. + function actingAsOwner(expectedGeneration = generation): boolean { + if (!generationOwnsLock(expectedGeneration)) return false; + if (activatedGeneration !== expectedGeneration) { + if (!releaseBranchLeases(expectedGeneration)) return false; + if (!generationOwnsLock(expectedGeneration)) return false; + markLoaded(); + activatedGeneration = expectedGeneration; + } + return generationOwnsLock(expectedGeneration); + } + + function runOutcomeScript(args: string[]): { ok: boolean; stdout: string; detail: string } { + try { + const result = spawnSync("bash", [outcomeScript, ...args], { + cwd: fmRoot, + encoding: "utf8", + env: scriptEnv, + }); + if (result.status === 0) return { ok: true, stdout: (result.stdout || "").trim(), detail: "" }; + return { + ok: false, + stdout: "", + detail: `fm-branch-outcome.sh exited ${result.status ?? "none"}: ${(result.stderr || "").trim()}`, + }; + } catch (error) { + return { ok: false, stdout: "", detail: error instanceof Error ? error.message : String(error) }; + } + } + + // Append-only merge into main. The store row is already durable when this + // runs; the note is a cache of it at main's tail. Delivery modes per the + // design: routine+idle appends now with no turn, routine+busy appends after + // the captain's next prompt, captain-relevant triggers exactly one turn + // (queued as a follow-up while main is busy) - that follow-up turn is + // itself the captain-visible outcome, so the captain-facing note is + // delivered silently (display: false) rather than printed or rendered a + // second time; routine notes stay rendered except an explicitly silent + // no-change heartbeat. The read cursor advances once the note is handed to + // Pi; a crash inside Pi's + // own delivery window leaves the outcome durable in the store, where + // main's fm_branch_outcomes tool still reads it on demand. + function mergeIntoMain( + expectedGeneration: number, + seq: string, + task: string, + verdict: Verdict, + summary: string, + silent: boolean, + ): boolean { + if (!actingAsOwner(expectedGeneration)) return false; + if (verdict === "captain") { + const message = { customType: "fm-branch-merge", content: `${task}: ${summary}`, display: false }; + pi.sendMessage(message, { triggerTurn: true, deliverAs: "followUp" }); + } else { + const message = { customType: "fm-branch-merge", content: `${MERGE_NOTE_BOAT} ${task}: ${summary}`, display: !(task === "fleet" && silent) }; + if (mainStreaming) { + pi.sendMessage(message, { deliverAs: "nextTurn" }); + } else { + pi.sendMessage(message, {}); + } + } + if (/^[0-9]+$/.test(seq)) { + if (!actingAsOwner(expectedGeneration)) return false; + return runOutcomeScript(["mark-read", "--through", seq]).ok; + } + return true; + } + + function createReportTool(toolGeneration: number): ToolDefinition { + return { + name: "fm_branch_report", + label: "Report supervision outcome", + description: + "Record the outcome of one handled fleet event: write it durably to the outcome store, then merge an append-only note into the captain-facing main conversation. verdict captain surfaces it to the captain in one turn; routine notes render unless silent marks a no-change heartbeat.", + parameters: Type.Object({ + task: Type.String({ description: "The task id the event belongs to (or 'fleet' for fleet-wide events)" }), + verdict: Type.Union([Type.Literal("routine"), Type.Literal("captain")], { + description: "captain only for what a human must see; routine otherwise", + }), + summary: Type.String({ + description: + "One or two sentences in captain outcome language; include the full https:// PR URL when a PR is involved", + }), + wake: Type.Optional(Type.String({ description: "The wake reason line this outcome answers" })), + silent: Type.Optional(Type.Boolean({ + description: "True only when a fleet-wide heartbeat review found literally nothing worth reporting; omit or use false whenever any action was taken or any routine result is worth a note", + })), + }), + execute: async (_toolCallId, params) => { + const task = String((params as { task: unknown }).task || "").trim(); + const verdictRaw = String((params as { verdict: unknown }).verdict || ""); + const summary = String((params as { summary: unknown }).summary || "").trim(); + const wake = String((params as { wake?: unknown }).wake ?? "").trim(); + const silent = (params as { silent?: unknown }).silent === true; + if (!task || !summary || (verdictRaw !== "routine" && verdictRaw !== "captain") || (silent && (task !== "fleet" || verdictRaw !== "routine"))) { + return { + content: [{ type: "text", text: "invalid report: task, verdict (routine|captain), and summary are required" }], + details: undefined, + isError: true, + }; + } + const verdict = verdictRaw as Verdict; + const appendArgs = ["append", "--task", task, "--verdict", verdict, "--summary", summary, "--silent", String(silent)]; + if (wake) appendArgs.push("--wake", wake); + if (!actingAsOwner(toolGeneration)) { + return { + content: [{ type: "text", text: "report refused: supervision session was replaced or lost lock ownership" }], + details: undefined, + isError: true, + }; + } + const appended = runOutcomeScript(appendArgs); + if (!appended.ok) { + return { + content: [{ type: "text", text: `outcome store append failed (nothing merged): ${appended.detail}` }], + details: undefined, + isError: true, + }; + } + if (!mergeIntoMain(toolGeneration, appended.stdout, task, verdict, summary, silent)) { + return { + content: [{ type: "text", text: `recorded seq ${appended.stdout}, but merge refused after supervision replacement or lock loss` }], + details: undefined, + isError: true, + }; + } + return { + content: [{ type: "text", text: `recorded seq ${appended.stdout} and merged [${verdict}] into main` }], + details: undefined, + }; + }, + }; + } + + async function createBranch(branchGeneration: number): Promise { + const prompt = spawnSync("bash", [promptScript], { + cwd: fmRoot, + encoding: "utf8", + env: scriptEnv, + maxBuffer: 4 * 1024 * 1024, + }); + if (prompt.status !== 0 || !prompt.stdout || prompt.stdout.length < 1024) { + throw new Error( + `fm-branch-prompt.sh did not produce a usable branch prompt (status=${prompt.status ?? "none"}): ${(prompt.stderr || "").trim()}`, + ); + } + if (!actingAsOwner(branchGeneration)) throw new Error("supervision session was replaced or lost lock ownership"); + mkdirSync(sessionsDir, { recursive: true }); + let sessionManager: SessionManager | null = null; + try { + const recorded = readFileSync(sessionPointer, "utf8").trim(); + if (recorded && existsSync(recorded)) { + sessionManager = SessionManager.open(recorded, sessionsDir); + } + } catch { + sessionManager = null; + } + if (!sessionManager) { + sessionManager = SessionManager.create(fmRoot, sessionsDir); + } + // The branch loads no project resources at all: extensions off (so it can + // never spawn its own branch), skills/context files off (they vary per + // home and would destabilize the byte-stable prefix). Its whole standing + // context is the generator's prompt. + const loader = new DefaultResourceLoader({ + cwd: fmRoot, + agentDir: getAgentDir(), + noExtensions: true, + noSkills: true, + noPromptTemplates: true, + noThemes: true, + noContextFiles: true, + systemPrompt: prompt.stdout, + extensionFactories: [ + { + name: "fm-branch-cache-key", + factory: (branchPi: ExtensionAPI) => { + branchPi.on("before_provider_request", (event) => { + const payload = event.payload; + // Only providers whose request already carries Pi's default + // per-session prompt_cache_key get the shared per-home override; + // any other provider payload passes through untouched. + if (payload && typeof payload === "object" && "prompt_cache_key" in payload) { + return { ...(payload as Record), prompt_cache_key: branchCacheKey }; + } + }); + }, + }, + ], + }); + await loader.reload(); + if (!actingAsOwner(branchGeneration)) throw new Error("supervision session was replaced or lost lock ownership"); + const leaseHolderPid = ownedLockPid; + const bashTool = createBashToolDefinition(fmRoot, { + spawnHook: (context) => { + if (!actingAsOwner(branchGeneration)) { + throw new Error("bash refused: supervision session was replaced or lost lock ownership"); + } + return { + ...context, + // Loud accidental-override guard (captain-decided): the actor + // variables are readonly inside the branch's own shell, so an + // accidental in-shell reassignment fails loudly instead of silently + // impersonating main. Confused-agent-grade by design; the threat + // model lives in bin/fm-lease-lib.sh. + command: `readonly FM_SUPERVISION_ACTOR FM_LEASE_HOLDER_PID +( +${context.command} +)`, + env: { + ...context.env, + ...scriptEnv, + FM_SUPERVISION_ACTOR: "branch", + FM_LEASE_HOLDER_PID: leaseHolderPid, + }, + }; + }, + }); + const created = await createAgentSession({ + cwd: fmRoot, + sessionManager, + resourceLoader: loader, + tools: [...BRANCH_TOOL_NAMES], + customTools: [bashTool as unknown as ToolDefinition, createReportTool(branchGeneration)], + }); + if (!actingAsOwner(branchGeneration)) { + try { + created.session.dispose(); + } catch {} + throw new Error("supervision session was replaced or lost lock ownership"); + } + try { + writeFileSync(sessionPointer, `${sessionManager.getSessionFile()}\n`); + } catch { + // Pointer write failure only costs cross-restart session reuse. + } + return created.session; + } + + async function ensureBranch(expectedGeneration: number): Promise { + if (!actingAsOwner(expectedGeneration)) throw new Error("supervision session was replaced or lost lock ownership"); + if (branch) return branch; + if (branchBroken) throw new Error(branchBroken); + try { + const created = await createBranch(expectedGeneration); + if (!actingAsOwner(expectedGeneration)) { + try { + created.dispose(); + } catch {} + throw new Error("supervision session was replaced or lost lock ownership"); + } + branch = created; + return created; + } catch (error) { + if (expectedGeneration === generation && !shuttingDown) { + branchBroken = error instanceof Error ? error.message : String(error); + } + throw error; + } + } + + async function flushMirror(session: AgentSession, expectedGeneration: number): Promise { + if (!actingAsOwner(expectedGeneration)) throw new Error("supervision session no longer owns the fleet lock"); + while (pendingMirror.length > 0) { + const item = pendingMirror[0]; + if (!actingAsOwner(expectedGeneration)) throw new Error("supervision session no longer owns the fleet lock"); + await session.sendCustomMessage( + { customType: "fm-main-mirror", content: `[${item.tag}] ${item.text}`, display: false }, + {}, + ); + if (!actingAsOwner(expectedGeneration)) throw new Error("supervision session was replaced during mirror delivery"); + pendingMirror.shift(); + } + if (mirrorCollection.pendingCursor) { + if (!actingAsOwner(expectedGeneration)) throw new Error("supervision session no longer owns the fleet lock"); + writeMirrorCursor(mirrorCollection.pendingCursor); + mirrorCollection.pendingCursor = null; + } + } + + async function fallbackToMain(message: string, detail: string): Promise { + const body = `FIRSTMATE WATCHER WAKE: ${message}\n\nRun bin/fm-wake-drain.sh first and handle the queued wake. (Supervision branch unavailable, falling back to main: ${detail})`; + let content = body; + try { + // Marked operational like every watcher injection, so the wake is never + // mistaken for captain input (away-mode return semantics, mirror filter). + content = encodeFirstmateOperationalInput("watcher", body); + } catch { + // An encoding failure must not lose the wake; deliver it unmarked. + } + await pi.sendUserMessage(content, { deliverAs: "followUp" }); + } + + function enqueueWake(message: string, acceptedGeneration: number): void { + branchChain = branchChain + .then(async () => { + if (shuttingDown || acceptedGeneration !== generation) { + throw new Error("supervision session was replaced before handling the accepted wake"); + } + if (!actingAsOwner(acceptedGeneration)) throw new Error("supervision session no longer owns the fleet lock"); + const session = await ensureBranch(acceptedGeneration); + await flushMirror(session, acceptedGeneration); + if (!actingAsOwner(acceptedGeneration)) throw new Error("supervision session no longer owns the fleet lock"); + const heartbeat = /^heartbeat($|:)/.test(message); + const scope = scopeForUnreadWake(state, heartbeat); + if (scope.status === "empty") return; + if (scope.status === "unsafe") { + throw new Error("unread wake queue now contains a main-owned row or could not be read safely"); + } + // A row can still arrive between this re-check and the model starting + // the drain; that residual is accepted by the confused-agent-grade boundary. + await session.prompt( + `FIRSTMATE SUPERVISION WAKE: ${message}\n\nHandle this per your operating procedure and finish with fm_branch_report.`, + ); + }) + .catch(async (error: unknown) => { + // Return the wake to main rather than losing it; the durable wake + // queue additionally re-presents anything never acknowledged. + try { + await fallbackToMain(message, error instanceof Error ? error.message : String(error)); + } catch {} + }); + } + + function enqueueMirrorFlush(): void { + if (!branch || pendingMirror.length === 0) return; + const flushGeneration = generation; + const flushSession = branch; + branchChain = branchChain + .then(async () => { + if (!actingAsOwner(flushGeneration)) return; + await flushMirror(flushSession, flushGeneration); + }) + .catch(() => { + // Mirror items stay queued in pendingMirror on failure; the next wake + // or flush retries them in order. + }); + } + + pi.events?.on?.(FM_BRANCH_DISPATCH_EVENT, (data) => { + const offer = data as BranchDispatchOffer; + if (!offer || typeof offer.accept !== "function") return; + // Check eligibility before ownership activation so an out-of-scope wake + // gets neither branch routing nor branch-owned state/lease cleanup side + // effects. + if (!offerEligible(offer)) return; + if (!actingAsOwner()) return; // cold start pre-lock, secondary session, or shutdown + if (afkActive()) return; // the away daemon owns supervision while afk + if (branchBroken) return; // fail back to today's wake-to-main path + offer.accept(); + enqueueWake(offer.message, generation); + }); + + pi.on?.("agent_start", () => { + mainStreaming = true; + }); + pi.on?.("agent_end", () => { + mainStreaming = false; + }); + pi.on?.("agent_settled", () => { + mainStreaming = false; + }); + + // Mirror at main's turn_end: collect the new captain/assistant dialog into + // the volatile queue, then deliver it through the serialized chain so it + // lands before any later wake. The durable cursor advances only in + // flushMirror after the complete pending batch reaches the branch. + pi.on?.("turn_end", (_event, ctx) => { + if (!actingAsOwner()) return; + try { + pendingMirror.push(...collectMainDialog(ctx.sessionManager, mirrorCollection)); + } catch { + return; + } + enqueueMirrorFlush(); + }); + + // Pi emits session_shutdown for ordinary same-process replacements (/new, + // /resume, /fork, reload) as well as terminal quit, exactly as the watcher + // extension documents. Shutdown quiesces this generation, clears the + // volatile mirror state so the replacement reconstructs from the durable + // cursor, and releases the branch session; a replacement session_start + // re-arms, and the next wake reopens the persistent branch from its + // recorded pointer. Terminal quit simply never fires another session_start. + pi.on?.("session_start", () => { + shuttingDown = false; + branchBroken = ""; + generation += 1; + actingAsOwner(generation); + }); + + pi.on?.("session_shutdown", () => { + shuttingDown = true; + generation += 1; + pendingMirror.length = 0; + mirrorCollection.collectAnchor = null; + mirrorCollection.pendingCursor = null; + if (branch) { + try { + branch.dispose(); + } catch { + // Already gone. + } + branch = null; + } + }); + + pi.registerTool?.({ + name: "fm_branch_outcomes", + label: "Read supervision branch outcomes", + description: + "Read the durable outcome store of the supervision branch: what fleet events it handled, each verdict, and each summary. Use when the captain asks what happened in the fleet.", + promptSnippet: "Read what the supervision branch handled (durable outcome store).", + parameters: Type.Object({ + recent: Type.Optional(Type.Number({ description: "How many most-recent outcomes to read (default 20)" })), + }), + execute: async (_toolCallId, params) => { + const recentRaw = (params as { recent?: unknown }).recent; + const recent = typeof recentRaw === "number" && recentRaw >= 1 ? String(Math.floor(recentRaw)) : "20"; + const listed = runOutcomeScript(["list", "--recent", recent]); + if (!listed.ok) { + return { + content: [{ type: "text", text: `could not read the outcome store: ${listed.detail}` }], + details: undefined, + isError: true, + }; + } + return { + content: [{ type: "text", text: listed.stdout || "(no branch outcomes recorded)" }], + details: undefined, + }; + }, + }); + + // Pi only calls this renderer for a message with display: true, which + // mergeIntoMain sets for every routine note except an explicitly silent + // fleet heartbeat; captain-facing notes are never printed or rendered here. + pi.registerMessageRenderer?.("fm-branch-merge", (message, _options, theme) => { + const note = textOfContent(message.content); + const hasGlyph = note.startsWith(MERGE_NOTE_BOAT); + const rest = hasGlyph ? note.slice(MERGE_NOTE_BOAT.length) : note; + const outputPad = 1; + return new Text( + `${hasGlyph ? theme.fg("customMessageText", MERGE_NOTE_BOAT) : ""}${theme.fg("dim", rest)}`, + outputPad, + 0, + ); + }); +} diff --git a/.pi/extensions/fm-primary-pi-watch.ts b/.pi/extensions/fm-primary-pi-watch.ts index 95a7eedd8d6..21c74442b88 100644 --- a/.pi/extensions/fm-primary-pi-watch.ts +++ b/.pi/extensions/fm-primary-pi-watch.ts @@ -16,6 +16,11 @@ import { fileURLToPath } from "node:url"; import type { ExtensionAPI, Theme } from "@earendil-works/pi-coding-agent"; import { Box, Container, Text, type Component } from "@earendil-works/pi-tui"; import { Type } from "typebox"; +import { + createBranchDispatchOffer, + FM_BRANCH_DISPATCH_EVENT, + scopeForUnreadWake, +} from "./lib/fm-branch-dispatch.ts"; import { type CalmPresentationState, calmTranscriptClassIsVisible, @@ -292,9 +297,18 @@ export default function (pi: ExtensionAPI) { return confirmHandlingDelivery(snapshot()); } + function offerWakeToBranch(message: string): boolean { + const heartbeat = /^heartbeat($|:)/.test(message); + const scope = scopeForUnreadWake(state, heartbeat); + const offer = createBranchDispatchOffer(message, scope.projects, heartbeat, scope.eligible); + pi.events?.emit?.(FM_BRANCH_DISPATCH_EVENT, offer); + return offer.accepted; + } + async function deliverActionableWake( owner: SessionGeneration, message: string, + repairFailed: boolean, recovery?: { generation: string; watcherPid: string }, ): Promise { if (!generationIsLive(owner)) return; @@ -309,6 +323,7 @@ export default function (pi: ExtensionAPI) { return; } } + if (!repairFailed && offerWakeToBranch(message)) return; await sendWake(owner, message); } @@ -506,7 +521,7 @@ export default function (pi: ExtensionAPI) { const restoration = await restoreAfterActionableClose(owner, predecessor); if (!generationIsLive(owner)) return; const message = restoration.failure ? `${classification.message}\n\n${restoration.failure}` : classification.message; - await deliverActionableWake(owner, message, restoration.recovery); + await deliverActionableWake(owner, message, Boolean(restoration.failure), restoration.recovery); } catch (error) { const detail = error instanceof Error ? error.message : String(error); surfaceFailure(owner, `watcher: FAILED - Pi extension could not deliver an actionable wake\n${detail}`); diff --git a/.pi/extensions/lib/fm-branch-dispatch.ts b/.pi/extensions/lib/fm-branch-dispatch.ts new file mode 100644 index 00000000000..e72cc09bce8 --- /dev/null +++ b/.pi/extensions/lib/fm-branch-dispatch.ts @@ -0,0 +1,110 @@ +import { readdirSync, readFileSync } from "node:fs"; + +// Shared wake-dispatch handshake between the Pi watcher extension (the +// dispatcher) and the supervision-branch extension (the handler), carried over +// pi.events so neither extension imports the other. +// +// Contract: the watcher builds one offer per actionable wake and emits it on +// FM_BRANCH_DISPATCH_EVENT. A live, enabled branch extension calls accept() +// SYNCHRONOUSLY inside its handler (the event bus invokes handlers +// synchronously up to their first await), so after emit returns the watcher +// reads `accepted`: true means the branch now owns delivering and handling the +// wake (including its own fallback back to main on a later failure); false +// means no branch took it and the watcher delivers to main exactly as it did +// before the branch existed. Watcher-failure alarms are never offered - only +// main can repair the watcher cycle (fm_watch_arm_pi lives on main). + +export const FM_BRANCH_DISPATCH_EVENT = "fm-branch-supervision:dispatch"; + +export type UnreadWakeScopeStatus = "safe" | "empty" | "unsafe"; + +export function scopeForUnreadWake(state: string, heartbeat: boolean): { + status: UnreadWakeScopeStatus; + eligible: boolean; + projects: string[]; +} { + let queue = ""; + try { + queue = readFileSync(`${state}/.wake-queue`, "utf8"); + } catch { + return { status: "unsafe", eligible: false, projects: [] }; + } + + const rows = queue.split(/\r?\n/).filter((line) => line.length > 0); + if (rows.length === 0) return { status: "empty", eligible: false, projects: [] }; + + const projects = new Set(); + const metadata = new Map(); + try { + for (const name of readdirSync(state)) { + if (!name.endsWith(".meta")) continue; + const task = name.slice(0, -5); + const fields = readFileSync(`${state}/${name}`, "utf8").split(/\r?\n/); + const project = fields.find((line) => line.startsWith("project="))?.slice(8) ?? ""; + const window = fields.find((line) => line.startsWith("window="))?.slice(7) ?? ""; + if (project) { + metadata.set(task, project); + if (window) metadata.set(window, project); + } + } + } catch { + return { status: "unsafe", eligible: false, projects: [] }; + } + + for (const line of rows) { + const fields = line.split("\t"); + if (fields.length < 4 || !/^[0-9]+$/.test(fields[1])) return { status: "unsafe", eligible: false, projects: [] }; + const kind = fields[2]; + const key = fields[3]; + if (kind === "heartbeat") continue; + let project = ""; + if (kind === "signal") { + const task = key.replace(/\.(?:status|turn-ended)$/, ""); + project = metadata.get(task) ?? ""; + } else if (kind === "stale") { + project = metadata.get(key) ?? metadata.get(key.replace(/^fm-/, "")) ?? ""; + } else { + return { status: "unsafe", eligible: false, projects: [] }; + } + if (!project) return { status: "unsafe", eligible: false, projects: [] }; + projects.add(project); + } + const eligible = heartbeat || projects.size > 0; + return { status: eligible ? "safe" : "unsafe", eligible, projects: [...projects] }; +} + +export interface BranchDispatchOffer { + /** The watcher's actionable close message (the wake reason line(s)). */ + message: string; + /** + * Exact project values from the unread task metadata this wake will drain. + * Empty means the wake is fleet-wide or could not be scoped safely. + */ + projects: readonly string[]; + /** True when the watcher classified this wake as a fleet-wide heartbeat scan. */ + heartbeat: boolean; + /** True only when every unread queue row is safe for branch handling. */ + eligible: boolean; + /** Set by accept(); read by the watcher after emit returns. */ + accepted: boolean; + accept(): void; +} + +export function createBranchDispatchOffer( + message: string, + projects: readonly string[] = [], + heartbeat = false, + eligible = false, +): BranchDispatchOffer { + const offer: BranchDispatchOffer = { + message, + projects: [...projects], + heartbeat, + eligible, + accepted: false, + accept() { + offer.accepted = true; + }, + }; + return offer; +} diff --git a/AGENTS.md b/AGENTS.md index 977dba22c18..fae141003df 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -104,6 +104,9 @@ state/ runtime records and signals; gitignored .pr-poll private validated data sidecar for the byte-static PR merge poll .pr-poll-registration private transactional provenance record binding the task, canonical metadata identity, sidecar, and static poll publication .pr-poll-retirement private identity-bound crash-recovery receipt for one exact validated merged result; removed after its poll artifacts retire + branch-outcomes.jsonl .branch-outcomes-cursor Pi supervision-branch durable outcome store and its read cursor; bin/fm-branch-outcome.sh owns the format + branch-session/ .branch-session .branch-mirror-cursor the branch's persistent conversation, its pointer, and the dialog-mirror cursor; extension-owned (docs/pi-supervision-branch.md) + .lease- per-task supervision lease naming which actor (main or branch) may change that task; bin/fm-lease-lib.sh owns the contract the guarded scripts enforce .pr-check-quarantine/ private non-runnable storage for checks neutralized by the non-executing migration .pr-check-migration.log private per-task outcomes distinguishing rebuilt or canonically registered replacement polls, quarantined unarmed polls, and incomplete migrations .pr-check-migration-scan-v1 private marker proving the non-executing scan disabled every unsafe legacy check; .pr-check-migration-v1 separately records completed private repairs @@ -156,7 +159,7 @@ If the session lock cannot be acquired and verified, report its exact diagnostic A lock-refused session must not spawn, steer, merge, drain the wake queue, repair supervision, repair a checkout, or perform any other fleet mutation. The digest itself makes no external-network call and never waits for one. -Every network check a session start owes - GitHub auth, dead-secondmate relaunch, secondmate convergence, pending handoff delivery, and project clone refresh - runs concurrently in a bounded worker owned by `bin/fm-startup-network.sh` and is reported in the digest's own `NETWORK CHECKS` section. +Every network check a session start owes - GitHub auth, dead-secondmate relaunch, secondmate convergence, pending handoff delivery, and project clone refresh - runs off the digest's blocking path in a bounded worker owned by `bin/fm-startup-network.sh` and is reported in the digest's own `NETWORK CHECKS` section. When that section reports its checks still in progress it names exactly what is unconfirmed; treat none of those as passed until the result lands, either from `bin/fm-startup-network.sh report` or as a `check: startup-network` wake. 1. **Lock** - acquires the per-home session lock first, before anything mutates shared state, then starts the deferred network stage above. diff --git a/bin/fm-bootstrap.sh b/bin/fm-bootstrap.sh index 180b92941cc..9829149dc96 100755 --- a/bin/fm-bootstrap.sh +++ b/bin/fm-bootstrap.sh @@ -89,13 +89,13 @@ # the fleet lock, so a second concurrent session never race-mutates # PR-check artifacts, secondmate homes, pending handoff outboxes, # X-mode artifacts, project clones, or repair instructions. -# Unset/0 (the default) runs every sweep exactly as before - this flag -# is purely additive. +# Unset/0 (the default) runs all six sweeps - this flag is purely +# additive. # Set FM_BOOTSTRAP_NETWORK to split this run by whether a step talks to # the network, so a session start can print its digest from local reads -# alone and run the network half concurrently: -# all (default, and any unrecognized value) - everything, exactly as -# before. Unrecognized values fall back here on purpose: a typo +# alone and run the network half off the digest's blocking path: +# all (default, and any unrecognized value) - every local and network +# step. Unrecognized values fall back here on purpose: a typo # must never silently skip a safety sweep. # skip - every LOCAL step, and none of the network ones. Skips # `gh auth status`, secondmate_liveness_sweep, secondmate_sync, @@ -108,7 +108,13 @@ # bin/fm-startup-network.sh owns the deferral: it runs the `only` phase # in a detached bounded worker and publishes the result. This file stays # the single owner of every sweep, and the split changes only WHEN each -# runs, never WHETHER. +# runs, never WHETHER. During the network phase, project clone refresh +# overlaps the independent secondmate work. Per-secondmate remote +# liveness workers run concurrently and finish before per-secondmate +# remote convergence workers run concurrently, because convergence +# consumes respawned ids. Worker output is captured separately and +# replayed in spawn order; failure to create that private capture +# directory selects the sequential fallback. # A relaunch that the liveness sweep performs during an `only` run is # always reported, because a digest composed before that run already # printed the superseded endpoint record. @@ -185,6 +191,55 @@ network_sweep_authorized() { return 1 } +# Concurrent per-item runner for the deferred network sweeps. Each worker's +# stdout and stderr are captured to private files and replayed in original +# order after every worker finishes, so concurrent probes cannot interleave +# or mis-attribute SECONDMATE_LIVENESS / SECONDMATE_SYNC lines. Respawned ids +# are collected from per-id files because background workers cannot mutate +# the parent's SECONDMATE_RESPAWNED_IDS. +bootstrap_parallel_begin() { + BOOTSTRAP_PAR_DIR=$(mktemp -d "${TMPDIR:-/tmp}/fm-bootstrap-par.XXXXXX") || return 1 + BOOTSTRAP_PAR_N=0 + FM_BOOTSTRAP_PARALLEL_DIR=$BOOTSTRAP_PAR_DIR + export FM_BOOTSTRAP_PARALLEL_DIR +} + +bootstrap_parallel_spawn() { + BOOTSTRAP_PAR_N=$((BOOTSTRAP_PAR_N + 1)) + ( + "$@" + ) >"$BOOTSTRAP_PAR_DIR/$BOOTSTRAP_PAR_N.out" 2>"$BOOTSTRAP_PAR_DIR/$BOOTSTRAP_PAR_N.err" & + printf '%s\n' "$!" > "$BOOTSTRAP_PAR_DIR/$BOOTSTRAP_PAR_N.pid" +} + +bootstrap_parallel_finish() { + local i pid f + i=1 + while [ "$i" -le "$BOOTSTRAP_PAR_N" ]; do + pid=$(cat "$BOOTSTRAP_PAR_DIR/$i.pid") + wait "$pid" || true + i=$((i + 1)) + done + i=1 + while [ "$i" -le "$BOOTSTRAP_PAR_N" ]; do + cat "$BOOTSTRAP_PAR_DIR/$i.out" + cat "$BOOTSTRAP_PAR_DIR/$i.err" >&2 + i=$((i + 1)) + done + for f in "$BOOTSTRAP_PAR_DIR"/respawned.*; do + [ -f "$f" ] || continue + SECONDMATE_RESPAWNED_IDS="$SECONDMATE_RESPAWNED_IDS $(tr -d '\n' < "$f")" + done + rm -rf "$BOOTSTRAP_PAR_DIR" + unset FM_BOOTSTRAP_PARALLEL_DIR BOOTSTRAP_PAR_DIR BOOTSTRAP_PAR_N +} + +secondmate_note_respawned() { # + SECONDMATE_RESPAWNED_IDS="$SECONDMATE_RESPAWNED_IDS $1" + [ -n "${FM_BOOTSTRAP_PARALLEL_DIR:-}" ] || return 0 + printf '%s\n' "$1" > "$FM_BOOTSTRAP_PARALLEL_DIR/respawned.$1" +} + fleet_sync_origin_backed_project_count() { local count proj count=0 @@ -549,17 +604,30 @@ secondmate_sync() { return 0 } + secondmate_sync_remote_one_timed() { # + local id=$1 home=$2 remote_host=$3 __fm_timing_stamp + __fm_timing_stamp=$(fm_timing_now_ms) + secondmate_sync_remote_one "$id" "$home" "$remote_host" + fm_timing_record secondmate convergence "$__fm_timing_stamp" "$id@$remote_host" + } + # Remote routes converge through the generic transport. Their code root and # inherited files are authoritative on that host; no local path probe or # local fast-forward is attempted for them. - local remote_host __fm_timing_stamp + local remote_host __fm_timing_stamp parallel=0 + if bootstrap_parallel_begin; then + parallel=1 + fi while IFS='|' read -r id _home _window meta; do remote_host=$(fm_meta_get "$meta" remote_host) [ -n "$remote_host" ] || continue - __fm_timing_stamp=$(fm_timing_now_ms) - secondmate_sync_remote_one "$id" "$_home" "$remote_host" - fm_timing_record secondmate convergence "$__fm_timing_stamp" "$id@$remote_host" + if [ "$parallel" -eq 1 ]; then + bootstrap_parallel_spawn secondmate_sync_remote_one_timed "$id" "$_home" "$remote_host" + else + secondmate_sync_remote_one_timed "$id" "$_home" "$remote_host" + fi done < <(live_secondmate_meta_records "$STATE" "$DATA/secondmates.md") + [ "$parallel" -eq 0 ] || bootstrap_parallel_finish return 0 } @@ -586,8 +654,11 @@ secondmate_liveness_sweep() { # primary-only no-op there. Mid-session liveness remains explicitly out of # scope and requires a separate periodic signal. [ -d "$STATE" ] || return 0 - local meta id remote_host label __fm_timing_stamp + local meta id remote_host label __fm_timing_stamp parallel=0 SECONDMATE_RESPAWNED_IDS="" + if bootstrap_parallel_begin; then + parallel=1 + fi for meta in "$STATE"/*.meta; do [ -f "$meta" ] || continue grep -q '^kind=secondmate$' "$meta" 2>/dev/null || continue @@ -597,18 +668,27 @@ secondmate_liveness_sweep() { remote_host=$(fm_meta_get "$meta" remote_host) label=$id [ -z "$remote_host" ] || label="$id@$remote_host" - __fm_timing_stamp=$(fm_timing_now_ms) - secondmate_liveness_one "$meta" "$id" - fm_timing_record secondmate liveness "$__fm_timing_stamp" "$label" + if [ "$parallel" -eq 1 ]; then + bootstrap_parallel_spawn secondmate_liveness_one_timed "$meta" "$id" "$label" + else + secondmate_liveness_one_timed "$meta" "$id" "$label" + fi done + [ "$parallel" -eq 0 ] || bootstrap_parallel_finish return 0 } +secondmate_liveness_one_timed() { #