diff --git a/src/engine/run-registry.ts b/src/engine/run-registry.ts index b8b9af3..f02be9e 100644 --- a/src/engine/run-registry.ts +++ b/src/engine/run-registry.ts @@ -28,6 +28,11 @@ export interface RunRecord { costTotal?: number; /** SPEC-6-1: latest context tokens (calcContextTokens(usage)) — live snapshot. */ contextTokens?: number; + /** #32: context-token snapshot at the end of turn 1 (the armory substrate baseline). + * Set once on the first assistant message_end; live-only (not journaled). The widget + * compares current contextTokens against this to label the tok/ctx% segment as + * "substrate" (flat across turns) vs "work" (growing) — see src/panel/widget-rows.ts. */ + substrateBaseline?: number; /** SPEC-6-1: the tier name this run used (for Tiers-view "used by" + per-tier spend). */ tier?: string; /** SPEC-6-2: the cwd this run belongs to (widget cross-cwd filter + reconcile ownership). */ diff --git a/src/engine/spawnSubagent.ts b/src/engine/spawnSubagent.ts index 6bdd23b..b8857fe 100644 --- a/src/engine/spawnSubagent.ts +++ b/src/engine/spawnSubagent.ts @@ -5,6 +5,7 @@ import type { MemoryHydratePort } from "../memory-hydrate/port.ts"; import type { VisionPort } from "../vision/port.ts"; import type { BackendRegistry } from "../backend/port.ts"; import { genRunId, RunRegistry } from "./run-registry.ts"; +import type { RunRecord } from "./run-registry.ts"; import { createTurnBudget, DEFAULT_MAX_TURNS } from "./turn-budget.ts"; import type { ForegroundLock } from "./concurrency-lock.ts"; import type { RunLog } from "../runtime/run-log.ts"; @@ -345,6 +346,10 @@ export async function spawnSubagent(opts: SpawnOptions): Promise { let costTotal = 0; let contextTokens = 0; let turnIdx = -1; + // #32: substrate baseline — the turn-1 context-token snapshot (armory substrate overhead). + // Captured once on the first assistant message_end (turnIdx === 0); threaded to the RunRecord + // so the widget can classify the tok/ctx% segment as "substrate" (flat) vs "work" (growing). + let substrateBaseline: number | undefined; // #26/#22: declared before subscribe() because some child sessions emit events // synchronously inside subscribe() (temporal-dead-zone guard). let modelError: string | undefined; // model-call failure surfaced via stopReason "error" @@ -403,7 +408,15 @@ export async function spawnSubagent(opts: SpawnOptions): Promise { const cost = u?.cost?.total ?? 0; costTotal += cost; contextTokens = calcContextTokens(u ?? {}); - opts.runRegistry.update(runId, { costTotal, contextTokens, tokenTotal }); + // #32: capture the substrate baseline at the end of turn 1 (the first assistant + // message_end). The turn-1 context is dominated by the armory substrate (system prompt + + // skills + memory); subsequent turns barely grow it unless real work adds tool results. + const patch: Partial = { costTotal, contextTokens, tokenTotal }; + if (substrateBaseline === undefined && turnIdx === 0 && contextTokens > 0) { + substrateBaseline = contextTokens; + patch.substrateBaseline = substrateBaseline; + } + opts.runRegistry.update(runId, patch); try { opts.runLog?.append(runId, { type: "message", role: "assistant", text, usage: { total: turnTokens, input: u?.input, output: u?.output, cacheRead: u?.cacheRead, cacheWrite: u?.cacheWrite, cost: u?.cost }, turnIndex: turnIdx }); } catch { /* best-effort */ } diff --git a/src/panel/widget-rows.ts b/src/panel/widget-rows.ts index 8e5d24c..fb1f124 100644 --- a/src/panel/widget-rows.ts +++ b/src/panel/widget-rows.ts @@ -16,6 +16,11 @@ import type { BgRunStatus } from "./rows.ts"; export const LIVENESS_THRESHOLD_MS = 30_000; /** #23: a run whose last event is older than this is flagged stale ("are events still arriving?"). */ export const STALE_THRESHOLD_MS = 60_000; +/** #32: once past turn 1, if contextTokens has grown by less than this fraction of the + * turn-1 substrate baseline, the tok/ctx% segment is labeled "substrate" (flat overhead) + * rather than "work" (growing from tool results). 5% — the dogfood evidence showed ~0.2%/turn + * growth on a substrate-dominated run vs tens-of-K (multi-%) once real tool output lands. */ +export const SUBSTRATE_GROWTH_THRESHOLD = 0.05; export interface WidgetRun { runId: string; @@ -34,6 +39,10 @@ export interface WidgetRun { task?: string; /** SPEC-6-1: latest context-token snapshot (for ctx% segment). */ contextTokens?: number; + /** #32: context-token baseline at end of turn 1 (armory substrate overhead). When the + * current contextTokens has grown little beyond this baseline across turns, the tok/ctx% + * segment is labeled "substrate" (flat overhead) vs "work" (growing from tool results). */ + substrateBaseline?: number; /** SPEC-6-1: max context window for the resolved model (set by controller — Task 7). */ maxContext?: number; /** SPEC-6-1: cumulative $ (for the $ segment). */ @@ -53,7 +62,7 @@ export function toWidgetRun(r: RunRecord): WidgetRun { runId: r.runId, agent: r.agent, status: r.status, startedAt: r.startedAt, endedAt: r.endedAt, tokenTotal: r.tokenTotal, kind: "fg", - task: r.task, costTotal: r.costTotal, contextTokens: r.contextTokens, + task: r.task, costTotal: r.costTotal, contextTokens: r.contextTokens, substrateBaseline: r.substrateBaseline, turnCount: r.turnCount, turnMax: r.turnMax, lastEventClass: r.lastEventClass, lastEventAt: r.lastEventAt, }; } @@ -114,7 +123,17 @@ function widgetLine(r: WidgetRun, now: number): string { const stale = (r.lastEventAt != null && now - r.lastEventAt > STALE_THRESHOLD_MS) ? " ⏰stale" : ""; liveness = `${turn}${ev}${stale}`; } - return `${glyph} ${label}${agentSeg}${dur}${liveness}${tok}${ctx}${cost}`; + // #32: substrate vs work — once past turn 1, classify the tok/ctx% segment. The armory substrate + // (system prompt + skills + memory) dominates turn-1 context; on substrate-dominated runs the + // ctx% barely moves across turns and reads as "frozen". Label it "substrate" (flat overhead) so + // that's distinguishable from "work" (context growing from tool results). Needs ≥2 turns of + // data (a baseline + a current snapshot); before that there's nothing to compare. + let substrate = ""; + if ((r.turnCount ?? 0) >= 2 && r.substrateBaseline != null && r.contextTokens != null && r.substrateBaseline > 0) { + const growth = (r.contextTokens - r.substrateBaseline) / r.substrateBaseline; + substrate = growth <= SUBSTRATE_GROWTH_THRESHOLD ? " substrate" : " work"; + } + return `${glyph} ${label}${agentSeg}${dur}${liveness}${tok}${ctx}${substrate}${cost}`; } /** Above-editor widget: one line per active run, cap 5, overflow → "+N more in /fleet". diff --git a/test/spawnSubagent.test.mts b/test/spawnSubagent.test.mts index 672b14f..54dadef 100644 --- a/test/spawnSubagent.test.mts +++ b/test/spawnSubagent.test.mts @@ -711,3 +711,61 @@ test("#23 liveness: turnCount + lastEventClass written to the RunRecord on event strictEqual(rec!.lastEventClass, "assistant", `lastEventClass = assistant (last meaningful event): ${rec!.lastEventClass}`); ok(typeof rec!.lastEventAt === "number", "lastEventAt timestamp set"); }); + +test("#32 substrate baseline: captured at end of turn 1 (first assistant message_end) and threaded to RunRecord", async () => { + // A child that completes turn 1 with a real usage block (substrate-dominated context). + // The RunRecord should carry substrateBaseline == contextTokens from that first turn. + const handlers: Array<(e: ChildSessionEvent) => void> = []; + const child: ChildSession = { + prompt: async () => { + for (const h of handlers) h({ type: "turn_start" }); // turnIdx -1 → 0 (turn 1) + for (const h of handlers) h({ + type: "message_end", + message: { + role: "assistant", + content: [{ type: "text", text: "turn 1 done" }], + usage: { input: 570_000, output: 48, cacheRead: 0, cacheWrite: 5_000, cost: { total: 0.001 } }, + }, + }); + }, + subscribe: (h) => { handlers.push(h); return () => {}; }, abort: async () => {}, dispose: () => {}, + }; + const factory: ChildSessionFactory = { create: async () => ({ session: child, model: "m" }) }; + const h = harness(factory); + const res = await spawnSubagent({ + agent: "g", task: "do", track: false, + registry: h.registry, todoSync: h.todoSync, runRegistry: h.runRegistry, lock: h.lock, backendRegistry: regWith(h.factory), + parentModel: PARENT, parentCwd: "/tmp", + }); + strictEqual(res.status, "completed"); + const rec = h.runRegistry.get(res.runId); + ok(rec, "run record exists"); + // contextTokens at turn 1 = input + output + cacheRead + cacheWrite = 575,048. + strictEqual(rec!.contextTokens, 575_048, `contextTokens = turn-1 calcContextTokens: ${rec!.contextTokens}`); + strictEqual(rec!.substrateBaseline, 575_048, `substrateBaseline captured == turn-1 contextTokens: ${rec!.substrateBaseline}`); +}); + +test("#32 substrate baseline: NOT captured when turn 1 produces no assistant message_end", async () => { + // Defensive: a child that aborts before emitting any assistant message_end must not set a + // baseline (there's no turn-1 context to anchor the substrate comparison against). + const handlers: Array<(e: ChildSessionEvent) => void> = []; + const child: ChildSession = { + prompt: async () => { + for (const h of handlers) h({ type: "turn_start" }); + // No message_end — simulates a silent abort / empty result. prompt() just resolves. + }, + subscribe: (h) => { handlers.push(h); return () => {}; }, abort: async () => {}, dispose: () => {}, + }; + const factory: ChildSessionFactory = { create: async () => ({ session: child, model: "m" }) }; + const h = harness(factory); + const res = await spawnSubagent({ + agent: "g", task: "do", track: false, + registry: h.registry, todoSync: h.todoSync, runRegistry: h.runRegistry, lock: h.lock, backendRegistry: regWith(h.factory), + parentModel: PARENT, parentCwd: "/tmp", + }); + // No assistant message_end → #22 EMPTY_RESULT failure. + strictEqual(res.status, "failed"); + const rec = h.runRegistry.get(res.runId); + ok(rec, "run record exists"); + ok(rec!.substrateBaseline === undefined, `no assistant message_end → no substrateBaseline: ${rec!.substrateBaseline}`); +}); diff --git a/test/widget-rows.test.mts b/test/widget-rows.test.mts index d04e837..0b9943f 100644 --- a/test/widget-rows.test.mts +++ b/test/widget-rows.test.mts @@ -217,3 +217,71 @@ test("#23 liveness: running fg run > threshold DOES trigger the abort-warning fo const lines = renderWidgetLines([w], now); ok(lines.some((l) => l.includes("aborts the foreground run")), `running fg run > threshold → footer fires: ${lines.join("|")}`); }); + +test("#32 substrate label: fg run past turn 1 with flat context growth → labeled 'substrate'", () => { + // Substrate-dominated run: turn-1 baseline 575K, turn 2+ barely grew (+0.2%). The tok/ctx% + // segment reads as frozen; the 'substrate' label explains it's flat overhead, not stuck work. + const w = toWidgetRun(fg({ + startedAt: 1000, task: "review the PR", agent: "coder", + turnCount: 3, turnMax: 20, + contextTokens: 576_000, substrateBaseline: 575_000, // +0.17% growth ≤ 5% threshold + })); + const lines = renderWidgetLines([w], 2000); + ok(lines[0]!.includes(" substrate"), `flat growth → 'substrate' label: ${lines[0]}`); + ok(!lines[0]!.includes(" work"), `flat growth → not 'work': ${lines[0]}`); +}); + +test("#32 substrate label: fg run past turn 1 with real context growth → labeled 'work'", () => { + // Work-growing run: turn-1 baseline 575K, but tool results added 80K (+13.9% > 5%) → 'work'. + const w = toWidgetRun(fg({ + startedAt: 1000, task: "refactor module", agent: "coder", + turnCount: 4, turnMax: 20, + contextTokens: 655_000, substrateBaseline: 575_000, // +13.9% growth > 5% threshold + })); + const lines = renderWidgetLines([w], 2000); + ok(lines[0]!.includes(" work"), `growing context → 'work' label: ${lines[0]}`); + ok(!lines[0]!.includes(" substrate"), `growing context → not 'substrate': ${lines[0]}`); +}); + +test("#32 substrate label: exactly at the 5% threshold → 'substrate' (≤ threshold)", () => { + const w = toWidgetRun(fg({ + startedAt: 1000, task: "edge", agent: "coder", + turnCount: 2, turnMax: 20, + contextTokens: 603_750, substrateBaseline: 575_000, // +5.0% exactly → ≤ threshold → substrate + })); + const lines = renderWidgetLines([w], 2000); + ok(lines[0]!.includes(" substrate"), `growth == threshold → 'substrate' (≤): ${lines[0]}`); +}); + +test("#32 substrate label: turn 1 only (turnCount < 2) → no label (baseline just established)", () => { + // Only one turn of data — nothing to compare against yet. No substrate/work label. + const w = toWidgetRun(fg({ + startedAt: 1000, task: "just started", agent: "coder", + turnCount: 1, turnMax: 20, + contextTokens: 575_000, substrateBaseline: 575_000, + })); + const lines = renderWidgetLines([w], 2000); + ok(!lines[0]!.includes(" substrate"), `turn 1 → no substrate label: ${lines[0]}`); + ok(!lines[0]!.includes(" work"), `turn 1 → no work label: ${lines[0]}`); +}); + +test("#32 substrate label: past turn 1 but no baseline captured → no label", () => { + // Defensive: if the baseline was never set (e.g. turn 1 produced no assistant message_end), + // there's no reference to classify against — no label rather than a misleading one. + const w = toWidgetRun(fg({ + startedAt: 1000, task: "no baseline", agent: "coder", + turnCount: 3, turnMax: 20, + contextTokens: 580_000, // substrateBaseline undefined + })); + const lines = renderWidgetLines([w], 2000); + ok(!lines[0]!.includes(" substrate"), `no baseline → no substrate label: ${lines[0]}`); + ok(!lines[0]!.includes(" work"), `no baseline → no work label: ${lines[0]}`); +}); + +test("#32 substrate label: bg runs never get a substrate/work label", () => { + // bg runs don't carry substrateBaseline (toWidgetRunFromBg doesn't set it) and have no turnCount; + // the label is a fg-only signal. + const w = toWidgetRunFromBg(bg({ runId: "fl-bgsub", status: "running" })); + const lines = renderWidgetLines([w], Date.now()); + ok(!lines.some((l) => l.includes(" substrate") || l.includes(" work")), `bg run → no substrate/work label: ${lines.join("|")}`); +});