Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions src/engine/run-registry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,11 @@ export interface RunRecord {
costTotal?: number;
/** SPEC-6-1: latest context tokens (calcContextTokens(usage)) — live snapshot. */
contextTokens?: number;
/** #32: context-token snapshot at the end of turn 1 (the armory substrate baseline).
* Set once on the first assistant message_end; live-only (not journaled). The widget
* compares current contextTokens against this to label the tok/ctx% segment as
* "substrate" (flat across turns) vs "work" (growing) — see src/panel/widget-rows.ts. */
substrateBaseline?: number;
/** SPEC-6-1: the tier name this run used (for Tiers-view "used by" + per-tier spend). */
tier?: string;
/** SPEC-6-2: the cwd this run belongs to (widget cross-cwd filter + reconcile ownership). */
Expand Down
15 changes: 14 additions & 1 deletion src/engine/spawnSubagent.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import type { MemoryHydratePort } from "../memory-hydrate/port.ts";
import type { VisionPort } from "../vision/port.ts";
import type { BackendRegistry } from "../backend/port.ts";
import { genRunId, RunRegistry } from "./run-registry.ts";
import type { RunRecord } from "./run-registry.ts";
import { createTurnBudget, DEFAULT_MAX_TURNS } from "./turn-budget.ts";
import type { ForegroundLock } from "./concurrency-lock.ts";
import type { RunLog } from "../runtime/run-log.ts";
Expand Down Expand Up @@ -345,6 +346,10 @@ export async function spawnSubagent(opts: SpawnOptions): Promise<SpawnResult> {
let costTotal = 0;
let contextTokens = 0;
let turnIdx = -1;
// #32: substrate baseline — the turn-1 context-token snapshot (armory substrate overhead).
// Captured once on the first assistant message_end (turnIdx === 0); threaded to the RunRecord
// so the widget can classify the tok/ctx% segment as "substrate" (flat) vs "work" (growing).
let substrateBaseline: number | undefined;
// #26/#22: declared before subscribe() because some child sessions emit events
// synchronously inside subscribe() (temporal-dead-zone guard).
let modelError: string | undefined; // model-call failure surfaced via stopReason "error"
Expand Down Expand Up @@ -403,7 +408,15 @@ export async function spawnSubagent(opts: SpawnOptions): Promise<SpawnResult> {
const cost = u?.cost?.total ?? 0;
costTotal += cost;
contextTokens = calcContextTokens(u ?? {});
opts.runRegistry.update(runId, { costTotal, contextTokens, tokenTotal });
// #32: capture the substrate baseline at the end of turn 1 (the first assistant
// message_end). The turn-1 context is dominated by the armory substrate (system prompt +
// skills + memory); subsequent turns barely grow it unless real work adds tool results.
const patch: Partial<RunRecord> = { costTotal, contextTokens, tokenTotal };
if (substrateBaseline === undefined && turnIdx === 0 && contextTokens > 0) {
substrateBaseline = contextTokens;
patch.substrateBaseline = substrateBaseline;
}
opts.runRegistry.update(runId, patch);
try {
opts.runLog?.append(runId, { type: "message", role: "assistant", text, usage: { total: turnTokens, input: u?.input, output: u?.output, cacheRead: u?.cacheRead, cacheWrite: u?.cacheWrite, cost: u?.cost }, turnIndex: turnIdx });
} catch { /* best-effort */ }
Expand Down
23 changes: 21 additions & 2 deletions src/panel/widget-rows.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,11 @@ import type { BgRunStatus } from "./rows.ts";
export const LIVENESS_THRESHOLD_MS = 30_000;
/** #23: a run whose last event is older than this is flagged stale ("are events still arriving?"). */
export const STALE_THRESHOLD_MS = 60_000;
/** #32: once past turn 1, if contextTokens has grown by less than this fraction of the
* turn-1 substrate baseline, the tok/ctx% segment is labeled "substrate" (flat overhead)
* rather than "work" (growing from tool results). 5% — the dogfood evidence showed ~0.2%/turn
* growth on a substrate-dominated run vs tens-of-K (multi-%) once real tool output lands. */
export const SUBSTRATE_GROWTH_THRESHOLD = 0.05;

export interface WidgetRun {
runId: string;
Expand All @@ -34,6 +39,10 @@ export interface WidgetRun {
task?: string;
/** SPEC-6-1: latest context-token snapshot (for ctx% segment). */
contextTokens?: number;
/** #32: context-token baseline at end of turn 1 (armory substrate overhead). When the
* current contextTokens has grown little beyond this baseline across turns, the tok/ctx%
* segment is labeled "substrate" (flat overhead) vs "work" (growing from tool results). */
substrateBaseline?: number;
/** SPEC-6-1: max context window for the resolved model (set by controller — Task 7). */
maxContext?: number;
/** SPEC-6-1: cumulative $ (for the $ segment). */
Expand All @@ -53,7 +62,7 @@ export function toWidgetRun(r: RunRecord): WidgetRun {
runId: r.runId, agent: r.agent, status: r.status,
startedAt: r.startedAt, endedAt: r.endedAt, tokenTotal: r.tokenTotal,
kind: "fg",
task: r.task, costTotal: r.costTotal, contextTokens: r.contextTokens,
task: r.task, costTotal: r.costTotal, contextTokens: r.contextTokens, substrateBaseline: r.substrateBaseline,
turnCount: r.turnCount, turnMax: r.turnMax, lastEventClass: r.lastEventClass, lastEventAt: r.lastEventAt,
};
}
Expand Down Expand Up @@ -114,7 +123,17 @@ function widgetLine(r: WidgetRun, now: number): string {
const stale = (r.lastEventAt != null && now - r.lastEventAt > STALE_THRESHOLD_MS) ? " ⏰stale" : "";
liveness = `${turn}${ev}${stale}`;
}
return `${glyph} ${label}${agentSeg}${dur}${liveness}${tok}${ctx}${cost}`;
// #32: substrate vs work — once past turn 1, classify the tok/ctx% segment. The armory substrate
// (system prompt + skills + memory) dominates turn-1 context; on substrate-dominated runs the
// ctx% barely moves across turns and reads as "frozen". Label it "substrate" (flat overhead) so
// that's distinguishable from "work" (context growing from tool results). Needs ≥2 turns of
// data (a baseline + a current snapshot); before that there's nothing to compare.
let substrate = "";
if ((r.turnCount ?? 0) >= 2 && r.substrateBaseline != null && r.contextTokens != null && r.substrateBaseline > 0) {
const growth = (r.contextTokens - r.substrateBaseline) / r.substrateBaseline;
substrate = growth <= SUBSTRATE_GROWTH_THRESHOLD ? " substrate" : " work";
}
return `${glyph} ${label}${agentSeg}${dur}${liveness}${tok}${ctx}${substrate}${cost}`;
}

/** Above-editor widget: one line per active run, cap 5, overflow → "+N more in /fleet".
Expand Down
58 changes: 58 additions & 0 deletions test/spawnSubagent.test.mts
Original file line number Diff line number Diff line change
Expand Up @@ -711,3 +711,61 @@ test("#23 liveness: turnCount + lastEventClass written to the RunRecord on event
strictEqual(rec!.lastEventClass, "assistant", `lastEventClass = assistant (last meaningful event): ${rec!.lastEventClass}`);
ok(typeof rec!.lastEventAt === "number", "lastEventAt timestamp set");
});

test("#32 substrate baseline: captured at end of turn 1 (first assistant message_end) and threaded to RunRecord", async () => {
// A child that completes turn 1 with a real usage block (substrate-dominated context).
// The RunRecord should carry substrateBaseline == contextTokens from that first turn.
const handlers: Array<(e: ChildSessionEvent) => void> = [];
const child: ChildSession = {
prompt: async () => {
for (const h of handlers) h({ type: "turn_start" }); // turnIdx -1 → 0 (turn 1)
for (const h of handlers) h({
type: "message_end",
message: {
role: "assistant",
content: [{ type: "text", text: "turn 1 done" }],
usage: { input: 570_000, output: 48, cacheRead: 0, cacheWrite: 5_000, cost: { total: 0.001 } },
},
});
},
subscribe: (h) => { handlers.push(h); return () => {}; }, abort: async () => {}, dispose: () => {},
};
const factory: ChildSessionFactory = { create: async () => ({ session: child, model: "m" }) };
const h = harness(factory);
const res = await spawnSubagent({
agent: "g", task: "do", track: false,
registry: h.registry, todoSync: h.todoSync, runRegistry: h.runRegistry, lock: h.lock, backendRegistry: regWith(h.factory),
parentModel: PARENT, parentCwd: "/tmp",
});
strictEqual(res.status, "completed");
const rec = h.runRegistry.get(res.runId);
ok(rec, "run record exists");
// contextTokens at turn 1 = input + output + cacheRead + cacheWrite = 575,048.
strictEqual(rec!.contextTokens, 575_048, `contextTokens = turn-1 calcContextTokens: ${rec!.contextTokens}`);
strictEqual(rec!.substrateBaseline, 575_048, `substrateBaseline captured == turn-1 contextTokens: ${rec!.substrateBaseline}`);
});

test("#32 substrate baseline: NOT captured when turn 1 produces no assistant message_end", async () => {
// Defensive: a child that aborts before emitting any assistant message_end must not set a
// baseline (there's no turn-1 context to anchor the substrate comparison against).
const handlers: Array<(e: ChildSessionEvent) => void> = [];
const child: ChildSession = {
prompt: async () => {
for (const h of handlers) h({ type: "turn_start" });
// No message_end — simulates a silent abort / empty result. prompt() just resolves.
},
subscribe: (h) => { handlers.push(h); return () => {}; }, abort: async () => {}, dispose: () => {},
};
const factory: ChildSessionFactory = { create: async () => ({ session: child, model: "m" }) };
const h = harness(factory);
const res = await spawnSubagent({
agent: "g", task: "do", track: false,
registry: h.registry, todoSync: h.todoSync, runRegistry: h.runRegistry, lock: h.lock, backendRegistry: regWith(h.factory),
parentModel: PARENT, parentCwd: "/tmp",
});
// No assistant message_end → #22 EMPTY_RESULT failure.
strictEqual(res.status, "failed");
const rec = h.runRegistry.get(res.runId);
ok(rec, "run record exists");
ok(rec!.substrateBaseline === undefined, `no assistant message_end → no substrateBaseline: ${rec!.substrateBaseline}`);
});
68 changes: 68 additions & 0 deletions test/widget-rows.test.mts
Original file line number Diff line number Diff line change
Expand Up @@ -217,3 +217,71 @@ test("#23 liveness: running fg run > threshold DOES trigger the abort-warning fo
const lines = renderWidgetLines([w], now);
ok(lines.some((l) => l.includes("aborts the foreground run")), `running fg run > threshold → footer fires: ${lines.join("|")}`);
});

test("#32 substrate label: fg run past turn 1 with flat context growth → labeled 'substrate'", () => {
// Substrate-dominated run: turn-1 baseline 575K, turn 2+ barely grew (+0.2%). The tok/ctx%
// segment reads as frozen; the 'substrate' label explains it's flat overhead, not stuck work.
const w = toWidgetRun(fg({
startedAt: 1000, task: "review the PR", agent: "coder",
turnCount: 3, turnMax: 20,
contextTokens: 576_000, substrateBaseline: 575_000, // +0.17% growth ≤ 5% threshold
}));
const lines = renderWidgetLines([w], 2000);
ok(lines[0]!.includes(" substrate"), `flat growth → 'substrate' label: ${lines[0]}`);
ok(!lines[0]!.includes(" work"), `flat growth → not 'work': ${lines[0]}`);
});

test("#32 substrate label: fg run past turn 1 with real context growth → labeled 'work'", () => {
// Work-growing run: turn-1 baseline 575K, but tool results added 80K (+13.9% > 5%) → 'work'.
const w = toWidgetRun(fg({
startedAt: 1000, task: "refactor module", agent: "coder",
turnCount: 4, turnMax: 20,
contextTokens: 655_000, substrateBaseline: 575_000, // +13.9% growth > 5% threshold
}));
const lines = renderWidgetLines([w], 2000);
ok(lines[0]!.includes(" work"), `growing context → 'work' label: ${lines[0]}`);
ok(!lines[0]!.includes(" substrate"), `growing context → not 'substrate': ${lines[0]}`);
});

test("#32 substrate label: exactly at the 5% threshold → 'substrate' (≤ threshold)", () => {
const w = toWidgetRun(fg({
startedAt: 1000, task: "edge", agent: "coder",
turnCount: 2, turnMax: 20,
contextTokens: 603_750, substrateBaseline: 575_000, // +5.0% exactly → ≤ threshold → substrate
}));
const lines = renderWidgetLines([w], 2000);
ok(lines[0]!.includes(" substrate"), `growth == threshold → 'substrate' (≤): ${lines[0]}`);
});

test("#32 substrate label: turn 1 only (turnCount < 2) → no label (baseline just established)", () => {
// Only one turn of data — nothing to compare against yet. No substrate/work label.
const w = toWidgetRun(fg({
startedAt: 1000, task: "just started", agent: "coder",
turnCount: 1, turnMax: 20,
contextTokens: 575_000, substrateBaseline: 575_000,
}));
const lines = renderWidgetLines([w], 2000);
ok(!lines[0]!.includes(" substrate"), `turn 1 → no substrate label: ${lines[0]}`);
ok(!lines[0]!.includes(" work"), `turn 1 → no work label: ${lines[0]}`);
});

test("#32 substrate label: past turn 1 but no baseline captured → no label", () => {
// Defensive: if the baseline was never set (e.g. turn 1 produced no assistant message_end),
// there's no reference to classify against — no label rather than a misleading one.
const w = toWidgetRun(fg({
startedAt: 1000, task: "no baseline", agent: "coder",
turnCount: 3, turnMax: 20,
contextTokens: 580_000, // substrateBaseline undefined
}));
const lines = renderWidgetLines([w], 2000);
ok(!lines[0]!.includes(" substrate"), `no baseline → no substrate label: ${lines[0]}`);
ok(!lines[0]!.includes(" work"), `no baseline → no work label: ${lines[0]}`);
});

test("#32 substrate label: bg runs never get a substrate/work label", () => {
// bg runs don't carry substrateBaseline (toWidgetRunFromBg doesn't set it) and have no turnCount;
// the label is a fg-only signal.
const w = toWidgetRunFromBg(bg({ runId: "fl-bgsub", status: "running" }));
const lines = renderWidgetLines([w], Date.now());
ok(!lines.some((l) => l.includes(" substrate") || l.includes(" work")), `bg run → no substrate/work label: ${lines.join("|")}`);
});
Loading