diff --git a/docs/specs/2026-08-28-harness-inspector-usage-context.md b/docs/specs/2026-08-28-harness-inspector-usage-context.md new file mode 100644 index 00000000..414faf9c --- /dev/null +++ b/docs/specs/2026-08-28-harness-inspector-usage-context.md @@ -0,0 +1,149 @@ +# Inspect session usage and context evidence + +## Traceability + +- Spec ID: harness-inspector-usage-context +- Status: Implemented + +## Intent + +Make Harness Inspector report the token and context evidence that supported +coding-agent sessions without confusing provider-specific accounting or +embedding sensitive system/developer instructions in its portable HTML. The +Inspector should expose what was actually observed, where it came from, and +which fields remain unavailable for Codex, Claude, and Cursor. + +This change also closes two correctness gaps found during the evidence audit: +Codex developer messages must not be projected as assistant dialogue, and +Cursor transcripts without native event timestamps must not silently disappear +from the default Inspector window. + +## Acceptance Scenarios + +- AC-1: Codex `response_item.message` records retain distinct user, assistant, + developer, and system roles. Only user and assistant content may enter the + Inspector dialogue projection; developer/system content contributes bounded + context-manifest counts without retaining text. +- AC-2: Codex `event_msg.token_count` records produce session usage from the + cumulative `total_token_usage` snapshots. Repeated snapshots are not summed, + and a decreasing cumulative total starts a new monotonic segment. The result + keeps input, output, cached-input, cache-write, reasoning-output, and explicit + total tokens without counting cached input twice. +- AC-3: The shared Inspector usage projection keeps optional cache-creation, + reasoning-output, explicit-total, source, basis, and coverage fields. Claude + cache-creation tokens survive adapter, session summary, report, and UI + projection. Providers without an explicit total continue to show a breakdown + without inventing a billing total. +- AC-4: Codex model, effort, model provider, CLI version, context-window usage, + compaction count, and context-layer counts are projected as bounded metadata. + Cursor Context Usage Canvas evidence may contribute window/category counts + only when its composer id matches the Session. Every context projection states + that raw text was omitted. +- AC-5: A Cursor transcript without native timestamps remains discoverable. Its + file modification time may supply a low-authority source timestamp only when + labelled `source-file-mtime`; it must not be presented as a native event time. +- AC-6: Standalone Inspector and native Studio Session Detail expose one docked, + progressively disclosed `Usage and context` facts section. It shows token + breakdown, context-window occupancy, provenance/coverage, runtime metadata, + and unavailability honestly at wide, compact, and narrow widths. +- AC-7: The self-contained report contains no raw base instructions, + developer/system message content, context item text, absolute home paths, + rate-limit/credit data, encrypted reasoning, or raw tool payloads. +- AC-8: Every retained model-inference event in a Turn exposes its own observed + token usage. When that same event includes a context-window observation, the + row also shows used tokens, window size, and percent full. Session totals, + hard-coded model limits, and later snapshots must never be substituted for + missing per-inference evidence. +- AC-9: Standalone Inspector and Studio render per-inference usage as compact, + ordered process-trace rows at wide, compact, and narrow widths. Claude or + Cursor rows without an observed context window say that it is unavailable; + Cursor Context Usage Canvas remains a session-current snapshot rather than + being repeated across historical responses. +- AC-10: Without an explicit time window, Cursor coverage treats every matched + workspace transcript as relevant, including terminal-only and unreadable + transcripts whose native event time is unobserved. A partial in-window subset + must not silently reduce the coverage denominator. + +## Non-goals + +- Treating local Codex, Claude, or Cursor files as stable public host APIs. +- Showing or exporting raw system prompts, developer messages, `AGENTS.md`, + `CLAUDE.md`, rules, skills, tool schemas, or conversation/context item text. +- Adding billing claims, account balances, rate limits, or estimated cost. +- Making Cursor transcript modification time authoritative Session chronology. +- Reconstructing historical per-response context occupancy from a current + Cursor Context Usage Canvas snapshot or a model-name lookup table. +- Unifying provider transcript/event IRs beyond the bounded shared usage and + context-evidence projection consumed by Inspector. + +## Plan and Tasks + +1. Correct Codex role normalization and add bounded usage/runtime/context + metadata events for current rollout records. +2. Extend session summarization with a provider-neutral usage evidence shape, + monotonic cumulative-snapshot aggregation, context manifest counts, and + explicit timestamp authority. +3. Preserve Claude cache-creation usage and attach a matching Cursor Context + Usage Canvas projection while keeping raw context text out of the report. +4. Extend `HarnessInspectorReportV1`, the standalone report, and native Studio + Session Detail with the same progressively disclosed facts. +5. Add behavioral fixtures for role privacy, cumulative usage resets, Claude + cache creation, Cursor timestamp fallback/context matching, report escaping, + and UI projection. +6. Preserve per-inference usage/context snapshots in Turn order, render them as + compact process evidence in both Inspector hosts, and keep missing provider + fields explicitly unavailable. +7. Keep Cursor coverage denominators independent of timestamp observability + when no time filter was requested, and retain the cross-platform fixture as + the behavioral guard. + +## Test and Review Evidence + +- AC-1/AC-2/AC-4: focused Codex adapter tests in + `test/sessions/session-analysis.test.mjs` and session-summary tests in + `test/sessions/commit-session-link.test.mjs`. +- AC-3/AC-5: provider fixtures in + `test/sessions/session-analysis-providers.test.mjs` plus Inspector report + projection coverage in `test/reporting/harness-inspector.test.mjs`. +- AC-6/AC-7: generated-report assertions, Studio component tests, Playwright at + 1440x900, 1024x768, and 390x844, console/page-error inspection, and saved + screenshots. +- AC-8/AC-9: Turn-folding tests for Codex invocation usage and Claude response + usage, report-projection privacy assertions, and the same three-width + Playwright/standalone visual contract with multiple usage rows expanded. +- AC-10: `Cursor facts distinguish absent, terminal-only, and unreadable + transcripts` must pass on Windows as well as POSIX hosts; the Windows PR job + is the authoritative receipt for filesystem behavior. +- Cross-platform evidence: focused tests must use `node:path` and temporary + directories, then the full Node 22/24 macOS, Windows, and Ubuntu PR jobs remain + authoritative for target-platform acceptance. +- Privacy risk: local host schemas may contain prompts, instructions, account + data, and raw tool payloads. Tests assert on normalized shapes and forbidden + secrets; portable output receives only enumerated metadata fields. +- Accounting risk: providers disagree on whether cached tokens are a subset of + input. The UI uses an explicit provider total when present and never adds + cached input to input as a generic fallback. +- Timestamp risk: Cursor source-file mtime is labelled as low-authority evidence + and never changes `eventTimestampCoverage` from `unobserved`. + +## Validation Evidence + +- The first follow-up Windows Node 22 PR run exposed AC-10 by reporting one + relevant Cursor transcript where the fixture contained two. After the + denominator fix, the focused Cursor/provider suite passed all 53 tests; the + refreshed Windows PR job remains the authoritative platform receipt. +- `npm run check` passed on supported Node 24.19.0: root Vitest reported + 1,549 passed and 2 skipped; Harness reported 172 passed; Harness UI reported + 31 passed; Studio reported 293 passed; generated-source and package/archive + verification also passed. +- The focused Studio Playwright scenario passed at 1440x900, 1024x768, and + 390x844 with no console/page errors or document-level horizontal overflow. +- `node scripts/harness-inspector/visual-contract-check.mjs` passed all 15 + standalone Inspector surface/viewport combinations with no clipped facts, + below-floor targets, page errors, or horizontal overflow. +- A live local Inspector render over Codex, Cursor, and Claude evidence produced + 58 sessions and 8,322 tool calls. It projected 5,341 per-inference usage rows + for 44 of 45 Codex sessions, all with same-event context-window evidence, and + 981 per-inference usage rows for all 13 Claude sessions, whose events supplied + no context-window field. The current workspace supplied no matching Cursor + session, so Cursor per-inference/context matching remains fixture-backed. diff --git a/packages/harness-studio/src/app/InspectorWorkbench.tsx b/packages/harness-studio/src/app/InspectorWorkbench.tsx index bb179dc1..9150e6d9 100644 --- a/packages/harness-studio/src/app/InspectorWorkbench.tsx +++ b/packages/harness-studio/src/app/InspectorWorkbench.tsx @@ -296,14 +296,14 @@ function SessionView({ workspaceName, session, commits, onClose }: { workspaceNa return
Harness Inspector
{session.platform ?? "agent"} · retained Session

{sessionTitle(session)}

{mode === "trace" ? : }
; } -type EvidenceKind = "prompts" | "responses" | "intermediate" | "commits" | "tools"; +type EvidenceKind = "prompts" | "responses" | "intermediate" | "usage" | "commits" | "tools"; function SessionTrace({ session, commits }: { session: Session; commits: Commit[] }): React.JSX.Element { const projection = useMemo(() => projectSessionTrace(session, commits), [session, commits]); const turns = projection.turns; const calls = session.toolActivity?.calls ?? []; const toolNames = useMemo(() => [...new Set(calls.map((call) => call.toolName ?? call.operation ?? "tool"))], [calls]); - const [kinds, setKinds] = useState>(() => new Set(["prompts", "responses", "intermediate", "commits", "tools"])); + const [kinds, setKinds] = useState>(() => new Set(["prompts", "responses", "intermediate", "usage", "commits", "tools"])); const [enabledTools, setEnabledTools] = useState>(() => new Set(toolNames)); const [showFiles, setShowFiles] = useState(true); const [openProcesses, setOpenProcesses] = useState>(new Set()); @@ -311,6 +311,7 @@ function SessionTrace({ session, commits }: { session: Session; commits: Commit[ const visibleCalls = filteredCallCount(calls, enabledTools); const responseCount = session.dialogue?.responseCount ?? turns.filter(({ turn }) => turn.response).length; const noteCount = session.dialogue?.noteCount ?? turns.reduce((sum, { turn }) => sum + (turn.steps ?? []).filter((step) => step.kind === "note").length, 0); + const usageCount = turns.reduce((sum, { turn }) => sum + (turn.usageEventCount ?? (turn.steps ?? []).filter((step) => step.kind === "usage").length), 0); function toggleKind(kind: EvidenceKind): void { setKinds((current) => toggle(current, kind)); } function toggleTool(name: string): void { setEnabledTools((current) => toggle(current, name)); } @@ -322,26 +323,57 @@ function SessionTrace({ session, commits }: { session: Session; commits: Commit[ const anchor = turn.anchorId ?? `turn-${turn.index}`; const processOpen = openProcesses.has(turn.index); const intermediateCount = turn.intermediateCount ?? (turn.steps ?? []).filter((step) => step.kind === "note").length; - const eventCount = turn.eventCount ?? intermediateCount + (turn.toolCallCount ?? turnCalls.length); + const usageEventCount = turn.usageEventCount ?? (turn.steps ?? []).filter((step) => step.kind === "usage").length; + const eventCount = turn.eventCount ?? intermediateCount + usageEventCount + (turn.toolCallCount ?? turnCalls.length); const shown = turn.shownEventCount ?? turn.steps?.length ?? 0; - const summary = `${turn.processTruncated ? `${shown} of ${eventCount}` : eventCount} process events · ${intermediateCount} intermediate response${intermediateCount === 1 ? "" : "s"} · ${turn.toolCallCount ?? turnCalls.length} tool calls${Number.isFinite(turn.durationMs) ? ` · ${formatDuration(turn.durationMs)}` : ""}`; - return
{ if (node) cellRefs.current.set(anchor, node); else cellRefs.current.delete(anchor); }}>
Turn {turn.index}{summary}
In [{turn.index}]
{kinds.has("prompts") &&

{turn.prompt?.text ?? "Prompt unavailable after privacy filtering."}

}
; + const summary = `${turn.processTruncated ? `${shown} of ${eventCount}` : eventCount} process events · ${usageEventCount} model response${usageEventCount === 1 ? "" : "s"} · ${intermediateCount} intermediate response${intermediateCount === 1 ? "" : "s"} · ${turn.toolCallCount ?? turnCalls.length} tool calls${Number.isFinite(turn.durationMs) ? ` · ${formatDuration(turn.durationMs)}` : ""}`; + return
{ if (node) cellRefs.current.set(anchor, node); else cellRefs.current.delete(anchor); }}>
Turn {turn.index}{summary}
In [{turn.index}]
{kinds.has("prompts") &&

{turn.prompt?.text ?? "Prompt unavailable after privacy filtering."}

}
; }) :
No retained dialogue turns. Unplaced evidence remains available below.
} {(projection.unplacedCalls.length > 0 || projection.unplacedFiles.length > 0) &&
{ if (node) cellRefs.current.set("unplaced", node); else cellRefs.current.delete("unplaced"); }}>
Unplaced evidence{projection.unplacedCalls.length} calls · {projection.unplacedFiles.length} files
[ ]
{kinds.has("tools") && enabledTools.has(call.toolName ?? call.operation ?? "tool"))} showFiles={showFiles} />}{showFiles && projection.unplacedFiles.length > 0 &&
{projection.unplacedFiles.length} attributed file pathsobserved tool evidence
{projection.unplacedFiles.map((file) => {file})}
}
} {kinds.has("commits") && projection.outsideCommits.length > 0 &&
{ if (node) cellRefs.current.set("outside", node); else cellRefs.current.delete("outside"); }}>
Commits outside Turn windows{projection.outsideCommits.length} commits
[ ]
Timestamps fall outside every observed Turn window.
{projection.outsideCommits.map(({ commit, relation }) => )}
} {calls.length > 0 && } - ; + ; +} + +function UsageContextFacts({ session }: { session: Session }): React.JSX.Element { + const usage = session.tokenUsage; + const context = session.contextManifest; + const runtime = session.runtime; + const layers = context?.layers?.map((layer) => `${layer.kind} ×${layer.itemCount}`).join(" · ") || "not observed"; + const categories = context?.categories?.map((category) => `${category.label} ${formatTokenCount(category.estimatedTokens)}`).join(" · ") || "not observed"; + const contextWindow = Number.isFinite(context?.windowTokens) && Number.isFinite(context?.usedTokens) + ? `${formatTokenCount(Number(context?.usedTokens))} / ${formatTokenCount(Number(context?.windowTokens))} (${context?.percentFull ?? 0}%)` + : "not observed"; + return
Usage and context{usage?.coverage ?? context?.status ?? "unobserved"}
+
Total
{Number.isFinite(usage?.totalTokens) ? formatTokenCount(Number(usage?.totalTokens)) : "not reported"}
+
Input
{formatObservedTokenCount(usage?.inputTokens)}
+
Output
{formatObservedTokenCount(usage?.outputTokens)}
+
Cache read
{formatObservedTokenCount(usage?.cacheReadInputTokens)}
+
Cache write
{formatObservedTokenCount(usage?.cacheCreationInputTokens)}
+
Reasoning
{formatObservedTokenCount(usage?.reasoningOutputTokens)}
+
Context window
{contextWindow}
+
Context layers
{layers}
+
Context categories
{categories}
+
Compactions
{context ? context.compactionCount ?? 0 : "not observed"}
+
Effort
{runtime?.effort ?? "not observed"}
+
Provider
{runtime?.modelProvider ?? "not observed"}
+
CLI
{runtime?.cliVersion ?? "not observed"}
+
Time basis
{session.timestampBasis ?? "unobserved"}
+
Evidence source
{usage?.source ?? context?.source ?? "not observed"}
+
Raw context
omitted
+
; } function Filter({ label, count, checked, subtype = false, onChange }: { label: string; count: number; checked: boolean; subtype?: boolean; onChange(): void }): React.JSX.Element { return ; } -function ProcessStream({ turn, calls, showIntermediate, showTools, enabledTools, showFiles }: { turn: Turn; calls: ToolCall[]; showIntermediate: boolean; showTools: boolean; enabledTools: ReadonlySet; showFiles: boolean }): React.JSX.Element { +function ProcessStream({ turn, calls, showIntermediate, showUsage, showTools, enabledTools, showFiles }: { turn: Turn; calls: ToolCall[]; showIntermediate: boolean; showUsage: boolean; showTools: boolean; enabledTools: ReadonlySet; showFiles: boolean }): React.JSX.Element { const byId = new Map(calls.map((call) => [call.id, call])); const rows: ReactNode[] = []; let pending: ToolCall[] = []; let noteIndex = 0; + let usageIndex = 0; const flush = (): void => { const visible = pending.filter((call) => enabledTools.has(call.toolName ?? call.operation ?? "tool")); pending = []; @@ -360,12 +392,21 @@ function ProcessStream({ turn, calls, showIntermediate, showTools, enabledTools, flush(); noteIndex += 1; if (showIntermediate) rows.push(
Intermediate {noteIndex}
); + } else if (step.kind === "usage") { + flush(); + usageIndex += 1; + if (showUsage) rows.push(); } } flush(); return
{rows.length ? rows :

No process events match the active evidence filters.

}
; } +function UsageEvidence({ step, index }: { step: ToolCall; index: number }): React.JSX.Element { + const source = step.source ?? "normalized model evidence"; + return
Model response {index}{step.model ?? source}
Tokens
{formatInvocationUsage(step.tokenUsage)}
Context
{formatContextWindowUsage(step.contextUsage)}
; +} + function ToolList({ calls, showFiles }: { calls: ToolCall[]; showFiles: boolean }): React.JSX.Element { const runs = groupToolRuns(calls); const [expanded, setExpanded] = useState(false); @@ -498,7 +539,32 @@ function formatDuration(value?: number | null): string { if (!Number.isFinite(va function formatToolRunDuration(calls: ToolCall[]): string { const duration = observedDurationTotal(calls); return duration === undefined ? "—" : `${formatDuration(duration)} total`; } function replayTiming(event: ReplayEvent): string { return event.timeBasis === "observed" ? `${formatStamp(event.atMs)} UTC · observed time` : event.timeBasis === "turn-boundary" ? `${formatStamp(event.atMs)} UTC · Turn boundary, not exact event time` : "Sequence only · timestamp unavailable"; } function replayLegend(events: ReplayEvent[]): Array<[string, string]> { const present = new Set(events.map((event) => event.type)); const labels: Array<[string, string]> = [["prompt", "Prompt"], ["intermediate", "Intermediate"], ["response", "Response"], ["tool-call", "Tool call"], ["commit", "Commit"]]; const entries = labels.filter(([type]) => present.has(type)); if (events.some((event) => event.status === "failed")) entries.push(["failed", "Failed"]); return entries; } -function formatTokenUsage(usage?: Session["tokenUsage"]): string { const total = (usage?.inputTokens ?? 0) + (usage?.outputTokens ?? 0) + (usage?.cacheReadInputTokens ?? 0); return total > 0 ? new Intl.NumberFormat("en", { notation: total >= 10_000 ? "compact" : "standard", maximumFractionDigits: 1 }).format(total) : "unavailable"; } +function formatTokenCount(value: number): string { return new Intl.NumberFormat("en", { notation: value >= 10_000 ? "compact" : "standard", maximumFractionDigits: 1 }).format(value); } +function formatObservedTokenCount(value?: number): string { return Number.isFinite(value) ? formatTokenCount(Number(value)) : "not reported"; } +function formatInvocationUsage(usage?: ToolCall["tokenUsage"]): string { + if (!usage) return "not observed"; + const parts: string[] = []; + if (Number.isFinite(usage.totalTokens)) parts.push(`${formatTokenCount(Number(usage.totalTokens))} total`); + if (Number.isFinite(usage.inputTokens)) parts.push(`${formatTokenCount(Number(usage.inputTokens))} input`); + if (Number.isFinite(usage.outputTokens)) parts.push(`${formatTokenCount(Number(usage.outputTokens))} output`); + if (Number.isFinite(usage.cacheReadInputTokens)) parts.push(`${formatTokenCount(Number(usage.cacheReadInputTokens))} cache read`); + if (Number.isFinite(usage.cacheCreationInputTokens)) parts.push(`${formatTokenCount(Number(usage.cacheCreationInputTokens))} cache write`); + if (Number.isFinite(usage.reasoningOutputTokens)) parts.push(`${formatTokenCount(Number(usage.reasoningOutputTokens))} reasoning`); + return parts.join(" · ") || "not observed"; +} +function formatContextWindowUsage(context?: ToolCall["contextUsage"]): string { + if (!context || !Number.isFinite(context.usedTokens) || !Number.isFinite(context.windowTokens)) return "not observed for this response"; + const percent = Number.isFinite(context.percentFull) + ? context.percentFull + : Math.min(100, Math.round((context.usedTokens / context.windowTokens) * 1_000) / 10); + return `${formatTokenCount(context.usedTokens)} / ${formatTokenCount(context.windowTokens)} · ${percent}% full`; +} +function formatTokenUsage(usage?: Session["tokenUsage"]): string { + if (!usage) return "unavailable"; + if (Number.isFinite(usage.totalTokens)) return `${formatTokenCount(Number(usage.totalTokens))} total`; + const inputOutput = (usage.inputTokens ?? 0) + (usage.outputTokens ?? 0); + return inputOutput > 0 ? `${formatTokenCount(inputOutput)} input + output` : "usage observed"; +} /** * The standalone report carries its own literal copy of the palette so it can * open offline. Studio owns the active *theme*, so the report's literal diff --git a/packages/harness-studio/src/app/inspector-session-model.ts b/packages/harness-studio/src/app/inspector-session-model.ts index ef8a4fb8..e893c7f4 100644 --- a/packages/harness-studio/src/app/inspector-session-model.ts +++ b/packages/harness-studio/src/app/inspector-session-model.ts @@ -1,7 +1,25 @@ +export interface InspectorTokenUsage { + inputTokens?: number; + outputTokens?: number; + cacheReadInputTokens?: number; + cacheCreationInputTokens?: number; + reasoningOutputTokens?: number; + totalTokens?: number; + basis?: string; + source?: string; + coverage?: "observed" | "partial" | "unobserved"; +} + +export interface InspectorContextWindowUsage { + usedTokens: number; + windowTokens: number; + percentFull: number; +} + export interface InspectorToolCall { id: string; callId?: string; - kind?: "note" | "tool"; + kind?: "note" | "tool" | "usage"; text?: string; toolName?: string; actionLabel?: string; @@ -15,6 +33,12 @@ export interface InspectorToolCall { detailKind?: string; filePath?: string | null; filePaths?: string[]; + tokenUsage?: InspectorTokenUsage; + contextUsage?: InspectorContextWindowUsage; + basis?: string; + source?: string; + model?: string; + timestamp?: string | null; } export interface InspectorTurn { @@ -24,6 +48,7 @@ export interface InspectorTurn { steps?: InspectorToolCall[]; toolCallCount?: number; intermediateCount?: number; + usageEventCount?: number; eventCount?: number; shownEventCount?: number; processTruncated?: boolean; @@ -81,7 +106,20 @@ export interface InspectorSession { files?: string[]; prompts?: Array<{ text: string; timestamp?: string | null; turnIndex?: number | null }>; models?: string[]; - tokenUsage?: { inputTokens?: number; outputTokens?: number; cacheReadInputTokens?: number }; + tokenUsage?: InspectorTokenUsage; + runtime?: { modelProvider?: string; cliVersion?: string; effort?: string } | null; + contextManifest?: { + status?: "observed" | "partial" | "unobserved"; + source?: string; + rawTextOmitted?: boolean; + usedTokens?: number; + windowTokens?: number; + percentFull?: number; + compactionCount?: number; + layers?: Array<{ kind: string; itemCount: number }>; + categories?: Array<{ kind: string; label: string; estimatedTokens: number }>; + } | null; + timestampBasis?: "native-event" | "native-metadata" | "source-file-mtime" | "unobserved"; toolActivity?: { totalCalls?: number; failedCalls?: number; diff --git a/packages/harness-studio/test/browser/artifact-host.spec.mjs b/packages/harness-studio/test/browser/artifact-host.spec.mjs index c8d72c1c..ed97973d 100644 --- a/packages/harness-studio/test/browser/artifact-host.spec.mjs +++ b/packages/harness-studio/test/browser/artifact-host.spec.mjs @@ -137,6 +137,30 @@ test.beforeAll(async () => { promptCount: 1, assistantMessageCount: 1, toolCallCount: record.toolCallCount, + tokenUsage: { + inputTokens: 180, + outputTokens: 18, + cacheReadInputTokens: 90, + cacheCreationInputTokens: 5, + reasoningOutputTokens: 6, + totalTokens: 198, + basis: "model-inference", + source: "fixture-session-usage", + coverage: "observed", + }, + runtime: { modelProvider: "fixture", cliVersion: "1.0.0", effort: "high" }, + timestampBasis: "native-event", + contextManifest: { + status: "observed", + source: "fixture-context-usage", + rawTextOmitted: true, + usedTokens: 25, + windowTokens: 100, + percentFull: 25, + compactionCount: 1, + layers: [{ kind: "developer-message", itemCount: 2 }], + categories: [{ kind: "rules", label: "Rules", estimatedTokens: 10 }], + }, toolActivity: { calls: record.timeline.filter((event) => event.kind === "tool-call").map((event, index) => ({ id: event.id, @@ -154,8 +178,27 @@ test.beforeAll(async () => { index: 1, anchorId: "turn-1", prompt: { text: record.prompt, timestamp: record.savedAt }, - steps: record.timeline.filter((event) => event.kind === "tool-call").map((event) => ({ kind: "tool", callId: event.id, toolName: event.name })), + steps: [ + ...record.timeline.filter((event) => event.kind === "tool-call").map((event) => ({ kind: "tool", callId: event.id, toolName: event.name })), + { + kind: "usage", + tokenUsage: { inputTokens: 90, outputTokens: 8, cacheReadInputTokens: 45, totalTokens: 98 }, + contextUsage: { usedTokens: 25, windowTokens: 100, percentFull: 25 }, + source: "fixture-response-usage", + model: "fixture-model", + }, + { + kind: "usage", + tokenUsage: { inputTokens: 90, outputTokens: 10, cacheReadInputTokens: 45, totalTokens: 100 }, + contextUsage: { usedTokens: 40, windowTokens: 100, percentFull: 40 }, + source: "fixture-response-usage", + model: "fixture-model", + }, + ], toolCallCount: record.toolCallCount, + usageEventCount: 2, + eventCount: record.toolCallCount + 2, + shownEventCount: record.toolCallCount + 2, response: `${record.prompt} complete`, responseStatus: "retained", }] }, @@ -912,8 +955,26 @@ test("opens a project workspace and compares Inspector-discovered Sessions", asy await expect(inspector.getByRole("region", { name: "Turn 1 outcome" })).toContainText("Outcome"); await inspector.getByRole("button", { name: "Expand process" }).click(); await expect(inspector.locator("details.session-process")).toHaveAttribute("open", ""); + await expect(inspector.locator(".session-event.usage")).toHaveCount(2); + await expect(inspector.locator(".session-event.usage").first()).toContainText("98 total"); + await expect(inspector.locator(".session-event.usage").first()).toContainText("25 / 100 · 25% full"); await inspector.locator("details.session-filter-disclosure > summary").click(); await expect(inspector.getByRole("checkbox", { name: /Tool calls/u })).toBeChecked(); + await inspector.locator("details.session-usage-disclosure > summary").click(); + await expect(inspector.locator("details.session-usage-disclosure")).toContainText("198"); + await expect(inspector.locator("details.session-usage-disclosure")).toContainText("25 / 100 (25%)"); + await expect(inspector.locator("details.session-usage-disclosure")).toContainText(/Raw context\s*omitted/u); + for (const layout of [ + { name: "wide", width: 1440, height: 900 }, + { name: "compact", width: 1024, height: 768 }, + { name: "narrow", width: 390, height: 844 }, + ]) { + await page.setViewportSize({ width: layout.width, height: layout.height }); + const overflow = await page.evaluate(() => document.documentElement.scrollWidth > document.documentElement.clientWidth + 1); + expect(overflow, `${layout.name} usage and context detail overflows horizontally`).toBe(false); + await page.screenshot({ path: `test-results/session-detail-usage-${layout.name}.png`, fullPage: true }); + } + await page.setViewportSize({ width: 1440, height: 900 }); await inspector.getByRole("tab", { name: "Replay" }).click(); await expect(inspector.getByRole("tab", { name: /Events/u })).toHaveAttribute("aria-selected", "true"); await expect(inspector.getByRole("tab", { name: /Files/u })).toBeVisible(); diff --git a/scripts/commit-session-link/session-source.mjs b/scripts/commit-session-link/session-source.mjs index 9ecb4745..507a7daf 100644 --- a/scripts/commit-session-link/session-source.mjs +++ b/scripts/commit-session-link/session-source.mjs @@ -97,6 +97,43 @@ function safeDialogueText(value, limit) { .replace(WINDOWS_PRIVATE_PATH_RE, ""); } +const TOKEN_USAGE_FIELDS = Object.freeze([ + "inputTokens", + "outputTokens", + "cacheReadInputTokens", + "cacheCreationInputTokens", + "reasoningOutputTokens", + "totalTokens", +]); + +function summarizeUsageStep(step) { + const tokenUsage = Object.fromEntries(TOKEN_USAGE_FIELDS.flatMap((field) => { + if (!Object.hasOwn(step?.tokenUsage ?? {}, field)) return []; + const value = Number(step.tokenUsage[field]); + return Number.isFinite(value) && value >= 0 ? [[field, Math.round(value)]] : []; + })); + const usedTokens = Number(step?.contextUsage?.usedTokens); + const windowTokens = Number(step?.contextUsage?.windowTokens); + const hasContext = Number.isFinite(usedTokens) && usedTokens >= 0 + && Number.isFinite(windowTokens) && windowTokens > 0; + if (Object.keys(tokenUsage).length === 0 && !hasContext) return null; + return { + kind: "usage", + ...(Object.keys(tokenUsage).length > 0 ? { tokenUsage } : {}), + ...(hasContext ? { + contextUsage: { + usedTokens: Math.round(usedTokens), + windowTokens: Math.round(windowTokens), + percentFull: Math.min(100, Math.round((usedTokens / windowTokens) * 1_000) / 10), + }, + } : {}), + ...(step?.basis ? { basis: String(step.basis).slice(0, 40) } : {}), + ...(step?.source ? { source: String(step.source).slice(0, 80) } : {}), + ...(step?.model ? { model: String(step.model).slice(0, 80) } : {}), + timestamp: step?.timestamp ?? null, + }; +} + function summarizeDialogue(events) { const { turns, truncated } = buildSessionTurns(events); let toolCallStep = 0; @@ -114,11 +151,13 @@ function summarizeDialogue(events) { toolCallStep += 1; return { kind: "tool", callStep: toolCallStep, toolName: String(step.toolName ?? "Unknown tool") }; } - return { kind: "note", text: safeDialogueText(step.text, 400) }; - }).filter((step) => step.kind === "tool" || step.text), + if (step.kind === "usage") return summarizeUsageStep(step); + return step.kind === "note" ? { kind: "note", text: safeDialogueText(step.text, 400) } : null; + }).filter(Boolean).filter((step) => step.kind !== "note" || step.text), toolCallCount: Number(turn.toolCallCount) || 0, messageCount: Number(turn.messageCount) || 0, intermediateCount: Number(turn.intermediateCount) || 0, + usageEventCount: Number(turn.usageEventCount) || 0, eventCount: Number(turn.eventCount) || 0, shownEventCount: Number(turn.shownEventCount) || 0, processTruncated: turn.processTruncated === true, @@ -133,6 +172,40 @@ function summarizeDialogue(events) { }; } +function addTokenUsage(target, usage, observedFields) { + if (!usage || typeof usage !== "object") return; + for (const field of TOKEN_USAGE_FIELDS) { + if (!Object.hasOwn(usage, field)) continue; + const value = Number(usage[field]); + if (!Number.isFinite(value) || value < 0) continue; + target[field] = (target[field] ?? 0) + value; + observedFields.add(field); + } +} + +function cumulativeCounter(usage) { + const explicit = Number(usage?.totalTokens); + if (Number.isFinite(explicit) && explicit >= 0) return explicit; + return (Number(usage?.inputTokens) || 0) + (Number(usage?.outputTokens) || 0); +} + +// Codex emits cumulative snapshots rather than one independent usage record per +// response. Repeated snapshots replace the prior point, while a decreasing +// counter starts a new accounting segment (for example after a resumed run). +function aggregateCumulativeUsage(snapshots) { + const aggregate = {}; + const observedFields = new Set(); + let previous = null; + for (const snapshot of snapshots) { + if (previous && cumulativeCounter(snapshot) < cumulativeCounter(previous)) { + addTokenUsage(aggregate, previous, observedFields); + } + previous = snapshot; + } + if (previous) addTokenUsage(aggregate, previous, observedFields); + return { aggregate, observedFields }; +} + // Reduce hydrated session events into the bounded, privacy-safe summary shape // consumed by correlate.mjs and render-html.mjs. Pure over events + repoRoot. export function summarizeSessionEvents(session, events = [], { @@ -157,6 +230,15 @@ export function summarizeSessionEvents(session, events = [], { const distinctUserTurns = new Set(); const seenToolInvocations = new Set(); const tokenTotals = { inputTokens: 0, outputTokens: 0, cacheReadInputTokens: 0 }; + const observedTokenFields = new Set(["inputTokens", "outputTokens", "cacheReadInputTokens"]); + const cumulativeUsageSnapshots = []; + const usageSources = new Set(); + const usageBases = new Set(); + const runtimeMetadata = {}; + const contextLayers = new Map(); + const contextCategories = new Map(); + let currentContextUsage = null; + let compactionCount = 0; let usageObserved = false; for (const event of attributedEvents) { @@ -203,11 +285,45 @@ export function summarizeSessionEvents(session, events = [], { } if (event?.modelUsage && typeof event.modelUsage === "object") { usageObserved = true; - tokenTotals.inputTokens += Number(event.modelUsage.inputTokens) || 0; - tokenTotals.outputTokens += Number(event.modelUsage.outputTokens) || 0; - tokenTotals.cacheReadInputTokens += Number(event.modelUsage.cacheReadInputTokens) || 0; + if (event.usageCumulative === true) cumulativeUsageSnapshots.push(event.modelUsage); + else addTokenUsage(tokenTotals, event.modelUsage, observedTokenFields); + if (event.usageSource) usageSources.add(String(event.usageSource)); + else if (event.sourceKind) usageSources.add(String(event.sourceKind)); + if (event.usageBasis) usageBases.add(String(event.usageBasis)); } if (event?.model && models.size < MAX_MODELS_PER_SESSION) models.add(String(event.model)); + if (event?.runtimeMetadata && typeof event.runtimeMetadata === "object") { + for (const field of ["modelProvider", "cliVersion", "effort"]) { + if (event.runtimeMetadata[field]) runtimeMetadata[field] = String(event.runtimeMetadata[field]); + } + } + for (const layer of Array.isArray(event?.contextLayers) ? event.contextLayers : []) { + const kind = String(layer?.kind ?? "").trim(); + const count = Number(layer?.itemCount); + if (!kind || !Number.isFinite(count) || count <= 0) continue; + const current = contextLayers.get(kind) ?? 0; + contextLayers.set(kind, layer.aggregation === "sum" ? current + count : Math.max(current, count)); + } + for (const category of Array.isArray(event?.contextCategories) ? event.contextCategories : []) { + const kind = String(category?.kind ?? "").trim(); + const estimatedTokens = Number(category?.estimatedTokens); + if (!kind || !Number.isFinite(estimatedTokens) || estimatedTokens < 0) continue; + contextCategories.set(kind, { + kind, + label: String(category?.label ?? kind), + estimatedTokens: Math.round(estimatedTokens), + }); + } + if (event?.currentContextUsage && typeof event.currentContextUsage === "object") { + currentContextUsage = event.currentContextUsage; + } + if (event?.compactionBoundary === true) compactionCount += 1; + } + + if (cumulativeUsageSnapshots.length > 0) { + const cumulative = aggregateCumulativeUsage(cumulativeUsageSnapshots); + addTokenUsage(tokenTotals, cumulative.aggregate, observedTokenFields); + for (const field of cumulative.observedFields) observedTokenFields.add(field); } const toolTrace = includeToolTrace @@ -222,6 +338,36 @@ export function summarizeSessionEvents(session, events = [], { toolTrace.calls = toolTrace.calls.map(({ transientInvocationKey: _transientInvocationKey, ...call }) => call); } + const contextWindowTokens = Number(currentContextUsage?.windowTokens); + const contextUsedTokens = Number(currentContextUsage?.usedTokens); + const hasContextWindow = Number.isFinite(contextWindowTokens) && contextWindowTokens > 0 + && Number.isFinite(contextUsedTokens) && contextUsedTokens >= 0; + const contextManifestObserved = contextLayers.size > 0 || contextCategories.size > 0 || hasContextWindow || compactionCount > 0; + const tokenUsage = usageObserved ? { + ...Object.fromEntries(TOKEN_USAGE_FIELDS + .filter((field) => observedTokenFields.has(field)) + .map((field) => [field, tokenTotals[field] ?? 0])), + basis: usageBases.size === 1 ? [...usageBases][0] : usageBases.size > 1 ? "mixed" : "model-inference", + source: usageSources.size === 1 ? [...usageSources][0] : usageSources.size > 1 ? "mixed-normalized-events" : "normalized-session-events", + coverage: "observed", + } : null; + const contextManifest = contextManifestObserved ? { + status: "observed", + source: currentContextUsage?.source ?? "normalized-context-events", + rawTextOmitted: true, + ...(hasContextWindow ? { + usedTokens: Math.round(contextUsedTokens), + windowTokens: Math.round(contextWindowTokens), + percentFull: Math.min(100, Math.round((contextUsedTokens / contextWindowTokens) * 100)), + } : {}), + compactionCount, + layers: [...contextLayers.entries()] + .map(([kind, itemCount]) => ({ kind, itemCount: Math.round(itemCount) })) + .sort((left, right) => left.kind.localeCompare(right.kind)), + categories: [...contextCategories.values()] + .sort((left, right) => right.estimatedTokens - left.estimatedTokens || left.kind.localeCompare(right.kind)), + } : null; + return { sessionId: session.sessionId, platform: platform ?? session.platform ?? null, @@ -243,7 +389,10 @@ export function summarizeSessionEvents(session, events = [], { ...(toolTrace ? { toolTrace, toolActivity } : {}), ...(dialogue ? { dialogue } : {}), models: [...models].sort(), - tokenUsage: usageObserved ? tokenTotals : null, + tokenUsage, + runtime: Object.keys(runtimeMetadata).length > 0 ? runtimeMetadata : null, + contextManifest, + timestampBasis: session.timestampBasis ?? (firstSeen !== null || lastSeen !== null ? "native-event" : "unobserved"), }; } @@ -480,6 +629,15 @@ export async function normalizeEntireCheckpointSession({ inputTokens: Number(metadata.tokenUsage.input_tokens ?? metadata.tokenUsage.inputTokens) || 0, outputTokens: Number(metadata.tokenUsage.output_tokens ?? metadata.tokenUsage.outputTokens) || 0, cacheReadInputTokens: Number(metadata.tokenUsage.cache_read_tokens ?? metadata.tokenUsage.cacheReadInputTokens) || 0, + ...(metadata.tokenUsage.cache_creation_input_tokens !== undefined || metadata.tokenUsage.cacheCreationInputTokens !== undefined + ? { cacheCreationInputTokens: Number(metadata.tokenUsage.cache_creation_input_tokens ?? metadata.tokenUsage.cacheCreationInputTokens) || 0 } + : {}), + ...(metadata.tokenUsage.total_tokens !== undefined || metadata.tokenUsage.totalTokens !== undefined + ? { totalTokens: Number(metadata.tokenUsage.total_tokens ?? metadata.tokenUsage.totalTokens) || 0 } + : {}), + basis: "model-inference", + source: "entire-checkpoint-metadata", + coverage: "observed", }; } if ((!Number.isFinite(summary.durationMs) || summary.durationMs === 0) && Number.isFinite(metadata?.sessionMetrics?.duration_ms)) { diff --git a/scripts/commit-session-link/session-view.mjs b/scripts/commit-session-link/session-view.mjs index 6e1996fa..4ec09e2a 100644 --- a/scripts/commit-session-link/session-view.mjs +++ b/scripts/commit-session-link/session-view.mjs @@ -85,6 +85,49 @@ function assistantText(event) { return cleaned.length > 0 ? cleaned : null; } +const TURN_TOKEN_USAGE_FIELDS = Object.freeze([ + "inputTokens", + "outputTokens", + "cacheReadInputTokens", + "cacheCreationInputTokens", + "reasoningOutputTokens", + "totalTokens", +]); + +function inferenceUsageStep(event) { + const sourceUsage = event?.modelInvocationUsage + ?? (event?.usageCumulative === true ? null : event?.modelUsage); + const tokenUsage = {}; + if (sourceUsage && typeof sourceUsage === "object") { + for (const field of TURN_TOKEN_USAGE_FIELDS) { + if (!Object.hasOwn(sourceUsage, field)) continue; + const value = Number(sourceUsage[field]); + if (Number.isFinite(value) && value >= 0) tokenUsage[field] = Math.round(value); + } + } + const usedTokens = Number(event?.currentContextUsage?.usedTokens); + const windowTokens = Number(event?.currentContextUsage?.windowTokens); + const hasContext = Number.isFinite(usedTokens) && usedTokens >= 0 + && Number.isFinite(windowTokens) && windowTokens > 0; + if (Object.keys(tokenUsage).length === 0 && !hasContext) return null; + const evidenceSource = event?.usageSource ?? event?.currentContextUsage?.source ?? null; + return { + kind: "usage", + ...(Object.keys(tokenUsage).length > 0 ? { tokenUsage } : {}), + ...(hasContext ? { + contextUsage: { + usedTokens: Math.round(usedTokens), + windowTokens: Math.round(windowTokens), + percentFull: Math.min(100, Math.round((usedTokens / windowTokens) * 1_000) / 10), + }, + } : {}), + ...(event?.usageBasis ? { basis: String(event.usageBasis) } : {}), + ...(evidenceSource ? { source: String(evidenceSource) } : {}), + ...(event?.model ? { model: String(event.model) } : {}), + timestamp: event?.timestamp ?? null, + }; +} + function toolDetail(event) { const detail = event?.commandText ?? event?.filePath ?? null; if (!detail) return null; @@ -106,6 +149,7 @@ function newTurn(index, promptEvent, promptText) { toolCallCount: 0, messageCount: 0, intermediateCount: 0, + usageEventCount: 0, eventCount: 0, response: null, responseStatus: "unavailable", @@ -117,6 +161,7 @@ function newTurn(index, promptEvent, promptText) { _lastObservedKind: null, _terminalAssistantText: null, _terminalAssistantStepRetained: false, + _terminalAssistantStep: null, }; } @@ -126,13 +171,16 @@ function closeTurn(turn) { if (hasTerminalResponse) { turn.response = redactTranscriptText(turn._terminalAssistantText, { limit: RESPONSE_TEXT_LIMIT }); turn.responseStatus = turn.response ? "retained" : "unavailable"; - if (turn._terminalAssistantStepRetained && turn.steps.at(-1)?.kind === "note") turn.steps.pop(); + if (turn._terminalAssistantStepRetained && turn._terminalAssistantStep) { + const retainedIndex = turn.steps.indexOf(turn._terminalAssistantStep); + if (retainedIndex >= 0) turn.steps.splice(retainedIndex, 1); + } } else if (turn._assistantMessageCount > 0) { turn.responseStatus = "incomplete"; } turn.intermediateCount = turn._assistantMessageCount - (turn.responseStatus === "retained" ? 1 : 0); turn.messageCount = turn.intermediateCount; - turn.eventCount = turn.intermediateCount + turn.toolCallCount; + turn.eventCount = turn.intermediateCount + turn.toolCallCount + turn.usageEventCount; turn.shownEventCount = turn.steps.length; turn.processTruncated = turn.shownEventCount < turn.eventCount; turn.durationMs = turn.startMs !== null && turn.endMs !== null && turn.endMs >= turn.startMs @@ -143,6 +191,7 @@ function closeTurn(turn) { delete turn._lastObservedKind; delete turn._terminalAssistantText; delete turn._terminalAssistantStepRetained; + delete turn._terminalAssistantStep; return turn; } @@ -202,6 +251,7 @@ export function buildSessionTurns(events = [], options = {}) { current._lastObservedKind = "tool"; current._lastAssistantText = null; current._terminalAssistantStepRetained = false; + current._terminalAssistantStep = null; if (current.steps.length < MAX_STEPS_PER_TURN) { current.steps.push({ kind: "tool", @@ -218,14 +268,22 @@ export function buildSessionTurns(events = [], options = {}) { current._lastObservedKind = "assistant"; current._terminalAssistantText = text; current._terminalAssistantStepRetained = false; + current._terminalAssistantStep = null; if (current.steps.length < MAX_STEPS_PER_TURN) { const note = redactTranscriptText(text, { limit: NOTE_TEXT_LIMIT }); if (note) { - current.steps.push({ kind: "note", text: note }); + const step = { kind: "note", text: note }; + current.steps.push(step); current._terminalAssistantStepRetained = true; + current._terminalAssistantStep = step; } } } + const usageStep = inferenceUsageStep(event); + if (usageStep) { + current.usageEventCount += 1; + if (current.steps.length < MAX_STEPS_PER_TURN) current.steps.push(usageStep); + } } if (current) turns.push(closeTurn(current)); diff --git a/scripts/harness-inspector/demo-report.mjs b/scripts/harness-inspector/demo-report.mjs index 18a5cb54..3c3129f3 100644 --- a/scripts/harness-inspector/demo-report.mjs +++ b/scripts/harness-inspector/demo-report.mjs @@ -82,10 +82,36 @@ function session({ }) { const files = [...new Set(calls.flatMap((item) => item.filePaths ?? []))]; const steps = []; + const usageStep = (index) => { + const inputTokens = 640 + (index * 220); + const outputTokens = 120 + (index * 40); + const usedTokens = 2_400 + (index * 950); + return { + kind: "usage", + tokenUsage: { + inputTokens, + outputTokens, + cacheReadInputTokens: 180 + (index * 30), + reasoningOutputTokens: 40 + (index * 12), + totalTokens: inputTokens + outputTokens, + }, + contextUsage: { + usedTokens, + windowTokens: 16_000, + percentFull: Math.round((usedTokens / 16_000) * 1_000) / 10, + }, + source: "public-demo-fixture", + model, + }; + }; calls.forEach((item, index) => { - if (notes[index]) steps.push({ kind: "note", text: notes[index] }); + if (notes[index]) { + steps.push({ kind: "note", text: notes[index] }); + steps.push(usageStep(index)); + } steps.push({ kind: "tool", callStep: item.step, toolName: item.toolName }); }); + steps.push(usageStep(notes.length)); return { sessionId, platform, @@ -102,7 +128,30 @@ function session({ toolCallCount: calls.length, fileEditCount: calls.filter((item) => item.operation === "edit-files").length, models: [model], - tokenUsage: { inputTokens: 2_400, outputTokens: 980, cacheReadInputTokens: 320 }, + tokenUsage: { + inputTokens: 2_400, + outputTokens: 980, + cacheReadInputTokens: 320, + cacheCreationInputTokens: 40, + reasoningOutputTokens: 180, + totalTokens: 3_380, + basis: "model-inference", + source: "public-demo-fixture", + coverage: "observed", + }, + runtime: { modelProvider: platform, cliVersion: "demo", effort: "medium" }, + timestampBasis: "native-event", + contextManifest: { + status: "observed", + source: "public-demo-fixture", + rawTextOmitted: true, + usedTokens: 2_400, + windowTokens: 16_000, + percentFull: 15, + compactionCount: 1, + layers: [{ kind: "project-instructions", itemCount: 1 }], + categories: [{ kind: "conversation", label: "Conversation", estimatedTokens: 1_600 }], + }, source: "public-demo-fixture", toolTrace: { schemaVersion: 2, @@ -133,6 +182,10 @@ function session({ prompt: { text: prompt, timestamp: firstSeen }, steps, toolCallCount: calls.length, + intermediateCount: notes.length, + usageEventCount: notes.length + 1, + eventCount: steps.length, + shownEventCount: steps.length, messageCount: notes.length + calls.length + 2, response, durationMs: at(lastSeen) - at(firstSeen), diff --git a/scripts/harness-inspector/report-model.mjs b/scripts/harness-inspector/report-model.mjs index ac1af7ce..00da71e9 100644 --- a/scripts/harness-inspector/report-model.mjs +++ b/scripts/harness-inspector/report-model.mjs @@ -259,6 +259,20 @@ function projectDialogue(dialogue, toolActivity) { const text = safeText(step.text, 400); return text ? { kind: "note", text } : null; } + if (step?.kind === "usage") { + const tokenUsage = projectInvocationTokenUsage(step.tokenUsage); + const contextUsage = projectContextWindowUsage(step.contextUsage); + if (!tokenUsage && !contextUsage) return null; + return { + kind: "usage", + ...(tokenUsage ? { tokenUsage } : {}), + ...(contextUsage ? { contextUsage } : {}), + ...(step.basis ? { basis: safeText(step.basis, 40) } : {}), + ...(step.source ? { source: safeText(step.source, 80) } : {}), + ...(step.model ? { model: safeText(step.model, 80) } : {}), + timestamp: step.timestamp ?? null, + }; + } if (step?.kind !== "tool") return null; const callStep = Number(step.callStep); const call = callsByStep.get(callStep); @@ -283,6 +297,12 @@ function projectDialogue(dialogue, toolActivity) { const intermediateCount = Number.isInteger(turn?.intermediateCount) ? turn.intermediateCount : steps.filter((step) => step.kind === "note").length; + const usageEventCount = Number.isInteger(turn?.usageEventCount) + ? turn.usageEventCount + : steps.filter((step) => step.kind === "usage").length; + const derivedEventCount = intermediateCount + + steps.filter((step) => step.kind === "tool").length + + usageEventCount; return { index: Number(turn?.index) || index + 1, anchorId: `turn-${Number(turn?.index) || index + 1}`, @@ -291,9 +311,10 @@ function projectDialogue(dialogue, toolActivity) { toolCallCount: Number(turn?.toolCallCount) || steps.filter((step) => step.kind === "tool").length, messageCount: Number(turn?.messageCount) || 0, intermediateCount, + usageEventCount, eventCount: Number.isInteger(turn?.eventCount) - ? turn.eventCount - : intermediateCount + steps.filter((step) => step.kind === "tool").length, + ? Math.max(turn.eventCount, derivedEventCount) + : derivedEventCount, shownEventCount: Number.isInteger(turn?.shownEventCount) ? turn.shownEventCount : steps.length, processTruncated: turn?.processTruncated === true, response, @@ -356,6 +377,88 @@ function turnCoverage(session, prompts, dialogue) { }; } +function nonNegativeNumber(value) { + const number = Number(value); + return Number.isFinite(number) && number >= 0 ? number : 0; +} + +function projectTokenUsage(usage) { + if (!usage || typeof usage !== "object") return null; + const projected = { + inputTokens: nonNegativeNumber(usage.inputTokens), + outputTokens: nonNegativeNumber(usage.outputTokens), + cacheReadInputTokens: nonNegativeNumber(usage.cacheReadInputTokens), + }; + for (const field of ["cacheCreationInputTokens", "reasoningOutputTokens", "totalTokens"]) { + if (Object.hasOwn(usage, field)) projected[field] = nonNegativeNumber(usage[field]); + } + projected.basis = safeText(usage.basis, 40, "model-inference"); + projected.source = safeText(usage.source, 80, "normalized-session-events"); + projected.coverage = ["observed", "partial", "unobserved"].includes(usage.coverage) + ? usage.coverage + : "observed"; + return projected; +} + +function projectInvocationTokenUsage(usage) { + if (!usage || typeof usage !== "object") return null; + const projected = {}; + for (const field of [ + "inputTokens", + "outputTokens", + "cacheReadInputTokens", + "cacheCreationInputTokens", + "reasoningOutputTokens", + "totalTokens", + ]) { + if (Object.hasOwn(usage, field)) projected[field] = nonNegativeNumber(usage[field]); + } + return Object.keys(projected).length > 0 ? projected : null; +} + +function projectContextWindowUsage(usage) { + if (!usage || typeof usage !== "object") return null; + const usedTokens = Number(usage.usedTokens); + const windowTokens = Number(usage.windowTokens); + if (!Number.isFinite(usedTokens) || usedTokens < 0 || !Number.isFinite(windowTokens) || windowTokens <= 0) return null; + return { + usedTokens: Math.round(usedTokens), + windowTokens: Math.round(windowTokens), + percentFull: Math.min(100, Math.round((usedTokens / windowTokens) * 1_000) / 10), + }; +} + +function projectRuntime(runtime) { + if (!runtime || typeof runtime !== "object") return null; + const projected = Object.fromEntries(["modelProvider", "cliVersion", "effort"] + .map((field) => [field, safeText(runtime[field], 80)]) + .filter(([_field, value]) => value)); + return Object.keys(projected).length > 0 ? projected : null; +} + +function projectContextManifest(manifest) { + if (!manifest || typeof manifest !== "object") return null; + const projected = { + status: ["observed", "partial", "unobserved"].includes(manifest.status) ? manifest.status : "partial", + source: safeText(manifest.source, 80, "normalized-context-events"), + rawTextOmitted: true, + compactionCount: Math.round(nonNegativeNumber(manifest.compactionCount)), + layers: (Array.isArray(manifest.layers) ? manifest.layers : []).slice(0, 16).map((layer) => ({ + kind: safeText(layer?.kind, 80, "other"), + itemCount: Math.round(nonNegativeNumber(layer?.itemCount)), + })), + categories: (Array.isArray(manifest.categories) ? manifest.categories : []).slice(0, 20).map((category) => ({ + kind: safeText(category?.kind, 80, "other"), + label: safeText(category?.label, 120, "Other"), + estimatedTokens: Math.round(nonNegativeNumber(category?.estimatedTokens)), + })), + }; + if (Number.isFinite(Number(manifest.usedTokens))) projected.usedTokens = Math.round(nonNegativeNumber(manifest.usedTokens)); + if (Number.isFinite(Number(manifest.windowTokens))) projected.windowTokens = Math.round(nonNegativeNumber(manifest.windowTokens)); + if (Number.isFinite(Number(manifest.percentFull))) projected.percentFull = Math.min(100, Math.round(nonNegativeNumber(manifest.percentFull))); + return projected; +} + function projectSession(session) { const sessionId = safeLocator(session.sessionId); if (!sessionId) return null; @@ -391,11 +494,12 @@ function projectSession(session) { assistantMessageCount: Number(session.assistantMessageCount) || 0, fileEditCount: Number(session.fileEditCount) || 0, models: (session.models ?? []).map((model) => safeText(model, 80)).filter(Boolean), - tokenUsage: session.tokenUsage && typeof session.tokenUsage === "object" ? { - inputTokens: Number(session.tokenUsage.inputTokens) || 0, - outputTokens: Number(session.tokenUsage.outputTokens) || 0, - cacheReadInputTokens: Number(session.tokenUsage.cacheReadInputTokens) || 0, - } : null, + tokenUsage: projectTokenUsage(session.tokenUsage), + runtime: projectRuntime(session.runtime), + contextManifest: projectContextManifest(session.contextManifest), + timestampBasis: ["native-event", "native-metadata", "source-file-mtime", "unobserved"].includes(session.timestampBasis) + ? session.timestampBasis + : "unobserved", toolTrace, toolActivity, dialogue, diff --git a/scripts/harness-inspector/ui/workbench.css b/scripts/harness-inspector/ui/workbench.css index 8d2524f8..fe261e13 100644 --- a/scripts/harness-inspector/ui/workbench.css +++ b/scripts/harness-inspector/ui/workbench.css @@ -356,6 +356,14 @@ .session-event.intermediate { padding:7px 9px; display:grid; grid-template-columns:96px minmax(0,1fr); gap:8px; background:var(--color-surface-subtle); } .session-event.intermediate .session-note-label { margin:0; color:var(--color-warning); font-size:12px; font-weight:700; } .session-event.intermediate p { margin:0; font-size:12px; line-height:1.45; white-space:pre-wrap; overflow-wrap:anywhere; } + .session-event.usage { padding:8px 9px; display:grid; gap:7px; background:var(--color-surface-subtle); } + .session-event.usage > header { display:flex; align-items:baseline; justify-content:space-between; gap:8px; } + .session-event.usage > header strong { color:var(--color-categorical-1); font-size:12px; font-weight:700; } + .session-event.usage > header span { min-width:0; overflow:hidden; color:var(--color-text-subtle); text-overflow:ellipsis; white-space:nowrap; font-size:12px; } + .session-event.usage dl { margin:0; display:grid; gap:5px; } + .session-event.usage dl div { display:grid; grid-template-columns:96px minmax(0,1fr); gap:8px; } + .session-event.usage dt { color:var(--color-text-subtle); font-size:12px; } + .session-event.usage dd { margin:0; color:var(--color-text); overflow-wrap:anywhere; font-size:12px; font-variant-numeric:tabular-nums; } .session-prose p { white-space:pre-wrap; overflow-wrap:anywhere; } .session-process { border-bottom:1px solid var(--color-border); background:var(--color-surface-subtle); } details.session-process > summary, .session-process-empty { min-height:35px; padding:7px 10px; display:flex; align-items:center; justify-content:space-between; gap:12px; color:var(--color-text-muted); list-style:none; } @@ -688,11 +696,15 @@ .session-bulk button { min-height:32px; font-size:12px; } .session-filter-list { gap:8px; } .session-filter { font-size:12px; } - .session-filter-disclosure { border-bottom:1px solid var(--color-border); } - .session-filter-disclosure > summary { min-height:48px; padding:10px 22px; display:flex; align-items:center; justify-content:space-between; gap:10px; color:var(--color-text-muted); cursor:pointer; list-style:none; font-size:12px; font-weight:700; } - .session-filter-disclosure > summary::-webkit-details-marker { display:none; } - .session-filter-disclosure > summary em { color:var(--color-text-subtle); font-size:12px; font-style:normal; font-weight:500; } + .session-filter-disclosure, .session-usage-disclosure { border-bottom:1px solid var(--color-border); } + .session-filter-disclosure > summary, .session-usage-disclosure > summary { min-height:48px; padding:10px 22px; display:flex; align-items:center; justify-content:space-between; gap:10px; color:var(--color-text-muted); cursor:pointer; list-style:none; font-size:12px; font-weight:700; } + .session-filter-disclosure > summary::-webkit-details-marker, .session-usage-disclosure > summary::-webkit-details-marker { display:none; } + .session-filter-disclosure > summary em, .session-usage-disclosure > summary em { color:var(--color-text-subtle); font-size:12px; font-style:normal; font-weight:500; } .session-filter-disclosure .session-filter-list { padding:0 22px 16px; } + .session-usage-disclosure dl { margin:0; padding:0 22px 16px; display:grid; gap:9px; } + .session-usage-disclosure dl div { display:flex; align-items:baseline; justify-content:space-between; gap:10px; } + .session-usage-disclosure dt { flex:0 0 108px; overflow:hidden; color:var(--color-text-subtle); text-overflow:ellipsis; white-space:nowrap; font-size:12px; } + .session-usage-disclosure dd { min-width:0; margin:0; overflow:hidden; color:var(--color-text); text-overflow:ellipsis; white-space:nowrap; font-size:12px; font-weight:600; } .session-outline-facts dl { margin:0; display:grid; gap:10px; } .session-outline-facts dl div { display:flex; align-items:baseline; justify-content:space-between; gap:10px; } .session-outline-facts dt { color:var(--color-text-subtle); font-size:12px; } @@ -723,6 +735,7 @@ .session-filter-disclosure > summary { min-height:42px; padding:9px; } .session-filter-disclosure > summary em { display:none; } .session-sidebar .session-outline-facts { grid-column:1 / -1; padding:12px; border-top:1px solid var(--color-border); } + .session-sidebar .session-usage-disclosure { grid-column:1 / -1; border-top:1px solid var(--color-border); border-bottom:0; } .session-filter-disclosure .session-filter-list { position:absolute; right:8px; z-index:4; width:min(280px,calc(100vw - 16px)); padding:10px; border:1px solid var(--color-border); border-radius:var(--radius-lg); background:var(--color-surface); box-shadow:0 12px 30px rgba(15,23,42,.12); } .session-cell { grid-template-columns:52px minmax(0,1fr); gap:0 7px; } .session-cell[data-session-cell="run"] > .session-turn-head { margin-left:59px; } diff --git a/scripts/harness-inspector/ui/workbench.js b/scripts/harness-inspector/ui/workbench.js index db836ea4..01c65d26 100644 --- a/scripts/harness-inspector/ui/workbench.js +++ b/scripts/harness-inspector/ui/workbench.js @@ -135,10 +135,69 @@ const time = new Date(value ?? NaN); return Number.isNaN(time.getTime()) ? null : pad(time.getUTCHours()) + ':' + pad(time.getUTCMinutes()) + ':' + pad(time.getUTCSeconds()); }; + const formatTokenCount = value => value >= 1000000 ? (Math.round(value / 100000) / 10) + 'M' + : value >= 1000 ? (Math.round(value / 100) / 10) + 'K' : String(value); + const formatObservedTokenCount = value => Number.isFinite(value) ? formatTokenCount(value) : 'not reported'; const formatTokens = usage => { - const total = (usage?.inputTokens ?? 0) + (usage?.outputTokens ?? 0) + (usage?.cacheReadInputTokens ?? 0); - if (!usage || total === 0) return 'token usage unavailable'; - return total >= 1000 ? (Math.round(total / 100) / 10) + 'K tokens' : total + ' tokens'; + if (!usage) return 'token usage unavailable'; + if (Number.isFinite(usage.totalTokens)) return formatTokenCount(usage.totalTokens) + ' total tokens'; + const inputOutput = (usage.inputTokens ?? 0) + (usage.outputTokens ?? 0); + return inputOutput > 0 ? formatTokenCount(inputOutput) + ' input + output tokens' : 'usage observed'; + }; + const formatInvocationUsage = usage => { + if (!usage) return 'not observed'; + const parts = []; + if (Number.isFinite(usage.totalTokens)) parts.push(formatTokenCount(usage.totalTokens) + ' total'); + if (Number.isFinite(usage.inputTokens)) parts.push(formatTokenCount(usage.inputTokens) + ' input'); + if (Number.isFinite(usage.outputTokens)) parts.push(formatTokenCount(usage.outputTokens) + ' output'); + if (Number.isFinite(usage.cacheReadInputTokens)) parts.push(formatTokenCount(usage.cacheReadInputTokens) + ' cache read'); + if (Number.isFinite(usage.cacheCreationInputTokens)) parts.push(formatTokenCount(usage.cacheCreationInputTokens) + ' cache write'); + if (Number.isFinite(usage.reasoningOutputTokens)) parts.push(formatTokenCount(usage.reasoningOutputTokens) + ' reasoning'); + return parts.join(' · ') || 'not observed'; + }; + const formatContextWindowUsage = context => { + if (!context || !Number.isFinite(context.usedTokens) || !Number.isFinite(context.windowTokens)) return 'not observed for this response'; + const percent = Number.isFinite(context.percentFull) + ? context.percentFull + : Math.min(100,Math.round((context.usedTokens / context.windowTokens) * 1000) / 10); + return formatTokenCount(context.usedTokens) + ' / ' + formatTokenCount(context.windowTokens) + ' · ' + percent + '% full'; + }; + const usageStepMarkup = (step,index) => { + const source = step.source ?? 'normalized model evidence'; + return '
Model response ' + index + '' + escape(step.model ?? source) + '
' + + '
Tokens
' + escape(formatInvocationUsage(step.tokenUsage)) + '
' + + '
Context
' + escape(formatContextWindowUsage(step.contextUsage)) + '
' + + '
'; + }; + const usageContextMarkup = session => { + const usage = session.tokenUsage; + const context = session.contextManifest; + const runtime = session.runtime; + const layers = context?.layers?.map(layer => layer.kind + ' ×' + layer.itemCount).join(' · ') || 'not observed'; + const categories = context?.categories?.map(category => category.label + ' ' + formatTokenCount(category.estimatedTokens)).join(' · ') || 'not observed'; + const contextWindow = Number.isFinite(context?.windowTokens) && Number.isFinite(context?.usedTokens) + ? formatTokenCount(context.usedTokens) + ' / ' + formatTokenCount(context.windowTokens) + ' (' + (context.percentFull ?? 0) + '%)' + : 'not observed'; + const row = (label,value,title = value) => '
' + escape(label) + '
' + escape(value) + '
'; + return '
Usage and context' + + escape(usage?.coverage ?? context?.status ?? 'unobserved') + '
' + + row('Total',Number.isFinite(usage?.totalTokens) ? formatTokenCount(usage.totalTokens) : 'not reported') + + row('Input',formatObservedTokenCount(usage?.inputTokens)) + + row('Output',formatObservedTokenCount(usage?.outputTokens)) + + row('Cache read',formatObservedTokenCount(usage?.cacheReadInputTokens)) + + row('Cache write',formatObservedTokenCount(usage?.cacheCreationInputTokens)) + + row('Reasoning',formatObservedTokenCount(usage?.reasoningOutputTokens)) + + row('Context window',contextWindow) + + row('Context layers',layers) + + row('Context categories',categories) + + row('Compactions',context ? String(context.compactionCount ?? 0) : 'not observed') + + row('Effort',runtime?.effort ?? 'not observed') + + row('Provider',runtime?.modelProvider ?? 'not observed') + + row('CLI',runtime?.cliVersion ?? 'not observed') + + row('Time basis',session.timestampBasis ?? 'unobserved') + + row('Evidence source',usage?.source ?? context?.source ?? 'not observed') + + row('Raw context','omitted') + + '
'; }; const evidence = (kind, label = kind) => '' + escape(label) + ''; const isDirectCommitLink = link => link?.evidenceKind === 'explicit' || link?.evidenceKind === 'observed-commit'; @@ -766,6 +825,7 @@ const rows = []; let pendingCalls = []; let noteIndex = 0; + let usageIndex = 0; const flushCalls = () => { if (!pendingCalls.length) return; const calls = pendingCalls; @@ -787,6 +847,9 @@ if (step.kind === 'note') { noteIndex += 1; rows.push('
Intermediate ' + noteIndex + '
' + renderSessionMarkdown(step.text) + '
'); + } else if (step.kind === 'usage') { + usageIndex += 1; + rows.push(usageStepMarkup(step,usageIndex)); } }); flushCalls(); @@ -855,10 +918,11 @@ const processTimeline = calls.length ? '
' : ''; const clock = Number.isFinite(turn.startMs) ? formatShortClock(turn.startMs) + (Number.isFinite(turn.endMs) ? '–' + formatShortClock(turn.endMs) : '') + ' UTC · ' : ''; const intermediateCount = Number.isInteger(turn.intermediateCount) ? turn.intermediateCount : turn.steps.filter(step => step.kind === 'note').length; - const eventCount = Number.isInteger(turn.eventCount) ? turn.eventCount : intermediateCount + turn.toolCallCount; + const usageEventCount = Number.isInteger(turn.usageEventCount) ? turn.usageEventCount : turn.steps.filter(step => step.kind === 'usage').length; + const eventCount = Number.isInteger(turn.eventCount) ? turn.eventCount : intermediateCount + usageEventCount + turn.toolCallCount; const shownEventCount = Number.isInteger(turn.shownEventCount) ? turn.shownEventCount : turn.steps.length; const truncationSummary = turn.processTruncated ? shownEventCount + ' of ' + eventCount + ' process events retained · ' : eventCount + ' process events · '; - const kindSummary = intermediateCount + ' intermediate response' + (intermediateCount === 1 ? '' : 's') + ' · ' + turn.toolCallCount + ' tool calls' + (turn.processTruncated ? ' observed' : ''); + const kindSummary = usageEventCount + ' model response' + (usageEventCount === 1 ? '' : 's') + ' · ' + intermediateCount + ' intermediate response' + (intermediateCount === 1 ? '' : 's') + ' · ' + turn.toolCallCount + ' tool calls' + (turn.processTruncated ? ' observed' : ''); const summary = clock + truncationSummary + kindSummary + (Number.isFinite(turn.durationMs) ? ' · ' + formatDuration(turn.durationMs) : ''); const turnCommits = placement.inTurn.get(turn.index) ?? []; const processStream = processStreamMarkup(session,turn,callsById); @@ -905,6 +969,7 @@ const coverage = turnCoverageOf(session); const responseCount = session.dialogue?.responseCount ?? turns.filter(turn => turn.response).length; const noteCount = session.dialogue?.noteCount ?? turns.reduce((sum,turn) => sum + turn.steps.filter(step => step.kind === 'note').length,0); + const usageCount = turns.reduce((sum,turn) => sum + (turn.usageEventCount ?? turn.steps.filter(step => step.kind === 'usage').length),0); const projectionFact = session.dialogue?.truncated ? '
Projection
Truncated
' : ''; const jumpOptions = turns.map(turn => '').join('') + (unplacedMarkup ? '' : '') @@ -919,7 +984,7 @@ + '
File edits
' + session.fileEditCount + '
' + '
Token usage
' + escape(formatTokens(session.tokenUsage)) + '
' + projectionFact; - const sessionOutline = ''; + const sessionOutline = ''; const overallActivity = session.toolActivity.totalCalls ? '
Overall session activity ' + session.toolActivity.totalCalls + ' callsAll retained Turns and unplaced calls
' : ''; const tracePanel = '
' + '
' + timeline + overallActivity + '
' + sessionOutline + '
'; diff --git a/scripts/harness-inspector/visual-contract-check.mjs b/scripts/harness-inspector/visual-contract-check.mjs index 131d0831..5b167c34 100644 --- a/scripts/harness-inspector/visual-contract-check.mjs +++ b/scripts/harness-inspector/visual-contract-check.mjs @@ -120,6 +120,20 @@ async function surfacesFor(page) { return undefined; }, }, + { + label: "session-usage", + enter: async () => { + const process = page.locator("details.session-process").first(); + if ((await process.count()) > 0 && !await process.evaluate((element) => element.open)) { + await process.locator(":scope > summary").click(); + } + const usage = page.locator(".session-usage-disclosure > summary").first(); + if ((await usage.count()) === 0) return "skip"; + await usage.click(); + await page.waitForTimeout(300); + return undefined; + }, + }, { label: "session-replay", enter: async () => { diff --git a/scripts/session-analysis/platforms/claude.mjs b/scripts/session-analysis/platforms/claude.mjs index 3f441752..cf16a2d8 100644 --- a/scripts/session-analysis/platforms/claude.mjs +++ b/scripts/session-analysis/platforms/claude.mjs @@ -203,7 +203,10 @@ function transcriptEvents(raw, sourceRef, options) { category: "model", model: model ?? null, modelUsage: usage, + modelInvocationUsage: usage, usageFieldsObserved: true, + usageBasis: "model-inference", + usageSource: "claude-project-transcript", responseId: raw?.message?.id ?? raw?.uuid ?? null, evidenceRef: evidenceRef(raw, sourceRef, "model.response.completed"), summary: "Claude model response completed", diff --git a/scripts/session-analysis/platforms/codex.mjs b/scripts/session-analysis/platforms/codex.mjs index 590739db..2f86ba80 100644 --- a/scripts/session-analysis/platforms/codex.mjs +++ b/scripts/session-analysis/platforms/codex.mjs @@ -82,11 +82,16 @@ function inferType(raw, fallback = "record") { const payload = raw?.payload ?? {}; if (outer === "event_msg") { if (payload.type === "user_message") return "user"; - if (payload.type === "agent_message" || payload.type === "agent_reasoning") return "assistant"; + if (payload.type === "agent_message") return "assistant"; + if (payload.type === "agent_reasoning") return "reasoning"; return payload.type ? `event.${payload.type}` : outer; } if (outer === "response_item") { - if (payload.type === "message") return payload.role === "user" ? "user" : "assistant"; + if (payload.type === "message") { + if (payload.role === "user" || payload.role === "assistant") return payload.role; + if (payload.role === "developer" || payload.role === "system") return `context.${payload.role}`; + return "response.message"; + } if (payload.type === "agent_message") return "assistant"; if (payload.type === "custom_tool_call" || payload.type === "function_call") return "tool.call"; if (payload.type === "custom_tool_call_output" || payload.type === "function_call_output") return "tool.result"; @@ -95,6 +100,59 @@ function inferType(raw, fallback = "record") { return outer ?? payload?.type ?? fallback; } +const CODEX_USAGE_FIELDS = Object.freeze({ + inputTokens: "input_tokens", + outputTokens: "output_tokens", + cacheReadInputTokens: "cached_input_tokens", + cacheCreationInputTokens: "cache_write_input_tokens", + reasoningOutputTokens: "reasoning_output_tokens", + totalTokens: "total_tokens", +}); + +function normalizeCodexUsage(value) { + if (!value || typeof value !== "object" || Array.isArray(value)) return null; + const usage = {}; + for (const [target, source] of Object.entries(CODEX_USAGE_FIELDS)) { + if (!Object.hasOwn(value, source)) continue; + const number = Number(value[source]); + if (Number.isFinite(number) && number >= 0) usage[target] = Math.round(number); + } + return Object.keys(usage).length > 0 ? usage : null; +} + +function contextItemCount(value) { + if (Array.isArray(value)) return value.length; + if (value && typeof value === "object") return Object.keys(value).length; + return typeof value === "string" ? Number(value.trim().length > 0) : Number(value !== null && value !== undefined); +} + +function codexContextLayers(raw) { + const outer = raw?.type ?? raw?.event ?? raw?._event ?? null; + const payload = raw?.payload ?? {}; + if (outer === "response_item" && payload.type === "message" + && (payload.role === "developer" || payload.role === "system")) { + return [{ kind: `${payload.role}-message`, itemCount: 1, aggregation: "sum" }]; + } + if (outer === "session_meta" && payload.base_instructions) { + return [{ kind: "base-instructions", itemCount: 1, aggregation: "max" }]; + } + if (outer !== "world_state" || !payload.state || typeof payload.state !== "object") return []; + const state = payload.state; + const definitions = [ + ["agents-md", ["agents_md"]], + ["managed-developer-instructions", ["managed_developer_instructions"]], + ["skills", ["skills", "host_skills", "orchestrator_skills"]], + ["plugins", ["plugins_instructions"]], + ["apps", ["apps_instructions"]], + ["permissions", ["permissions"]], + ]; + return definitions.map(([kind, keys]) => ({ + kind, + itemCount: keys.reduce((sum, key) => sum + contextItemCount(state[key]), 0), + aggregation: "max", + })).filter((layer) => layer.itemCount > 0); +} + function userVisibleAssistantText(raw) { const message = raw?.message ?? raw?.content ?? raw?.payload?.message ?? raw?.payload?.content ?? null; const extract = (value) => { @@ -816,6 +874,7 @@ export class CodexSessionAnalyzer extends SessionAnalyzer { normalizeEvent(raw, sourceRef, options = {}) { const type = inferType(raw); const text = messageText(raw); + const retainsDialogueText = ["user", "assistant", "last-prompt", "UserPromptSubmit"].includes(type); const event = { sessionId: inferSessionId(raw, sourceRef.sessionId), type, @@ -830,8 +889,47 @@ export class CodexSessionAnalyzer extends SessionAnalyzer { seq: raw?.seq ?? null, type, }, - summary: text ? `${type} message (${text.length} chars)` : type, + summary: retainsDialogueText && text ? `${type} message (${text.length} chars)` : type, }; + const outer = raw?.type ?? raw?.event ?? raw?._event ?? null; + const payload = raw?.payload ?? {}; + const contextLayers = codexContextLayers(raw); + if (contextLayers.length > 0) event.contextLayers = contextLayers; + if (outer === "session_meta") { + event.runtimeMetadata = { + ...(payload.model_provider ? { modelProvider: String(payload.model_provider) } : {}), + ...(payload.cli_version ? { cliVersion: String(payload.cli_version) } : {}), + }; + } + if (outer === "turn_context") { + if (payload.model) event.model = String(payload.model); + event.runtimeMetadata = { + ...(payload.effort ? { effort: String(payload.effort) } : {}), + }; + } + if (outer === "event_msg" && payload.type === "token_count") { + const info = payload.info && typeof payload.info === "object" ? payload.info : {}; + const modelUsage = normalizeCodexUsage(info.total_token_usage); + const lastModelUsage = normalizeCodexUsage(info.last_token_usage); + const contextWindowTokens = Number(info.model_context_window); + if (modelUsage) { + event.modelUsage = modelUsage; + event.usageFieldsObserved = true; + event.usageCumulative = true; + event.usageBasis = "model-inference"; + event.usageSource = "codex-rollout-token-count"; + } + if (lastModelUsage) event.modelInvocationUsage = lastModelUsage; + if (lastModelUsage && Number.isFinite(contextWindowTokens) && contextWindowTokens > 0) { + event.currentContextUsage = { + usedTokens: Number(lastModelUsage.inputTokens) || 0, + windowTokens: Math.round(contextWindowTokens), + source: "codex-rollout-token-count", + rawTextOmitted: true, + }; + } + } + if (outer === "compacted") event.compactionBoundary = true; const cwd = inferCwd(raw); if (cwd) event.cwd = cwd; if (type === "user" || type === "last-prompt" || type === "UserPromptSubmit") { @@ -928,13 +1026,13 @@ export class CodexSessionAnalyzer extends SessionAnalyzer { if (raw?.permission_escalated === true || raw?.permissionEscalated === true || raw?.payload?.permission_escalated === true || raw?.payload?.permissionEscalated === true) event.permissionEscalated = true; } - if (text) { + if (retainsDialogueText && text) { event.contentLength = text.length; } - if (options.includeContent && text) { + if (options.includeContent && retainsDialogueText && text) { event.content = text; } - if (options.includeUserText && (type === "user" || type === "last-prompt" || type === "response_item" || type === "UserPromptSubmit")) { + if (options.includeUserText && (type === "user" || type === "last-prompt" || type === "UserPromptSubmit")) { const promptText = userText(raw); if (promptText) { event.userText = promptText; diff --git a/scripts/session-analysis/platforms/cursor.mjs b/scripts/session-analysis/platforms/cursor.mjs index 354f060c..80c6554a 100644 --- a/scripts/session-analysis/platforms/cursor.mjs +++ b/scripts/session-analysis/platforms/cursor.mjs @@ -277,6 +277,14 @@ function transcriptEvents(raw, sourceRef, options) { if (options.includeUserText && text) event.userText = text; } if (rowType === "assistant" && text) event.userVisibleAssistantMessage = true; + const usage = rowType === "assistant" ? inferUsage(raw) : null; + if (usage) { + event.modelUsage = usage; + event.modelInvocationUsage = usage; + event.usageFieldsObserved = true; + event.usageBasis = "model-inference"; + event.usageSource = "cursor-project-transcript"; + } if (options.includeContent && text) event.content = text; events.push(event); } else if (rowType === "turn_ended") { @@ -351,6 +359,32 @@ function metaEvents(raw, sourceRef) { ]; } +function contextUsageEvents(sourceRef) { + const usage = sourceRef.contextUsage; + if (!usage || usage.status !== "observed") return []; + return [{ + sessionId: sourceRef.sessionId, + type: "context.usage", + category: "context", + timestamp: usage.capturedAt ?? null, + sourceKind: sourceRef.kind, + planningScope: "workspace", + evidenceRef: evidenceRef(sourceRef, "context.usage"), + summary: "Cursor context usage snapshot", + currentContextUsage: { + usedTokens: usage.totalTokensUsed, + windowTokens: usage.contextWindowSize, + source: usage.evidence, + rawTextOmitted: true, + }, + contextCategories: usage.categories.map((category) => ({ + kind: category.id, + label: category.label, + estimatedTokens: category.estimatedTokens, + })), + }]; +} + function cursorLifecycle(raw) { const value = String(raw?._event ?? raw?.event ?? "audit"); const lower = value.toLowerCase(); @@ -407,6 +441,9 @@ function auditEvents(raw, sourceRef, options) { if (usage) { event.modelUsage = usage; event.usageFieldsObserved = true; + event.usageBasis = "model-inference"; + event.usageSource = "cursor-hook-audit"; + if (/afteragentresponse/i.test(auditType)) event.modelInvocationUsage = usage; } if (/subagentstart/i.test(auditType)) event.isSubagent = true; if (/afteragentresponse/i.test(auditType)) event.userVisibleAssistantMessage = true; @@ -423,6 +460,7 @@ function addRef(sessions, sessionId, workspace, ref) { sourceKinds: new Set(), sourceRefs: [], eventTimestampCoverage: "unobserved", + sourceTimestampFallback: null, workspaceCwdCandidates: new Map(), }; if (typeof ref.cwd === "string" && ref.cwd.length > 0) { @@ -435,6 +473,12 @@ function addRef(sessions, sessionId, workspace, ref) { if (!session.sourceRefs.some((existing) => existing.kind === ref.kind && existing.path === ref.path)) { session.sourceRefs.push(ref); } + const timeBasisPriority = { "native-metadata": 2, "native-event": 3 }; + if (ref.sourceTimestampFallback) session.sourceTimestampFallback = ref.sourceTimestampFallback; + if (ref.timestampBasis && ref.timestampBasis !== "source-file-mtime" + && (timeBasisPriority[ref.timestampBasis] ?? 0) > (timeBasisPriority[session.timestampBasis] ?? 0)) { + session.timestampBasis = ref.timestampBasis; + } session.sourceKinds.add(ref.kind); mergeTimeRange(session, ref.firstSeen ?? ref.timestamp); mergeTimeRange(session, ref.lastSeen ?? ref.timestamp); @@ -443,7 +487,12 @@ function addRef(sessions, sessionId, workspace, ref) { } function finalizeSession(session) { - const { workspaceCwdCandidates, ...publicSession } = session; + const { workspaceCwdCandidates, sourceTimestampFallback, ...publicSession } = session; + if (!publicSession.firstSeen && !publicSession.lastSeen && sourceTimestampFallback) { + publicSession.firstSeen = sourceTimestampFallback; + publicSession.lastSeen = sourceTimestampFallback; + publicSession.timestampBasis = "source-file-mtime"; + } const finalized = { ...publicSession, sourceKinds: [...session.sourceKinds].sort() }; const priorities = [...workspaceCwdCandidates.values()]; const strongest = priorities.length > 0 ? Math.max(...priorities) : null; @@ -501,18 +550,19 @@ function sourceJoinCoverage(roots, sessions, relevantIds, predicate) { function buildCursorSourceCoverage({ scope, roots, sessions, inWindowSessions, transcriptCoverage }) { const workspaceSessions = sessions.length; - const timeObservedSessions = sessions.filter((session) => session.firstSeen || session.lastSeen); + const timeObservedSessions = sessions.filter((session) => + session.timestampBasis !== "source-file-mtime" && (session.firstSeen || session.lastSeen)); const timeUnobservedSessions = workspaceSessions - timeObservedSessions.length; const inWindowIds = new Set(inWindowSessions.map((session) => session.sessionId)); const unknownTimeIds = new Set(sessions - .filter((session) => !session.firstSeen && !session.lastSeen) + .filter((session) => session.timestampBasis === "source-file-mtime" || (!session.firstSeen && !session.lastSeen)) .map((session) => session.sessionId)); const requestedWindow = scope.sinceTime !== null || scope.untilTime !== null; - const relevantIds = inWindowIds.size > 0 - ? inWindowIds - : requestedWindow - ? unknownTimeIds - : new Set(sessions.map((session) => session.sessionId)); + const relevantIds = requestedWindow + ? inWindowIds.size > 0 + ? inWindowIds + : unknownTimeIds + : new Set(sessions.map((session) => session.sessionId)); const relevantSummaries = [...relevantIds] .map((sessionId) => transcriptCoverage.get(sessionId)) .filter(Boolean); @@ -684,10 +734,13 @@ export class CursorSessionAnalyzer extends SessionAnalyzer { const files = await walkFiles(rootPath, { maxDepth: 2, limit: 20_000, match: (file) => file.endsWith(".jsonl") }); for (const filePath of files) { const sessionId = transcriptSessionId(filePath); + let sourceTimestamp = null; + try { sourceTimestamp = new Date((await stat(filePath)).mtimeMs).toISOString(); } catch { /* leave unobserved */ } addRef(sessions, sessionId, scope.workspace, { kind: transcriptRoot.kind, role: transcriptRoot.role, path: filePath, + sourceTimestampFallback: sourceTimestamp, }); if (scope._command === "facts") { const inspected = await inspectTranscriptCoverage(filePath); @@ -699,6 +752,23 @@ export class CursorSessionAnalyzer extends SessionAnalyzer { } } const knownIds = new Set(sessions.keys()); + const contextSessions = new Set(); + for (const candidate of await findCursorContextUsageSnapshots(scope)) { + if (!withinTimeRange(candidate.capturedAt, scope)) continue; + let raw; + try { raw = await readJson(candidate.filePath); } catch { continue; } + const usage = projectContextUsageSnapshot(raw, { capturedAt: candidate.capturedAt, scope }); + const sessionId = usage?.actions?.openAgentId; + if (!sessionId || !knownIds.has(sessionId) || contextSessions.has(sessionId)) continue; + contextSessions.add(sessionId); + addRef(sessions, sessionId, scope.workspace, { + kind: "cursor-context-usage-canvas", + role: "context-usage-snapshot", + path: candidate.filePath, + sourceTimestampFallback: candidate.capturedAt, + contextUsage: usage, + }); + } const metaRoot = roots.find((root) => root.kind === "cursor-chat-meta"); if (metaRoot?.exists) { const metaFiles = await walkFiles(metaRoot.path, { maxDepth: 3, limit: 20_000, match: (file) => path.basename(file) === "meta.json" }); @@ -716,6 +786,7 @@ export class CursorSessionAnalyzer extends SessionAnalyzer { path: filePath, firstSeen, lastSeen, + timestampBasis: firstSeen || lastSeen ? "native-metadata" : null, ...(meta.cwd ? { cwd: meta.cwd, cwdPriority: 4 } : {}), }); } @@ -739,16 +810,13 @@ export class CursorSessionAnalyzer extends SessionAnalyzer { path: root.path, firstSeen: range.firstSeen, lastSeen: range.lastSeen, + timestampBasis: range.firstSeen || range.lastSeen ? "native-event" : null, }); } } const workspaceSessions = [...sessions.values()].map(finalizeSession); const inWindowSessions = workspaceSessions - .filter((session) => { - const timestamp = session.lastSeen ?? session.firstSeen; - if ((scope.sinceTime !== null || scope.untilTime !== null) && !timestamp) return false; - return withinTimeRange(timestamp, scope); - }) + .filter((session) => withinTimeRange(session.lastSeen ?? session.firstSeen, scope)) .sort((left, right) => (timestampMillis(right.lastSeen) ?? 0) - (timestampMillis(left.lastSeen) ?? 0)); if (scope._command === "facts") { scope._cursorSourceCoverage = buildCursorSourceCoverage({ @@ -768,6 +836,7 @@ export class CursorSessionAnalyzer extends SessionAnalyzer { normalizeEvents(raw, sourceRef, options = {}) { if (sourceRef.kind === "cursor-chat-meta") return metaEvents(raw, sourceRef); + if (sourceRef.kind === "cursor-context-usage-canvas") return contextUsageEvents(sourceRef); if (sourceRef.kind.includes("audit")) return auditEvents(raw, sourceRef, options); return transcriptEvents(raw, sourceRef, options); } @@ -799,6 +868,11 @@ export class CursorSessionAnalyzer extends SessionAnalyzer { .filter((event) => withinTimeRange(event.timestamp, scope))); continue; } + if (ref.kind === "cursor-context-usage-canvas") { + events.push(...contextUsageEvents({ ...ref, sessionId: session.sessionId }) + .filter((event) => withinTimeRange(event.timestamp, scope))); + continue; + } const readCoverage = await forEachJsonLine(ref.path, (raw, line) => { if (ref.kind.includes("audit")) { const identities = [raw?.session_id, raw?.conversation_id, diff --git a/test/reporting/harness-inspector.test.mjs b/test/reporting/harness-inspector.test.mjs index b7a087b5..e6660332 100644 --- a/test/reporting/harness-inspector.test.mjs +++ b/test/reporting/harness-inspector.test.mjs @@ -574,6 +574,98 @@ test("Inspector final HTML redacts credentials and omits absolute roots (AC-7)", assert.match(html, /absolute-path/u); }); +test("Inspector projects usage and context metadata without raw context text", () => { + const secret = "private-system-context-fixture"; + const report = buildHarnessInspectorReport({ + repoRoot: "/workspace/repo", + featureTree: parseFeatureTreeMarkdown(FEATURE_TREE), + sessions: [fixtureSession({ + dialogue: { + truncated: false, + turns: [{ + index: 1, + prompt: { text: "Inspect usage", timestamp: "2026-08-12T08:00:00.000Z" }, + steps: [{ + kind: "usage", + tokenUsage: { inputTokens: 90, outputTokens: 8, totalTokens: 98, raw: secret }, + contextUsage: { usedTokens: 25, windowTokens: 100, percentFull: 25, raw: secret }, + source: "fixture-response-usage", + raw: secret, + }], + usageEventCount: 1, + eventCount: 1, + shownEventCount: 1, + toolCallCount: 0, + response: "Measured.", + responseStatus: "retained", + }], + }, + tokenUsage: { + inputTokens: 180, + outputTokens: 18, + cacheReadInputTokens: 90, + cacheCreationInputTokens: 5, + reasoningOutputTokens: 6, + totalTokens: 198, + basis: "model-inference", + source: "codex-rollout-token-count", + coverage: "observed", + raw: secret, + }, + runtime: { modelProvider: "openai", cliVersion: "fixture-cli", effort: "high", raw: secret }, + timestampBasis: "native-event", + contextManifest: { + status: "observed", + source: "codex-rollout-token-count", + rawTextOmitted: false, + usedTokens: 25, + windowTokens: 100, + percentFull: 25, + compactionCount: 1, + layers: [{ kind: "developer-message", itemCount: 2, text: secret }], + categories: [{ kind: "rules", label: "Rules", estimatedTokens: 10, text: secret }], + rawText: secret, + }, + })], + correlation: fixtureCorrelation(), + }); + const session = report.sessions[0]; + assert.deepEqual(session.tokenUsage, { + inputTokens: 180, + outputTokens: 18, + cacheReadInputTokens: 90, + cacheCreationInputTokens: 5, + reasoningOutputTokens: 6, + totalTokens: 198, + basis: "model-inference", + source: "codex-rollout-token-count", + coverage: "observed", + }); + assert.deepEqual(session.runtime, { modelProvider: "openai", cliVersion: "fixture-cli", effort: "high" }); + assert.deepEqual(session.contextManifest, { + status: "observed", + source: "codex-rollout-token-count", + rawTextOmitted: true, + compactionCount: 1, + layers: [{ kind: "developer-message", itemCount: 2 }], + categories: [{ kind: "rules", label: "Rules", estimatedTokens: 10 }], + usedTokens: 25, + windowTokens: 100, + percentFull: 25, + }); + assert.deepEqual(session.dialogue.turns[0].steps[0], { + kind: "usage", + tokenUsage: { inputTokens: 90, outputTokens: 8, totalTokens: 98 }, + contextUsage: { usedTokens: 25, windowTokens: 100, percentFull: 25 }, + source: "fixture-response-usage", + timestamp: null, + }); + const html = renderHarnessInspectorHtml(report); + assert.match(html, /Usage and context/u); + assert.match(html, /Model response/u); + assert.doesNotMatch(html, new RegExp(secret, "u")); +}); + test("declared refs keep platform identity and reject ambiguous commit prefixes", () => { const tree = parseFeatureTreeMarkdown(`# Feature: Refs {#refs}\n## Story: Strict refs {#strict-refs}\n- session: qoder/session-a\n- commit: aaaaaaa\n`); const correlation = fixtureCorrelation(); diff --git a/test/sessions/commit-session-link.test.mjs b/test/sessions/commit-session-link.test.mjs index cc5ef840..71420dfb 100644 --- a/test/sessions/commit-session-link.test.mjs +++ b/test/sessions/commit-session-link.test.mjs @@ -508,11 +508,80 @@ test("summarizeSessionEvents extracts repo-relative files, prompts, tools, and t assert.equal(summary.toolCallCount, 2); assert.equal(summary.promptCount, 1); assert.equal(summary.prompts.length, 1); - assert.deepEqual(summary.tokenUsage, { inputTokens: 100, outputTokens: 50, cacheReadInputTokens: 10 }); + assert.deepEqual(summary.tokenUsage, { + inputTokens: 100, + outputTokens: 50, + cacheReadInputTokens: 10, + basis: "model-inference", + source: "normalized-session-events", + coverage: "observed", + }); assert.equal(summary.firstSeen, new Date("2026-08-02T10:00:00+08:00").toISOString()); assert.equal(summary.toolTrace.totalCalls, 2); }); +test("summarizeSessionEvents aggregates cumulative usage segments and bounded context metadata", () => { + const snapshots = [ + { inputTokens: 100, outputTokens: 10, cacheReadInputTokens: 50, cacheCreationInputTokens: 2, reasoningOutputTokens: 2, totalTokens: 110 }, + { inputTokens: 100, outputTokens: 10, cacheReadInputTokens: 50, cacheCreationInputTokens: 2, reasoningOutputTokens: 2, totalTokens: 110 }, + { inputTokens: 150, outputTokens: 15, cacheReadInputTokens: 80, cacheCreationInputTokens: 4, reasoningOutputTokens: 5, totalTokens: 165 }, + { inputTokens: 20, outputTokens: 2, cacheReadInputTokens: 5, cacheCreationInputTokens: 0, reasoningOutputTokens: 0, totalTokens: 22 }, + { inputTokens: 30, outputTokens: 3, cacheReadInputTokens: 10, cacheCreationInputTokens: 1, reasoningOutputTokens: 1, totalTokens: 33 }, + ]; + const events = snapshots.map((modelUsage, index) => ({ + type: "model.usage.snapshot", + category: "model", + timestamp: `2026-08-28T10:00:0${index}.000Z`, + modelUsage, + usageCumulative: true, + usageBasis: "model-inference", + usageSource: "codex-rollout-token-count", + })); + events.push( + { type: "context.developer", contextLayers: [{ kind: "developer-message", itemCount: 1, aggregation: "sum" }] }, + { type: "context.developer", contextLayers: [{ kind: "developer-message", itemCount: 1, aggregation: "sum" }] }, + { type: "session_meta", contextLayers: [{ kind: "base-instructions", itemCount: 1, aggregation: "max" }], runtimeMetadata: { modelProvider: "openai", cliVersion: "fixture-cli" } }, + { type: "turn_context", model: "fixture-model", runtimeMetadata: { effort: "high" } }, + { type: "compacted", compactionBoundary: true }, + { type: "context.usage", currentContextUsage: { usedTokens: 25, windowTokens: 100, source: "codex-rollout-token-count", rawTextOmitted: true } }, + ); + + const summary = summarizeSessionEvents( + { sessionId: "usage-session", firstSeen: null, lastSeen: null }, + events, + { repoRoot: path.join(os.tmpdir(), "fixture-repo"), platform: "codex", includeDialogue: true }, + ); + + assert.deepEqual(summary.tokenUsage, { + inputTokens: 180, + outputTokens: 18, + cacheReadInputTokens: 90, + cacheCreationInputTokens: 5, + reasoningOutputTokens: 6, + totalTokens: 198, + basis: "model-inference", + source: "codex-rollout-token-count", + coverage: "observed", + }); + assert.deepEqual(summary.runtime, { modelProvider: "openai", cliVersion: "fixture-cli", effort: "high" }); + assert.deepEqual(summary.contextManifest, { + status: "observed", + source: "codex-rollout-token-count", + rawTextOmitted: true, + usedTokens: 25, + windowTokens: 100, + percentFull: 25, + compactionCount: 1, + layers: [ + { kind: "base-instructions", itemCount: 1 }, + { kind: "developer-message", itemCount: 2 }, + ], + categories: [], + }); + assert.deepEqual(summary.models, ["fixture-model"]); + assert.equal(JSON.stringify(summary.dialogue).includes("developer"), false); +}); + test("summarizeSessionEvents projects Entire-style dialogue without raw command payloads", () => { const repoRoot = path.resolve("/tmp/fixture-repo"); const secret = "ghp_abcdefghijklmnop123456"; @@ -686,6 +755,38 @@ test("buildSessionTurns folds prompts, steps, and responses into turns (AC-7)", assert.equal(turns[1].response, "Committed."); }); +test("buildSessionTurns keeps per-inference usage and only same-event context windows (AC-8)", () => { + const { turns } = buildSessionTurns([ + { type: "user", userPrompt: true, userText: "measure each response", timestamp: "2026-08-02T10:00:00Z" }, + { type: "assistant", content: "I will inspect it.", timestamp: "2026-08-02T10:01:00Z" }, + { + type: "model.response.completed", + modelInvocationUsage: { inputTokens: 80, outputTokens: 8, totalTokens: 88 }, + usageBasis: "model-inference", + usageSource: "claude-project-transcript", + timestamp: "2026-08-02T10:01:00Z", + }, + { type: "assistant", content: "Measured.", timestamp: "2026-08-02T10:02:00Z" }, + { + type: "event.token_count", + modelInvocationUsage: { inputTokens: 120, outputTokens: 12, cacheReadInputTokens: 60, totalTokens: 132 }, + currentContextUsage: { usedTokens: 120, windowTokens: 400, source: "codex-rollout-token-count" }, + usageBasis: "model-inference", + usageSource: "codex-rollout-token-count", + timestamp: "2026-08-02T10:02:01Z", + }, + ]); + + assert.equal(turns.length, 1); + assert.deepEqual(turns[0].steps.map((step) => step.kind), ["note", "usage", "usage"]); + assert.equal(turns[0].usageEventCount, 2); + assert.equal(turns[0].eventCount, 3); + assert.equal(turns[0].response, "Measured."); + assert.deepEqual(turns[0].steps[1].tokenUsage, { inputTokens: 80, outputTokens: 8, totalTokens: 88 }); + assert.equal(turns[0].steps[1].contextUsage, undefined); + assert.deepEqual(turns[0].steps[2].contextUsage, { usedTokens: 120, windowTokens: 400, percentFull: 30 }); +}); + test("buildSessionTurns preserves interleaved assistant and tool evidence and marks an unfinished Turn", () => { const { turns } = buildSessionTurns([ { type: "user", userPrompt: true, userText: "refactor it", timestamp: "2026-08-02T10:00:00Z" }, diff --git a/test/sessions/session-analysis-providers.test.mjs b/test/sessions/session-analysis-providers.test.mjs index 4b90ed5e..0bd9e6ae 100644 --- a/test/sessions/session-analysis-providers.test.mjs +++ b/test/sessions/session-analysis-providers.test.mjs @@ -248,7 +248,7 @@ test("Claude provider expands nested tool requests and results without using gen message: { role: "assistant", model: "claude-fixture", - usage: { input_tokens: 10, output_tokens: 4 }, + usage: { input_tokens: 10, output_tokens: 4, cache_read_input_tokens: 5, cache_creation_input_tokens: 3 }, content: [ { type: "text", text: "I will inspect and validate it." }, { type: "tool_use", id: "tool-1", name: "Bash", input: { command: "npm test" } }, @@ -309,11 +309,15 @@ test("Claude provider expands nested tool requests and results without using gen assert.equal(events.filter((event) => event.type === "tool.call").length, 2); assert.equal(events.filter((event) => event.type === "tool.result").length, 2); assert.equal(events.find((event) => event.model === "claude-fixture")?.modelUsage.inputTokens, 10); + assert.equal(events.find((event) => event.model === "claude-fixture")?.modelInvocationUsage.outputTokens, 4); + assert.equal(events.find((event) => event.model === "claude-fixture")?.modelUsage.cacheCreationInputTokens, 3); + assert.equal(events.find((event) => event.model === "claude-fixture")?.usageSource, "claude-project-transcript"); assert.equal(events.find((event) => event.toolInvocationId === "tool-2")?.filePath, path.join(workspace, "package.json")); assert.equal(events.find((event) => event.toolInvocationId === "tool-2" && event.type === "tool.result")?.success, false); const insights = await analyzer.analyze({ command: "insights", workspace, home, selection: "all-eligible" }); assert.equal(insights.insights.keySignals.usageEfficiency.coverage.responseCount, 1); assert.equal(insights.insights.keySignals.usageEfficiency.tokenTotals.inputTokens, 10); + assert.equal(insights.insights.keySignals.usageEfficiency.tokenTotals.cacheCreationInputTokens, 3); const facts = await analyzer.analyze({ command: "facts", workspace, home, limit: 1 }); assert.equal(facts.kind, "session-core-facts"); assert.equal(facts.scope.platform, "claude"); @@ -348,6 +352,8 @@ test("Cursor provider joins transcript, metadata, and only matching audit sessio { role: "user", message: { content: [{ type: "text", text: "Fix the Cursor adapter" }] } }, { role: "assistant", + input_tokens: 18, + output_tokens: 4, message: { content: [ { type: "text", text: "I will edit and test it." }, { type: "tool_use", name: "Write", input: { file_path: path.join(workspace, "adapter.mjs"), content: "x" } }, @@ -416,7 +422,13 @@ test("Cursor provider joins transcript, metadata, and only matching audit sessio assert.equal(events.filter((event) => event.type === "tool.call").length, 2); assert.equal(events.filter((event) => event.type === "tool.result").length, 1); assert.equal(events.some((event) => event.sessionId === "foreign-session"), false); - assert.equal(events.find((event) => event.usageFieldsObserved)?.modelUsage.inputTokens, 20); + assert.deepEqual(events.find((event) => event.type === "assistant")?.modelInvocationUsage, { + inputTokens: 18, + outputTokens: 4, + cacheReadInputTokens: 0, + cacheCreationInputTokens: 0, + }); + assert.equal(events.find((event) => event.usageSource === "cursor-hook-audit")?.modelUsage.inputTokens, 20); const duration = measureLongSessionRows(discovery.sessions, events).rows[0]; assert.equal(duration.activeTimeObserved, true); const facts = await analyzer.analyze({ command: "facts", workspace, home, limit: 1 }); @@ -576,6 +588,46 @@ test("Cursor time filters use metadata before excluding out-of-window transcript assert.equal(facts.sourceCoverage.transcript.relevantSessions, 0); }); +test("Cursor keeps timestamp-unobserved transcripts with labelled source time and matching context usage", async () => { + const root = await fixtureRoot("session-cursor-source-time-"); + const home = path.join(root, ".cursor"); + const workspace = path.join(root, "workspace", "project"); + const sessionId = "77777777-7777-4777-8777-777777777777"; + const slug = workspaceToCursorSlugVariants(workspace)[0]; + await writeJsonl(path.join(home, "projects", slug, "agent-transcripts", sessionId, `${sessionId}.jsonl`), [ + { role: "user", message: { content: [{ type: "text", text: "Inspect context usage" }] } }, + { role: "assistant", message: { content: [{ type: "text", text: "Done" }] } }, + ]); + const canvasPath = path.join(home, "projects", slug, "canvases", "context-usage-fixture.canvas.data.json"); + await mkdir(path.dirname(canvasPath), { recursive: true }); + await writeFile(canvasPath, JSON.stringify({ + contextUsage: { + composerId: sessionId, + totalTokensUsed: 250, + contextWindowSize: 1_000, + categories: [{ id: "rules", label: "Rules", tokens: 80 }], + items: [{ categoryId: "rules", label: "Private rule text", estimatedTokens: 80, characterCount: 320 }], + }, + })); + + const analyzer = new CursorSessionAnalyzer(); + const result = await analyzer.analyze({ command: "sources", workspace, home, since: "2026-08-20" }); + assert.equal(result.sessions.length, 1); + assert.equal(result.sessions[0].timestampBasis, "source-file-mtime"); + assert.equal(result.sessions[0].eventTimestampCoverage, "unobserved"); + const scope = await analyzer.resolveScope({ workspace, home, since: "2026-08-20" }); + const events = await analyzer.readSession(result.sessions[0], scope, { includeContent: true, includeUserText: true }); + const context = events.find((event) => event.type === "context.usage"); + assert.deepEqual(context?.currentContextUsage, { + usedTokens: 250, + windowTokens: 1_000, + source: "cursor-native-context-usage-canvas", + rawTextOmitted: true, + }); + assert.deepEqual(context?.contextCategories, [{ kind: "rules", label: "Rules", estimatedTokens: 80 }]); + assert.equal(JSON.stringify(context).includes("Private rule text"), false); +}); + test("Qwen provider expands function calls and tool results from parts", async () => { const root = await fixtureRoot("session-qwen-provider-"); const home = path.join(root, ".qwen"); diff --git a/test/sessions/session-analysis.test.mjs b/test/sessions/session-analysis.test.mjs index 3767c864..cc491af4 100644 --- a/test/sessions/session-analysis.test.mjs +++ b/test/sessions/session-analysis.test.mjs @@ -542,6 +542,28 @@ test("Qoder and Codex preserve only explicit user-visible handoff and delivery f type: "event_msg", payload: { type: "agent_message", message: "done" }, }, sourceRef); + const codexDeveloper = codexAnalyzer.normalizeEvent({ + type: "response_item", + payload: { type: "message", role: "developer", content: [{ type: "input_text", text: "private developer context" }] }, + }, sourceRef, { includeContent: true }); + const codexUsage = codexAnalyzer.normalizeEvent({ + type: "event_msg", + payload: { + type: "token_count", + info: { + total_token_usage: { + input_tokens: 120, + cached_input_tokens: 80, + cache_write_input_tokens: 4, + output_tokens: 30, + reasoning_output_tokens: 8, + total_tokens: 150, + }, + last_token_usage: { input_tokens: 40, output_tokens: 10, total_tokens: 50 }, + model_context_window: 200, + }, + }, + }, sourceRef); const qoderToolUseOnly = qoderAnalyzer.normalizeEvent({ type: "assistant", stop_reason: "tool_use", @@ -553,11 +575,25 @@ test("Qoder and Codex preserve only explicit user-visible handoff and delivery f assert.equal(normalized.taskCompleted, true); assert.equal(normalized.userCorrection, true); } - assert.equal(codexReasoning.type, "assistant"); + assert.equal(codexReasoning.type, "reasoning"); assert.equal(codexReasoning.userVisibleAssistantMessage, undefined); + assert.equal(codexReasoning.content, undefined); assert.equal(codexReasoning.taskCompleted, undefined); assert.equal(codexReasoning.userCorrection, undefined); assert.equal(codexAgentMessage.userVisibleAssistantMessage, true); + assert.equal(codexDeveloper.type, "context.developer"); + assert.equal(codexDeveloper.content, undefined); + assert.deepEqual(codexDeveloper.contextLayers, [{ kind: "developer-message", itemCount: 1, aggregation: "sum" }]); + assert.equal(codexUsage.usageCumulative, true); + assert.equal(codexUsage.modelUsage.totalTokens, 150); + assert.equal(codexUsage.modelUsage.cacheReadInputTokens, 80); + assert.deepEqual(codexUsage.modelInvocationUsage, { inputTokens: 40, outputTokens: 10, totalTokens: 50 }); + assert.deepEqual(codexUsage.currentContextUsage, { + usedTokens: 40, + windowTokens: 200, + source: "codex-rollout-token-count", + rawTextOmitted: true, + }); assert.equal(qoderToolUseOnly.userVisibleAssistantMessage, undefined); });