diff --git a/src/cli/commands/deploy/__tests__/utils.test.ts b/src/cli/commands/deploy/__tests__/utils.test.ts index 8bf07311a..2ba22c15a 100644 --- a/src/cli/commands/deploy/__tests__/utils.test.ts +++ b/src/cli/commands/deploy/__tests__/utils.test.ts @@ -24,7 +24,7 @@ describe('computeDeployAttrs', () => { gateway_target_count: 3, policy_engine_count: 2, policy_count: 3, - mode: 'diff', + deploy_mode: 'diff', }); }); @@ -39,7 +39,7 @@ describe('computeDeployAttrs', () => { gateway_target_count: 0, policy_engine_count: 0, policy_count: 0, - mode: 'deploy', + deploy_mode: 'deploy', }); }); @@ -49,6 +49,6 @@ describe('computeDeployAttrs', () => { expect(attrs.runtime_count).toBe(1); expect(attrs.memory_count).toBe(0); - expect(attrs.mode).toBe('dry-run'); + expect(attrs.deploy_mode).toBe('dry-run'); }); }); diff --git a/src/cli/commands/deploy/utils.ts b/src/cli/commands/deploy/utils.ts index d0db1ec08..c866ded9e 100644 --- a/src/cli/commands/deploy/utils.ts +++ b/src/cli/commands/deploy/utils.ts @@ -1,6 +1,5 @@ import type { AgentCoreProjectSpec } from '../../../schema'; - -export type DeployMode = 'deploy' | 'dry-run' | 'diff'; +import type { DeployMode } from '../../telemetry/schemas/common-shapes'; export const DEFAULT_DEPLOY_ATTRS = { runtime_count: 0, @@ -12,7 +11,7 @@ export const DEFAULT_DEPLOY_ATTRS = { gateway_target_count: 0, policy_engine_count: 0, policy_count: 0, - mode: 'deploy' as DeployMode, + deploy_mode: 'deploy' as DeployMode, }; export function computeDeployAttrs(projectSpec: Partial, mode: DeployMode) { @@ -28,6 +27,6 @@ export function computeDeployAttrs(projectSpec: Partial, m gateway_target_count: gateways.reduce((sum, g) => sum + (g.targets ?? []).length, 0), policy_engine_count: policyEngines.length, policy_count: policyEngines.reduce((sum, pe) => sum + (pe.policies ?? []).length, 0), - mode, + deploy_mode: mode, }; } diff --git a/src/cli/commands/dev/command.tsx b/src/cli/commands/dev/command.tsx index b6cb0a7ba..926cf3045 100644 --- a/src/cli/commands/dev/command.tsx +++ b/src/cli/commands/dev/command.tsx @@ -479,20 +479,21 @@ export const registerDev = (program: Command) => { // Default: launch web UI in browser // NOTE: Do not copy this pattern. runBrowserMode blocks forever (internal // await new Promise(() => {})) so we cannot use withCommandRunTelemetry here. - // We emit telemetry eagerly before the blocking call. If startup fails, the - // error propagates to the outer catch. Prefer withCommandRunTelemetry for - // commands that return. + // We emit telemetry eagerly before the blocking call. { const client = await TelemetryClientAccessor.get().catch(() => undefined); - const devAttrs = { - action: 'server' as const, - ui_mode: 'browser' as const, - has_stream: false, - protocol: standardize(Protocol, (targetDevAgent?.protocol ?? 'http').toLowerCase()), - invoke_count: 0, - }; if (client) { - await client.withCommandRun('dev', () => devAttrs); + client.emit('cli.command_run', 0, { + command_group: 'dev', + command: 'dev', + exit_reason: 'success', + action: 'server', + ui_mode: 'browser', + has_stream: false, + protocol: standardize(Protocol, (targetDevAgent?.protocol ?? 'http').toLowerCase()), + invoke_count: 0, + }); + await client.flush(); } await runBrowserMode({ workingDir, diff --git a/src/cli/commands/help/command.tsx b/src/cli/commands/help/command.tsx index 338684d7e..b4ab65a70 100644 --- a/src/cli/commands/help/command.tsx +++ b/src/cli/commands/help/command.tsx @@ -1,4 +1,4 @@ -import { TelemetryClientAccessor } from '../../telemetry/client-accessor.js'; +import { withCommandRunTelemetry } from '../../telemetry/cli-command-run.js'; import type { Command } from '@commander-js/extra-typings'; const MODES_HELP = ` @@ -43,11 +43,10 @@ export const registerHelp = (program: Command) => { .command('help') .description('Display help topics') .action(async () => { - const client = await TelemetryClientAccessor.get(); - await client.withCommandRun('help', () => { + await withCommandRunTelemetry('help', {}, () => { console.log('Available help topics: modes'); console.log('Run `agentcore help ` for details.'); - return {}; + return { success: true as const }; }); }); @@ -55,10 +54,9 @@ export const registerHelp = (program: Command) => { .command('modes') .description('Explain interactive vs non-interactive modes') .action(async () => { - const client = await TelemetryClientAccessor.get(); - await client.withCommandRun('help.modes', () => { + await withCommandRunTelemetry('help.modes', {}, () => { console.log(MODES_HELP); - return {}; + return { success: true as const }; }); }); }; diff --git a/src/cli/telemetry/README.md b/src/cli/telemetry/README.md index 49f98bbcb..6750c3108 100644 --- a/src/cli/telemetry/README.md +++ b/src/cli/telemetry/README.md @@ -1,121 +1,97 @@ -# Adding New Telemetry Metrics +# Telemetry -## Overview +## Adding a New Metric -Every CLI command emits a `command_run` metric with a command key, exit reason, and command-specific attributes. This -guide shows how to add telemetry to a new command. +### 1. Define attributes in `schemas/common-shapes.ts` -## Step 1: Register the command in `schemas/command-run.ts` - -Add an entry to `COMMAND_SCHEMAS`: +Skip if reusing existing attributes. ```ts -// No attributes: -'remove.widget': NoAttrs, +export const ToolName = z.enum(['read_file', 'write_file', 'search']); +``` -// With attributes: -'add.widget': safeSchema({ - widget_type: WidgetType, // z.enum(), z.boolean(), z.number(), or z.literal() only - count: Count, -}), +Add to the `ATTRIBUTES` object using the field name as the key: + +```ts +export const ATTRIBUTES = { + // ...existing + tool_name: ToolName, +} as const; ``` -`safeSchema` enforces allowed field types at compile time. No `z.string()` fields. +### 2. Register the metric in `schemas/registry.ts` -## Step 2: Add enums to `schemas/common-shapes.ts` +Add an entry to `METRICS` with a description, and a corresponding `MetricAttrs` branch: ```ts -export const WidgetType = z.enum(['basic', 'advanced']); +export const METRICS = { + 'cli.command_run': { description: 'CLI/TUI Command Execution' }, + 'cli.mcp_tool_call': { description: 'MCP tool invocation' }, +} as const satisfies MetricRegistry; + +export type MetricAttrs = M extends 'cli.command_run' + ? CommandRunAttrs + : M extends 'cli.mcp_tool_call' + ? { tool_name: z.infer; success: boolean } + : never; ``` -Use `standardize()` to normalize input before recording: +### 3. Emit it ```ts -import { WidgetType, standardize } from '../telemetry/schemas/common-shapes.js'; - -const type = standardize(WidgetType, userInput); +client.emit('cli.mcp_tool_call', durationMs, { tool_name: 'read_file', success: true }); ``` -## Step 3: Instrument the command handler +Wrong metric name or missing attrs = compile error. -Use **`withCommandRunTelemetry`** — the primary helper for recording telemetry: +--- -```ts -import { withCommandRunTelemetry } from '../telemetry/cli-command-run.js'; +## Adding a New Command (to `cli.command_run`) -const result = await withCommandRunTelemetry('remove.gateway', {}, () => this.remove(name)); -``` - -**Signature:** +### 1. Define the command's attribute schema in `schemas/command-run.ts` ```ts -async function withCommandRunTelemetry( - command: C, - attrs: CommandAttrs, - fn: () => Promise -): Promise; +const AddWidgetAttrs = safeSchema({ + widget_type: WidgetType, + count: Count, +}); ``` -- `command` — the registered command key (e.g. `'add.widget'`) -- `attrs` — attribute object matching the schema registered in Step 1 -- `fn` — async callback returning `Result` (from `src/lib/result.ts`) +Add to `COMMAND_SCHEMAS`: -**Behavior:** +```ts +'add.widget': AddWidgetAttrs, +``` -- On success: records `attrs` with `exit_reason: 'success'`, returns the result. -- On failure/throw: records `attrs` with `exit_reason: 'failure'`, returns `{ success: false, error }`. -- If telemetry is unavailable: runs `fn()` untracked. +The `Command` type and optional fields in `MetricAttrs<'cli.command_run'>` are derived automatically from +`COMMAND_SCHEMAS`. -Since `attrs` are passed upfront, they are always recorded — even on failure. +### 2. Instrument the handler -**Example with attributes:** +Use `withCommandRunTelemetry`: ```ts const result = await withCommandRunTelemetry( 'add.widget', - { widget_type: standardize(WidgetType, config.type), count: config.items.length }, + { widget_type: standardize(WidgetType, input), count: items.length }, () => widgetPrimitive.add(config) ); - -if (!result.success) { - console.error(result.error); - process.exit(1); -} ``` -### `runCliCommand` (alternative for top-level CLI handlers) - -For CLI handlers that own `process.exit`, use `runCliCommand` instead. The callback throws on failure and returns attrs -on success: +Or `runCliCommand` for top-level CLI handlers that own `process.exit`: ```ts -await runCliCommand('add.widget', !!options.json, async () => { - const result = await widgetPrimitive.add(options); - if (!result.success) throw new Error(result.error); - return { widget_type: standardize(WidgetType, options.type), count: options.items.length }; +await runCliCommand('add.widget', !!opts.json, async () => { + await widgetPrimitive.add(opts); + return { widget_type: standardize(WidgetType, opts.type), count: opts.items.length }; }); ``` -To record attrs on failure, pass `knownAttrs` as the fourth argument: - -```ts -const knownAttrs = { widget_type: standardize(WidgetType, options.type), count: options.items.length }; -await runCliCommand( - 'add.widget', - !!options.json, - async () => { - const result = await widgetPrimitive.add(options); - if (!result.success) throw new Error(result.error); - return knownAttrs; - }, - knownAttrs -); -``` +--- -## Key Points +## Key Rules -- Telemetry never crashes the CLI — `standardize()` falls back gracefully, `resilientParse` defaults invalid fields to - `'unknown'`. -- Prefer `withCommandRunTelemetry` for new code — it returns the `Result` for the caller to handle output and control - flow. -- Use `runCliCommand` only when the handler owns `process.exit` and prints its own output. +- `safeSchema` only allows `z.enum()`, `z.boolean()`, `z.number()`, `z.literal()`. No `z.string()`. +- `standardize(schema, value)` lowercases and validates enum values. Invalid values fall through gracefully. +- `resilientParse` validates each field independently — one bad field defaults to `'unknown'`, never drops the metric. +- Telemetry never crashes the CLI. diff --git a/src/cli/telemetry/__tests__/client.test.ts b/src/cli/telemetry/__tests__/client.test.ts index b66111849..ddcca6168 100644 --- a/src/cli/telemetry/__tests__/client.test.ts +++ b/src/cli/telemetry/__tests__/client.test.ts @@ -1,213 +1,169 @@ /* eslint-disable @typescript-eslint/require-await */ import { AccessDeniedError, DependencyCheckError } from '../../../lib/errors/types'; -import { CANCELLED, TelemetryClient } from '../client'; +import { withCommandRunTelemetry } from '../cli-command-run'; +import { TelemetryClient } from '../client'; +import { TelemetryClientAccessor } from '../client-accessor'; import { InMemorySink } from '../sinks/in-memory-sink'; -import { describe, expect, it } from 'vitest'; - -describe('TelemetryClient', () => { - describe('withCommandRun', () => { - it('records success with returned attrs', async () => { - const sink = new InMemorySink(); - const client = new TelemetryClient(sink); - - await client.withCommandRun('update', async () => ({ check_only: true })); - - expect(sink.metrics).toHaveLength(1); - expect(sink.metrics[0]!.attrs).toMatchObject({ - command_group: 'update', - command: 'update', - exit_reason: 'success', - check_only: 'true', - }); - }); - - it('accepts sync callbacks', async () => { - const sink = new InMemorySink(); - const client = new TelemetryClient(sink); - - await client.withCommandRun('telemetry.disable', () => ({})); - - expect(sink.metrics).toHaveLength(1); - expect(sink.metrics[0]!.attrs).toMatchObject({ exit_reason: 'success' }); - }); +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; - it('records failure and re-throws on error', async () => { - const sink = new InMemorySink(); - const client = new TelemetryClient(sink); - - await expect( - client.withCommandRun('deploy', async () => { - throw new Error('boom'); - }) - ).rejects.toThrow('boom'); - - expect(sink.metrics).toHaveLength(1); - expect(sink.metrics[0]!.attrs).toMatchObject({ - command_group: 'deploy', - exit_reason: 'failure', - error_name: 'UnknownError', - }); - }); +let sink: InMemorySink; - it('classifies DependencyCheckError correctly', async () => { - const sink = new InMemorySink(); - const client = new TelemetryClient(sink); +beforeEach(() => { + sink = new InMemorySink(); + vi.spyOn(TelemetryClientAccessor, 'get').mockResolvedValue(new TelemetryClient(sink)); +}); - await expect( - client.withCommandRun('deploy', async () => { - throw new DependencyCheckError(['missing docker']); - }) - ).rejects.toThrow(); +afterEach(() => { + vi.restoreAllMocks(); +}); - expect(sink.metrics[0]!.attrs).toMatchObject({ - error_name: 'DependencyCheckError', - error_source: 'user', - }); +describe('withCommandRunTelemetry', () => { + it('records success with returned attrs', async () => { + await withCommandRunTelemetry('update', { check_only: true }, async () => ({ success: true })); + + expect(sink.metrics).toHaveLength(1); + expect(sink.metrics[0]!.metric).toBe('cli.command_run'); + expect(sink.metrics[0]!.attrs).toMatchObject({ + command_group: 'update', + command: 'update', + exit_reason: 'success', + check_only: 'true', }); + }); - it('marks credential errors as user errors', async () => { - const sink = new InMemorySink(); - const client = new TelemetryClient(sink); - - await expect( - client.withCommandRun('invoke', async () => { - throw new AccessDeniedError('creds expired'); - }) - ).rejects.toThrow(); - - expect(sink.metrics[0]!.attrs).toMatchObject({ - error_name: 'AccessDeniedError', - error_source: 'user', - }); + it('records failure when callback returns failure result', async () => { + const result = await withCommandRunTelemetry('deploy', {} as never, async () => ({ + success: false as const, + error: new Error('boom'), + })); + + expect(result.success).toBe(false); + expect(sink.metrics).toHaveLength(1); + expect(sink.metrics[0]!.attrs).toMatchObject({ + command_group: 'deploy', + exit_reason: 'failure', + error_name: 'UnknownError', }); + }); - it('records duration as a non-negative integer', async () => { - const sink = new InMemorySink(); - const client = new TelemetryClient(sink); - - await client.withCommandRun('telemetry.disable', async () => { - await new Promise(r => globalThis.setTimeout(r, 5)); - return {}; - }); + it('classifies DependencyCheckError correctly', async () => { + await withCommandRunTelemetry('deploy', {} as never, async () => ({ + success: false as const, + error: new DependencyCheckError(['missing docker']), + })); - expect(sink.metrics[0]!.value).toBeGreaterThanOrEqual(0); - expect(Number.isInteger(sink.metrics[0]!.value)).toBe(true); + expect(sink.metrics[0]!.attrs).toMatchObject({ + error_name: 'DependencyCheckError', + error_source: 'user', }); + }); - it('converts boolean attrs to strings', async () => { - const sink = new InMemorySink(); - const client = new TelemetryClient(sink); - - await client.withCommandRun('update', async () => ({ check_only: true })); + it('marks credential errors as user errors', async () => { + await withCommandRunTelemetry('invoke', {} as never, async () => ({ + success: false as const, + error: new AccessDeniedError('creds expired'), + })); - expect(sink.metrics[0]!.attrs.check_only).toBe('true'); + expect(sink.metrics[0]!.attrs).toMatchObject({ + error_name: 'AccessDeniedError', + error_source: 'user', }); + }); - it('publishes metric with unknown defaults for incomplete success payloads', async () => { - const sink = new InMemorySink(); - const client = new TelemetryClient(sink); - - // Missing required attrs for 'create' — should still publish with 'unknown' defaults - await client.withCommandRun( - 'create', - // @ts-expect-error — intentionally incomplete - async () => ({ language: 'python' }) - ); - - expect(sink.metrics).toHaveLength(1); - expect(sink.metrics[0]!.attrs).toMatchObject({ - exit_reason: 'success', - language: 'python', - framework: 'unknown', - model_provider: 'unknown', - }); + it('records duration as a non-negative integer', async () => { + await withCommandRunTelemetry('telemetry.disable', {}, async () => { + await new Promise(r => globalThis.setTimeout(r, 5)); + return { success: true as const }; }); - it('defaults invalid attrs to unknown while preserving valid ones', async () => { - const sink = new InMemorySink(); - const client = new TelemetryClient(sink); - - await client.withCommandRun( - 'create', - // @ts-expect-error — intentionally invalid enum value - async () => ({ - language: 'rust', // invalid enum - framework: 'strands', - model_provider: 'bedrock', - memory: 'shortterm', - protocol: 'mcp', - build: 'codezip', - agent_type: 'create', - network_mode: 'public', - has_agent: true, - }) - ); - - expect(sink.metrics).toHaveLength(1); - expect(sink.metrics[0]!.attrs.language).toBe('unknown'); - expect(sink.metrics[0]!.attrs.framework).toBe('strands'); - }); + expect(sink.metrics[0]!.value).toBeGreaterThanOrEqual(0); + expect(Number.isInteger(sink.metrics[0]!.value)).toBe(true); + }); - it('records cancel when callback returns CANCELLED', async () => { - const sink = new InMemorySink(); - const client = new TelemetryClient(sink); + it('converts boolean attrs to strings', async () => { + await withCommandRunTelemetry('update', { check_only: true }, async () => ({ success: true })); - await client.withCommandRun('deploy', () => CANCELLED); + expect(sink.metrics[0]!.attrs.check_only).toBe('true'); + }); - expect(sink.metrics).toHaveLength(1); - expect(sink.metrics[0]!.attrs).toMatchObject({ - command_group: 'deploy', - exit_reason: 'cancel', - }); - }); + it('defaults invalid attrs to unknown while preserving valid ones', async () => { + await withCommandRunTelemetry( + 'create', + { + language: 'rust' as never, + framework: 'strands', + model_provider: 'bedrock', + memory: 'shortterm', + protocol: 'mcp', + build: 'codezip', + agent_type: 'create', + network_mode: 'public', + has_agent: true, + }, + async () => ({ success: true }) + ); + + expect(sink.metrics).toHaveLength(1); + expect(sink.metrics[0]!.attrs.language).toBe('unknown'); + expect(sink.metrics[0]!.attrs.framework).toBe('strands'); + }); - it('records fallbackAttrs on failure when provided', async () => { - const sink = new InMemorySink(); - const client = new TelemetryClient(sink); - - await expect( - client.withCommandRun( - 'create', - async () => { - throw new Error('validation failed'); - }, - { - language: 'python', - framework: 'strands', - model_provider: 'bedrock', - memory: 'none', - protocol: 'http', - build: 'codezip', - agent_type: 'create', - network_mode: 'public', - has_agent: true, - } - ) - ).rejects.toThrow('validation failed'); - - expect(sink.metrics).toHaveLength(1); - expect(sink.metrics[0]!.attrs).toMatchObject({ - exit_reason: 'failure', - error_name: 'UnknownError', + it('records fallbackAttrs on failure', async () => { + await withCommandRunTelemetry( + 'create', + { language: 'python', framework: 'strands', model_provider: 'bedrock', - has_agent: 'true', - }); + memory: 'none', + protocol: 'http', + build: 'codezip', + agent_type: 'create', + network_mode: 'public', + has_agent: true, + }, + async () => ({ success: false as const, error: new Error('validation failed') }) + ); + + expect(sink.metrics).toHaveLength(1); + expect(sink.metrics[0]!.attrs).toMatchObject({ + exit_reason: 'failure', + error_name: 'UnknownError', + language: 'python', + framework: 'strands', + model_provider: 'bedrock', + has_agent: 'true', }); + }); + + it('runs untracked when telemetry client is unavailable', async () => { + vi.spyOn(TelemetryClientAccessor, 'get').mockRejectedValue(new Error('no client')); - it('records empty attrs on failure when fallbackAttrs not provided', async () => { - const sink = new InMemorySink(); - const client = new TelemetryClient(sink); + const result = await withCommandRunTelemetry('deploy', {} as never, async () => ({ success: true })); - await expect( - client.withCommandRun('deploy', async () => { - throw new Error('boom'); - }) - ).rejects.toThrow('boom'); + expect(result).toEqual({ success: true }); + expect(sink.metrics).toHaveLength(0); + }); - expect(sink.metrics).toHaveLength(1); - expect(sink.metrics[0]!.attrs.language).toBeUndefined(); + it('records failure and returns error result when callback throws', async () => { + type R = { success: true } | { success: false; error: Error }; + const result = await withCommandRunTelemetry<'telemetry.disable', R>( + 'telemetry.disable', + {}, + async (): Promise => { + throw new Error('network timeout'); + } + ); + + expect(result.success).toBe(false); + if (!result.success) { + expect(result.error.message).toBe('network timeout'); + } + expect(sink.metrics).toHaveLength(1); + expect(sink.metrics[0]!.attrs).toMatchObject({ + command: 'telemetry.disable', + exit_reason: 'failure', + error_name: 'UnknownError', }); }); }); diff --git a/src/cli/telemetry/__tests__/composite-sink.test.ts b/src/cli/telemetry/__tests__/composite-sink.test.ts index e07b2d65f..a3a7afb6e 100644 --- a/src/cli/telemetry/__tests__/composite-sink.test.ts +++ b/src/cli/telemetry/__tests__/composite-sink.test.ts @@ -8,7 +8,7 @@ describe('CompositeSink', () => { const b = new InMemorySink(); const composite = new CompositeSink([a, b]); - composite.record(100, { command: 'deploy' }); + composite.record('cli.command_run', 100, { command: 'deploy' }); expect(a.metrics).toHaveLength(1); expect(b.metrics).toHaveLength(1); @@ -26,7 +26,7 @@ describe('CompositeSink', () => { const good = new InMemorySink(); const composite = new CompositeSink([bad, good]); - composite.record(100, { command: 'deploy' }); + composite.record('cli.command_run', 100, { command: 'deploy' }); expect(good.metrics).toHaveLength(1); }); diff --git a/src/cli/telemetry/__tests__/filesystem-sink.test.ts b/src/cli/telemetry/__tests__/filesystem-sink.test.ts index 50d8d4620..87865063c 100644 --- a/src/cli/telemetry/__tests__/filesystem-sink.test.ts +++ b/src/cli/telemetry/__tests__/filesystem-sink.test.ts @@ -28,12 +28,13 @@ describe('FileSystemSink', () => { it('writes each record as a JSONL line on disk', async () => { const sink = createSink(); - sink.record(42, { command_group: 'deploy', command: 'deploy', exit_reason: 'success' }); + sink.record('cli.command_run', 42, { command_group: 'deploy', command: 'deploy', exit_reason: 'success' }); await sink.flush(); const entries = await readJsonl(join(outputDir, 'test-session.json')); expect(entries).toHaveLength(1); expect(entries[0]).toMatchObject({ + metric: 'cli.command_run', value: 42, attrs: { command_group: 'deploy', command: 'deploy', exit_reason: 'success' }, }); @@ -41,8 +42,8 @@ describe('FileSystemSink', () => { it('appends multiple records as separate lines', async () => { const sink = createSink(); - sink.record(10, { command_group: 'add', command: 'add.agent' }); - sink.record(20, { command_group: 'add', command: 'add.memory' }); + sink.record('cli.command_run', 10, { command_group: 'add', command: 'add.agent' }); + sink.record('cli.command_run', 20, { command_group: 'add', command: 'add.memory' }); await sink.flush(); const entries = await readJsonl(join(outputDir, 'test-session.json')); @@ -55,7 +56,7 @@ describe('FileSystemSink', () => { const nested = join(tmp.testDir, 'deep', 'nested', 'telemetry'); const filePath = join(nested, 'test.json'); const sink = new FileSystemSink({ filePath }); - sink.record(1, { command_group: 'status', command: 'status' }); + sink.record('cli.command_run', 1, { command_group: 'status', command: 'status' }); await sink.flush(); const entries = await readJsonl(filePath); @@ -70,7 +71,7 @@ describe('FileSystemSink', () => { it('shutdown logs audit message when records were written', async () => { const logged: string[] = []; const sink = createSink({ log: msg => logged.push(msg) }); - sink.record(99, { command_group: 'invoke', command: 'invoke' }); + sink.record('cli.command_run', 99, { command_group: 'invoke', command: 'invoke' }); await sink.shutdown(); expect(logged).toHaveLength(1); diff --git a/src/cli/telemetry/cli-command-run.ts b/src/cli/telemetry/cli-command-run.ts index 2fabd39fe..0ed290027 100644 --- a/src/cli/telemetry/cli-command-run.ts +++ b/src/cli/telemetry/cli-command-run.ts @@ -1,9 +1,11 @@ import type { Result } from '../../lib/result'; import { getErrorMessage } from '../errors'; import { TelemetryClientAccessor } from './client-accessor.js'; -import type { Command, CommandAttrs } from './schemas/command-run.js'; - -export type OperationResult = Result; +import { TelemetryClient } from './client.js'; +import { classifyError } from './error.js'; +import { COMMAND_SCHEMAS, type Command, type CommandAttrs, deriveCommandGroup } from './schemas/command-run.js'; +import { type CommandResult, CommandResultSchema, resilientParse } from './schemas/common-shapes.js'; +import { performance } from 'perf_hooks'; async function getTelemetryClient() { try { @@ -13,25 +15,82 @@ async function getTelemetryClient() { } } +function recordCommandRun( + client: TelemetryClient, + command: C, + result: CommandResult, + attrs: CommandAttrs | Partial>, + durationMs: number +): void { + try { + CommandResultSchema.parse(result); + + const validatedAttrs = + Object.keys(attrs as Record).length > 0 + ? resilientParse(COMMAND_SCHEMAS[command], attrs as Record) + : attrs; + + client.emit('cli.command_run', durationMs, { + command_group: deriveCommandGroup(command), + command, + ...result, + ...validatedAttrs, + }); + } catch { + // Telemetry must never affect CLI behavior + } +} + +/** + * Return attrs on success + * Unhandled throws are classified as failures and re-thrown. + */ +async function trackCommandRun( + client: TelemetryClient, + command: C, + fn: () => CommandAttrs | Promise>, + fallbackAttrs?: Partial> +): Promise { + const start = performance.now(); + try { + const result = await fn(); + const durationMs = Math.round(performance.now() - start); + recordCommandRun(client, command, { exit_reason: 'success' }, result, durationMs); + } catch (err) { + const { category, source } = classifyError(err); + const failureResult: CommandResult & { exit_reason: 'failure' } = { + exit_reason: 'failure', + error_name: category, + error_source: source, + }; + recordCommandRun(client, command, failureResult, fallbackAttrs ?? {}, Math.round(performance.now() - start)); + throw err; + } finally { + await client.flush(); + } +} + /** * Record telemetry for an operation and return its result. * Use in TUI hooks and CLI paths where the caller handles output and control flow. * * If the callback returns a failure result, telemetry is recorded and the result * is returned to the caller. If the callback throws, telemetry is recorded and - * the exception propagates. If telemetry is unavailable, the callback runs untracked. + * the exception is converted to a result type such that callers do not need to handle result + try/catch. + * If telemetry is unavailable, the callback runs untracked. */ -export async function withCommandRunTelemetry( +export async function withCommandRunTelemetry( command: C, attrs: CommandAttrs, - fn: () => Promise + fn: () => R | Promise ): Promise { const client = await getTelemetryClient(); - if (!client) return fn(); let result: R | undefined; try { - await client.withCommandRun( + if (!client) return fn(); + await trackCommandRun( + client, command, async () => { result = await fn(); @@ -41,7 +100,7 @@ export async function withCommandRunTelemetry( await fn(); process.exit(0); } - await client.withCommandRun(command, fn, knownAttrs); + await trackCommandRun(client, command, fn, knownAttrs); process.exit(0); } catch (error) { if (json) { diff --git a/src/cli/telemetry/client.ts b/src/cli/telemetry/client.ts index 14082ac42..10157a465 100644 --- a/src/cli/telemetry/client.ts +++ b/src/cli/telemetry/client.ts @@ -1,105 +1,41 @@ -import { classifyError } from './error.js'; -import { COMMAND_SCHEMAS, type Command, type CommandAttrs, deriveCommandGroup } from './schemas/command-run.js'; -import { type CommandResult, CommandResultSchema, resilientParse } from './schemas/common-shapes.js'; +import type { MetricAttrs, MetricName } from './schemas/registry.js'; import type { MetricSink } from './sinks/metric-sink.js'; -import { performance } from 'perf_hooks'; - -/** Return this from the withCommandRun callback to record a cancellation. */ -export const CANCELLED = Symbol('cancelled'); +/** + * Generic metric emitter. + */ export class TelemetryClient { constructor(private readonly sink: MetricSink) {} - /** - * Wrap a command action with telemetry recording. - * - * Return attrs on success, or CANCELLED on user cancellation. - * Unhandled throws are classified as failures and re-thrown. - * - * ```ts - * await client.withCommandRun('deploy', async () => { - * if (userCancelled) return CANCELLED; - * const result = await runDeploy(options); - * return { runtime_count: result.runtimes.length, ... }; - * }); - * ``` - */ - async withCommandRun( - command: C, - fn: () => CommandAttrs | typeof CANCELLED | Promise | typeof CANCELLED>, - fallbackAttrs?: Partial> - ): Promise { - const start = performance.now(); + emit(metricName: M, value: number, attrs: MetricAttrs): void { try { - const result = await fn(); - const durationMs = Math.round(performance.now() - start); - if (result === CANCELLED) { - this.recordCommandRun(command, { exit_reason: 'cancel' }, {}, durationMs); - } else { - this.recordCommandRun(command, { exit_reason: 'success' }, result, durationMs); - } - } catch (err) { - const { category, source } = classifyError(err); - const failureResult: CommandResult & { exit_reason: 'failure' } = { - exit_reason: 'failure', - error_name: category, - error_source: source, - }; - this.recordCommandRun(command, failureResult, fallbackAttrs ?? {}, Math.round(performance.now() - start)); - throw err; - } finally { - try { - await this.sink.flush(); - } catch { - /* telemetry must not mask command errors */ + const otelAttrs: Record = {}; + for (const [k, v] of Object.entries(attrs)) { + if (typeof v === 'boolean') { + otelAttrs[k] = String(v); + } else if (typeof v === 'string' || typeof v === 'number') { + otelAttrs[k] = v; + } } + this.sink.record(metricName, value, otelAttrs); + } catch { + // Telemetry must never affect CLI behavior } } - async shutdown(): Promise { + async flush(): Promise { try { - await this.sink.shutdown(); + await this.sink.flush(); } catch { - /* telemetry must not affect CLI behavior */ + /* telemetry must not mask command errors */ } } - private recordCommandRun( - command: C, - result: CommandResult, - attrs: CommandAttrs | Partial>, - durationMs: number - ): void { + async shutdown(): Promise { try { - // CommandResult is built internally — hard parse is intentional since - // a metric without a valid exit_reason is meaningless. - CommandResultSchema.parse(result); - - // Validate command attrs resiliently: invalid fields default to 'unknown' - // instead of dropping the entire metric. - const validatedAttrs = - Object.keys(attrs as Record).length > 0 - ? resilientParse(COMMAND_SCHEMAS[command], attrs as Record) - : attrs; - - const otelAttrs: Record = { - command_group: deriveCommandGroup(command), - command, - }; - - for (const obj of [result, validatedAttrs]) { - for (const [k, v] of Object.entries(obj)) { - if (typeof v === 'boolean') { - otelAttrs[k] = String(v); - } else if (typeof v === 'string' || typeof v === 'number') { - otelAttrs[k] = v; - } - } - } - - this.sink.record(durationMs, otelAttrs); + await this.sink.shutdown(); } catch { - // Telemetry must never affect CLI behavior + /* telemetry must not affect CLI behavior */ } } } diff --git a/src/cli/telemetry/index.ts b/src/cli/telemetry/index.ts index 911141c6e..16395562f 100644 --- a/src/cli/telemetry/index.ts +++ b/src/cli/telemetry/index.ts @@ -1,7 +1,7 @@ export { resolveTelemetryPreference, resolveResourceAttributes, resolveAuditFilePath } from './config.js'; export type { TelemetryPreference } from './config.js'; export { TelemetryClientAccessor } from './client-accessor.js'; -export { TelemetryClient, CANCELLED } from './client.js'; -export { type MetricSink, CompositeSink } from './sinks/metric-sink.js'; +export { TelemetryClient } from './client.js'; +export { type MetricSink } from './sinks/metric-sink.js'; export { OtelMetricSink, type OtelMetricSinkConfig } from './sinks/otel-metric-sink.js'; export { FileSystemSink, type FileSystemSinkConfig } from './sinks/filesystem-sink.js'; diff --git a/src/cli/telemetry/schemas/__tests__/command-run.test.ts b/src/cli/telemetry/schemas/__tests__/command-run.test.ts index e0ad17f25..f6994021c 100644 --- a/src/cli/telemetry/schemas/__tests__/command-run.test.ts +++ b/src/cli/telemetry/schemas/__tests__/command-run.test.ts @@ -47,7 +47,7 @@ describe('COMMAND_SCHEMAS', () => { gateway_target_count: 3, policy_engine_count: 0, policy_count: 0, - mode: 'diff', + deploy_mode: 'diff', }; expect(COMMAND_SCHEMAS.deploy.parse(attrs)).toEqual(attrs); }); @@ -64,7 +64,7 @@ describe('COMMAND_SCHEMAS', () => { gateway_target_count: 0, policy_engine_count: 0, policy_count: 0, - mode: 'deploy', + deploy_mode: 'deploy', }) ).toThrow(); }); @@ -81,7 +81,7 @@ describe('COMMAND_SCHEMAS', () => { gateway_target_count: 0, policy_engine_count: 0, policy_count: 0, - mode: 'deploy', + deploy_mode: 'deploy', }) ).toThrow(); }); diff --git a/src/cli/telemetry/schemas/command-run.ts b/src/cli/telemetry/schemas/command-run.ts index fae8f8394..d5fd1cd48 100644 --- a/src/cli/telemetry/schemas/command-run.ts +++ b/src/cli/telemetry/schemas/command-run.ts @@ -7,6 +7,7 @@ import { Build, Count, CredentialType, + DeployModeSchema, EvaluatorType, FilterState, FilterType, @@ -30,11 +31,6 @@ import { } from './common-shapes.js'; import { z } from 'zod'; -// --------------------------------------------------------------------------- -// Per-command attribute schemas -// All schemas use safeSchema() which rejects z.string() at compile time. -// --------------------------------------------------------------------------- - const CreateAttrs = safeSchema({ language: Language, framework: Framework, @@ -101,7 +97,7 @@ const DeployAttrs = safeSchema({ gateway_target_count: Count, policy_engine_count: Count, policy_count: Count, - mode: z.enum(['deploy', 'dry-run', 'diff']), + deploy_mode: DeployModeSchema, }); const DevAttrs = safeSchema({ @@ -141,15 +137,12 @@ const PauseResumeOnlineEvalAttrs = safeSchema({ ref_type: RefType }); const NoAttrs = safeSchema({}); -// --------------------------------------------------------------------------- -// Command schema registry — single source of truth -// --------------------------------------------------------------------------- - +/* + Mapping of commands to required attributes. + This is chosen over discriminated unions to avoid complexity in the root-level definition. +*/ export const COMMAND_SCHEMAS = { - // create create: CreateAttrs, - - // add 'add.agent': AddAgentAttrs, 'add.memory': AddMemoryAttrs, 'add.credential': AddCredentialAttrs, @@ -160,33 +153,17 @@ export const COMMAND_SCHEMAS = { 'add.policy-engine': AddPolicyEngineAttrs, 'add.policy': AddPolicyAttrs, 'add.runtime-endpoint': NoAttrs, - - // deploy deploy: DeployAttrs, - - // dev / invoke dev: DevAttrs, invoke: InvokeAttrs, - - // status / logs status: StatusAttrs, logs: LogsAttrs, 'logs.evals': LogsEvalsAttrs, - - // run 'run.eval': RunEvalAttrs, - - // fetch 'fetch.access': FetchAccessAttrs, - - // update update: UpdateAttrs, - - // pause / resume 'pause.online-eval': PauseResumeOnlineEvalAttrs, 'resume.online-eval': PauseResumeOnlineEvalAttrs, - - // no command-specific attributes 'traces.list': NoAttrs, 'traces.get': NoAttrs, 'evals.history': NoAttrs, @@ -220,26 +197,13 @@ export const COMMAND_SCHEMAS = { // --------------------------------------------------------------------------- export type Command = keyof typeof COMMAND_SCHEMAS; +export type CommandGroup = { [C in Command]: C extends `${infer G}.${string}` ? G : C }[Command]; export type CommandAttrs = z.infer<(typeof COMMAND_SCHEMAS)[C]>; -/** Extract the command group prefix from a dotted command key (e.g. 'add' from 'add.agent'). */ -type CommandGroup = { - [C in Command]: C extends `${infer G}.${string}` ? G : C; -}[Command]; - -/** - * Type-safe lookup of a subcommand under a command group. - * Produces a compile-time error if `${G}.${S}` is not a registered command. - * - * @example - * SubCommand<'remove', 'agent'> // → 'remove.agent' - * SubCommand<'add', 'memory'> // → 'add.memory' - * SubCommand<'remove', 'bogus'> // → never (compile error at call site) - */ export type SubCommand = Extract; /** Derive command_group from command key (e.g. 'add.agent' → 'add') */ -export function deriveCommandGroup(command: Command): string { +export function deriveCommandGroup(command: Command): CommandGroup { const dot = command.indexOf('.'); - return dot === -1 ? command : command.slice(0, dot); + return (dot === -1 ? command : command.slice(0, dot)) as CommandGroup; } diff --git a/src/cli/telemetry/schemas/common-attributes.ts b/src/cli/telemetry/schemas/common-attributes.ts index 3085db7cc..4be71a5a5 100644 --- a/src/cli/telemetry/schemas/common-attributes.ts +++ b/src/cli/telemetry/schemas/common-attributes.ts @@ -9,11 +9,6 @@ const MAX_ATTR_LENGTH = 64; /** * Resource attributes attached to every metric datapoint. * Set once per session, not per-event. - * - * Constraints are intentionally strict to prevent PII leakage: - * - IDs must be UUID format (no user-chosen strings) - * - Version strings are pattern-constrained - * - All free-text fields are length-bounded */ export const ResourceAttributesSchema = z.object({ 'service.name': z.literal('agentcore-cli'), diff --git a/src/cli/telemetry/schemas/common-shapes.ts b/src/cli/telemetry/schemas/common-shapes.ts index d9622512a..fcacb7370 100644 --- a/src/cli/telemetry/schemas/common-shapes.ts +++ b/src/cli/telemetry/schemas/common-shapes.ts @@ -30,7 +30,7 @@ export function resilientParse( * Lowercase a CLI value and parse it through a Zod enum, returning the narrowed type. * The `as` cast on the failure branch is intentional: invalid values pass through to * recordCommandRun, where COMMAND_SCHEMAS[command].parse(attrs) validates the full - * attr object in a try/catch — silently dropping the metric if any field is invalid. + * attr object with resilient parsing. * This ensures telemetry never crashes the CLI while keeping the happy-path type-safe. */ // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -55,7 +55,7 @@ export const AuthorizerType = z.enum(['aws_iam', 'custom_jwt', 'none']); export const Build = z.enum(['codezip', 'container']); export const CredentialType = z.enum(['api-key', 'oauth']); export const EvaluatorType = z.enum(['llm-as-a-judge', 'code-based']); -export const ExitReason = z.enum(['success', 'failure', 'cancel']); +export const ExitReason = z.enum(['success', 'failure']); export const FilterState = z.enum(['deployed', 'local-only', 'pending-removal', 'none']); export const FilterType = z.enum([ 'agent', @@ -133,11 +133,79 @@ export const ErrorSource = z.enum(['user', 'client', 'service', 'unknown']); // Common result shapes — reusable across metrics export const SuccessResult = z.object({ exit_reason: z.literal('success') }); -export const CancelResult = z.object({ exit_reason: z.literal('cancel') }); export const FailureResult = z.object({ exit_reason: z.literal('failure'), error_name: ErrorName, error_source: ErrorSource, }); -export const CommandResultSchema = z.discriminatedUnion('exit_reason', [SuccessResult, CancelResult, FailureResult]); +export const CommandResultSchema = z.discriminatedUnion('exit_reason', [SuccessResult, FailureResult]); export type CommandResult = z.infer; + +export const DeployModeSchema = z.enum(['deploy', 'dry-run', 'diff']); +export type DeployMode = z.infer; + +/* + All attributes the CLI may attach to a metric. + Keys are the field names as they appear in emitted metrics. +*/ +export const ATTRIBUTES = { + action: Action, + agent_type: AgentType, + attach_gateway_count: Count, + attach_mode: AttachMode, + auth_type: AuthType, + authorizer_type: AuthorizerType, + build: Build, + check_only: z.boolean(), + credential_count: Count, + credential_type: CredentialType, + deploy_mode: DeployModeSchema, + enable_on_create: z.boolean(), + error_name: ErrorName, + evaluator_count: Count, + evaluator_type: EvaluatorType, + exit_reason: ExitReason, + filter_state: FilterState, + filter_type: FilterType, + framework: Framework, + gateway_count: Count, + gateway_target_count: Count, + has_agent: z.boolean(), + has_assertions: z.boolean(), + has_expected_response: z.boolean(), + has_expected_trajectory: z.boolean(), + has_follow: z.boolean(), + has_level_filter: z.boolean(), + has_policy_engine: z.boolean(), + has_query: z.boolean(), + has_session_id: z.boolean(), + has_stream: z.boolean(), + host: GatewayTargetHost, + invoke_count: Count, + error_source: ErrorSource, + language: Language, + level: Level, + memory: Memory, + memory_count: Count, + model_provider: ModelProvider, + network_mode: NetworkMode, + online_eval_count: Count, + outbound_auth: OutboundAuth, + policy_count: Count, + policy_engine_count: Count, + policy_engine_mode: PolicyEngineMode, + protocol: Protocol, + ref_type: RefType, + resource_type: ResourceType, + runtime_count: Count, + semantic_search: z.boolean(), + source_type: SourceType, + strategy_count: Count, + strategy_episodic: z.boolean(), + strategy_semantic: z.boolean(), + strategy_summarization: z.boolean(), + strategy_user_preference: z.boolean(), + target_type: GatewayTargetType, + ui_mode: UiMode, + validation_mode: ValidationMode, +} as const; diff --git a/src/cli/telemetry/schemas/index.ts b/src/cli/telemetry/schemas/index.ts index dd921df7a..61ead5c3b 100644 --- a/src/cli/telemetry/schemas/index.ts +++ b/src/cli/telemetry/schemas/index.ts @@ -1,4 +1,5 @@ export { + ATTRIBUTES, CommandResultSchema, Count, ErrorName, @@ -7,8 +8,8 @@ export { FailureResult, Mode, SuccessResult, - CancelResult, type CommandResult, } from './common-shapes.js'; export { ResourceAttributesSchema, type ResourceAttributes } from './common-attributes.js'; export { COMMAND_SCHEMAS, deriveCommandGroup, type Command, type CommandAttrs } from './command-run.js'; +export { METRICS, type MetricName, type MetricAttrs } from './registry.js'; diff --git a/src/cli/telemetry/schemas/registry.ts b/src/cli/telemetry/schemas/registry.ts new file mode 100644 index 000000000..5f547b0e8 --- /dev/null +++ b/src/cli/telemetry/schemas/registry.ts @@ -0,0 +1,37 @@ +import { COMMAND_SCHEMAS, type Command, type CommandGroup } from './command-run.js'; +import { ATTRIBUTES } from './common-shapes.js'; +import { z } from 'zod'; + +/** + * Metric registry — single source of truth for all metrics the CLI can emit. + * + * Per-command optional fields are derived from COMMAND_SCHEMAS automatically. + * Adding a new command's attrs to COMMAND_SCHEMAS is sufficient — no manual + * update here needed. + */ + +// Merge all per-command schemas into a single partial type +type AllCommandSchemas = (typeof COMMAND_SCHEMAS)[keyof typeof COMMAND_SCHEMAS]; +type MergedCommandAttrs = Partial>; + +type CommandRunAttrs = { + command: Command; + command_group: CommandGroup; + exit_reason: z.infer; + error_name?: z.infer; + error_source?: z.infer; +} & MergedCommandAttrs; + +interface MetricRegistryItem { + description?: string; +} +type MetricRegistry = Record; + +export const METRICS = { + 'cli.command_run': { + description: 'CLI/TUI Command Execution', + }, +} as const satisfies MetricRegistry; + +export type MetricName = keyof typeof METRICS; +export type MetricAttrs = M extends 'cli.command_run' ? CommandRunAttrs : never; diff --git a/src/cli/telemetry/sinks/filesystem-sink.ts b/src/cli/telemetry/sinks/filesystem-sink.ts index a9868f2b9..43638b91a 100644 --- a/src/cli/telemetry/sinks/filesystem-sink.ts +++ b/src/cli/telemetry/sinks/filesystem-sink.ts @@ -1,3 +1,4 @@ +import type { MetricName } from '../schemas/registry.js'; import type { MetricSink } from './metric-sink.js'; import { appendFile, mkdir } from 'fs/promises'; import { dirname } from 'path'; @@ -20,10 +21,10 @@ export class FileSystemSink implements MetricSink { this.log = config.log ?? (msg => console.log(msg)); } - record(value: number, attrs: Record): void { + record(metricName: MetricName, value: number, attrs: Record): void { this.hasRecords = true; this.pendingWrite = this.pendingWrite.then(() => - this.appendEntry({ value, attrs: { ...this.resource, ...attrs } }) + this.appendEntry({ metric: metricName, value, attrs: { ...this.resource, ...attrs } }) ); } @@ -38,10 +39,13 @@ export class FileSystemSink implements MetricSink { } } - // Promise chain that serializes async writes so record() can stay synchronous. private pendingWrite: Promise = Promise.resolve(); - private async appendEntry(entry: { value: number; attrs: Record }): Promise { + private async appendEntry(entry: { + metric: string; + value: number; + attrs: Record; + }): Promise { await mkdir(dirname(this.filePath), { recursive: true }); await appendFile(this.filePath, JSON.stringify(entry) + '\n'); } diff --git a/src/cli/telemetry/sinks/in-memory-sink.ts b/src/cli/telemetry/sinks/in-memory-sink.ts index aab23680c..7fa5c3398 100644 --- a/src/cli/telemetry/sinks/in-memory-sink.ts +++ b/src/cli/telemetry/sinks/in-memory-sink.ts @@ -1,6 +1,8 @@ +import type { MetricName } from '../schemas/registry.js'; import type { MetricSink } from './metric-sink.js'; export interface RecordedMetric { + metric: MetricName; value: number; attrs: Record; } @@ -8,8 +10,8 @@ export interface RecordedMetric { export class InMemorySink implements MetricSink { readonly metrics: RecordedMetric[] = []; - record(value: number, attrs: Record): void { - this.metrics.push({ value, attrs }); + record(metricName: MetricName, value: number, attrs: Record): void { + this.metrics.push({ metric: metricName, value, attrs }); } // eslint-disable-next-line @typescript-eslint/no-empty-function diff --git a/src/cli/telemetry/sinks/metric-sink.ts b/src/cli/telemetry/sinks/metric-sink.ts index 6622a34e7..38ac48f43 100644 --- a/src/cli/telemetry/sinks/metric-sink.ts +++ b/src/cli/telemetry/sinks/metric-sink.ts @@ -1,8 +1,10 @@ +import type { MetricName } from '../schemas/registry.js'; + /** * A destination for metric data. Implementations handle transport (OTel, file, etc.). */ export interface MetricSink { - record(value: number, attrs: Record): void; + record(metricName: MetricName, value: number, attrs: Record): void; flush(timeoutMs?: number): Promise; shutdown(): Promise; } @@ -14,10 +16,10 @@ export interface MetricSink { export class CompositeSink implements MetricSink { constructor(private readonly sinks: MetricSink[]) {} - record(value: number, attrs: Record): void { + record(metricName: MetricName, value: number, attrs: Record): void { for (const sink of this.sinks) { try { - sink.record(value, attrs); + sink.record(metricName, value, attrs); } catch { // Individual sink failure must not affect others } diff --git a/src/cli/telemetry/sinks/otel-metric-sink.ts b/src/cli/telemetry/sinks/otel-metric-sink.ts index 734220e8c..2fe44edbb 100644 --- a/src/cli/telemetry/sinks/otel-metric-sink.ts +++ b/src/cli/telemetry/sinks/otel-metric-sink.ts @@ -1,6 +1,7 @@ import type { ResourceAttributes } from '../schemas/common-attributes.js'; +import type { MetricName } from '../schemas/registry.js'; import type { MetricSink } from './metric-sink.js'; -import type { Histogram } from '@opentelemetry/api'; +import type { Histogram, Meter } from '@opentelemetry/api'; import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http'; import { resourceFromAttributes } from '@opentelemetry/resources'; import { AggregationTemporality, MeterProvider, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'; @@ -13,7 +14,8 @@ export interface OtelMetricSinkConfig { export class OtelMetricSink implements MetricSink { private readonly meterProvider: MeterProvider; - private readonly histogram: Histogram; + private readonly meter: Meter; + private readonly histograms = new Map(); constructor(config: OtelMetricSinkConfig) { const resource = resourceFromAttributes(config.resource); @@ -34,13 +36,16 @@ export class OtelMetricSink implements MetricSink { }), ], }); - this.histogram = this.meterProvider - .getMeter('agentcore-cli') - .createHistogram('cli.command_run', { description: 'CLI command execution' }); + this.meter = this.meterProvider.getMeter('agentcore-cli'); } - record(value: number, attrs: Record): void { - this.histogram.record(value, attrs); + record(metricName: MetricName, value: number, attrs: Record): void { + let histogram = this.histograms.get(metricName); + if (!histogram) { + histogram = this.meter.createHistogram(metricName, { description: metricName }); + this.histograms.set(metricName, histogram); + } + histogram.record(value, attrs); } async flush(timeoutMs = 5_000): Promise { diff --git a/src/cli/tui/screens/deploy/useDeployFlow.ts b/src/cli/tui/screens/deploy/useDeployFlow.ts index e65aa77f5..cdeff0915 100644 --- a/src/cli/tui/screens/deploy/useDeployFlow.ts +++ b/src/cli/tui/screens/deploy/useDeployFlow.ts @@ -747,7 +747,7 @@ export function useDeployFlow(options: DeployFlowOptions = {}): DeployFlowState const attrs = context ? computeDeployAttrs(context.projectSpec, 'diff') - : { ...DEFAULT_DEPLOY_ATTRS, mode: 'diff' as const }; + : { ...DEFAULT_DEPLOY_ATTRS, deploy_mode: 'diff' as const }; const run = async (): Promise<{ success: true } | { success: false; error: Error }> => { setDiffStep(prev => ({ ...prev, status: 'running' }));