diff --git a/.changeset/input-schema-field-strip-notice.md b/.changeset/input-schema-field-strip-notice.md new file mode 100644 index 000000000..7fea2d0f6 --- /dev/null +++ b/.changeset/input-schema-field-strip-notice.md @@ -0,0 +1,5 @@ +--- +'@adcp/sdk': patch +--- + +Surface input-schema field stripping as structured runner notices so compliance JSON output no longer hides stripped request fields in console warnings only. diff --git a/src/lib/core/SingleAgentClient.ts b/src/lib/core/SingleAgentClient.ts index 2d9dc6a9e..1e099ffb7 100644 --- a/src/lib/core/SingleAgentClient.ts +++ b/src/lib/core/SingleAgentClient.ts @@ -1637,7 +1637,8 @@ export class SingleAgentClient { // Adapt request for v2 servers if needed const serverVersion = await this.detectServerVersion(); - const adaptedParams = await this.adaptRequestForServerVersion(taskType, normalizedParams); + const inputSchemaStripLogs: any[] = []; + const adaptedParams = await this.adaptRequestForServerVersion(taskType, normalizedParams, inputSchemaStripLogs); // Symmetric to the pre-adapter v3 pass above: when the adapter // rewrote the request for a v2 server, warn-validate the adapted @@ -1664,8 +1665,9 @@ export class SingleAgentClient { // the executor's own logs. On error paths the executor may not surface // result.debug_logs at all — drift collected before the failure is // dropped, matching the executor's own debug-log behavior. - if (v25DriftLogs.length > 0) { - result.debug_logs = [...(result.debug_logs ?? []), ...v25DriftLogs]; + const postAdapterLogs = [...inputSchemaStripLogs, ...v25DriftLogs]; + if (postAdapterLogs.length > 0) { + result.debug_logs = [...(result.debug_logs ?? []), ...postAdapterLogs]; } // Normalize response to v3 format @@ -2051,7 +2053,7 @@ export class SingleAgentClient { * * Converts v3-style requests to v2 format when talking to v2 servers. */ - private async adaptRequestForServerVersion(taskType: string, params: any): Promise { + private async adaptRequestForServerVersion(taskType: string, params: any, debugLogs?: any[]): Promise { // Get server version (cached after first call) const version = await this.detectServerVersion(); @@ -2138,6 +2140,17 @@ export class SingleAgentClient { console.warn( `[AdCP] Stripping fields not declared in agent "${this.agent.id}" schema for ${taskType}: ${stripped.join(', ')}` ); + debugLogs?.push({ + type: 'warning', + message: `Stripped fields not declared in agent tool input schema for ${taskType}: ${stripped.join(', ')}`, + timestamp: new Date().toISOString(), + details: { + code: 'input_schema_field_stripped', + task: taskType, + fields: stripped, + agent_id: this.agent.id, + }, + }); } return filtered; @@ -2991,7 +3004,8 @@ export class SingleAgentClient { // Adapt request for the server's protocol version (e.g. strip v3-only // fields like buying_mode when talking to v2 agents). const serverVersion = await this.detectServerVersion(); - const adaptedParams = await this.adaptRequestForServerVersion(taskName, normalizedParams); + const inputSchemaStripLogs: any[] = []; + const adaptedParams = await this.adaptRequestForServerVersion(taskName, normalizedParams, inputSchemaStripLogs); // Symmetric warn-only post-adapter pass against the v2.5 schema bundle. // Drift gets surfaced via result.metadata.debug_logs so adapter @@ -3010,8 +3024,9 @@ export class SingleAgentClient { serverVersion ); - if (v25DriftLogs.length > 0) { - result.debug_logs = [...(result.debug_logs ?? []), ...v25DriftLogs]; + const postAdapterLogs = [...inputSchemaStripLogs, ...v25DriftLogs]; + if (postAdapterLogs.length > 0) { + result.debug_logs = [...(result.debug_logs ?? []), ...postAdapterLogs]; } // Normalize response to v3 format for consistent API surface diff --git a/src/lib/testing/storyboard/runner.ts b/src/lib/testing/storyboard/runner.ts index 182fc6d24..21ae0f17b 100644 --- a/src/lib/testing/storyboard/runner.ts +++ b/src/lib/testing/storyboard/runner.ts @@ -1783,6 +1783,56 @@ function collectCapabilityNotices(storyboard: Storyboard, rawCaps: unknown): Run return notices; } +function collectInputSchemaFieldStripNotices(debugLogs: unknown, storyboardId: string): RunnerNotice[] { + if (!Array.isArray(debugLogs)) return []; + const notices: RunnerNotice[] = []; + const seen = new Set(); + for (const entry of debugLogs) { + if (!entry || typeof entry !== 'object') continue; + const details = (entry as { details?: unknown }).details; + if (!details || typeof details !== 'object') continue; + const record = details as Record; + if (record.code !== 'input_schema_field_stripped') continue; + const task = typeof record.task === 'string' ? record.task : 'unknown_task'; + const fields = Array.isArray(record.fields) + ? record.fields.filter((field): field is string => typeof field === 'string') + : []; + if (fields.length === 0) continue; + const key = `${task}\u0000${fields.join('\u0000')}`; + if (seen.has(key)) continue; + seen.add(key); + notices.push({ + severity: 'info', + code: 'input_schema_field_stripped', + message: + `Runner stripped fields not declared in the agent's tool input schema for ${task}: ` + + `${fields.join(', ')}. Fix the tool schema declaration or avoid sending unsupported fields.`, + docs_url: 'https://github.com/adcontextprotocol/adcp/issues/5495', + storyboard_ids: [storyboardId], + }); + } + return notices; +} + +function mergeRunnerNotices(notices: RunnerNotice[]): RunnerNotice[] { + const byCode = new Map(); + for (const notice of notices) { + const existing = byCode.get(notice.code); + if (existing) { + for (const sid of notice.storyboard_ids) { + if (!existing.storyboard_ids.includes(sid)) existing.storyboard_ids.push(sid); + } + } else { + byCode.set(notice.code, { ...notice, storyboard_ids: [...notice.storyboard_ids] }); + } + } + return [...byCode.values()]; +} + +function collectStepNotices(phases: StoryboardPhaseResult[]): RunnerNotice[] { + return phases.flatMap(phase => phase.steps.flatMap(step => step.notices ?? [])); +} + /** * Execute a single pass of the storyboard against the supplied replica URLs * using round-robin dispatch starting at `dispatchOffset`. Called directly @@ -2553,19 +2603,28 @@ async function executeStoryboardPass( phasePassed = false; continue; } - const rawResult = await executeStep(assignment.client, step, phase.id, context, allSteps, options, { - contributions, - priorStepResults, - priorProbes, - agentUrl: assignment.agentUrl, - webhookReceiver, - runnerVars, - contextProvenance, - priorA2aEnvelopes, - stepRequestStarts, - responseDerivedNotApplicableContextKeys, - agentLibraryVersion: profile?.library_version, - }); + const rawResult = await executeStep( + assignment.client, + step, + storyboard.id, + phase.id, + context, + allSteps, + options, + { + contributions, + priorStepResults, + priorProbes, + agentUrl: assignment.agentUrl, + webhookReceiver, + runnerVars, + contextProvenance, + priorA2aEnvelopes, + stepRequestStarts, + responseDerivedNotApplicableContextKeys, + agentLibraryVersion: profile?.library_version, + } + ); const result: StoryboardStepResult = { ...rawResult, storyboard_id: storyboard.id }; if (isMultiInstance || useRouting) { // Echo per-step routing on the result so JUnit/CI consumers and @@ -2994,7 +3053,10 @@ async function executeStoryboardPass( // Use the fully-fetched profile for notice detection; fall back to pre-flight // notices (which used options._profile) when profile was not re-fetched in // this pass (standalone runner with options._profile pre-set skips the fetch). - const notices = collectCapabilityNotices(storyboard, profile?.raw_capabilities ?? options._profile?.raw_capabilities); + const notices = mergeRunnerNotices([ + ...collectCapabilityNotices(storyboard, profile?.raw_capabilities ?? options._profile?.raw_capabilities), + ...collectStepNotices(phaseResults), + ]); const result: StoryboardResult = { storyboard_id: storyboard.id, storyboard_title: storyboard.title, @@ -3376,7 +3438,7 @@ async function runStoryboardStepBody( const responseDerivedNotApplicableContextKeys = new Map( Object.entries(options.response_derived_not_applicable_context_keys ?? {}) ); - const result = await executeStep(client, found.step, found.phaseId, context, allSteps, options, { + const result = await executeStep(client, found.step, storyboard.id, found.phaseId, context, allSteps, options, { contributions: new Set(), priorStepResults: new Map(), priorProbes: new Map(), @@ -3461,6 +3523,7 @@ async function executeStep( // eslint-disable-next-line @typescript-eslint/no-explicit-any -- client type varies (TestClient) client: any, step: StoryboardStep, + storyboardId: string, phaseId: string, context: StoryboardContext, allSteps: FlatStep[], @@ -4024,6 +4087,10 @@ async function executeStep( payload: redactSecrets(request), ...(runState.agentUrl ? { url: runState.agentUrl } : {}), }; + const inputSchemaStripNotices = collectInputSchemaFieldStripNotices( + (taskResult as { debug_logs?: unknown } | undefined)?.debug_logs, + storyboardId + ); // AdCP 3.0.12 runner-output-contract `force_scenario_unsupported`: when a // comply_test_controller step calls a force_* scenario that the agent @@ -4070,6 +4137,7 @@ async function executeStep( context, next, extraction: extractionFromTaskResult(taskResult), + ...(inputSchemaStripNotices.length > 0 && { notices: inputSchemaStripNotices }), }; } } @@ -4098,6 +4166,7 @@ async function executeStep( error: stepResult.error, next, extraction: { path: 'none' }, + ...(inputSchemaStripNotices.length > 0 && { notices: inputSchemaStripNotices }), }; } @@ -4137,6 +4206,7 @@ async function executeStep( request: requestRecord, ...(responseRecord && { response_record: responseRecord }), extraction: extractionFromTaskResult(taskResult), + ...(inputSchemaStripNotices.length > 0 && { notices: inputSchemaStripNotices }), }; } @@ -4532,6 +4602,7 @@ async function executeStep( request: requestRecord, ...(responseRecord && { response_record: responseRecord }), extraction: extractionFromTaskResult(taskResult), + ...(inputSchemaStripNotices.length > 0 && { notices: inputSchemaStripNotices }), ...(hints.length > 0 && { hints }), }; } diff --git a/src/lib/testing/storyboard/task-map.ts b/src/lib/testing/storyboard/task-map.ts index 5beb5e587..a745c660a 100644 --- a/src/lib/testing/storyboard/task-map.ts +++ b/src/lib/testing/storyboard/task-map.ts @@ -179,6 +179,8 @@ export async function executeStoryboardTask( // submitted), use that data. Only poll when there's no data at all. const hasData = result.data !== undefined && result.data !== null; const isAsync = result.status === 'submitted' || result.status === 'working'; + const prePollingDebugLogs = Array.isArray(result.debug_logs) ? [...result.debug_logs] : []; + let replacedByPolling = false; if (!hasData && isAsync && result.submitted?.waitForCompletion) { try { const timeout = new Promise((_, reject) => @@ -188,6 +190,7 @@ export async function executeStoryboardTask( Promise.race([result.submitted.waitForCompletion(2000, opts.signal), timeout]), opts.signal ); + replacedByPolling = true; } catch (err) { if (opts.signal?.aborted) throw err; // Polling failed or timed out — return the intermediate result as-is @@ -204,12 +207,15 @@ export async function executeStoryboardTask( const success = normalizeStoryboardTaskSuccess(result, taskName, terminalDataError, adcpError); const error = result.error ?? (!success ? errorMessageFrom(adcpError, undefined) : undefined); const extractionPath = readExtractionPath(data); + const debugLogs = Array.isArray(result.debug_logs) ? result.debug_logs : []; + const mergedDebugLogs = replacedByPolling ? [...prePollingDebugLogs, ...debugLogs] : debugLogs; return { success, data, error, ...(adcpError && { adcp_error: adcpError }), ...(extractionPath !== undefined && { _extraction_path: extractionPath }), + ...(mergedDebugLogs.length > 0 && { debug_logs: mergedDebugLogs }), }; } diff --git a/src/lib/testing/storyboard/types.ts b/src/lib/testing/storyboard/types.ts index c2748bacf..39fdcee56 100644 --- a/src/lib/testing/storyboard/types.ts +++ b/src/lib/testing/storyboard/types.ts @@ -1890,7 +1890,9 @@ export type NoticeCode = | 'request_signing.required' /** Agent advertises `webhook_signing.legacy_hmac_fallback: true`. Removed in * AdCP 4.0 per `effective_version`. */ - | 'webhook_signing.legacy_hmac_fallback.removed'; + | 'webhook_signing.legacy_hmac_fallback.removed' + /** Runner stripped request fields missing from the agent's advertised tool input schema. */ + | 'input_schema_field_stripped'; /** * Severity of a runner notice. Deliberately separate from `ObservationSeverity` @@ -2138,6 +2140,11 @@ export interface StoryboardStepResult { response_record?: RunnerResponseRecord; /** Which extraction path produced the parsed response (required per contract). */ extraction: RunnerExtractionRecord; + /** + * Informational notices scoped to this step. These do not affect pass/fail. + * Storyboard-level `notices` aggregates step notices by code. + */ + notices?: RunnerNotice[]; /** * Non-fatal diagnostic hints the runner emitted alongside this step — * currently surfaces `context_value_rejected` when a request value came diff --git a/test/lib/request-validation.test.js b/test/lib/request-validation.test.js index 9e7edad53..3a117ea14 100644 --- a/test/lib/request-validation.test.js +++ b/test/lib/request-validation.test.js @@ -863,6 +863,7 @@ describe('v3 partial-schema field stripping', () => { ]); const capturedCalls = []; + let result; const originalCallTool = ProtocolClient.callTool; ProtocolClient.callTool = async (_agentConfig, toolName, args) => { capturedCalls.push({ toolName, args }); @@ -870,7 +871,7 @@ describe('v3 partial-schema field stripping', () => { }; try { - await agent.getProducts({ + result = await agent.getProducts({ brand: { domain: 'fanta.com' }, brief: 'love chocolate and have 20k to spend', }); @@ -891,6 +892,10 @@ describe('v3 partial-schema field stripping', () => { 'brand should be stripped when not declared in agent schema' ); assert.ok(getProductsCall.args.brief, 'brief should be preserved'); + const stripLog = result.debug_logs.find(log => log.details?.code === 'input_schema_field_stripped'); + assert.ok(stripLog, 'stripped fields should be surfaced in structured debug_logs'); + assert.strictEqual(stripLog.details.task, 'get_products'); + assert.deepStrictEqual(stripLog.details.fields, ['brand']); }); test('should pass through all fields when v3 agent schema declares them', async () => { diff --git a/test/lib/storyboard-notices.test.js b/test/lib/storyboard-notices.test.js index ccae6d7b8..e815ceb17 100644 --- a/test/lib/storyboard-notices.test.js +++ b/test/lib/storyboard-notices.test.js @@ -347,6 +347,135 @@ describe('RunnerNotice: webhook_signing.legacy_hmac_fallback.removed (#1704)', ( }); }); +// ───────────────────────────────────────────────────────────────────────────── +// Tests: input_schema_field_stripped +// ───────────────────────────────────────────────────────────────────────────── + +describe('RunnerNotice: input_schema_field_stripped (#5495)', () => { + test('promotes structured field-strip debug logs to step and storyboard notices', async () => { + const sb = buildMinimalStoryboard({ id: 'input_schema_strip_notice' }); + const client = { + executeTask: async taskName => ({ + success: true, + status: 'completed', + data: { products: [] }, + metadata: { + taskId: 'task-strip', + taskName, + agent: { id: 'partial-agent', name: 'Partial Agent', protocol: 'mcp' }, + responseTimeMs: 1, + timestamp: new Date().toISOString(), + clarificationRounds: 0, + status: 'completed', + }, + debug_logs: [ + { + type: 'warning', + message: 'Stripped fields not declared in agent tool input schema for get_products: max_width, max_height', + timestamp: new Date().toISOString(), + details: { + code: 'input_schema_field_stripped', + task: 'get_products', + fields: ['max_width', 'max_height'], + agent_id: 'partial-agent', + }, + }, + ], + }), + resetContext: () => {}, + }; + + const result = await runStoryboard('http://fake-local-99999', sb, { + _client: client, + _profile: profileClean, + agentTools: ['get_products'], + }); + + const step = result.phases[0].steps[0]; + const stepNotice = step.notices?.find(n => n.code === 'input_schema_field_stripped'); + assert.ok(stepNotice, 'step_result.notices should include the stripped-field notice'); + assert.equal(stepNotice.severity, 'info'); + assert.match(stepNotice.message, /get_products/); + assert.match(stepNotice.message, /max_width/); + assert.match(stepNotice.message, /max_height/); + + const storyboardNotice = result.notices.find(n => n.code === 'input_schema_field_stripped'); + assert.ok(storyboardNotice, 'StoryboardResult.notices should aggregate the step notice'); + assert.deepEqual(storyboardNotice.storyboard_ids, ['input_schema_strip_notice']); + assert.equal(result.overall_passed, true, 'notice should not affect pass/fail'); + }); + + test('preserves field-strip notices through async waitForCompletion polling path', async () => { + const sb = buildMinimalStoryboard({ id: 'input_schema_strip_async' }); + let polled = false; + const client = { + executeTask: async taskName => ({ + success: true, + status: 'submitted', + data: undefined, + metadata: { + taskId: 'task-async-strip', + taskName, + agent: { id: 'partial-agent', name: 'Partial Agent', protocol: 'mcp' }, + responseTimeMs: 1, + timestamp: new Date().toISOString(), + clarificationRounds: 0, + status: 'submitted', + }, + debug_logs: [ + { + type: 'warning', + message: 'Stripped fields not declared in agent tool input schema for get_products: max_width', + timestamp: new Date().toISOString(), + details: { + code: 'input_schema_field_stripped', + task: 'get_products', + fields: ['max_width'], + agent_id: 'partial-agent', + }, + }, + ], + submitted: { + waitForCompletion: async () => { + polled = true; + return { + success: true, + status: 'completed', + data: { products: [] }, + metadata: { + taskId: 'task-async-strip', + taskName, + agent: { id: 'partial-agent', name: 'Partial Agent', protocol: 'mcp' }, + responseTimeMs: 2, + timestamp: new Date().toISOString(), + clarificationRounds: 0, + status: 'completed', + }, + }; + }, + }, + }), + resetContext: () => {}, + }; + + const result = await runStoryboard('http://fake-local-99999', sb, { + _client: client, + _profile: profileClean, + agentTools: ['get_products'], + }); + + assert.ok(polled, 'waitForCompletion should have been called'); + const step = result.phases[0].steps[0]; + const stepNotice = step.notices?.find(n => n.code === 'input_schema_field_stripped'); + assert.ok(stepNotice, 'step_result.notices must survive the async polling replacement'); + + const storyboardNotice = result.notices.find(n => n.code === 'input_schema_field_stripped'); + assert.ok(storyboardNotice, 'StoryboardResult.notices must aggregate pre-polling strip notices'); + assert.deepEqual(storyboardNotice.storyboard_ids, ['input_schema_strip_async']); + assert.equal(result.overall_passed, true, 'notice must not affect pass/fail'); + }); +}); + // ───────────────────────────────────────────────────────────────────────────── // Tests: multiple notices in one run // ─────────────────────────────────────────────────────────────────────────────