diff --git a/.changeset/thread-step-contributions.md b/.changeset/thread-step-contributions.md new file mode 100644 index 000000000..d65e60ed4 --- /dev/null +++ b/.changeset/thread-step-contributions.md @@ -0,0 +1,5 @@ +--- +'@adcp/sdk': patch +--- + +Thread storyboard contribution flags through `runStoryboardStep()` and the `adcp storyboard step` CLI so step-by-step runners preserve branch-set state for synthetic `assert_contribution` checks. diff --git a/bin/adcp.js b/bin/adcp.js index d61d9e586..e1b7e698b 100755 --- a/bin/adcp.js +++ b/bin/adcp.js @@ -873,6 +873,31 @@ function parseJsonFlag(flagName, value) { } } +function parseStringListFlag(flagName, value) { + const parseValue = raw => { + const trimmed = raw.trim(); + if (trimmed.startsWith('[') || trimmed.startsWith('@')) { + const parsed = trimmed.startsWith('@') ? parseJsonFlag(flagName, trimmed) : JSON.parse(trimmed); + if (!Array.isArray(parsed) || parsed.some(v => typeof v !== 'string')) { + console.error(`${flagName} must be a JSON array of strings or a comma-separated string`); + process.exit(2); + } + return parsed; + } + return trimmed + .split(',') + .map(v => v.trim()) + .filter(Boolean); + }; + + try { + return parseValue(value); + } catch (e) { + console.error(`Invalid value for ${flagName}: ${e.message}`); + process.exit(2); + } +} + function closestFlag(input, known) { let best = null; let bestDist = Infinity; @@ -927,13 +952,23 @@ function parseAgentOptions(args) { brief = args[briefIndex + 1]; } - // Storyboard-specific flags (--context, --request) with JSON values + // Storyboard-specific flags with step-threaded state values. const contextIndex = args.indexOf('--context'); let contextValue = null; if (contextIndex !== -1 && contextIndex + 1 < args.length && !args[contextIndex + 1].startsWith('--')) { contextValue = args[contextIndex + 1]; } + const contributionsIndex = args.indexOf('--contributions'); + let contributionsValue = null; + if ( + contributionsIndex !== -1 && + contributionsIndex + 1 < args.length && + !args[contributionsIndex + 1].startsWith('--') + ) { + contributionsValue = args[contributionsIndex + 1]; + } + const requestIndex = args.indexOf('--request'); let requestValue = null; if (requestIndex !== -1 && requestIndex + 1 < args.length && !args[requestIndex + 1].startsWith('--')) { @@ -1110,6 +1145,7 @@ function parseAgentOptions(args) { protocolFlag, brief, contextValue, + contributionsValue, requestValue, tracksValue, storyboardsValue, @@ -1816,6 +1852,8 @@ WEBHOOK OPTIONS: OPTIONS: --context JSON Pass context from previous step (step only) + --contributions JSON|CSV + Pass contribution flags from previous step (step only) --request JSON Override sample_request for the step (step only) --json JSON output (recommended for LLM consumption) --auth TOKEN Authentication token (or 'user:pass' with --auth-scheme basic) @@ -4473,13 +4511,18 @@ async function handleStoryboardStepCmd(args) { oauthClientCredentials: resolvedOauthClientCredentials, } = await resolveAgent(agentArg, authToken, protocolFlag, jsonOutput, authScheme); - // Parse --context and --request flags (supports inline JSON or @file.json) + // Parse --context, --contributions, and --request flags (supports inline JSON or @file.json) let context = {}; + let contributions; let request; const contextIndex = args.indexOf('--context'); if (contextIndex !== -1 && args[contextIndex + 1]) { context = parseJsonFlag('--context', args[contextIndex + 1]); } + const contributionsIndex = args.indexOf('--contributions'); + if (contributionsIndex !== -1 && args[contributionsIndex + 1]) { + contributions = parseStringListFlag('--contributions', args[contributionsIndex + 1]); + } const requestIndex = args.indexOf('--request'); if (requestIndex !== -1 && args[requestIndex + 1]) { request = parseJsonFlag('--request', args[requestIndex + 1]); @@ -4488,6 +4531,7 @@ async function handleStoryboardStepCmd(args) { const options = { protocol, context, + ...(contributions && { contributions }), request, ...(complianceVersion && { adcpVersion: complianceVersion }), ...(schemaRoot && { schemaRoot }), diff --git a/src/lib/testing/storyboard/runner.ts b/src/lib/testing/storyboard/runner.ts index cd559ce8b..cc61a1ba5 100644 --- a/src/lib/testing/storyboard/runner.ts +++ b/src/lib/testing/storyboard/runner.ts @@ -3481,8 +3481,9 @@ async function runStoryboardStepBody( const responseDerivedNotApplicableContextKeys = new Map( Object.entries(options.response_derived_not_applicable_context_keys ?? {}) ); + const contributions = new Set(options.contributions ?? []); const result = await executeStep(client, found.step, found.phaseId, context, allSteps, options, { - contributions: new Set(), + contributions, priorStepResults: new Map(), priorProbes: new Map(), agentUrl, @@ -3495,13 +3496,19 @@ async function runStoryboardStepBody( agentLibraryVersion: profile?.library_version, }); + if (!result.skipped && result.passed && found.step.contributes_to) { + if (evalContributesIf(found.step.contributes_if, new Map())) { + contributions.add(found.step.contributes_to); + } + } + if (!clientResolution.reusedShared) { await closeConnections(options.protocol); } if (ownsWebhookReceiver && webhookReceiver) await webhookReceiver.close(); - return result; + return { ...result, contributions: Array.from(contributions) }; } // ──────────────────────────────────────────────────────────── diff --git a/src/lib/testing/storyboard/types.ts b/src/lib/testing/storyboard/types.ts index 37f4c03f5..a43f556bd 100644 --- a/src/lib/testing/storyboard/types.ts +++ b/src/lib/testing/storyboard/types.ts @@ -1363,6 +1363,13 @@ export interface StoryboardRunOptions extends TestOptions { * semantics as full `runStoryboard` runs. */ response_derived_not_applicable_context_keys?: Record; + /** + * Contribution flags accumulated by prior `runStoryboardStep` invocations. + * Thread this from `StoryboardStepResult.contributions` so synthetic + * aggregate steps such as `assert_contribution` see the same branch-set + * state they would see inside a full `runStoryboard` execution. + */ + contributions?: string[]; /** Override the step's sample_request with a custom request */ request?: Record; /** Agent's available tools for storyboard/step-level tool gates. */ @@ -2127,6 +2134,12 @@ export interface StoryboardStepResult { * `not_applicable` rather than `prerequisite_failed`. */ response_derived_not_applicable_context_keys?: Record; + /** + * Contribution flags accumulated after this step. Thread back into + * `StoryboardRunOptions.contributions` on the next `runStoryboardStep` + * invocation when orchestrating a storyboard step-by-step. + */ + contributions?: string[]; error?: string; /** * Structured AdCP error forwarded from the transport layer when the step failed. diff --git a/test/lib/storyboard-branch-set-grading.test.js b/test/lib/storyboard-branch-set-grading.test.js index 7621da89c..a00e06af6 100644 --- a/test/lib/storyboard-branch-set-grading.test.js +++ b/test/lib/storyboard-branch-set-grading.test.js @@ -2,7 +2,7 @@ const { describe, it } = require('node:test'); const assert = require('node:assert'); const http = require('http'); -const { runStoryboard } = require('../../dist/lib/testing/storyboard/runner'); +const { runStoryboard, runStoryboardStep } = require('../../dist/lib/testing/storyboard/runner'); const { parseStoryboard } = require('../../dist/lib/testing/storyboard/loader'); /** @@ -239,6 +239,134 @@ phases: } }); + it('normalizes programmatic `contributes: true` storyboards before recording contributions', async () => { + const { server, url } = await startStub(500, {}); + try { + const storyboard = { + id: 'programmatic_shorthand_sb', + version: '1.0.0', + title: 'Programmatic branch set shorthand', + category: 'test', + summary: '', + narrative: '', + agent: { interaction_model: '*', capabilities: [] }, + caller: { role: 'buyer_agent' }, + phases: [ + { + id: 'past_start_reject', + title: 'Reject past start', + optional: true, + branch_set: { id: 'past_start_handled', semantics: 'any_of' }, + steps: [ + { + id: 'reject_step', + title: 'reject', + task: 'list_creatives', + auth: 'none', + expect_error: true, + contributes: true, + validations: [{ check: 'http_status', value: 500, description: '' }], + }, + ], + }, + { + id: 'gate', + title: 'gate', + steps: [ + { + id: 'assert_past_start_handled', + title: 'Require past_start_handled from either path', + task: 'assert_contribution', + validations: [ + { + check: 'any_of', + allowed_values: ['past_start_handled'], + description: '', + }, + ], + }, + ], + }, + ], + }; + + const result = await runStoryboard(url, storyboard, runOpts); + + assert.strictEqual(storyboard.phases[0].steps[0].contributes_to, 'past_start_handled'); + assert.strictEqual(storyboard.phases[0].steps[0].contributes, undefined); + assert.strictEqual(result.phases[0].steps[0].passed, true); + assert.strictEqual(result.phases[1].steps[0].passed, true); + assert.strictEqual(result.overall_passed, true); + } finally { + server.close(); + } + }); + + it('threads contributions through stateless step-by-step runs for assert_contribution', async () => { + const { server, url } = await startStub(500, {}); + try { + const storyboard = { + id: 'stepwise_contributions_sb', + version: '1.0.0', + title: 'Stepwise branch set contribution', + category: 'test', + summary: '', + narrative: '', + agent: { interaction_model: '*', capabilities: [] }, + caller: { role: 'buyer_agent' }, + phases: [ + { + id: 'past_start_reject', + title: 'Reject past start', + optional: true, + branch_set: { id: 'past_start_handled', semantics: 'any_of' }, + steps: [ + { + id: 'reject_step', + title: 'reject', + task: 'list_creatives', + auth: 'none', + expect_error: true, + contributes: true, + validations: [{ check: 'http_status', value: 500, description: '' }], + }, + ], + }, + { + id: 'gate', + title: 'gate', + steps: [ + { + id: 'assert_past_start_handled', + title: 'Require past_start_handled from either path', + task: 'assert_contribution', + validations: [ + { + check: 'any_of', + allowed_values: ['past_start_handled'], + description: '', + }, + ], + }, + ], + }, + ], + }; + + const first = await runStoryboardStep(url, storyboard, 'reject_step', runOpts); + assert.deepStrictEqual(first.contributions, ['past_start_handled']); + + const gate = await runStoryboardStep(url, storyboard, 'assert_past_start_handled', { + ...runOpts, + contributions: first.contributions, + }); + assert.strictEqual(gate.passed, true); + assert.deepStrictEqual(gate.contributions, ['past_start_handled']); + } finally { + server.close(); + } + }); + it('implicit detection: storyboard without branch_set declarations still re-grades peers', async () => { // Pre-adcp#2646 shape — no `branch_set:` keyword on the phases; runner // must infer branch-set membership from the shared `contributes_to`