diff --git a/__tests__/e2e/skip-tools.e2e.ts b/__tests__/e2e/skip-tools.e2e.ts index d463a11..98fd448 100644 --- a/__tests__/e2e/skip-tools.e2e.ts +++ b/__tests__/e2e/skip-tools.e2e.ts @@ -23,7 +23,9 @@ describe('when the user skips connecting tools', () => { await session.press('Enter'); await session.waitForText('onboarding complete', { timeout: 30_000 }); + session.checkpoint(); await session.waitForText('Confidence is ready'); + await session.waitForText("What's next?"); expect(session.snapshot()).toMatchSnapshot('done-tools-skipped'); }); }); diff --git a/__tests__/features/onboarding/build-prompt.test.ts b/__tests__/features/onboarding/build-prompt.test.ts index 83b0910..3ffea73 100644 --- a/__tests__/features/onboarding/build-prompt.test.ts +++ b/__tests__/features/onboarding/build-prompt.test.ts @@ -1,3 +1,4 @@ +import { basename, resolve } from 'node:path'; import { buildOnboardingPrompt } from '@features/onboarding/index.js'; describe('buildOnboardingPrompt', () => { @@ -45,6 +46,22 @@ describe('buildOnboardingPrompt', () => { expect(sut).toContain('Read `.claude/skills/analyze-project/SKILL.md`'); }); + + it('asks the summary to list Confidence resources next to file changes', () => { + const sut = buildOnboardingPrompt(baseOpts); + + expect(sut).toContain('List the resources you created in Confidence as well'); + expect(sut).toContain('not only files and packages'); + expect(sut).toContain('use that wording so the same change is not listed twice'); + }); + + it('requires a persisted identity for feature flag evaluation', () => { + const sut = buildOnboardingPrompt(baseOpts); + + expect(sut).toContain("first entity field from the client's context schema"); + expect(sut).toContain('`localStorage` only in a browser entrypoint'); + expect(sut).toContain("don't fabricate them or mint a new ID per page load"); + }); }); describe('event tracking skill reference', () => { @@ -115,4 +132,176 @@ describe('buildOnboardingPrompt', () => { expect(sut).toContain('Invoke the `/analyze-project` skill as a **methodology reference**'); }); }); + + describe('session recordings MCP setup', () => { + const recordingOpts = { + ...baseOpts, + goals: ['session-recordings' as const], + }; + + it('instructs the agent to create a recording policy and targeting-key rule', () => { + const sut = buildOnboardingPrompt(recordingOpts); + + expect(sut).toContain('mcp__confidence-flags__createRecordingPolicy'); + expect(sut).toContain('mcp__confidence-flags__addRecordingRule'); + expect(sut).toContain('targetingKeySelector'); + expect(sut).toContain('`enabled`: true'); + expect(sut).toContain('STATUS: Setting up recording policy...'); + }); + + it('includes Confidence resources in the final change summary', () => { + const sut = buildOnboardingPrompt(recordingOpts); + + expect(sut).toContain('Created recording policy with targeting key'); + expect(sut).toContain( + 'Created recording rule (Record all visitors, 100% audience, 100% sessions, enabled)', + ); + }); + + it('instructs the agent to pass a 100% audience instead of sending 0', () => { + const sut = buildOnboardingPrompt(recordingOpts); + + expect(sut).toContain('`stableAudiencePercentage`: 100'); + expect(sut).toContain('`sessionSampleRate`: 1'); + expect(sut).toContain('agents often send `0`'); + }); + + it('matches recording policies to the client resource name, not display name', () => { + const sut = buildOnboardingPrompt(recordingOpts); + + expect(sut).toContain('never reuse a policy because its display name looks similar'); + expect(sut).toContain( + "Reuse a policy only when its `clients` list contains this client's resource name", + ); + expect(sut).toContain('pass each non-empty `nextPageToken` back as `pageToken`'); + expect(sut).toContain("`clientName` set to this client's resource name"); + expect(sut).toContain('Keep the returned resource name (`clients/` from `name:`)'); + expect(sut).not.toContain('cannot match an existing policy to a client'); + }); + + it('treats goal selection as confirmation to enable recording', () => { + const sut = buildOnboardingPrompt(recordingOpts); + + expect(sut).toContain('explicit confirmation to start recording'); + expect(sut).toContain('enable the rule immediately without asking another question'); + expect(sut).toContain('Tell the user afterward that the rule is enabled'); + }); + + it('names the client and policy after the resolved project dir, not the framework', () => { + const sut = buildOnboardingPrompt({ ...recordingOpts, projectDir: '/tmp/checkout-web' }); + + expect(sut).toContain('with the display name "checkout-web"'); + expect(sut).toContain('with `displayName` "checkout-web Session Recording"'); + expect(sut).toContain('never after the framework'); + expect(sut).not.toContain('react Session Recording'); + }); + + it('resolves --dir . so the client is not named "."', () => { + const sut = buildOnboardingPrompt({ ...recordingOpts, projectDir: '.' }); + const name = basename(resolve('.')); + + expect(sut).toContain(`with the display name "${name}"`); + expect(sut).not.toContain('with the display name "."'); + }); + + it('does not reuse a colliding client from another project', () => { + const sut = buildOnboardingPrompt(recordingOpts); + + expect(sut).toContain('Do not reuse a client solely because its display name already exists'); + expect(sut).toContain( + 'unless the colliding client is the one created earlier in this same run', + ); + expect(sut).toContain( + 'Never call `mcp__confidence-flags__getClientSecret` for a colliding client', + ); + expect(sut).not.toContain( + 'If the tool says that display name already exists, keep the resource name', + ); + }); + + it('refers to targeting key and policy by name instead of numbered steps', () => { + const sut = buildOnboardingPrompt(recordingOpts); + + expect(sut).toContain('from the **Targeting key** item in'); + expect(sut).toContain('the resource name from the **Policy** item in'); + expect(sut).not.toContain('from step 2'); + expect(sut).not.toContain('from step 3'); + }); + + it('uses the framework public env var so the browser can read the client secret', () => { + const sut = buildOnboardingPrompt(recordingOpts); + + expect(sut).toContain('VITE_CONFIDENCE_CLIENT_SECRET'); + expect(sut).toContain('NEXT_PUBLIC_CONFIDENCE_CLIENT_SECRET'); + expect(sut).toContain('REACT_APP_CONFIDENCE_CLIENT_SECRET'); + expect(sut).toContain('clientSecret: '); + expect(sut).toContain('fill ``'); + }); + + it('gates recording behind an existing consent tool when one is found', () => { + const sut = buildOnboardingPrompt(recordingOpts); + + expect(sut).toContain('OneTrust, Cookiebot, Usercentrics, Didomi'); + expect(sut).toContain("`mode: 'manual'`"); + expect(sut).toContain('only after analytics or recording consent is granted'); + expect(sut).toContain('Fill ``'); + }); + + it('tells the agent what recording-rule status to write for each path', () => { + const sut = buildOnboardingPrompt(recordingOpts); + + expect(sut).toContain('Fill `` with "No recording rule was created'); + expect(sut).toContain('Fill `` with "The recording rule is enabled'); + expect(sut).toContain( + 'Fill `` with "The existing recording rule records nobody', + ); + }); + + it('flags a reused rule that has no audience segment', () => { + const sut = buildOnboardingPrompt(recordingOpts); + + expect(sut).toContain('STATUS: Existing recording rule records nobody'); + expect(sut).toContain('An audience segment (`segments/`) is the healthy'); + expect(sut).toContain('including rules with no targeting conditions'); + expect(sut).toContain('records nobody and no MCP tool can repair'); + }); + + it('keeps the client secret out of output and source', () => { + const sut = buildOnboardingPrompt(recordingOpts); + + expect(sut).toContain('write the Frontend client secret to `.env` under that exact name'); + expect(sut).toContain('Ensure `.env` is in `.gitignore`'); + expect(sut).toContain( + 'never echo the secret in STATUS lines, the report, or generated source', + ); + }); + + it('picks the targeting key from the context schema like flags', () => { + const sut = buildOnboardingPrompt(recordingOpts); + + expect(sut).toContain('mcp__confidence-flags__getContextSchema'); + expect(sut).toContain('first available entity field'); + expect(sut).toContain('Do not assume `user_id` or `targeting_key`'); + expect(sut).toContain('`localStorage` only in a browser entrypoint'); + expect(sut).not.toContain("inspect the app's auth/session code"); + }); + + it('reuses the feature flag identity when both goals are selected', () => { + const sut = buildOnboardingPrompt({ + ...recordingOpts, + goals: ['feature-flags', 'session-recordings'], + }); + + expect(sut).toContain('If feature flags were integrated earlier, reuse that entity field'); + expect(sut).toContain('reuse the flag identity if present'); + }); + + it('uses Codex MCP tool names for Codex', () => { + const sut = buildOnboardingPrompt({ ...recordingOpts, ide: 'codex' }); + + expect(sut).toContain('confidence-flags:createRecordingPolicy'); + expect(sut).toContain('confidence-flags:addRecordingRule'); + expect(sut).not.toContain('mcp__confidence-flags__createRecordingPolicy'); + }); + }); }); diff --git a/__tests__/features/onboarding/recording.test.ts b/__tests__/features/onboarding/recording.test.ts new file mode 100644 index 0000000..c7f0c69 --- /dev/null +++ b/__tests__/features/onboarding/recording.test.ts @@ -0,0 +1,34 @@ +import { basename, resolve } from 'node:path'; +import { projectDisplayName, projectParentName } from '@features/onboarding/sections/recording.js'; + +describe('projectDisplayName', () => { + it('uses the last segment of an absolute project dir', () => { + const sut = projectDisplayName('/tmp/checkout-web'); + expect(sut).toBe('checkout-web'); + }); + + it('resolves a relative project dir instead of using "." as the client name', () => { + const sut = projectDisplayName('.'); + + expect(sut).toBe(basename(resolve('.'))); + expect(sut).not.toBe('.'); + }); + + it('resolves ".." to the parent folder name', () => { + const sut = projectDisplayName('..'); + + expect(sut).toBe(basename(resolve('..'))); + expect(sut).not.toBe('..'); + }); + + it('falls back to "project" when the resolved path has no basename', () => { + expect(projectDisplayName('/')).toBe('project'); + }); +}); + +describe('projectParentName', () => { + it('uses the parent folder of the resolved project dir', () => { + const sut = projectParentName('/tmp/checkout-web'); + expect(sut).toBe('tmp'); + }); +}); diff --git a/__tests__/features/onboarding/report-templates.test.ts b/__tests__/features/onboarding/report-templates.test.ts index d7ffec3..feb1d55 100644 --- a/__tests__/features/onboarding/report-templates.test.ts +++ b/__tests__/features/onboarding/report-templates.test.ts @@ -23,6 +23,50 @@ function depEntries(goals: OnboardingGoal[]): string[] { : []; } +function usageEntries(goals: OnboardingGoal[]): string[] { + const { end } = buildReportTemplate(goals); + const section = end.split('## How to use it')[1]?.split('## Before you merge')[0]; + return section + ? section + .split('\n') + .map((l) => l.trim()) + .filter((l) => l.startsWith('- ')) + : []; +} + +function checklistEntries(goals: OnboardingGoal[]): string[] { + const { end } = buildReportTemplate(goals); + const section = end.split('## Before you merge')[1]?.split('## Next steps')[0]; + return section + ? section + .split('\n') + .map((l) => l.trim()) + .filter((l) => l.startsWith('- [ ]')) + : []; +} + +function undoEntries(goals: OnboardingGoal[]): string[] { + const { end } = buildReportTemplate(goals); + const section = end.split('## To undo everything')[1]?.split('```')[0]; + return section + ? section + .split('\n') + .map((l) => l.trim()) + .filter((l) => l.startsWith('- ')) + : []; +} + +function tableRows(goals: OnboardingGoal[]): string[] { + const { start } = buildReportTemplate(goals); + const section = start.split('| | |')[1]?.split('## What changed')[0]; + return section + ? section + .split('\n') + .map((l) => l.trim()) + .filter((l) => l.startsWith('|') && !l.startsWith('|---')) + : []; +} + describe('when only feature-flags is selected', () => { const goals: OnboardingGoal[] = ['feature-flags']; @@ -30,7 +74,7 @@ describe('when only feature-flags is selected', () => { const sut = fileEntries(goals); expect(sut).toEqual([ - '- `<.env file>` — added `CONFIDENCE_CLIENT_SECRET`', + '- `<.env file>` — added ``', '- `` — added SDK initialization', '- `` — added flag evaluation', ]); @@ -40,6 +84,13 @@ describe('when only feature-flags is selected', () => { const sut = depEntries(goals); expect(sut).toEqual(['- ``']); }); + + it('explains that flags keep their default variant without mentioning recordings', () => { + const sut = usageEntries(goals).join('\n'); + + expect(sut).toContain('default variant'); + expect(sut).not.toContain('recording'); + }); }); describe('when only session-recordings is selected', () => { @@ -49,7 +100,7 @@ describe('when only session-recordings is selected', () => { const sut = fileEntries(goals); expect(sut).toEqual([ - '- `<.env file>` — added `CONFIDENCE_CLIENT_SECRET`', + '- `<.env file>` — added ``', '- `` — added SDK initialization', '- `` — added session recording provider', ]); @@ -59,6 +110,55 @@ describe('when only session-recordings is selected', () => { const sut = depEntries(goals); expect(sut).toEqual(['- ``']); }); + + it('lists the client, recording policy, and targeting key', () => { + const sut = tableRows(goals); + + expect(sut).toEqual([ + '| Client | |', + '| Recording policy | |', + '| Targeting key | |', + ]); + }); + + it('leaves recording rule and consent status for the agent to fill', () => { + const sut = usageEntries(goals).join('\n'); + + expect(sut).toContain(''); + expect(sut).toContain(''); + expect(sut).not.toContain('The recording rule is enabled'); + expect(sut).not.toContain('nothing changes'); + }); + + it('asks the user to verify captured sessions instead of unchanged behavior', () => { + const sut = checklistEntries(goals).join('\n'); + + expect(sut).toContain('confirm a session appears'); + expect(sut).not.toContain('default behavior is unchanged'); + }); + + it('asks the user to cover privacy, consent, and production sampling', () => { + const sut = checklistEntries(goals).join('\n'); + + expect(sut).toContain('Mention session recording in your privacy policy'); + expect(sut).toContain('gate it behind user consent where required'); + expect(sut).toContain("Lower the rule's session sample rate"); + expect(sut).toContain('before rolling out to production traffic'); + }); + + it('describes the recorder context rather than only flag evaluation', () => { + const sut = checklistEntries(goals).join('\n'); + + expect(sut).toContain('flag evaluation / recorder context'); + expect(sut).toContain(''); + }); + + it('explains how to undo the recording resources', () => { + const sut = undoEntries(goals).join('\n'); + + expect(sut).toContain('Disable the recording rule or archive the recording policy'); + expect(sut).not.toContain('Archive the flag'); + }); }); describe('when only event-tracking is selected', () => { @@ -68,7 +168,7 @@ describe('when only event-tracking is selected', () => { const sut = fileEntries(goals); expect(sut).toEqual([ - '- `<.env file>` — added `CONFIDENCE_CLIENT_SECRET`', + '- `<.env file>` — added ``', '- `` — added SDK initialization', '- `` — added event tracking calls', ]); @@ -108,4 +208,13 @@ describe('when all goals are selected', () => { const sut = depEntries(goals); expect(sut).toHaveLength(3); }); + + it('lists one client row plus recording policy details', () => { + const sut = tableRows(goals); + const clientRows = sut.filter((l) => l.includes('Client')); + + expect(clientRows).toHaveLength(1); + expect(sut).toContainEqual('| Recording policy | |'); + expect(sut).toContainEqual('| Targeting key | |'); + }); }); diff --git a/__tests__/integrations/utils.test.ts b/__tests__/integrations/utils.test.ts index ae8b4af..5e3c1b7 100644 --- a/__tests__/integrations/utils.test.ts +++ b/__tests__/integrations/utils.test.ts @@ -1,4 +1,68 @@ -import { formatOnboardingError, spawnErrorMessage } from '@integrations/utils.js'; +import { + extractCodeChanges, + formatOnboardingError, + spawnErrorMessage, +} from '@integrations/utils.js'; + +describe('extractCodeChanges', () => { + it('keeps only lines describing a change', () => { + const sut = extractCodeChanges([ + 'Analyzing the project structure', + 'Created confidence.config.ts', + 'Added @spotify-confidence/sdk', + 'Modified src/main.tsx', + ]); + + expect(sut).toEqual([ + 'Created confidence.config.ts', + 'Added @spotify-confidence/sdk', + 'Modified src/main.tsx', + ]); + }); + + it('reports a change once when the agent repeats its summary', () => { + const sut = extractCodeChanges([ + 'Created confidence.config.ts', + 'Added @spotify-confidence/sdk', + 'Created confidence.config.ts', + 'Added @spotify-confidence/sdk', + ]); + + expect(sut).toEqual(['Created confidence.config.ts', 'Added @spotify-confidence/sdk']); + }); + + it('ignores status lines already shown as progress', () => { + const sut = extractCodeChanges(['STATUS: Created recording policy: my-app']); + expect(sut).toEqual([]); + }); + + it('strips list markers and markdown emphasis', () => { + const sut = extractCodeChanges(['- **Created** `confidence.config.ts`']); + expect(sut).toEqual(['Created confidence.config.ts']); + }); + + it('keeps checkmark-prefixed change lines', () => { + const sut = extractCodeChanges(['✓ Created recording policy with targeting key']); + expect(sut).toEqual(['Created recording policy with targeting key']); + }); + + it('drops change lines wrapped in other prefixes', () => { + const sut = extractCodeChanges(['> Created foo']); + expect(sut).toEqual([]); + }); + + it('ignores prose that merely mentions a change verb', () => { + const sut = extractCodeChanges(['I have Created the config for you']); + expect(sut).toEqual([]); + }); + + it('truncates long descriptions', () => { + const sut = extractCodeChanges([`Created ${'a'.repeat(80)}`]); + + expect(sut[0]).toHaveLength(60); + expect(sut[0]).toMatch(/…$/); + }); +}); describe('spawnErrorMessage', () => { it('returns a helpful message for ENOENT', () => { diff --git a/src/features/onboarding/build-prompt.ts b/src/features/onboarding/build-prompt.ts index e6bed28..d90264b 100644 --- a/src/features/onboarding/build-prompt.ts +++ b/src/features/onboarding/build-prompt.ts @@ -45,7 +45,14 @@ export function buildOnboardingPrompt({ ), addIf(withRecordings, () => determineRecordingSDK(framework, steps.next(), tools)), - addIf(withRecordings, () => integrateRecording(steps.next(), isEmptyProject)), + addIf(withRecordings, () => + integrateRecording({ + step: steps.next(), + isEmptyProject, + projectDir, + toolVars: tools, + }), + ), addIf(withEventTracking, () => instrumentEvents(framework, steps.next(), isEmptyProject, ide, pluginInstallMethod), diff --git a/src/features/onboarding/report-templates.ts b/src/features/onboarding/report-templates.ts index b1746b4..6f43611 100644 --- a/src/features/onboarding/report-templates.ts +++ b/src/features/onboarding/report-templates.ts @@ -3,20 +3,26 @@ import type { OnboardingGoal } from '@shared-kernel/types.js'; export type ReportTemplate = { start: string; end: string }; export function buildReportTemplate(goals: OnboardingGoal[]): ReportTemplate { - return { start: buildTemplateStart(goals), end: TEMPLATE_END }; + return { start: buildTemplateStart(goals), end: buildTemplateEnd(goals) }; } function buildTemplateStart(goals: OnboardingGoal[]): string { const tableRowEntries: string[] = []; const dependencyEntries: string[] = []; const fileChangeEntries = [ - '- `<.env file>` — added `CONFIDENCE_CLIENT_SECRET`', + '- `<.env file>` — added ``', '- `` — added SDK initialization', ]; - if (goals.includes('feature-flags')) { + const withFlags = goals.includes('feature-flags'); + const withRecordings = goals.includes('session-recordings'); + + if (withFlags || withRecordings) { + tableRowEntries.push('| Client | |'); + } + + if (withFlags) { tableRowEntries.push( - '| Client | |', '| Flag | |', '| Variants | |', '| Default | (100% allocation) |', @@ -25,8 +31,11 @@ function buildTemplateStart(goals: OnboardingGoal[]): string { dependencyEntries.push('- ``'); } - if (goals.includes('session-recordings')) { - tableRowEntries.push('| Session Recording | Enabled |'); + if (withRecordings) { + tableRowEntries.push( + '| Recording policy | |', + '| Targeting key | |', + ); fileChangeEntries.push('- `` — added session recording provider'); dependencyEntries.push('- ``'); } @@ -62,19 +71,61 @@ ${fileChangeEntries.join('\n')} ${dependencyEntries.join('\n')}`; } -const TEMPLATE_END = `\ +function buildTemplateEnd(goals: OnboardingGoal[]): string { + const usageEntries = ['- Manage your setup at https://app.confidence.spotify.com']; + const checklistEntries = [ + '- [ ] Check that `.env` is in `.gitignore` (so the secret stays out of git)', + '- [ ] Add `` to your CI/staging/prod environment', + ]; + const undoEntries = ['- Revert the changed files (`git checkout` / `git stash`)']; + + if (goals.includes('feature-flags')) { + usageEntries.push('- Flags stay on their default variant until you change them in Confidence'); + undoEntries.push('- Archive the flag in the Confidence UI'); + } + + if (goals.includes('session-recordings')) { + usageEntries.push( + '- ', + '- Recordings show up under **Recordings** in the Confidence UI', + '- ', + ); + checklistEntries.push( + '- [ ] Run the app and confirm a session appears under **Recordings**', + '- [ ] Mention session recording in your privacy policy and gate it behind user consent where required (e.g. EU)', + "- [ ] Lower the rule's session sample rate in Confidence before rolling out to production traffic", + ); + undoEntries.push( + '- Disable the recording rule or archive the recording policy in the Confidence UI', + ); + } + + if (goals.includes('event-tracking')) { + usageEntries.push('- Events are sent once the app runs with the client secret'); + checklistEntries.push('- [ ] Run the app and confirm events appear in Confidence'); + undoEntries.push('- Archive generated event definitions in the Confidence UI (if applicable)'); + } + + if (goals.includes('feature-flags') || goals.includes('session-recordings')) { + checklistEntries.push( + '- [ ] Verify the flag evaluation / recorder context supplies a stable value for the selected targeting key', + ); + } + + if (goals.includes('feature-flags')) { + checklistEntries.push('- [ ] Confirm flags still resolve to their intended default variants'); + } + + checklistEntries.push('- [ ] Review the diff — make sure nothing unexpected was modified'); + + return `\ ## How to use it -- Manage your setup at https://app.confidence.spotify.com -- The default configuration is safe to merge — nothing changes until you flip a flag or enable recording +${usageEntries.join('\n')} ## Before you merge -- [ ] Check that \`.env\` is in \`.gitignore\` (so the secret stays out of git) -- [ ] Add \`CONFIDENCE_CLIENT_SECRET\` to your CI/staging/prod environment -- [ ] Verify the evaluation context sets a stable \`targeting_key\` for consistent variant assignment -- [ ] Run the app locally and confirm the default behavior is unchanged -- [ ] Review the diff — make sure nothing unexpected was modified +${checklistEntries.join('\n')} ## Next steps @@ -88,6 +139,6 @@ const TEMPLATE_END = `\ ## To undo everything -- Revert the changed files (\`git checkout\` / \`git stash\`) -- Archive the flag in the Confidence UI (if applicable) +${undoEntries.join('\n')} \`\`\``; +} diff --git a/src/features/onboarding/sections/recording.ts b/src/features/onboarding/sections/recording.ts index 8567b3e..c23f438 100644 --- a/src/features/onboarding/sections/recording.ts +++ b/src/features/onboarding/sections/recording.ts @@ -1,6 +1,24 @@ +import { basename, dirname, resolve } from 'node:path'; import { CONFIDENCE_DOCS_URL } from '@lib/constants.js'; import { loadStep } from '../steps/load.js'; +const FALLBACK_PROJECT_NAME = 'project'; + +type IntegrateRecordingParams = { + step: number; + isEmptyProject: boolean; + projectDir: string; + toolVars: Record; +}; + +export function projectDisplayName(projectDir: string): string { + return basename(resolve(projectDir)) || FALLBACK_PROJECT_NAME; +} + +export function projectParentName(projectDir: string): string { + return basename(dirname(resolve(projectDir))) || FALLBACK_PROJECT_NAME; +} + export function determineRecordingSDK( framework: string, step: number, @@ -14,11 +32,20 @@ export function determineRecordingSDK( }); } -export function integrateRecording(step: number, isEmptyProject: boolean): string { +export function integrateRecording({ + step, + isEmptyProject, + projectDir, + toolVars, +}: IntegrateRecordingParams): string { return loadStep('integrate-recording.md', { STEP: step, + PROJECT_NAME: projectDisplayName(projectDir), + PARENT_NAME: projectParentName(projectDir), + DOCS_URL: CONFIDENCE_DOCS_URL, ANALYSIS_CONTEXT: isEmptyProject ? "The project was just scaffolded — configure recording on the sample app's main view." : "Identify the app's entry point or root layout where the session recorder should be initialized.", + ...toolVars, }); } diff --git a/src/features/onboarding/steps/integrate-recording.md b/src/features/onboarding/steps/integrate-recording.md index 144eaa5..cb0b82d 100644 --- a/src/features/onboarding/steps/integrate-recording.md +++ b/src/features/onboarding/steps/integrate-recording.md @@ -10,7 +10,42 @@ Print "STATUS: Analyzing project for session recording..." **Detect the source root** — check for `src`, `app`, `lib`, `pages`, `server` and use the first match (or `.`). Exclude `node_modules`, `.venv`, `vendor`, `target`, `build`, `dist`, `.next`, `__pycache__` from scans. -### {{STEP}}b. Install the session recording SDK +**Consent tool** — look for an existing consent or cookie-banner implementation: OneTrust, Cookiebot, Usercentrics, Didomi, `react-cookie-consent`, a custom consent context, or a cookie-consent state hook. Note whether analytics or recording consent is already modeled. You will use this in {{STEP}}d. + +### {{STEP}}b. Resolve client and recording policy + +Print "STATUS: Setting up recording policy..." + +If flag management is unavailable from preflight, skip MCP calls in this substep. Write placeholders for the client secret and document in the report that the user must create a recording policy under Recordings > Settings. Fill `` with "No recording rule was created — create a policy and rule under Recordings > Settings before sessions will be captured". If they need setup details, search {{DOCS_URL}}. + +If flag management is available: + +1. **Client** — reuse the Confidence client from an earlier feature-flags step if one was created in this same run. Otherwise call `{{FLAGS_createClient}}` with the display name "{{PROJECT_NAME}}" and `clientType` `Frontend`. Name it after this project, never after the framework — a framework name collides with clients from unrelated projects and attaches this policy to the wrong app. Keep the returned resource name (`clients/` from `name:`). + + Do not reuse a client solely because its display name already exists. A folder named `app`, `web`, or `frontend` often belongs to a different project. If `{{FLAGS_createClient}}` reports that display name is taken, call it again with a unique name: "{{PROJECT_NAME}} ({{PARENT_NAME}})", then "{{PROJECT_NAME}}-2", then "-3", until creation succeeds — unless the colliding client is the one created earlier in this same run, in which case keep that resource name. Never call `{{FLAGS_getClientSecret}}` for a colliding client you did not create in this run. + +2. **Targeting key** — same as flags. Call `{{FLAGS_getContextSchema}}` with the client's display name. Use the first available entity field (typically `visitor_id`). Do not assume `user_id` or `targeting_key`. If feature flags were integrated earlier, reuse that entity field. If the schema has no entity field, call `{{FLAGS_addContextField}}` with `fieldName` `visitor_id`, `fieldType` `string`, and `isEntity` `"true"` (string, not boolean). Fill it with a persisted visitor ID: reuse the flag identity if present, otherwise an existing anonymous/device ID, or generate once and store where the app already persists client state (`localStorage` only in a browser entrypoint). + +3. **Policy** — never reuse a policy because its display name looks similar. Call `{{FLAGS_listRecordingPolicies}}` and inspect every page: pass each non-empty `nextPageToken` back as `pageToken` until `nextPageToken` is empty. Reuse a policy only when its `clients` list contains this client's resource name (`clients/`). If nothing matches, call `{{FLAGS_createRecordingPolicy}}` with `displayName` "{{PROJECT_NAME}} Session Recording" and `clientName` set to this client's resource name. Keep the returned policy resource name. + +4. **Rule** — call `{{FLAGS_getRecordingPolicy}}` with `recordingPolicy` set to that resource name. If the policy has no rule yet, call `{{FLAGS_addRecordingRule}}` like this: + + Good: `targetingKeySelector` from the **Targeting key** item in {{STEP}}b, omit `targetingJson`, `stableAudiencePercentage`: 100, `sessionSampleRate`: 1, `enabled`: true + Bad: omitting the percentages (agents often send `0`, which records nobody) + + Pass `recordingPolicy` (the resource name from the **Policy** item in {{STEP}}b) and `displayName` "Record all visitors". + + Selecting Session Recordings in the wizard is explicit confirmation to start recording, so enable the rule immediately without asking another question. The MCP creates an unrestricted `segments/` audience even though `targetingJson` is omitted. Tell the user afterward that the rule is enabled and records 100% of visitors and sessions. Fill `` with "The recording rule is enabled — sessions are captured once the app runs with the client secret". + + If the policy already has a rule that is not enabled, call `{{FLAGS_setRecordingRuleEnabled}}` with that rule's resource name and `enabled` true. + + When reusing an existing rule, read its audience from the `{{FLAGS_getRecordingPolicy}}` output. An audience segment (`segments/`) is the healthy 100%-of-visitors representation, including rules with no targeting conditions. An audience of "all users" means the pre-fix rule has no segment, which records nobody and no MCP tool can repair — print "STATUS: Existing recording rule records nobody" and add a "Before you merge" item telling the user to delete that rule under Recordings > Settings and add a new one. Fill `` with "The existing recording rule records nobody — delete it under Recordings > Settings and add a new one". + +Print "STATUS: Created recording policy: " after a new policy, or "STATUS: Reusing recording policy: " when reusing. Print "STATUS: Enabled recording rule" after the rule is active. + +In the final change summary, include "Created recording policy with targeting key" and "Created recording rule (Record all visitors, 100% audience, 100% sessions, enabled)" when those resources were created. + +### {{STEP}}c. Install the session recording SDK Print "STATUS: Installing session recording SDK..." @@ -19,26 +54,40 @@ npm install @spotify-confidence/session-recording # or: yarn add / pnpm add ``` -### {{STEP}}c. Initialize the recorder +### {{STEP}}d. Initialize the recorder Print "STATUS: Adding session recording provider..." +Pick the env var the browser can actually read, then write the Frontend client secret to `.env` under that exact name (exposing it to the browser is intended). Use the same name in generated code and fill `` in the report: + +- Vite: `VITE_CONFIDENCE_CLIENT_SECRET` via `import.meta.env.VITE_CONFIDENCE_CLIENT_SECRET` +- Next.js client code: `NEXT_PUBLIC_CONFIDENCE_CLIENT_SECRET` via `process.env.NEXT_PUBLIC_CONFIDENCE_CLIENT_SECRET` +- Create React App: `REACT_APP_CONFIDENCE_CLIENT_SECRET` via `process.env.REACT_APP_CONFIDENCE_CLIENT_SECRET` +- Other browser bundlers: follow that framework's public-env convention +- Server-only entrypoints: `CONFIDENCE_CLIENT_SECRET` via `process.env.CONFIDENCE_CLIENT_SECRET` + +Ensure `.env` is in `.gitignore`, and never echo the secret in STATUS lines, the report, or generated source. + Add to the app's entry point (e.g. `main.ts`, `index.tsx`, root layout): ```ts import { initSessionRecorder } from '@spotify-confidence/session-recording'; const recorder = initSessionRecorder({ - clientSecret: process.env.CONFIDENCE_CLIENT_SECRET, + clientSecret: , context: { visitor_id: '', }, }); ``` -The function always returns a `SessionRecorder` — safe to call, never throws. Recording starts automatically by default. For manual control, pass `mode: 'manual'` and call `recorder.start()`. +Use the same field name as `targetingKeySelector`, filled with the identity from the **Targeting key** item in {{STEP}}b. Rename `visitor_id` in this snippet if the schema's first entity field is different. + +The function always returns a `SessionRecorder` — safe to call, never throws. + +**Start mode** — if {{STEP}}a found a consent tool, pass `mode: 'manual'` and call `recorder.start()` only after analytics or recording consent is granted. Fill `` with "Recording starts only after the user grants analytics or recording consent". If none was found, keep the SDK default (recording starts automatically) and fill `` with "No consent tool was found — recording starts when the app loads; mention session recording in the privacy policy and gate it behind consent where required (e.g. EU)". -### {{STEP}}d. Configure privacy and capture settings +### {{STEP}}e. Configure privacy and capture settings Print "STATUS: Configuring privacy and capture settings..." @@ -69,9 +118,9 @@ Analyze the project to decide: Merge the chosen settings into the `initSessionRecorder` call from the previous step. Only include options that differ from defaults — don't add `maskInputs: true` or `captureRouteChanges: true` since they're already on. -If the project already uses Confidence feature flags, pass the same `targeting_key` / `visitor_id` in `context` so sessions correlate with flag evaluations. +If the project already uses Confidence feature flags, pass the same identity field in `context` so sessions correlate with flag evaluations. -### {{STEP}}e. Verify the project builds +### {{STEP}}f. Verify the project builds Print "STATUS: Verifying project builds..." diff --git a/src/features/onboarding/steps/integrate-via-skill.md b/src/features/onboarding/steps/integrate-via-skill.md index 1ef9d9b..b549f6a 100644 --- a/src/features/onboarding/steps/integrate-via-skill.md +++ b/src/features/onboarding/steps/integrate-via-skill.md @@ -39,7 +39,7 @@ Note: when naming project files, use file name only, no path. - Read the client secret from CONFIDENCE_CLIENT_SECRET env var in all generated code — never hardcode it. Write the secret to `.env` and ensure `.env` is listed in `.gitignore`. - Use the OpenFeature API with local resolve where supported. Access flag values via dot notation: `flag-name.property`. -- Set up the evaluation context with a stable `targeting_key` (user ID, session ID, or anonymous ID) and any attributes the app already has (`country`, `plan`, `device`). Don't fabricate attributes. +- Set the evaluation context to the first entity field from the client's context schema and a stable value. Reuse an existing Confidence identity if the app has one. For `user_id`, use the authenticated user ID. For `visitor_id` or another anonymous field, reuse a persisted anonymous/device ID, or generate one UUID and persist it where the app already stores client state (`localStorage` only in a browser entrypoint). Include attributes the app already has (`country`, `plan`, `device`); don't fabricate them or mint a new ID per page load. - Use only SDK APIs from the docs integration guide — do not improvise method names or signatures from memory. {{FLAG_GUIDANCE}} {{REACT_GOTCHAS}} diff --git a/src/features/onboarding/steps/rules.md b/src/features/onboarding/steps/rules.md index 50f0d3c..70cf1e6 100644 --- a/src/features/onboarding/steps/rules.md +++ b/src/features/onboarding/steps/rules.md @@ -4,7 +4,7 @@ - **Timing: print a STATUS line at least every 30–60 seconds.** If a sub-task takes more than a minute (reading files, calling MCP tools, writing code, running builds), print intermediate STATUS lines describing what you're currently doing — e.g. "STATUS: Reading layout.tsx...", "STATUS: Querying docs for SDK setup...", "STATUS: Writing flag evaluation code...". The user has no other way to know you're still working. - Keep STATUS text short **(~60 characters max)**. - Never show raw JSON payloads, MCP tool names, or secrets in output. -- Read the client secret from CONFIDENCE_CLIENT_SECRET env var in all generated code. +- Read the client secret from the env var chosen for this project (`CONFIDENCE_CLIENT_SECRET`, or the framework's public prefix such as `VITE_`, `NEXT_PUBLIC_`, or `REACT_APP_` in browser code). - Use the OpenFeature API with local resolve where supported. Access flag values via dot notation: `flag-name.property`. - Only create or modify files inside the project directory. Never write to paths outside it (e.g. home directory dotfiles, global configs, `/tmp`). - If a step fails, print the error and continue with remaining steps where possible. The report file must always be generated — if steps failed, document what succeeded and what needs to be completed manually. diff --git a/src/features/onboarding/steps/summary.md b/src/features/onboarding/steps/summary.md index 1d0f2c0..64c017a 100644 --- a/src/features/onboarding/steps/summary.md +++ b/src/features/onboarding/steps/summary.md @@ -9,4 +9,6 @@ Then list every change on its own line using exactly one of these prefixes: - "Modified " for changed functionality or files - "Added " for installed packages or new capabilities +List the resources you created in Confidence as well — the client, flags, recording policy and rule, event definitions — not only files and packages. Anything you leave out looks to the user like it never happened. When an earlier step prescribed the exact line for a resource, use that wording so the same change is not listed twice. + Keep each description to a few words — e.g. "Added @spotify-confidence/sdk", "Modified app/routes/home.tsx — flag evaluation loader", "Created welcome-subtitle flag". Include file names when they fit, but never repeat what the prefix already says. No bullets, no markdown — one change per line. Omit mentioning the CONFIDENCE_QUICKSTART.md file, it will be handled by the CLI tool itself. diff --git a/src/features/onboarding/tool-vars.ts b/src/features/onboarding/tool-vars.ts index ad1537b..7aeb834 100644 --- a/src/features/onboarding/tool-vars.ts +++ b/src/features/onboarding/tool-vars.ts @@ -45,10 +45,17 @@ export function buildToolVars(ide: IdeId): Record { FLAGS_listClients: flags('listClients'), FLAGS_createClient: flags('createClient'), FLAGS_getClientSecret: flags('getClientSecret'), + FLAGS_getContextSchema: flags('getContextSchema'), + FLAGS_addContextField: flags('addContextField'), FLAGS_listFlags: flags('listFlags'), FLAGS_createFlag: flags('createFlag'), FLAGS_addTargetingRule: flags('addTargetingRule'), FLAGS_resolveFlag: flags('resolveFlag'), + FLAGS_listRecordingPolicies: flags('listRecordingPolicies'), + FLAGS_createRecordingPolicy: flags('createRecordingPolicy'), + FLAGS_getRecordingPolicy: flags('getRecordingPolicy'), + FLAGS_addRecordingRule: flags('addRecordingRule'), + FLAGS_setRecordingRuleEnabled: flags('setRecordingRuleEnabled'), DOCS_searchDocumentation: docs('searchDocumentation'), DOCS_getLocalResolveIntegrationGuide: docs('getLocalResolveIntegrationGuide'), DOCS_getCodeSnippetAndSdkIntegrationTips: docs('getCodeSnippetAndSdkIntegrationTips'), diff --git a/src/integrations/index.ts b/src/integrations/index.ts index 232cd4a..e12a4fb 100644 --- a/src/integrations/index.ts +++ b/src/integrations/index.ts @@ -7,7 +7,7 @@ export type { } from './types.js'; export { getIntegrations, getIntegration } from './registry.js'; -export { normalizeStatusLine, normalizeReportLine } from './utils.js'; +export { normalizeStatusLine, extractCodeChanges } from './utils.js'; export { launchChatSession } from './chat.js'; export { diff --git a/src/integrations/utils.ts b/src/integrations/utils.ts index 5b0e55c..4a8af3c 100644 --- a/src/integrations/utils.ts +++ b/src/integrations/utils.ts @@ -11,13 +11,34 @@ export function normalizeStatusLine(line: StatusLine) { } const MAX_REPORT_LINE_LENGTH = 60; +const CHANGE_PREFIXES = ['Created', 'Modified', 'Added']; -export function normalizeReportLine(line: string) { - const stripped = isStatusLine(line) ? normalizeStatusLine(line) : line; - const clean = stripMarkdown(stripped); - return clean.length > MAX_REPORT_LINE_LENGTH - ? clean.slice(0, MAX_REPORT_LINE_LENGTH - 1) + '…' - : clean; +// Agents repeat their final summary (once while streaming, once in the closing +// event), so the same change can arrive several times. +export function extractCodeChanges(lines: string[]): string[] { + const changes = new Map(); + + for (const line of lines) { + if (isStatusLine(line)) continue; + + const change = stripMarkdown(stripListMarker(line)).trim(); + if (!CHANGE_PREFIXES.some((prefix) => change.startsWith(prefix))) continue; + + const key = change.toLowerCase(); + if (!changes.has(key)) changes.set(key, truncate(change)); + } + + return [...changes.values()]; +} + +function truncate(text: string) { + return text.length > MAX_REPORT_LINE_LENGTH + ? text.slice(0, MAX_REPORT_LINE_LENGTH - 1) + '…' + : text; +} + +function stripListMarker(text: string) { + return text.replace(/^\s*(?:[-*•✓]|\d+\.)\s+/, ''); } function stripMarkdown(text: string) { diff --git a/src/ui/tui/screens/onboard-project/useOnboardingProcess.ts b/src/ui/tui/screens/onboard-project/useOnboardingProcess.ts index 734f74a..0037754 100644 --- a/src/ui/tui/screens/onboard-project/useOnboardingProcess.ts +++ b/src/ui/tui/screens/onboard-project/useOnboardingProcess.ts @@ -4,7 +4,7 @@ import { buildOnboardingPrompt } from '@features/onboarding/index.js'; import { detectFramework } from '@frameworks/index.js'; import type { IdeId, OnboardingGoal } from '@shared-kernel/types.js'; import { ScreenId } from '@lib/session.js'; -import { getIntegration, normalizeReportLine } from '@integrations/index.js'; +import { getIntegration, extractCodeChanges } from '@integrations/index.js'; import { useLogger } from '../../hooks/useLog.js'; import { $session, store, isStaleSession } from '../../store.js'; import { useInitialOnboarding } from './useInitialOnboarding.js'; @@ -58,16 +58,7 @@ export function useOnboardingProcess(): OnboardingProcess { addStatus('', 'blank'); addStatus('Project onboarding complete!', 'success'); store.setReportFile('CONFIDENCE_QUICKSTART.md'); - store.setCodeChanges( - lines - ? lines - .filter( - (line) => - line.includes('Created') || line.includes('Modified') || line.includes('Added'), - ) - .map(normalizeReportLine) - : dryRunCodeChanges(goals), - ); + store.setCodeChanges(lines ? extractCodeChanges(lines) : dryRunCodeChanges(goals)); setPhase('done'); }, [addStatus], @@ -227,6 +218,7 @@ const DRY_RUN_STEPS: Record = { 'Creating feature flag example...', ], 'session-recordings': [ + 'Setting up recording policy...', 'Installing session recording SDK...', 'Adding session recording provider...', 'Configuring privacy settings...', @@ -256,6 +248,7 @@ const DRY_RUN_CODE_CHANGES: Record = { 'Created feature flag example', ], 'session-recordings': [ + 'Created recording policy with targeting key', 'Added @spotify-confidence/session-recording', 'Added session recording provider', ],