From 91f330ac7c58ae3d0dc4dda41d39210b2b17472d Mon Sep 17 00:00:00 2001 From: Ryan Albrecht Date: Thu, 3 Sep 2026 11:19:33 -0700 Subject: [PATCH] feat(seer): render autofix embed detail through the live-run components MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `{% autofix %}` embed rendered its whole step as one markdown blob, so a root cause Seer wrote up itself looked nothing like the same root cause shown by `{% autofixRef %}`, which reads the run state and lays it out as titled sections. Both embeds now share `RootCauseBody` and `SolutionBody`, so either path shows "Why did this happen?", "Reproduction Steps", and "Steps to Resolve" identically. To make that possible the `autofix` payload gained optional `fiveWhys`, `reproductionSteps`, and `steps` fields mirroring the artifact shapes the API returns, and its description tells Seer to send detail there rather than folding it into `result`. The fields are optional, so a step that arrives as the write-up alone still renders as before — it collapses to the summary rather than showing empty sections. Two smaller consequences worth a look: - `AutofixRefBody` now falls through to the step's error text when a completed section carries a null artifact, where it previously rendered nothing. A completed step with no data is a real failure, so saying so beats a blank. - The step bodies take plain values instead of the artifact wrapper, which is what lets the embed that has no artifact feed them. The storybook fixtures were a single hand-written sentence per step; they now carry a full causal chain, reproduction steps, and plan steps so the stories exercise the sections they are meant to demonstrate. Claude-Session: https://claude.ai/code/session_01PYSENxo4inE3WN5X1Hm7FB --- .../seer/agent/embed_widgets.generated.json | 67 ++++++++++++- .../events/autofix/useExplorerAutofix.tsx | 2 +- .../embeds/components/autofix.spec.tsx | 78 ++++++++++++++- .../embeds/components/autofix.stories.tsx | 72 ++++++++++++-- .../markdown/embeds/components/autofix.tsx | 96 +++++++++++++------ .../seer/markdown/embeds/schemas.ts | 59 +++++++++++- 6 files changed, 330 insertions(+), 44 deletions(-) diff --git a/src/sentry/seer/agent/embed_widgets.generated.json b/src/sentry/seer/agent/embed_widgets.generated.json index c4706cfec7a3..10a2f3202bef 100644 --- a/src/sentry/seer/agent/embed_widgets.generated.json +++ b/src/sentry/seer/agent/embed_widgets.generated.json @@ -452,7 +452,7 @@ }, { "name": "autofix", - "description": "Render one step of a Seer Autofix run (root cause, solution, or code changes) as a collapsible block linking back to the issue. Emit this embed whenever the user signals intent to fix, solve, debug, or resolve a problem — e.g. \"fix this issue\", \"solve the problem\", \"find the root cause\", \"why is this happening\", \"how do I resolve this error\" — or asks for the status/result of an autofix run already in progress. `id` and `shortId` are the issue the run belongs to, exactly as the issue API returns them. `step` is the autofix step identifier exactly as the autofix API reports it — the UI renders the human-readable label, so do not send a display string. `result` is the full markdown write-up for that step. Prefer this embed over a plaintext explanation whenever an issue can be autofixed, and emit one embed per step rather than combining multiple steps into one.", + "description": "Render one step of a Seer Autofix run (root cause, solution, or code changes) as a collapsible block linking back to the issue. Emit this embed whenever the user signals intent to fix, solve, debug, or resolve a problem — e.g. \"fix this issue\", \"solve the problem\", \"find the root cause\", \"why is this happening\", \"how do I resolve this error\" — or asks for the status/result of an autofix run already in progress. `id` and `shortId` are the issue the run belongs to, exactly as the issue API returns them. `step` is the autofix step identifier exactly as the autofix API reports it — the UI renders the human-readable label, so do not send a display string. `result` is the markdown summary for that step. Send the step's detail in the structured fields rather than folding it into `result`, so it renders as the same sections a live run shows: `fiveWhys` and `reproductionSteps` for `root_cause`, `steps` for `solution`. Prefer this embed over a plaintext explanation whenever an issue can be autofixed, and emit one embed per step rather than combining multiple steps into one.", "level": ["block"], "body": { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -470,6 +470,37 @@ }, "shortId": { "type": "string" + }, + "fiveWhys": { + "description": "root_cause only: the causal chain, most immediate cause first.", + "type": "array", + "items": { + "type": "string" + } + }, + "reproductionSteps": { + "description": "root_cause only: ordered steps that reproduce the error.", + "type": "array", + "items": { + "type": "string" + } + }, + "steps": { + "description": "solution only: the ordered steps needed to resolve the issue.", + "type": "array", + "items": { + "type": "object", + "properties": { + "title": { + "type": "string" + }, + "description": { + "type": "string" + } + }, + "required": ["title", "description"], + "additionalProperties": false + } } }, "required": ["step", "result", "id", "shortId"], @@ -481,9 +512,41 @@ "data": { "id": "1234567890", "shortId": "EXMPL-123", - "result": "The root cause of the issue is that the code is not working correctly.", + "result": "`CartService.total()` reduces the line items without an initial accumulator, so an empty cart throws instead of totalling to zero.", + "fiveWhys": [ + "`POST /checkout` returned a 500 for every request with an empty cart.", + "`CartService.total()` threw `TypeError: Reduce of empty array with no initial value`.", + "`items.reduce((sum, item) => sum + item.price)` was called without a second argument.", + "With no initial value `reduce` uses the first element as the seed, which an empty array does not have.", + "The empty cart path was never covered — every test seeded at least one line item." + ], + "reproductionSteps": [ + "Sign in and add a single item to the cart.", + "Remove that item, leaving the cart empty.", + "Open `/checkout`, which calls `POST /api/checkout/quote`.", + "The request 500s and the page renders the generic error state." + ], "step": "root_cause" } + }, + { + "label": "Plan", + "data": { + "id": "1234567890", + "shortId": "EXMPL-123", + "result": "Seed the reduction with `0` so an empty cart totals to zero, and cover the path with a test.", + "steps": [ + { + "title": "Pass an initial accumulator to `CartService.total()`", + "description": "Change `items.reduce((sum, item) => sum + item.price)` to pass `0` as the second argument." + }, + { + "title": "Add a regression test for the empty cart", + "description": "Assert `total()` returns `0` for `[]` in `src/checkout/cartService.test.ts`." + } + ], + "step": "solution" + } } ], "featureFlag": "organizations:seer-agent-autofix" diff --git a/static/app/components/events/autofix/useExplorerAutofix.tsx b/static/app/components/events/autofix/useExplorerAutofix.tsx index 882156850b1c..33ae4d875eb0 100644 --- a/static/app/components/events/autofix/useExplorerAutofix.tsx +++ b/static/app/components/events/autofix/useExplorerAutofix.tsx @@ -88,7 +88,7 @@ export function isRootCauseArtifact( ); } -interface SolutionStep { +export interface SolutionStep { description: string; title: string; } diff --git a/static/app/components/seer/markdown/embeds/components/autofix.spec.tsx b/static/app/components/seer/markdown/embeds/components/autofix.spec.tsx index 1eb3e3bc9f1d..aee12b7c38c8 100644 --- a/static/app/components/seer/markdown/embeds/components/autofix.spec.tsx +++ b/static/app/components/seer/markdown/embeds/components/autofix.spec.tsx @@ -1,9 +1,10 @@ import {useInfiniteQuery} from '@tanstack/react-query'; import {OrganizationFixture} from 'sentry-fixture/organization'; -import {render, screen, waitFor} from 'sentry-test/reactTestingLibrary'; +import {render, screen, userEvent, waitFor} from 'sentry-test/reactTestingLibrary'; import type {ExplorerAutofixState} from 'sentry/components/events/autofix/useExplorerAutofix'; +import {SeerMarkdown} from 'sentry/components/seer/markdown'; import {AutofixRef} from 'sentry/components/seer/markdown/embeds/components/autofix'; import type {Group} from 'sentry/types/group'; import {apiOptions} from 'sentry/utils/api/apiOptions'; @@ -209,3 +210,78 @@ describe('AutofixRef embed', () => { await waitFor(() => expect(issuesMock).toHaveBeenCalledTimes(3)); }, 20_000); }); + +const ISSUE = {id: '6789012345', shortId: 'CHECKOUT-42'}; + +function renderAutofixEmbed(data: Record) { + const tag = `{% autofix %}${JSON.stringify({...ISSUE, ...data})}{% /autofix %}`; + return render(); +} + +async function expand(name: string) { + await userEvent.click(screen.getByRole('button', {name: new RegExp(name)})); +} + +describe('autofix embed', () => { + it('renders the root cause sections a live run shows', async () => { + renderAutofixEmbed({ + step: 'root_cause', + result: '`CartService.total()` reduces line items without an initial accumulator.', + fiveWhys: [ + '`POST /api/checkout/quote` returned a 500 for every empty cart.', + 'The empty-cart path was never exercised by a test.', + ], + reproductionSteps: ['Empty the cart.', 'Open `/checkout`.'], + }); + + await expand('Root Cause'); + + expect( + screen.getByText(/reduces line items without an initial accumulator/) + ).toBeInTheDocument(); + + expect(screen.getByText('Why did this happen?')).toBeInTheDocument(); + expect(screen.getByText(/returned a 500 for every empty cart/)).toBeInTheDocument(); + + expect(screen.getByText('Reproduction Steps')).toBeInTheDocument(); + expect(screen.getByText('Empty the cart.')).toBeInTheDocument(); + }); + + it('renders the plan steps', async () => { + renderAutofixEmbed({ + step: 'solution', + result: 'Seed the reduction with `0`.', + steps: [ + { + title: 'Pass an initial accumulator', + description: 'Pass `0` as the second argument to `reduce`.', + }, + ], + }); + + await expand('Plan'); + + expect(screen.getByText('Steps to Resolve')).toBeInTheDocument(); + expect(screen.getByText('Pass an initial accumulator')).toBeInTheDocument(); + expect( + screen.getByText('Pass `0` as the second argument to `reduce`.') + ).toBeInTheDocument(); + }); + + // Seer writes this embed itself, so the structured fields can be absent even + // on a step that normally carries them. + it('renders the summary alone when no structured detail is sent', async () => { + renderAutofixEmbed({ + step: 'root_cause', + result: 'The cart total throws on an empty cart.', + }); + + await expand('Root Cause'); + + expect( + screen.getByText('The cart total throws on an empty cart.') + ).toBeInTheDocument(); + expect(screen.queryByText('Why did this happen?')).not.toBeInTheDocument(); + expect(screen.queryByText('Reproduction Steps')).not.toBeInTheDocument(); + }); +}); diff --git a/static/app/components/seer/markdown/embeds/components/autofix.stories.tsx b/static/app/components/seer/markdown/embeds/components/autofix.stories.tsx index 8e5e6d741346..e6c693bc3959 100644 --- a/static/app/components/seer/markdown/embeds/components/autofix.stories.tsx +++ b/static/app/components/seer/markdown/embeds/components/autofix.stories.tsx @@ -20,6 +20,7 @@ import { NEXT_STEP, STEP_LABELS, } from 'sentry/components/seer/markdown/embeds/components/autofix'; +import type {EmbedOutput} from 'sentry/components/seer/markdown/embeds/utils'; import {IconArrow} from 'sentry/icons'; import * as Storybook from 'sentry/stories'; import type {Group} from 'sentry/types/group'; @@ -34,8 +35,21 @@ import type { const ISSUE = {id: '6789012345', shortId: 'CHECKOUT-42'}; -function autofix(step: AutofixExplorerStep, result: string): string { - return `{% autofix %}${JSON.stringify({...ISSUE, step, result})}{% /autofix %}`; +/** + * Taken from the embed schema so these fixtures fail to compile rather than + * silently drop a field if the structured payload changes shape. + */ +type AutofixDetails = Pick< + EmbedOutput<'autofix'>, + 'fiveWhys' | 'reproductionSteps' | 'steps' +>; + +function autofix( + step: AutofixExplorerStep, + result: string, + details: AutofixDetails = {} +): string { + return `{% autofix %}${JSON.stringify({...ISSUE, step, result, ...details})}{% /autofix %}`; } function autofixRefTag( @@ -48,23 +62,69 @@ function autofixRefTag( const ROOT_CAUSE = autofix( 'root_cause', - '`CartService.total()` calls `items.reduce((sum, item) => sum + item.price)` without an initial accumulator. When a customer empties their cart the array is empty, so `reduce` throws `TypeError: Reduce of empty array with no initial value` and the checkout request 500s.' + '`CartService.total()` reduces the line items without an initial accumulator, so an empty cart throws `TypeError: Reduce of empty array with no initial value` and `POST /api/checkout/quote` 500s.', + { + fiveWhys: [ + '`POST /api/checkout/quote` returned a 500 for every request carrying an empty cart.', + '`CartService.total()` threw `TypeError: Reduce of empty array with no initial value`.', + '`items.reduce((sum, item) => sum + item.price)` is called without a second argument.', + 'Without an initial value `reduce` seeds itself from the first element, which an empty array does not have.', + 'The empty-cart path was never exercised — every fixture in `cartService.test.ts` seeds at least one line item.', + ], + reproductionSteps: [ + 'Sign in as any customer and add one item to the cart.', + 'Remove that item, leaving the cart empty.', + 'Navigate to `/checkout`, which calls `POST /api/checkout/quote` on mount.', + 'The request 500s and the page falls back to the generic error state.', + ], + } ); const SOLUTION = autofix( 'solution', - 'Seed the reduction with `0` so an empty cart totals to zero instead of throwing: `items.reduce((sum, item) => sum + item.price, 0)`.' + 'Seed the reduction with `0` so an empty cart totals to zero instead of throwing.', + { + steps: [ + { + title: 'Pass an initial accumulator to `CartService.total()`', + description: + 'Change `items.reduce((sum, item) => sum + item.price)` to pass `0` as the second argument.', + }, + { + title: 'Cover the empty cart in `cartService.test.ts`', + description: 'Assert `total()` returns `0` for an empty line-item array.', + }, + ], + } ); const CODE_CHANGES = autofix( 'code_changes', - 'Updated `src/checkout/cartService.ts` to pass the initial value and added a regression test covering the empty-cart path.' + '2 files changed in 1 repo — `src/checkout/cartService.ts` now passes the initial value, and `src/checkout/cartService.test.ts` covers the empty-cart path.' ); // Autofix has no "plan" step — a plan is the write-up of the solution step. const PLANNED_SOLUTION = autofix( 'solution', - 'Guard `CartService.total()` with an initial accumulator of `0`, add a regression test covering the empty-cart path, then backfill a smoke test that renders the checkout page with zero items.' + 'Guard `CartService.total()` against an empty cart, then close the coverage gap that let this ship.', + { + steps: [ + { + title: 'Pass an initial accumulator to `CartService.total()`', + description: + 'Change `items.reduce((sum, item) => sum + item.price)` to pass `0` as the second argument.', + }, + { + title: 'Cover the empty cart in `cartService.test.ts`', + description: 'Assert `total()` returns `0` for an empty line-item array.', + }, + { + title: 'Add a checkout smoke test with zero items', + description: + 'Render `/checkout` with an empty cart and assert the quote renders `$0.00` instead of the error state.', + }, + ], + } ); function User({children}: {children: ReactNode}) { diff --git a/static/app/components/seer/markdown/embeds/components/autofix.tsx b/static/app/components/seer/markdown/embeds/components/autofix.tsx index c6db39a1ae89..653b0e68540b 100644 --- a/static/app/components/seer/markdown/embeds/components/autofix.tsx +++ b/static/app/components/seer/markdown/embeds/components/autofix.tsx @@ -21,8 +21,7 @@ import { useExplorerAutofix, type AutofixExplorerStep, type AutofixSection, - type RootCauseArtifact, - type SolutionArtifact, + type SolutionStep, } from 'sentry/components/events/autofix/useExplorerAutofix'; import {useRefreshAutofixProgressQueries} from 'sentry/components/events/autofix/useRefreshAutofixProgressQueries'; import {ArtifactDetails} from 'sentry/components/events/autofix/v3/artifactDetails'; @@ -36,7 +35,6 @@ import {IconOpen} from 'sentry/icons/iconOpen'; import {IconPullRequest} from 'sentry/icons/iconPullRequest'; import {t, tn} from 'sentry/locale'; import type {Group} from 'sentry/types/group'; -import {MarkedText} from 'sentry/utils/marked/markedText'; import {useOrganization} from 'sentry/utils/useOrganization'; import {FileDiffViewer} from 'sentry/views/seerExplorer/components/fileDiffViewer'; @@ -92,14 +90,46 @@ interface AutofixContentProps extends Pick { */ result: string; step: AutofixExplorerStep; + fiveWhys?: string[]; + reproductionSteps?: string[]; + steps?: SolutionStep[]; +} + +/** + * The structured fields are optional because Seer writes this embed itself + * rather than echoing back run state, so a step can arrive as the write-up + * alone. Missing detail collapses to the summary rather than an empty section. + */ +function AutofixStepBody({ + fiveWhys, + reproductionSteps, + result, + step, + steps, +}: Omit) { + if (step === 'root_cause') { + return ( + + ); + } + + if (step === 'solution') { + return ; + } + + return ; } export const Autofix = defineSeerEmbed({ name: 'autofix', - render({id, shortId, result, step}: AutofixContentProps) { + render({id, shortId, ...content}: AutofixContentProps) { return ( - - + + ); }, @@ -271,12 +301,23 @@ function AutofixRefBody({isLoading, section, step}: AutofixRefBodyProps) { const artifact = getAutofixArtifactFromSection(section); - if (step === 'root_cause' && isRootCauseArtifact(artifact)) { - return ; + if (step === 'root_cause' && isRootCauseArtifact(artifact) && artifact.data) { + return ( + + ); } - if (step === 'solution' && isSolutionArtifact(artifact)) { - return ; + if (step === 'solution' && isSolutionArtifact(artifact) && artifact.data) { + return ( + + ); } if (isCodeChangesArtifact(artifact)) { @@ -290,22 +331,20 @@ function AutofixRefBody({isLoading, section, step}: AutofixRefBodyProps) { } interface RootCauseBodyProps { - data: RootCauseArtifact | null; + description: string; + fiveWhys: string[]; + reproductionSteps?: string[]; } -function RootCauseBody({data}: RootCauseBodyProps) { - if (!data) { - return ; - } - +function RootCauseBody({description, fiveWhys, reproductionSteps}: RootCauseBodyProps) { return ( - - {data.five_whys.length > 0 && ( + + {fiveWhys.length > 0 && ( {t('Why did this happen?')} - {data.five_whys.map((why, index) => ( + {fiveWhys.map((why, index) => (
    • @@ -313,11 +352,11 @@ function RootCauseBody({data}: RootCauseBodyProps) {     )} - {data.reproduction_steps && data.reproduction_steps.length > 0 && ( + {reproductionSteps && reproductionSteps.length > 0 && ( {t('Reproduction Steps')} - {data.reproduction_steps.map((step, index) => ( + {reproductionSteps.map((step, index) => (
    • @@ -330,22 +369,19 @@ function RootCauseBody({data}: RootCauseBodyProps) { } interface SolutionBodyProps { - data: SolutionArtifact | null; + steps: SolutionStep[]; + summary: string; } -function SolutionBody({data}: SolutionBodyProps) { - if (!data) { - return ; - } - +function SolutionBody({steps, summary}: SolutionBodyProps) { return ( - - {data.steps.length > 0 && ( + + {steps.length > 0 && ( {t('Steps to Resolve')} - {data.steps.map((step, index) => ( + {steps.map((step, index) => (
  • diff --git a/static/app/components/seer/markdown/embeds/schemas.ts b/static/app/components/seer/markdown/embeds/schemas.ts index 8f9f369d7ec3..ebdbd353ef31 100644 --- a/static/app/components/seer/markdown/embeds/schemas.ts +++ b/static/app/components/seer/markdown/embeds/schemas.ts @@ -338,9 +338,12 @@ export const SEER_EMBED_SCHEMAS = { 'belongs to, exactly as the issue API returns them. `step` is the ' + 'autofix step identifier exactly as the autofix API reports it — the ' + 'UI renders the human-readable label, so do not send a display ' + - 'string. `result` is the full markdown write-up for that step. ' + - 'Prefer this embed over a plaintext explanation whenever an issue ' + - 'can be autofixed, and emit one embed per step rather than ' + + 'string. `result` is the markdown summary for that step. Send the ' + + "step's detail in the structured fields rather than folding it into " + + '`result`, so it renders as the same sections a live run shows: ' + + '`fiveWhys` and `reproductionSteps` for `root_cause`, `steps` for ' + + '`solution`. Prefer this embed over a plaintext explanation whenever ' + + 'an issue can be autofixed, and emit one embed per step rather than ' + 'combining multiple steps into one.', level: ['block'], schema: z.object({ @@ -348,6 +351,18 @@ export const SEER_EMBED_SCHEMAS = { result: z.string(), id: z.string(), shortId: z.string(), + fiveWhys: z + .array(z.string()) + .optional() + .describe('root_cause only: the causal chain, most immediate cause first.'), + reproductionSteps: z + .array(z.string()) + .optional() + .describe('root_cause only: ordered steps that reproduce the error.'), + steps: z + .array(z.object({title: z.string(), description: z.string()})) + .optional() + .describe('solution only: the ordered steps needed to resolve the issue.'), }), examples: [ { @@ -356,10 +371,46 @@ export const SEER_EMBED_SCHEMAS = { id: '1234567890', shortId: 'EXMPL-123', result: - 'The root cause of the issue is that the code is not working correctly.', + '`CartService.total()` reduces the line items without an initial ' + + 'accumulator, so an empty cart throws instead of totalling to zero.', + fiveWhys: [ + '`POST /checkout` returned a 500 for every request with an empty cart.', + '`CartService.total()` threw `TypeError: Reduce of empty array with no initial value`.', + '`items.reduce((sum, item) => sum + item.price)` was called without a second argument.', + 'With no initial value `reduce` uses the first element as the seed, which an empty array does not have.', + 'The empty cart path was never covered — every test seeded at least one line item.', + ], + reproductionSteps: [ + 'Sign in and add a single item to the cart.', + 'Remove that item, leaving the cart empty.', + 'Open `/checkout`, which calls `POST /api/checkout/quote`.', + 'The request 500s and the page renders the generic error state.', + ], step: 'root_cause' as const, }, }, + { + label: 'Plan', + data: { + id: '1234567890', + shortId: 'EXMPL-123', + result: + 'Seed the reduction with `0` so an empty cart totals to zero, and cover the path with a test.', + steps: [ + { + title: 'Pass an initial accumulator to `CartService.total()`', + description: + 'Change `items.reduce((sum, item) => sum + item.price)` to pass `0` as the second argument.', + }, + { + title: 'Add a regression test for the empty cart', + description: + 'Assert `total()` returns `0` for `[]` in `src/checkout/cartService.test.ts`.', + }, + ], + step: 'solution' as const, + }, + }, ], }, alert: {