diff --git a/static/app/components/core/markdown/markdown.spec.tsx b/static/app/components/core/markdown/markdown.spec.tsx index 3187247ab7eb..93e165f98237 100644 --- a/static/app/components/core/markdown/markdown.spec.tsx +++ b/static/app/components/core/markdown/markdown.spec.tsx @@ -402,6 +402,91 @@ describe('Markdown', () => { }); }); + describe('tag index', () => { + function IndexProbe({name, index}: {name: string; index?: number}) { + return {`${name}=${index}`}; + } + + const indexes = () => screen.getAllByRole('log').map(el => el.textContent); + + it('numbers tags in document order across blocks', () => { + render( + + ); + expect(indexes()).toEqual(['a=0', 'b=1', 'c=2']); + }); + + it('numbers two inline tags in the same paragraph separately', () => { + render( + + ); + expect(indexes()).toEqual(['a=0', 'b=1']); + }); + + it('numbers identical tags separately', () => { + render( + + ); + expect(indexes()).toEqual(['a=0', 'a=1']); + }); + + it('numbers tags nested in lists', () => { + render( + + ); + expect(indexes()).toEqual(['a=0', 'b=1']); + }); + + it('numbers tags in table headers before table rows', () => { + render( + + ); + expect(indexes()).toEqual(['a=0', 'b=1']); + }); + + it('keeps existing indexes when content is appended', () => { + const {rerender} = render( + + ); + expect(indexes()).toEqual(['a=0']); + + rerender( + + ); + expect(indexes()).toEqual(['a=0', 'b=1']); + }); + + it('does not count a tag whose closing marker has not arrived', () => { + const {rerender} = render( + + ); + expect(indexes()).toEqual(['a=0']); + + rerender( + + ); + expect(indexes()).toEqual(['a=0', 'b=1']); + }); + }); + describe('token caching', () => { it('renders correctly when raw prop changes', () => { const {rerender} = render(); diff --git a/static/app/components/core/markdown/markdown.tsx b/static/app/components/core/markdown/markdown.tsx index 17127111a48d..3cedcd30853f 100644 --- a/static/app/components/core/markdown/markdown.tsx +++ b/static/app/components/core/markdown/markdown.tsx @@ -50,6 +50,11 @@ export type MarkdownComponents = Partial<{ name: string; /** Original `{% tag %}` source, including body and closing tag. */ raw: string; + /** + * Position of this tag among all tags in the message, in document order. + * Counts tags only, so two inline tags in one paragraph get 0 and 1. + */ + index?: number; }> >; TaskList: ComponentType>; @@ -64,19 +69,75 @@ export interface MarkdownProps { variant?: 'static' | 'streaming'; } +/** + * Stamps every tag token with its position among all tags in the message, in + * document order. + * + * Runs after lexing rather than inside the tokenizer because marked defers + * inline tokenization to a second pass: a tokenizer counter would number an + * inline tag in the first paragraph after a block tag in the second. + * + * The result is stable while streaming. Content only ever grows by appending, + * so a newly closed tag can only appear after the existing ones and never + * shifts their index -- and a tag whose closing marker has not arrived yet is + * not a tag token at all, so it claims no index early. + */ +function assignTagIndexes(tokens: ExtendedToken[]): void { + let nextIndex = 0; + + function visitAll(list: readonly ExtendedToken[]): void { + for (const token of list) { + visit(token); + } + } + + function visit(token: ExtendedToken): void { + if (token.type === 'tag') { + // A tag body is JSON, never markdown, so it has no child tokens. + token.index = nextIndex++; + return; + } + if ('tokens' in token && token.tokens) { + visitAll(token.tokens as ExtendedToken[]); + } + if ('items' in token && token.items) { + visitAll(token.items as ExtendedToken[]); + } + // Tables hold their cells outside `tokens`; header precedes rows on screen. + if ('header' in token && token.header) { + for (const cell of token.header) { + visitAll(cell.tokens as ExtendedToken[]); + } + } + if ('rows' in token && token.rows) { + for (const row of token.rows) { + for (const cell of row) { + visitAll(cell.tokens as ExtendedToken[]); + } + } + } + } + + visitAll(tokens); +} + export function Markdown({raw, components = {}, variant = 'static'}: MarkdownProps) { const containerRef = useRef(null); const prevTextLensRef = useRef(new Map()); const isStreaming = variant === 'streaming'; - const tokens = useMemo(() => MarkedLexer.lex(raw), [raw]); + const tokens = useMemo(() => { + const lexed = MarkedLexer.lex(raw) as ExtendedToken[]; + assignTagIndexes(lexed); + return lexed; + }, [raw]); const elements = useMemo( () => tokens.map((token, i) => ( )), diff --git a/static/app/components/core/markdown/token.tsx b/static/app/components/core/markdown/token.tsx index bf7c4b4dbabf..9c5d46e0f48c 100644 --- a/static/app/components/core/markdown/token.tsx +++ b/static/app/components/core/markdown/token.tsx @@ -295,6 +295,7 @@ export function Token({ attrs={token.attrs} data={token.data} raw={token.raw} + index={token.index} /> ); } diff --git a/static/app/components/seer/markdown/embeds/registry.tsx b/static/app/components/seer/markdown/embeds/registry.tsx index 544f394c7424..14c7c0017c80 100644 --- a/static/app/components/seer/markdown/embeds/registry.tsx +++ b/static/app/components/seer/markdown/embeds/registry.tsx @@ -8,6 +8,11 @@ export interface SeerEmbedProps { data: unknown; level: 'block' | 'inline'; name: string; + /** + * Position among all embeds in the message, in document order. Assigned by + * `Markdown` while lexing; see `renderTracking` for what it is used for. + */ + index?: number; } export type SeerEmbedComponent = (props: SeerEmbedProps) => ReactNode; diff --git a/static/app/components/seer/markdown/embeds/renderTracking.spec.tsx b/static/app/components/seer/markdown/embeds/renderTracking.spec.tsx new file mode 100644 index 000000000000..de42d37b2040 --- /dev/null +++ b/static/app/components/seer/markdown/embeds/renderTracking.spec.tsx @@ -0,0 +1,146 @@ +import {GEN_AI_CONVERSATION_ID} from '@sentry/conventions/attributes'; +import * as Sentry from '@sentry/react'; + +import {render} from 'sentry-test/reactTestingLibrary'; + +import {SeerMarkdown} from 'sentry/components/seer/markdown'; + +import type {SeerEmbedScope} from './renderTracking'; + +const timestamp = (value: string) => + `{% timestamp %}${JSON.stringify({value, format: 'absolute'})}{% /timestamp %}`; + +/** + * Renders already reported are suppressed for the life of the page, so every + * test needs a scope no earlier test has used. + */ +let nextConversation = 0; +function scope(overrides: Partial = {}): SeerEmbedScope { + nextConversation += 1; + return { + conversationId: `run-${nextConversation}`, + messageId: 'block-1', + surface: 'seer_explorer', + ...overrides, + }; +} + +describe('seer embed render tracking', () => { + let info!: jest.SpyInstance; + + beforeEach(() => { + info = jest.spyOn(Sentry.logger, 'info').mockImplementation(() => {}); + }); + + afterEach(() => { + info.mockRestore(); + }); + + const attributesOf = (call: unknown[]) => call[1] as Record; + + it('records a render with its conversation, message and index', () => { + const current = scope(); + render( + + ); + + expect(info).toHaveBeenCalledTimes(1); + expect(attributesOf(info.mock.calls[0]!)).toEqual( + expect.objectContaining({ + 'seer_embed.name': 'timestamp', + 'seer_embed.level': 'inline', + 'seer_embed.index': 0, + 'seer_embed.surface': 'seer_explorer', + [GEN_AI_CONVERSATION_ID]: current.conversationId, + 'seer_embed.conversation_id': current.conversationId, + 'seer_embed.message_id': 'block-1', + 'seer_embed.message_key': `${current.conversationId}:block-1`, + 'seer_embed.embed_key': `${current.conversationId}:block-1:0`, + }) + ); + }); + + it('records each embed in a message separately', () => { + render( + + ); + + expect(info).toHaveBeenCalledTimes(2); + expect(info.mock.calls.map(call => attributesOf(call)['seer_embed.index'])).toEqual([ + 0, 1, + ]); + }); + + it('records identical embeds in one message separately', () => { + const same = timestamp('2025-07-15T14:30:00Z'); + render(); + + expect(info).toHaveBeenCalledTimes(2); + expect(info.mock.calls.map(call => attributesOf(call)['seer_embed.index'])).toEqual([ + 0, 1, + ]); + }); + + it('records an embed once across re-renders of the same message', () => { + const current = scope(); + const raw = `at ${timestamp('2025-07-15T14:30:00Z')}`; + const {rerender} = render(); + expect(info).toHaveBeenCalledTimes(1); + + // Streaming remounts the paragraph holding an inline embed on every chunk. + rerender(); + rerender(); + + expect(info).toHaveBeenCalledTimes(1); + }); + + it('records the same embed in a different message', () => { + const raw = `at ${timestamp('2025-07-15T14:30:00Z')}`; + const conversationId = scope().conversationId; + + render( + + ); + render( + + ); + + expect(info).toHaveBeenCalledTimes(2); + expect( + info.mock.calls.map(call => attributesOf(call)['seer_embed.message_id']) + ).toEqual(['block-1', 'block-2']); + }); + + it('records nothing without a scope', () => { + render(); + expect(info).not.toHaveBeenCalled(); + }); + + it('records nothing for an embed whose props are invalid', () => { + const warn = jest.spyOn(console, 'warn').mockImplementation(() => {}); + const captureException = jest + .spyOn(Sentry, 'captureException') + .mockImplementation(() => ''); + + render( + + ); + + expect(info).not.toHaveBeenCalled(); + + warn.mockRestore(); + captureException.mockRestore(); + }); +}); diff --git a/static/app/components/seer/markdown/embeds/renderTracking.tsx b/static/app/components/seer/markdown/embeds/renderTracking.tsx new file mode 100644 index 000000000000..e4e21b83d078 --- /dev/null +++ b/static/app/components/seer/markdown/embeds/renderTracking.tsx @@ -0,0 +1,107 @@ +import {createContext, useContext, useEffect} from 'react'; +import {GEN_AI_CONVERSATION_ID} from '@sentry/conventions/attributes'; +import * as Sentry from '@sentry/react'; + +/** + * Identifies where a Seer embed was rendered, so a render can be attributed to + * a conversation and a message rather than just a page load. + * + * Supplied by the surface rendering the markdown. A surface that cannot name + * both ids supplies nothing and its embeds go untracked -- better a missing + * surface than rows that cannot be deduplicated. + */ +export interface SeerEmbedScope { + /** Run the embed was rendered in. */ + conversationId: string; + /** + * Message within the run. Must be the server-assigned id: it is what makes a + * render deduplicable across viewers and reloads. Optimistic client-side ids + * change once the server responds and would double count. + */ + messageId: string; + /** Product surface, so one embed type can be compared across surfaces. */ + surface: string; +} + +export const SeerEmbedScopeContext = createContext(null); + +/** + * Embeds already reported this page load. + * + * Seer markdown re-lexes and re-renders on every streamed chunk, and a + * paragraph holding an inline embed remounts each time text is appended after + * it, so an embed would otherwise report once per chunk. Keyed on the same + * composite the query counts distinct on, which makes this an optimisation + * rather than a correctness requirement: a missed dedup (a reopened + * conversation, a second tab) collapses again at query time. + */ +const reportedEmbeds = new Set(); + +interface TrackEmbedRenderedOptions { + /** + * Position among all embeds in the message, in document order. Undefined when + * the markdown was not rendered through `Markdown` (which assigns it). + */ + index: number | undefined; + level: 'block' | 'inline'; + name: string; + /** + * False when the embed's props failed validation. Such an embed renders + * nothing, so counting it would overstate what users actually saw. + */ + rendered: boolean; +} + +/** + * Records that an embed was rendered, once per embed instance per page load. + * + * Emitted as a log rather than a metric because the question it answers is + * "how many distinct embeds", which needs `count_unique` over an identifier -- + * an aggregate the logs dataset offers and a pre-aggregated counter cannot. + */ +export function useTrackEmbedRendered({ + index, + level, + name, + rendered, +}: TrackEmbedRenderedOptions): void { + const scope = useContext(SeerEmbedScopeContext); + + useEffect(() => { + if (!rendered || !scope || index === undefined) { + return; + } + + const messageKey = `${scope.conversationId}:${scope.messageId}`; + const embedKey = `${messageKey}:${index}`; + if (reportedEmbeds.has(embedKey)) { + return; + } + reportedEmbeds.add(embedKey); + + Sentry.logger.info('Seer embed rendered', { + 'seer_embed.name': name, + 'seer_embed.level': level, + 'seer_embed.index': index, + 'seer_embed.surface': scope.surface, + // The conversation is written twice on purpose. The convention name is + // what correlates this render with everything else describing the same + // conversation -- spans, other producers -- while the `seer_embed.` + // copy keeps every attribute of this log under one prefix, so a query + // for embeds does not have to know that one of its fields is namespaced + // somewhere else. + // + // The message has no such pair: `gen_ai.response.id` means the + // provider's completion id, not a Seer block id, so writing a block id + // there would put two meanings behind one key. + [GEN_AI_CONVERSATION_ID]: scope.conversationId, + 'seer_embed.conversation_id': scope.conversationId, + 'seer_embed.message_id': scope.messageId, + // Pre-composed because the query layer cannot concatenate attributes: + // `count_unique(seer_embed.message_key)` counts messages that showed an + // embed of a type, `count_unique(seer_embed.embed_key)` counts embeds. + 'seer_embed.message_key': messageKey, + 'seer_embed.embed_key': embedKey, + }); + }, [index, level, name, rendered, scope]); +} diff --git a/static/app/components/seer/markdown/embeds/utils.tsx b/static/app/components/seer/markdown/embeds/utils.tsx index 0f78cc862dd7..786c1df58eb5 100644 --- a/static/app/components/seer/markdown/embeds/utils.tsx +++ b/static/app/components/seer/markdown/embeds/utils.tsx @@ -5,6 +5,7 @@ import type {z} from 'zod'; import {NODE_ENV} from 'sentry/constants/env'; import type {SeerEmbedProps} from './registry'; +import {useTrackEmbedRendered} from './renderTracking'; import {ALL_SEER_EMBED_SCHEMAS, type SeerEmbedName} from './schemas'; export type EmbedOutput = z.output< @@ -51,8 +52,11 @@ export function defineSeerEmbed({ }: DefineSeerEmbedOptions) { const {schema} = ALL_SEER_EMBED_SCHEMAS[name]; - function Embed({data, level}: SeerEmbedProps) { + function Embed({data, level, index}: SeerEmbedProps) { const parsed = schema.safeParse(data); + // Called before the early return so the hook stays unconditional; it + // no-ops for an embed that failed validation and renders nothing. + useTrackEmbedRendered({name, level, index, rendered: parsed.success}); if (!parsed.success) { reportInvalidEmbed(name, parsed.error.issues); return null; diff --git a/static/app/components/seer/markdown/index.tsx b/static/app/components/seer/markdown/index.tsx index d53993485f20..8e411f646e19 100644 --- a/static/app/components/seer/markdown/index.tsx +++ b/static/app/components/seer/markdown/index.tsx @@ -8,6 +8,7 @@ import {Link} from '@sentry/scraps/link'; import {Markdown, type MarkdownProps} from '@sentry/scraps/markdown'; import {Heading} from '@sentry/scraps/text'; +import {type SeerEmbedScope, SeerEmbedScopeContext} from './embeds/renderTracking'; import {STRUCTURED_SEER_EMBED_SCHEMAS} from './embeds/schemas'; import {SeerEmbedRegistry} from './embeds'; @@ -85,7 +86,7 @@ function reportUnhandledTag( } const SEER_EMBED_COMPONENTS: MarkdownProps['components'] = { - Tag: function SeerTag({name, data, level, attrs}) { + Tag: function SeerTag({name, data, level, attrs, index}) { const structuredContent = useContext(StructuredContentContext); const Embed = SeerEmbedRegistry.get(name); if (Embed) { @@ -95,7 +96,7 @@ const SEER_EMBED_COMPONENTS: MarkdownProps['components'] = { : data === undefined ? structuredContent?.[name] : data; - const embed = ; + const embed = ; if (level === 'inline') { return embed; } @@ -159,13 +160,25 @@ const SEER_EMBED_COMPONENTS: MarkdownProps['components'] = { export function SeerMarkdown({ components, structuredContent = null, + scope = null, ...props }: MarkdownProps & { + /** + * Conversation and message this markdown belongs to. Supply it to record + * embed renders; omit it (stories, demos, previews) to render untracked. + * + * Scoped to this call rather than to the message, so a surface that renders + * one message through two `SeerMarkdown` calls would give both embeds the + * same index. Pass the whole message in one call. + */ + scope?: SeerEmbedScope | null; structuredContent?: Record | null; }) { return ( - - - + + + + + ); } diff --git a/static/app/utils/marked/extensions/tag.ts b/static/app/utils/marked/extensions/tag.ts index 3ef551935841..4c9f76909d9b 100644 --- a/static/app/utils/marked/extensions/tag.ts +++ b/static/app/utils/marked/extensions/tag.ts @@ -7,6 +7,15 @@ export interface TagToken { name: string; raw: string; type: 'tag'; + /** + * Position of this tag among all tags in the message, in document order. + * + * Assigned by `Markdown` after lexing rather than here, because marked defers + * inline tokenization to a second pass -- so the order tokenizers run in does + * not match the order tags appear in. Undefined for tokens that were lexed + * without going through `Markdown`. + */ + index?: number; } const TAG_START_RE = /\{%\s+[\w-]/; diff --git a/static/app/views/seerExplorer/components/chat/assistant.tsx b/static/app/views/seerExplorer/components/chat/assistant.tsx index 47491d9730d6..afc54ded2ff1 100644 --- a/static/app/views/seerExplorer/components/chat/assistant.tsx +++ b/static/app/views/seerExplorer/components/chat/assistant.tsx @@ -4,6 +4,7 @@ import {css} from '@emotion/react'; import {AssistantActions, AssistantMessage, MessageRow} from '@sentry/scraps/chat'; import {SeerMarkdown} from 'sentry/components/seer/markdown'; +import type {SeerEmbedScope} from 'sentry/components/seer/markdown/embeds/renderTracking'; import {trackAnalytics} from 'sentry/utils/analytics'; import {useOrganization} from 'sentry/utils/useOrganization'; import {useSessionStorage} from 'sentry/utils/useSessionStorage'; @@ -25,6 +26,19 @@ export function AssistantBlock({ const content = block.message.content ?? ''; const isStreamingEnabled = organization.features.includes('seer-explorer-stream'); + // Only the settled render carries a scope. While `block.loading`, the id is + // still the optimistic client-side one (`loading-N-optimistic`), which the + // server replaces on the next poll -- tracking both would count one embed + // twice. The settled render fires immediately after, so nothing is lost. + const embedScope: SeerEmbedScope | null = + runId === undefined + ? null + : { + conversationId: String(runId), + messageId: block.id, + surface: 'seer_explorer', + }; + if (block.loading) { if (isStreamingEnabled && hasValidContent(content)) { return ( @@ -43,7 +57,7 @@ export function AssistantBlock({ {hasValidContent(content) && ( - + )}