test(e2e): Update live ingestion verification for span streaming (#24165)

Verify streamed spans against Sentry ingestion using the captured trace
ID, since streaming no longer produces a transaction event to look up.

Preserve the original transaction and error ingestion checks in a
separate node-express-send-to-sentry-static app with the static trace
lifecycle. Both apps remain in the optional E2E suite.

part of #23800

---------

Co-authored-by: GPT-6 <codex@openai.com>

Author: msonnb

dev-packages/e2e-tests/test-applications/node-express-send-to-sentry-static/.gitignore

@@ -0,0 +1 @@
+dist

dev-packages/e2e-tests/test-applications/node-express-send-to-sentry-static/package.json

@@ -0,0 +1,29 @@
+{
+  "name": "node-express-send-to-sentry-static-app",
+  "version": "1.0.0",
+  "private": true,
+  "scripts": {
+    "build": "tsc",
+    "start": "node dist/app.js",
+    "test": "playwright test",
+    "clean": "npx rimraf node_modules pnpm-lock.yaml",
+    "test:build": "pnpm install && pnpm build",
+    "test:assert": "pnpm test"
+  },
+  "dependencies": {
+    "@sentry/node": "file:../../packed/sentry-node-packed.tgz",
+    "@types/express": "4.17.17",
+    "@types/node": "^18.19.1",
+    "express": "^4.21.2",
+    "typescript": "~5.0.0"
+  },
+  "devDependencies": {
+    "@playwright/test": "~1.56.0"
+  },
+  "volta": {
+    "extends": "../../package.json"
+  },
+  "sentryTest": {
+    "optional": true
+  }
+}

dev-packages/e2e-tests/test-applications/node-express-send-to-sentry-static/playwright.config.mjs

@@ -0,0 +1,71 @@
+import { devices } from '@playwright/test';
+
+const expressPort = 3030;
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+const config = {
+  testDir: './tests',
+  /* Maximum time one test can run for. Spans take ~2min to become queryable via the trace endpoint. */
+  timeout: 210_000,
+  expect: {
+    /**
+     * Maximum time expect() should wait for the condition to be met.
+     * For example in `await expect(locator).toHaveText();`
+     */
+    timeout: 5000,
+  },
+  /* Run tests in files in parallel */
+  fullyParallel: true,
+  /* Fail the build on CI if you accidentally left test.only in the source code. */
+  forbidOnly: !!process.env.CI,
+  /* Retry on CI only */
+  retries: 0,
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: process.env.CI ? [['list'], ['junit', { outputFile: 'results.junit.xml' }]] : 'list',
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    /* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */
+    actionTimeout: 0,
+
+    /* Base URL to use in actions like `await page.goto('/')`. */
+    baseURL: `http://localhost:${expressPort}`,
+
+    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+    trace: 'on-first-retry',
+  },
+
+  /* Configure projects for major browsers */
+  projects: [
+    {
+      name: 'chromium',
+      use: {
+        ...devices['Desktop Chrome'],
+      },
+    },
+    // For now we only test Chrome!
+    // {
+    //   name: 'firefox',
+    //   use: {
+    //     ...devices['Desktop Firefox'],
+    //   },
+    // },
+    // {
+    //   name: 'webkit',
+    //   use: {
+    //     ...devices['Desktop Safari'],
+    //   },
+    // },
+  ],
+
+  /* Run your local dev server before starting the tests */
+  webServer: [
+    {
+      command: 'pnpm start',
+      port: expressPort,
+    },
+  ],
+};
+
+export default config;

dev-packages/e2e-tests/test-applications/node-express-send-to-sentry-static/src/app.ts

@@ -0,0 +1,94 @@
+import * as Sentry from '@sentry/node';
+
+let lastTransactionId: string | undefined;
+let lastTransactionTraceId: string | undefined;
+let lastErrorTraceId: string | undefined;
+
+Sentry.init({
+  traceLifecycle: 'static',
+  environment: 'qa', // dynamic sampling bias to keep transactions
+  dsn: process.env.E2E_TEST_DSN,
+  includeLocalVariables: true,
+  tracesSampleRate: 1,
+  beforeSend(event) {
+    lastErrorTraceId = event.contexts?.trace?.trace_id;
+    return event;
+  },
+  beforeSendTransaction(event) {
+    lastTransactionId = event.event_id;
+    lastTransactionTraceId = event.contexts?.trace?.trace_id;
+    return event;
+  },
+});
+
+import express from 'express';
+
+const app = express();
+const port = 3030;
+
+app.get('/test-success', function (req, res) {
+  res.send({ version: 'v1' });
+});
+
+app.get('/test-param/:param', function (req, res) {
+  res.send({ paramWas: req.params.param });
+});
+
+app.get('/test-transaction', function (req, res) {
+  Sentry.withActiveSpan(null, async () => {
+    Sentry.startSpan({ name: 'test-transaction', op: 'e2e-test' }, () => {
+      Sentry.startSpan({ name: 'test-span' }, () => undefined);
+    });
+
+    await Sentry.flush();
+
+    res.send({
+      transactionId: lastTransactionId,
+      traceId: lastTransactionTraceId,
+    });
+  });
+});
+
+app.get('/test-error', async function (req, res) {
+  const exceptionId = Sentry.captureException(new Error('This is an error'));
+
+  await Sentry.flush(2000);
+
+  res.send({ exceptionId, traceId: lastErrorTraceId });
+});
+
+app.get('/test-exception/:id', function (req, _res) {
+  throw new Error(`This is an exception with id ${req.params.id}`);
+});
+
+app.get('/test-local-variables-uncaught', function (req, res) {
+  const randomVariableToRecord = Math.random();
+  throw new Error(`Uncaught Local Variable Error - ${JSON.stringify({ randomVariableToRecord })}`);
+});
+
+app.get('/test-local-variables-caught', function (req, res) {
+  const randomVariableToRecord = Math.random();
+
+  let exceptionId: string;
+  try {
+    throw new Error('Local Variable Error');
+  } catch (e) {
+    exceptionId = Sentry.captureException(e);
+  }
+
+  res.send({ exceptionId, randomVariableToRecord });
+});
+
+Sentry.setupExpressErrorHandler(app);
+
+// @ts-ignore
+app.use(function onError(err, req, res, next) {
+  // The error id is attached to `res.sentry` to be returned
+  // and optionally displayed to the user for support.
+  res.statusCode = 500;
+  res.end(res.sentry + '\n');
+});
+
+app.listen(port, () => {
+  console.log(`Example app listening on port ${port}`);
+});

dev-packages/e2e-tests/test-applications/node-express-send-to-sentry-static/tests/send-to-sentry.test.ts

@@ -0,0 +1,22 @@
+import { expect, test } from '@playwright/test';
+import { EVENT_POLLING_OPTIONS, findErrorInTrace, findTransactionInTrace } from './utils/sentry-api';
+
+test('Sends exception to Sentry', async ({ baseURL }) => {
+  const response = await fetch(`${baseURL}/test-error`);
+  const { exceptionId, traceId } = await response.json();
+
+  console.log(`Polling for error eventId: ${exceptionId} in trace: ${traceId}`);
+
+  await expect.poll(() => findErrorInTrace(traceId, exceptionId), EVENT_POLLING_OPTIONS).toBeDefined();
+});
+
+test('Sends transaction to Sentry', async ({ baseURL }) => {
+  const response = await fetch(`${baseURL}/test-transaction`);
+  const { transactionId, traceId } = await response.json();
+
+  console.log(`Polling for transaction eventId: ${transactionId} in trace: ${traceId}`);
+
+  await expect
+    .poll(() => findTransactionInTrace(traceId, transactionId), EVENT_POLLING_OPTIONS)
+    .toMatchObject({ op: 'e2e-test' });
+});

dev-packages/e2e-tests/test-applications/node-express-send-to-sentry-static/tests/utils/sentry-api.ts

@@ -0,0 +1,72 @@
+const authToken = process.env.E2E_TEST_AUTH_TOKEN;
+const sentryTestOrgSlug = process.env.E2E_TEST_SENTRY_ORG_SLUG;
+
+/**
+ * Spans only become queryable once they have made it through to EAP, which takes
+ * noticeably longer than the error pipeline (~2min vs ~20s when this was measured).
+ */
+export const EVENT_POLLING_OPTIONS = { timeout: 180_000, intervals: [5_000] };
+
+/**
+ * A node of the span tree returned by the organization trace endpoint. Spans, errors and
+ * occurrences all share this shape and are discriminated by `event_type`.
+ */
+export interface TraceItem {
+  event_id?: string;
+  /** On spans this is the event id of the transaction the span belongs to. */
+  transaction_id?: string;
+  event_type?: 'span' | 'error' | 'occurrence' | 'uptime_check';
+  op?: string;
+  is_transaction?: boolean;
+  children?: TraceItem[];
+  errors?: TraceItem[];
+  occurrences?: TraceItem[];
+}
+
+export async function fetchTrace(traceId: string): Promise<TraceItem[]> {
+  const response = await fetch(
+    `https://sentry.io/api/0/organizations/${sentryTestOrgSlug}/trace/${traceId}/?statsPeriod=1h`,
+    { headers: { Authorization: `Bearer ${authToken}` } },
+  );
+
+  // The trace endpoint is org scoped, so the auth token needs `org:read` on top of the
+  // project scopes the other assertions rely on. That never resolves by waiting, so fail
+  // loudly instead of polling until the timeout and reporting it as a missing event.
+  if (response.status === 401 || response.status === 403) {
+    throw new Error(
+      `Trace lookup for ${traceId} was rejected with ${response.status}: ${await response.text()}. ` +
+        'E2E_TEST_AUTH_TOKEN needs the `org:read` scope.',
+    );
+  }
+
+  // Empty traces and the occasional rate limit are expected while polling, so treat anything
+  // else that is not a success as "not there yet" -- but log it, since a rejected request and
+  // a trace that has not landed are otherwise indistinguishable.
+  if (!response.ok) {
+    console.log(`Trace lookup for ${traceId} returned ${response.status}: ${await response.text()}`);
+    return [];
+  }
+
+  return await response.json();
+}
+
+/**
+ * Errors attach to whichever span was active when they were captured, and relocate from the
+ * top level into that span once it lands, so a given event can surface at any depth.
+ */
+export function flattenTrace(items: TraceItem[]): TraceItem[] {
+  return items.flatMap(item => [
+    item,
+    ...flattenTrace(item.children ?? []),
+    ...flattenTrace(item.errors ?? []),
+    ...flattenTrace(item.occurrences ?? []),
+  ]);
+}
+
+export async function findErrorInTrace(traceId: string, eventId: string): Promise<TraceItem | undefined> {
+  return flattenTrace(await fetchTrace(traceId)).find(item => item.event_type === 'error' && item.event_id === eventId);
+}
+
+export async function findTransactionInTrace(traceId: string, eventId: string): Promise<TraceItem | undefined> {
+  return flattenTrace(await fetchTrace(traceId)).find(item => item.is_transaction && item.transaction_id === eventId);
+}

dev-packages/e2e-tests/test-applications/node-express-send-to-sentry-static/tsconfig.json

@@ -0,0 +1,11 @@
+{
+  "compilerOptions": {
+    "types": ["node"],
+    "esModuleInterop": true,
+    "lib": ["es2018"],
+    "strict": true,
+    "outDir": "dist",
+    "skipLibCheck": true
+  },
+  "include": ["src/**/*.ts"]
+}

dev-packages/e2e-tests/test-applications/node-express-send-to-sentry/src/app.ts

@@ -1,23 +1,22 @@
 import * as Sentry from '@sentry/node';
 
-let lastTransactionId: string | undefined;
-let lastTransactionTraceId: string | undefined;
+let lastSpanTraceId: string | undefined;
 let lastErrorTraceId: string | undefined;
 
 Sentry.init({
-  traceLifecycle: 'static',
-  environment: 'qa', // dynamic sampling bias to keep transactions
+  environment: 'qa', // dynamic sampling bias to keep traces
   dsn: process.env.E2E_TEST_DSN,
   includeLocalVariables: true,
   tracesSampleRate: 1,
   beforeSend(event) {
     lastErrorTraceId = event.contexts?.trace?.trace_id;
     return event;
   },
-  beforeSendTransaction(event) {
-    lastTransactionId = event.event_id;
-    lastTransactionTraceId = event.contexts?.trace?.trace_id;
-    return event;
+  beforeSendSpan(span) {
+    if (span.name === 'test-span') {
+      lastSpanTraceId = span.trace_id;
+    }
+    return span;
   },
 });
 
@@ -34,17 +33,16 @@ app.get('/test-param/:param', function (req, res) {
   res.send({ paramWas: req.params.param });
 });
 
-app.get('/test-transaction', function (req, res) {
+app.get('/test-span', function (req, res) {
   Sentry.withActiveSpan(null, async () => {
-    Sentry.startSpan({ name: 'test-transaction', op: 'e2e-test' }, () => {
-      Sentry.startSpan({ name: 'test-span' }, () => undefined);
+    Sentry.startSpan({ name: 'test-span', op: 'e2e-test' }, () => {
+      Sentry.startSpan({ name: 'test-child-span' }, () => undefined);
     });
 
     await Sentry.flush();
 
     res.send({
-      transactionId: lastTransactionId,
-      traceId: lastTransactionTraceId,
+      traceId: lastSpanTraceId,
     });
   });
 });

dev-packages/e2e-tests/test-applications/node-express-send-to-sentry/tests/send-to-sentry.test.ts

@@ -1,5 +1,5 @@
 import { expect, test } from '@playwright/test';
-import { EVENT_POLLING_OPTIONS, findErrorInTrace, findTransactionInTrace } from './utils/sentry-api';
+import { EVENT_POLLING_OPTIONS, findErrorInTrace, findSpanInTrace } from './utils/sentry-api';
 
 test('Sends exception to Sentry', async ({ baseURL }) => {
   const response = await fetch(`${baseURL}/test-error`);
@@ -10,13 +10,13 @@ test('Sends exception to Sentry', async ({ baseURL }) => {
   await expect.poll(() => findErrorInTrace(traceId, exceptionId), EVENT_POLLING_OPTIONS).toBeDefined();
 });
 
-test('Sends transaction to Sentry', async ({ baseURL }) => {
-  const response = await fetch(`${baseURL}/test-transaction`);
-  const { transactionId, traceId } = await response.json();
+test('Sends streamed span to Sentry', async ({ baseURL }) => {
+  const response = await fetch(`${baseURL}/test-span`);
+  const { traceId } = await response.json();
 
-  console.log(`Polling for transaction eventId: ${transactionId} in trace: ${traceId}`);
+  console.log(`Polling for streamed span in trace: ${traceId}`);
 
   await expect
-    .poll(() => findTransactionInTrace(traceId, transactionId), EVENT_POLLING_OPTIONS)
+    .poll(() => findSpanInTrace(traceId, 'e2e-test'), EVENT_POLLING_OPTIONS)
     .toMatchObject({ op: 'e2e-test' });
 });