feat(evals): add OTEL tracer-provider module + deps (unwired)

.env.example

@@ -22,5 +22,10 @@ LANGSMITH_API_KEY=""
 LANGSMITH_TRACING="false"
 # Trace transport: how spans are shipped ("native" | "otel").
 EVAL_TRACE_TRANSPORT="native"
+# Optional Braintrust OTEL parent override (defaults to project_name:stagehand-dev,
+# or project_name:stagehand in CI).
+# BRAINTRUST_OTEL_PARENT="project_name:stagehand-dev"
+# Optional Braintrust OTEL traces endpoint override.
+# BRAINTRUST_OTEL_URL="https://api.braintrust.dev/otel/v1/traces"
 # Primary trace sink when multiple are configured ("braintrust" | "langsmith").
 EVAL_TRACE_PRIMARY="braintrust"

.gitignore

@@ -34,6 +34,7 @@ ctrf/
 packages/evals/playwright-mcp-screenshot-*.png
 packages/evals/chrome-devtools-mcp-screenshot-*.png
 .trajectories/
+.rubric-cache/
 
 # The browse CLI has a command source dir named "downloads" that must be tracked
 # despite the broad downloads/ rule above.

packages/evals/framework/otel.ts

@@ -0,0 +1,149 @@
+import { ProxyTracerProvider, type Tracer } from "@opentelemetry/api";
+import {
+  BatchSpanProcessor,
+  type SpanProcessor,
+} from "@opentelemetry/sdk-trace-base";
+import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node";
+
+import { hasLangSmithApiKey, resolveTraceTransport } from "./langsmith.js";
+
+const TRACER_NAME = "stagehand-evals";
+const SHUTDOWN_TIMEOUT_MS = 10_000;
+const NOOP_TRACER = new ProxyTracerProvider().getTracer(TRACER_NAME);
+
+// Provider state lives on globalThis, NOT in module scope: the CLI bundle
+// (esbuild --bundle) inlines this module once per import site, so module-level
+// variables would give each importer its own copy — the runner would register
+// the provider in one copy while tracedSpan reads NOOP from another, silently
+// dropping every span. A Symbol.for-keyed global is shared across all copies.
+type TracingState = {
+  provider: NodeTracerProvider | null;
+  providerPromise: Promise<NodeTracerProvider | null> | null;
+};
+const STATE_KEY = Symbol.for("stagehand.evals.otel.state");
+function state(): TracingState {
+  const g = globalThis as { [STATE_KEY]?: TracingState };
+  return (g[STATE_KEY] ??= { provider: null, providerPromise: null });
+}
+
+/** Test-only: clear shared provider state between vitest module resets. */
+export function resetTracingStateForTests(): void {
+  const g = globalThis as { [STATE_KEY]?: TracingState };
+  delete g[STATE_KEY];
+}
+
+export async function buildTracerProvider(options?: {
+  braintrustParent?: string;
+}): Promise<NodeTracerProvider | null> {
+  if (resolveTraceTransport() !== "otel") {
+    return null;
+  }
+
+  const s = state();
+  if (s.provider) {
+    return s.provider;
+  }
+
+  // The provider is initialized once per process, so the first call's options win.
+  const pendingProvider =
+    s.providerPromise ??
+    (s.providerPromise = initializeTracerProvider(options));
+
+  try {
+    return await pendingProvider;
+  } finally {
+    if (s.providerPromise === pendingProvider) {
+      s.providerPromise = null;
+    }
+  }
+}
+
+async function initializeTracerProvider(options?: {
+  braintrustParent?: string;
+}): Promise<NodeTracerProvider | null> {
+  const spanProcessors: SpanProcessor[] = [];
+
+  const braintrustApiKey = process.env.BRAINTRUST_API_KEY;
+  if (braintrustApiKey) {
+    const { OTLPTraceExporter } = await import(
+      "@opentelemetry/exporter-trace-otlp-proto"
+    );
+    const braintrustProjectName =
+      process.env.CI === "true" ? "stagehand" : "stagehand-dev";
+    const parent =
+      options?.braintrustParent ??
+      process.env.BRAINTRUST_OTEL_PARENT ??
+      `project_name:${braintrustProjectName}`;
+    spanProcessors.push(
+      new BatchSpanProcessor(
+        new OTLPTraceExporter({
+          url:
+            process.env.BRAINTRUST_OTEL_URL ??
+            "https://api.braintrust.dev/otel/v1/traces",
+          headers: {
+            Authorization: `Bearer ${braintrustApiKey}`,
+            "x-bt-parent": parent,
+          },
+        }),
+      ),
+    );
+  }
+
+  if (hasLangSmithApiKey() && process.env.LANGSMITH_TRACING === "true") {
+    const { LangSmithOTLPTraceExporter } = await import(
+      "langsmith/experimental/otel/exporter"
+    );
+    spanProcessors.push(
+      new BatchSpanProcessor(new LangSmithOTLPTraceExporter()),
+    );
+  }
+
+  if (spanProcessors.length === 0) {
+    return null;
+  }
+
+  const nextProvider = new NodeTracerProvider({ spanProcessors });
+  nextProvider.register();
+  state().provider = nextProvider;
+  return nextProvider;
+}
+
+export function getTracer(): Tracer {
+  return state().provider?.getTracer(TRACER_NAME) ?? NOOP_TRACER;
+}
+
+export async function shutdownTracing(): Promise<void> {
+  const s = state();
+  if (resolveTraceTransport() !== "otel" || !s.provider) {
+    return;
+  }
+
+  const activeProvider = s.provider;
+  s.provider = null;
+
+  let timeout: ReturnType<typeof setTimeout> | undefined;
+  const timeoutPromise = new Promise<never>((_, reject) => {
+    timeout = setTimeout(() => {
+      reject(
+        new Error(
+          `Timed out shutting down tracing after ${SHUTDOWN_TIMEOUT_MS}ms.`,
+        ),
+      );
+    }, SHUTDOWN_TIMEOUT_MS);
+    timeout.unref?.();
+  });
+
+  try {
+    await Promise.race([
+      activeProvider
+        .forceFlush()
+        .catch(() => {})
+        .then(() => activeProvider.shutdown()),
+      timeoutPromise,
+    ]);
+  } finally {
+    if (timeout) {
+      clearTimeout(timeout);
+    }
+  }
+}

packages/evals/package.json

@@ -26,6 +26,10 @@
     "@anthropic-ai/claude-agent-sdk": "^0.2.141",
     "@browserbasehq/stagehand": "workspace:*",
     "@openai/codex-sdk": "0.125.0",
+    "@opentelemetry/api": "^1.9.0",
+    "@opentelemetry/exporter-trace-otlp-proto": "^0.220.0",
+    "@opentelemetry/sdk-trace-base": "^2.0.0",
+    "@opentelemetry/sdk-trace-node": "^2.0.0",
     "ai": "^5.0.133",
     "dotenv": "^17.3.1",
     "langsmith": "^0.5.16",

packages/evals/tests/framework/otel.test.ts

@@ -0,0 +1,166 @@
+import { afterAll, beforeEach, describe, expect, it, vi } from "vitest";
+
+const mocks = vi.hoisted(() => {
+  const braintrustExporter = vi.fn(function () {
+    return {
+      export: vi.fn(),
+      shutdown: vi.fn().mockResolvedValue(undefined),
+    };
+  });
+  const langSmithExporter = vi.fn(function () {
+    return {
+      export: vi.fn(),
+      shutdown: vi.fn().mockResolvedValue(undefined),
+    };
+  });
+  const nodeTracerProvider = vi.fn(function (options: unknown) {
+    return {
+      forceFlush: vi.fn().mockResolvedValue(undefined),
+      getTracer: vi.fn(),
+      options,
+      register: vi.fn(),
+      shutdown: vi.fn().mockResolvedValue(undefined),
+    };
+  });
+
+  return {
+    braintrustExporter,
+    langSmithExporter,
+    nodeTracerProvider,
+  };
+});
+
+vi.mock("@opentelemetry/exporter-trace-otlp-proto", () => ({
+  OTLPTraceExporter: mocks.braintrustExporter,
+}));
+
+vi.mock("langsmith/experimental/otel/exporter", () => ({
+  LangSmithOTLPTraceExporter: mocks.langSmithExporter,
+}));
+
+vi.mock("@opentelemetry/sdk-trace-node", () => ({
+  NodeTracerProvider: mocks.nodeTracerProvider,
+}));
+
+const originalEnv = { ...process.env };
+
+function constructedSpanProcessors(): unknown[] {
+  const options = mocks.nodeTracerProvider.mock.calls[0]?.[0] as
+    | { spanProcessors: unknown[] }
+    | undefined;
+  return options?.spanProcessors ?? [];
+}
+
+describe("buildTracerProvider", () => {
+  beforeEach(async () => {
+    process.env = { ...originalEnv };
+    delete process.env.BRAINTRUST_API_KEY;
+    delete process.env.BRAINTRUST_OTEL_PARENT;
+    delete process.env.BRAINTRUST_OTEL_URL;
+    delete process.env.CI;
+    delete process.env.EVAL_TRACE_TRANSPORT;
+    delete process.env.LANGSMITH_API_KEY;
+    delete process.env.LANGSMITH_TRACING;
+    vi.clearAllMocks();
+    vi.resetModules();
+    // Provider state is globalThis-backed (bundle-duplication proof), so a
+    // module reset alone no longer isolates tests — use the module's own
+    // test-reset helper to clear the shared slot.
+    const { resetTracingStateForTests } = await import(
+      "../../framework/otel.js"
+    );
+    resetTracingStateForTests();
+  });
+
+  afterAll(() => {
+    process.env = originalEnv;
+  });
+
+  it("does not construct a provider in native mode", async () => {
+    const { buildTracerProvider } = await import("../../framework/otel.js");
+
+    await expect(buildTracerProvider()).resolves.toBeNull();
+    expect(mocks.nodeTracerProvider).not.toHaveBeenCalled();
+  });
+
+  it("constructs a provider with both backend processors", async () => {
+    process.env.EVAL_TRACE_TRANSPORT = "otel";
+    process.env.BRAINTRUST_API_KEY = "braintrust-test-key";
+    process.env.LANGSMITH_API_KEY = "langsmith-test-key";
+    process.env.LANGSMITH_TRACING = "true";
+    const { buildTracerProvider } = await import("../../framework/otel.js");
+
+    await expect(buildTracerProvider()).resolves.not.toBeNull();
+    expect(mocks.braintrustExporter).toHaveBeenCalledWith({
+      url: "https://api.braintrust.dev/otel/v1/traces",
+      headers: {
+        Authorization: "Bearer braintrust-test-key",
+        "x-bt-parent": "project_name:stagehand-dev",
+      },
+    });
+    expect(mocks.langSmithExporter).toHaveBeenCalledOnce();
+    expect(constructedSpanProcessors()).toHaveLength(2);
+  });
+
+  it("constructs a provider with only the Braintrust processor", async () => {
+    process.env.EVAL_TRACE_TRANSPORT = "otel";
+    process.env.BRAINTRUST_API_KEY = "braintrust-test-key";
+    const { buildTracerProvider } = await import("../../framework/otel.js");
+
+    await expect(buildTracerProvider()).resolves.not.toBeNull();
+    expect(mocks.braintrustExporter).toHaveBeenCalledWith({
+      url: "https://api.braintrust.dev/otel/v1/traces",
+      headers: {
+        Authorization: "Bearer braintrust-test-key",
+        "x-bt-parent": "project_name:stagehand-dev",
+      },
+    });
+    expect(mocks.langSmithExporter).not.toHaveBeenCalled();
+    expect(constructedSpanProcessors()).toHaveLength(1);
+  });
+
+  it("uses the provided Braintrust parent over the environment default", async () => {
+    process.env.EVAL_TRACE_TRANSPORT = "otel";
+    process.env.BRAINTRUST_API_KEY = "braintrust-test-key";
+    process.env.BRAINTRUST_OTEL_PARENT = "project_name:from-env";
+    const { buildTracerProvider } = await import("../../framework/otel.js");
+
+    await expect(
+      buildTracerProvider({ braintrustParent: "project_name:custom" }),
+    ).resolves.not.toBeNull();
+    expect(mocks.braintrustExporter).toHaveBeenCalledWith({
+      url: "https://api.braintrust.dev/otel/v1/traces",
+      headers: {
+        Authorization: "Bearer braintrust-test-key",
+        "x-bt-parent": "project_name:custom",
+      },
+    });
+  });
+
+  it("constructs a provider with only the LangSmith processor", async () => {
+    process.env.EVAL_TRACE_TRANSPORT = "otel";
+    process.env.LANGSMITH_API_KEY = "langsmith-test-key";
+    process.env.LANGSMITH_TRACING = "true";
+    const { buildTracerProvider } = await import("../../framework/otel.js");
+
+    await expect(buildTracerProvider()).resolves.not.toBeNull();
+    expect(mocks.braintrustExporter).not.toHaveBeenCalled();
+    expect(mocks.langSmithExporter).toHaveBeenCalledOnce();
+    expect(constructedSpanProcessors()).toHaveLength(1);
+  });
+
+  it("shuts down the provider when forceFlush rejects", async () => {
+    process.env.EVAL_TRACE_TRANSPORT = "otel";
+    process.env.BRAINTRUST_API_KEY = "braintrust-test-key";
+    const { buildTracerProvider, shutdownTracing } = await import(
+      "../../framework/otel.js"
+    );
+
+    await buildTracerProvider();
+    const activeProvider = mocks.nodeTracerProvider.mock.results[0]?.value;
+    activeProvider.forceFlush.mockRejectedValueOnce(new Error("export failed"));
+
+    await expect(shutdownTracing()).resolves.toBeUndefined();
+    expect(activeProvider.shutdown).toHaveBeenCalledOnce();
+  });
+});