feat(evals): bump braintrust to 3.x; first-party BraintrustSpanProcessor OTEL sink

.env.example

@@ -25,7 +25,8 @@ EVAL_TRACE_TRANSPORT="native"
 # Optional Braintrust OTEL parent override (defaults to project_name:stagehand-dev,
 # or project_name:stagehand in CI).
 # BRAINTRUST_OTEL_PARENT="project_name:stagehand-dev"
-# Optional Braintrust OTEL traces endpoint override.
+# Optional Braintrust OTEL traces endpoint override. The first-party processor
+# derives its Braintrust API base URL from this endpoint.
 # BRAINTRUST_OTEL_URL="https://api.braintrust.dev/otel/v1/traces"
 # Primary trace sink when multiple are configured ("braintrust" | "langsmith").
 EVAL_TRACE_PRIMARY="braintrust"

packages/evals/framework/otel.ts

@@ -65,27 +65,23 @@ async function initializeTracerProvider(options?: {
 
   const braintrustApiKey = process.env.BRAINTRUST_API_KEY;
   if (braintrustApiKey) {
-    const { OTLPTraceExporter } = await import(
-      "@opentelemetry/exporter-trace-otlp-proto"
-    );
+    const { BraintrustSpanProcessor } = await import("@braintrust/otel");
     const braintrustProjectName =
       process.env.CI === "true" ? "stagehand" : "stagehand-dev";
     const parent =
       options?.braintrustParent ??
       process.env.BRAINTRUST_OTEL_PARENT ??
       `project_name:${braintrustProjectName}`;
+    const braintrustOtelUrl = process.env.BRAINTRUST_OTEL_URL;
     spanProcessors.push(
-      new BatchSpanProcessor(
-        new OTLPTraceExporter({
-          url:
-            process.env.BRAINTRUST_OTEL_URL ??
-            "https://api.braintrust.dev/otel/v1/traces",
-          headers: {
-            Authorization: `Bearer ${braintrustApiKey}`,
-            "x-bt-parent": parent,
-          },
+      new BraintrustSpanProcessor({
+        apiKey: braintrustApiKey,
+        parent,
+        filterAISpans: false,
+        ...(braintrustOtelUrl && {
+          apiUrl: braintrustOtelUrl.replace(/otel\/v1\/traces\/?$/, ""),
         }),
-      ),
+      }),
     );
   }
 

packages/evals/lib/AISdkClientWrapped.ts

@@ -21,24 +21,27 @@ import {
   toJsonSchema,
 } from "@browserbasehq/stagehand";
 
-type WrappedAI = ReturnType<(typeof import("braintrust"))["wrapAISDK"]>;
+type WrappedAI = Pick<
+  typeof ai,
+  "generateText" | "generateObject" | "streamText" | "streamObject"
+>;
 
 let wrappedAiPromise: Promise<WrappedAI> | undefined;
 
 async function loadWrappedAISDK(): Promise<WrappedAI> {
   wrappedAiPromise ??= (async () => {
     if (!process.env.BRAINTRUST_API_KEY) {
-      return ai as unknown as WrappedAI;
+      return ai;
     }
     const { wrapAISDK } = await import("braintrust");
-    return wrapAISDK(ai);
+    return wrapAISDK(ai) as WrappedAI;
   })();
   return wrappedAiPromise;
 }
 
 async function loadAISDK(): Promise<WrappedAI> {
   if (process.env.EVAL_TRACE_TRANSPORT === "otel") {
-    return ai as unknown as WrappedAI;
+    return ai;
   }
   return loadWrappedAISDK();
 }

packages/evals/package.json

@@ -25,9 +25,9 @@
     "@ai-sdk/provider": "^2.0.0",
     "@anthropic-ai/claude-agent-sdk": "^0.2.141",
     "@browserbasehq/stagehand": "workspace:*",
+    "@braintrust/otel": "^0.2.1",
     "@openai/codex-sdk": "0.125.0",
     "@opentelemetry/api": "^1.9.0",
-    "@opentelemetry/exporter-trace-otlp-proto": "^0.220.0",
     "@opentelemetry/sdk-trace-base": "^2.0.0",
     "@opentelemetry/sdk-trace-node": "^2.0.0",
     "ai": "^5.0.133",
@@ -41,7 +41,7 @@
   },
   "devDependencies": {
     "@types/node": "^25.6.2",
-    "braintrust": "^0.4.10",
+    "braintrust": "^3.21.0",
     "chalk": "^5.4.1",
     "gray-matter": "^4.0.3",
     "string-comparison": "^1.3.0",

packages/evals/tests/framework/otel.test.ts

@@ -1,9 +1,11 @@
 import { afterAll, beforeEach, describe, expect, it, vi } from "vitest";
 
 const mocks = vi.hoisted(() => {
-  const braintrustExporter = vi.fn(function () {
+  const braintrustSpanProcessor = vi.fn(function () {
     return {
-      export: vi.fn(),
+      forceFlush: vi.fn().mockResolvedValue(undefined),
+      onEnd: vi.fn(),
+      onStart: vi.fn(),
       shutdown: vi.fn().mockResolvedValue(undefined),
     };
   });
@@ -24,14 +26,14 @@ const mocks = vi.hoisted(() => {
   });
 
   return {
-    braintrustExporter,
+    braintrustSpanProcessor,
     langSmithExporter,
     nodeTracerProvider,
   };
 });
 
-vi.mock("@opentelemetry/exporter-trace-otlp-proto", () => ({
-  OTLPTraceExporter: mocks.braintrustExporter,
+vi.mock("@braintrust/otel", () => ({
+  BraintrustSpanProcessor: mocks.braintrustSpanProcessor,
 }));
 
 vi.mock("langsmith/experimental/otel/exporter", () => ({
@@ -91,12 +93,10 @@ describe("buildTracerProvider", () => {
     const { buildTracerProvider } = await import("../../framework/otel.js");
 
     await expect(buildTracerProvider()).resolves.not.toBeNull();
-    expect(mocks.braintrustExporter).toHaveBeenCalledWith({
-      url: "https://api.braintrust.dev/otel/v1/traces",
-      headers: {
-        Authorization: "Bearer braintrust-test-key",
-        "x-bt-parent": "project_name:stagehand-dev",
-      },
+    expect(mocks.braintrustSpanProcessor).toHaveBeenCalledWith({
+      apiKey: "braintrust-test-key",
+      parent: "project_name:stagehand-dev",
+      filterAISpans: false,
     });
     expect(mocks.langSmithExporter).toHaveBeenCalledOnce();
     expect(constructedSpanProcessors()).toHaveLength(2);
@@ -108,12 +108,10 @@ describe("buildTracerProvider", () => {
     const { buildTracerProvider } = await import("../../framework/otel.js");
 
     await expect(buildTracerProvider()).resolves.not.toBeNull();
-    expect(mocks.braintrustExporter).toHaveBeenCalledWith({
-      url: "https://api.braintrust.dev/otel/v1/traces",
-      headers: {
-        Authorization: "Bearer braintrust-test-key",
-        "x-bt-parent": "project_name:stagehand-dev",
-      },
+    expect(mocks.braintrustSpanProcessor).toHaveBeenCalledWith({
+      apiKey: "braintrust-test-key",
+      parent: "project_name:stagehand-dev",
+      filterAISpans: false,
     });
     expect(mocks.langSmithExporter).not.toHaveBeenCalled();
     expect(constructedSpanProcessors()).toHaveLength(1);
@@ -128,12 +126,26 @@ describe("buildTracerProvider", () => {
     await expect(
       buildTracerProvider({ braintrustParent: "project_name:custom" }),
     ).resolves.not.toBeNull();
-    expect(mocks.braintrustExporter).toHaveBeenCalledWith({
-      url: "https://api.braintrust.dev/otel/v1/traces",
-      headers: {
-        Authorization: "Bearer braintrust-test-key",
-        "x-bt-parent": "project_name:custom",
-      },
+    expect(mocks.braintrustSpanProcessor).toHaveBeenCalledWith({
+      apiKey: "braintrust-test-key",
+      parent: "project_name:custom",
+      filterAISpans: false,
+    });
+  });
+
+  it("derives the Braintrust apiUrl from BRAINTRUST_OTEL_URL", async () => {
+    process.env.EVAL_TRACE_TRANSPORT = "otel";
+    process.env.BRAINTRUST_API_KEY = "braintrust-test-key";
+    process.env.BRAINTRUST_OTEL_URL =
+      "https://custom.example.com/otel/v1/traces";
+    const { buildTracerProvider } = await import("../../framework/otel.js");
+
+    await expect(buildTracerProvider()).resolves.not.toBeNull();
+    expect(mocks.braintrustSpanProcessor).toHaveBeenCalledWith({
+      apiKey: "braintrust-test-key",
+      parent: "project_name:stagehand-dev",
+      filterAISpans: false,
+      apiUrl: "https://custom.example.com/",
     });
   });
 
@@ -144,7 +156,7 @@ describe("buildTracerProvider", () => {
     const { buildTracerProvider } = await import("../../framework/otel.js");
 
     await expect(buildTracerProvider()).resolves.not.toBeNull();
-    expect(mocks.braintrustExporter).not.toHaveBeenCalled();
+    expect(mocks.braintrustSpanProcessor).not.toHaveBeenCalled();
     expect(mocks.langSmithExporter).toHaveBeenCalledOnce();
     expect(constructedSpanProcessors()).toHaveLength(1);
   });