From 51cf53e9fd4394d90a4dfbc0063bbca9f97c279f Mon Sep 17 00:00:00 2001 From: miguel Date: Thu, 30 Jul 2026 16:10:38 -0700 Subject: [PATCH] evals: wire opt-in OTEL transport (EVAL_TRACE_TRANSPORT=otel; native unchanged) Reconciled with main's refactored runEvals: OTEL provider init/shutdown wraps the eval body in try/finally while preserving the run-scoped trajectory-group stamping and writeExperimentLink cross-link. --- packages/evals/cli.ts | 6 + packages/evals/framework/braintrust.ts | 151 +++++++++ packages/evals/framework/runner.ts | 307 ++++++++++-------- packages/evals/lib/AISdkClientWrapped.ts | 31 +- .../tests/framework/tracedspan-otel.test.ts | 125 +++++++ 5 files changed, 476 insertions(+), 144 deletions(-) create mode 100644 packages/evals/tests/framework/tracedspan-otel.test.ts diff --git a/packages/evals/cli.ts b/packages/evals/cli.ts index 1cd9f85525..2a22fca44a 100644 --- a/packages/evals/cli.ts +++ b/packages/evals/cli.ts @@ -89,6 +89,12 @@ const args = process.argv.slice(2); } catch { // ignore } + try { + const { shutdownTracing } = await import("./framework/otel.js"); + await shutdownTracing(); + } catch { + // ignore + } process.exit(code); }; process.on("SIGINT", () => void handleSignal("SIGINT")); diff --git a/packages/evals/framework/braintrust.ts b/packages/evals/framework/braintrust.ts index ffd83757dd..dbd6778f95 100644 --- a/packages/evals/framework/braintrust.ts +++ b/packages/evals/framework/braintrust.ts @@ -8,6 +8,14 @@ * (e.g., `bench verify`). */ import type { Span, StartSpanArgs } from "braintrust"; +import type { + Attributes, + AttributeValue, + Span as OtelSpan, +} from "@opentelemetry/api"; +import { SpanStatusCode } from "@opentelemetry/api"; + +import { resolveTraceTransport } from "./langsmith.js"; let braintrustPromise: Promise | undefined; @@ -36,10 +44,153 @@ const NOOP_SPAN: SpanLike = { log: () => {}, }; +function toAttributeValue(value: unknown): AttributeValue | undefined { + if ( + typeof value === "string" || + typeof value === "number" || + typeof value === "boolean" + ) { + return value; + } + if (value === undefined) { + return undefined; + } + try { + return JSON.stringify(value); + } catch { + return String(value); + } +} + +function toAttributes( + values: Record, + prefix?: string, +): Attributes { + const attributes: Attributes = {}; + for (const [key, value] of Object.entries(values)) { + const attribute = toAttributeValue(value); + if (attribute !== undefined) { + attributes[prefix ? `${prefix}.${key}` : key] = attribute; + } + } + return attributes; +} + +function jsonStringify(value: unknown): string | undefined { + try { + return JSON.stringify(value); + } catch { + return undefined; + } +} + +function outputAttributes(output: unknown): Attributes { + const json = jsonStringify(output); + if (json === undefined) return {}; + return { + "output.value": typeof output === "string" ? output : json, + "braintrust.output_json": json, + }; +} + +function setSpanAttributes(span: OtelSpan, attributes: Attributes): void { + for (const [key, value] of Object.entries(attributes)) { + if (value !== undefined) { + span.setAttribute(key, value); + } + } +} + export async function tracedSpan( fn: TracedFn, options: TracedSpanOptions, ): Promise { + if (resolveTraceTransport() === "otel") { + const { getTracer } = await import("./otel.js"); + return getTracer().startActiveSpan(options.name, async (span) => { + span.setAttribute( + "langsmith.span.kind", + options.type === "llm" + ? "LLM" + : options.type === "tool" + ? "TOOL" + : "CHAIN", + ); + if (options.type !== undefined) { + span.setAttribute("type", options.type); + } + const input = options.event?.input; + if (input && typeof input === "object" && !Array.isArray(input)) { + setSpanAttributes( + span, + toAttributes(input as Record, "input"), + ); + } else if (input !== undefined) { + const value = toAttributeValue(input); + if (value !== undefined) { + span.setAttribute("input", value); + } + } + if (input !== undefined) { + const json = jsonStringify(input); + if (json !== undefined) { + span.setAttribute( + "input.value", + typeof input === "string" ? input : json, + ); + span.setAttribute("braintrust.input_json", json); + } + } + + const adapter: SpanLike = { + log: ({ output, scores, metrics, metadata, ...fields }) => { + if (metadata) { + setSpanAttributes( + span, + toAttributes(metadata, "langsmith.metadata"), + ); + } + if (metrics) { + setSpanAttributes(span, toAttributes(metrics, "metrics")); + } + if (scores) { + const attributes = toAttributes(scores, "scores"); + span.addEvent("scores", attributes); + setSpanAttributes( + span, + toAttributes( + Object.fromEntries( + Object.entries(scores) + .filter( + (entry): entry is [string, number] => + typeof entry[1] === "number", + ) + .map(([key, value]) => [`score_${key}`, value]), + ), + "langsmith.metadata", + ), + ); + } + setSpanAttributes(span, toAttributes(fields)); + if (output !== undefined) { + setSpanAttributes(span, outputAttributes(output)); + } + }, + }; + try { + return await fn(adapter); + } catch (error) { + span.setStatus({ + code: SpanStatusCode.ERROR, + message: error instanceof Error ? error.message : String(error), + }); + span.recordException(error instanceof Error ? error : String(error)); + throw error; + } finally { + span.end(); + } + }); + } if (!hasBraintrustApiKey()) { return fn(NOOP_SPAN); } diff --git a/packages/evals/framework/runner.ts b/packages/evals/framework/runner.ts index 7b8d191a88..0e8d3c5f29 100644 --- a/packages/evals/framework/runner.ts +++ b/packages/evals/framework/runner.ts @@ -37,6 +37,8 @@ import { } from "./braintrust.js"; import { onceAsync, registerActiveRunCleanup } from "./activeRunCleanup.js"; import { loadTaskModuleFromPath } from "./taskLoader.js"; +import { resolveTraceTransport } from "./langsmith.js"; +import { buildTracerProvider, shutdownTracing } from "./otel.js"; export { discoverTasks, resolveTarget } from "./discovery.js"; export { @@ -311,6 +313,7 @@ function formatProgressError(error: unknown): string | undefined { export async function runEvals( options: RunEvalsOptions, ): Promise { + const traceTransport = resolveTraceTransport(); const concurrency = options.concurrency ?? 3; const trials = options.trials ?? 3; const environment = options.environment ?? "LOCAL"; @@ -377,164 +380,182 @@ export async function runEvals( ? "stagehand" : "stagehand-dev"; - const scores = hasCoreOnly - ? [passRate, errorMatch] - : [exactMatch, errorMatch]; - - const { Eval, flush } = await loadBraintrust(); - const sendLogs = hasBraintrustApiKey(); - - // Aggressive abort: when the caller flips signal.reason to "aggressive", - // close every active session so any in-flight task throws on its next - // page operation. The cleanup path inside executeBenchTask handles the - // throw; finished tasks' cleanup is a no-op via onceAsync. - const onAggressiveAbort = async (): Promise => { - if (readAbortMode(options.signal) !== "aggressive") return; - const { cleanupActiveRunResources } = await import("./activeRunCleanup.js"); - await cleanupActiveRunResources(); - }; - options.signal?.addEventListener("abort", () => { - void onAggressiveAbort(); - }); + if (traceTransport === "otel") { + await buildTracerProvider({ + braintrustParent: `project_name:${braintrustProjectName}`, + }); + } + + try { + const scores = hasCoreOnly + ? [passRate, errorMatch] + : [exactMatch, errorMatch]; + + const { Eval, flush } = await loadBraintrust(); + const sendLogs = hasBraintrustApiKey(); + + // Aggressive abort: when the caller flips signal.reason to "aggressive", + // close every active session so any in-flight task throws on its next + // page operation. The cleanup path inside executeBenchTask handles the + // throw; finished tasks' cleanup is a no-op via onceAsync. + const onAggressiveAbort = async (): Promise => { + if (readAbortMode(options.signal) !== "aggressive") return; + const { cleanupActiveRunResources } = await import( + "./activeRunCleanup.js" + ); + await cleanupActiveRunResources(); + }; + options.signal?.addEventListener("abort", () => { + void onAggressiveAbort(); + }); + + const evalResult = await Eval( + braintrustProjectName, + { + experimentName, + metadata: { + environment, + tier: hasCoreOnly ? "core" : "bench", + ...(effectiveCoreToolSurface && { + toolSurface: effectiveCoreToolSurface, + }), + ...(effectiveCoreStartupProfile && { + startupProfile: effectiveCoreStartupProfile, + }), + ...(effectiveBenchHarness && { harness: effectiveBenchHarness }), + ...(options.provider && { provider: options.provider }), + ...(options.modelOverride && { model: options.modelOverride }), + ...(options.useApi && { api: true }), + }, + data: () => testcases, + task: async (input: EvalInput): Promise => { + // Cooperative abort: skip any testcase that hasn't started yet + // when the signal has flipped. The in-flight task at the moment of + // abort still finishes its current step; this stops the next one + // from spinning up. + if (options.signal?.aborted) { + options.onProgress?.({ + type: "failed", + taskName: input.name, + modelName: input.modelName, + error: "aborted", + }); + return { + _success: false, + error: "aborted by user", + logs: [], + }; + } + + const resolvedTask = + options.registry.byName.get(input.name) ?? + (input.name.includes("/") + ? undefined + : options.registry.byName.get(`agent/${input.name}`)); + + if (!resolvedTask) { + throw new EvalsError(`Task "${input.name}" not found in registry.`); + } - const evalResult = await Eval( - braintrustProjectName, - { - experimentName, - metadata: { - environment, - tier: hasCoreOnly ? "core" : "bench", - ...(effectiveCoreToolSurface && { - toolSurface: effectiveCoreToolSurface, - }), - ...(effectiveCoreStartupProfile && { - startupProfile: effectiveCoreStartupProfile, - }), - ...(effectiveBenchHarness && { harness: effectiveBenchHarness }), - ...(options.provider && { provider: options.provider }), - ...(options.modelOverride && { model: options.modelOverride }), - ...(options.useApi && { api: true }), - }, - data: () => testcases, - task: async (input: EvalInput): Promise => { - // Cooperative abort: skip any testcase that hasn't started yet - // when the signal has flipped. The in-flight task at the moment of - // abort still finishes its current step; this stops the next one - // from spinning up. - if (options.signal?.aborted) { options.onProgress?.({ - type: "failed", + type: "started", taskName: input.name, modelName: input.modelName, - error: "aborted", }); - return { - _success: false, - error: "aborted by user", - logs: [], - }; - } - const resolvedTask = - options.registry.byName.get(input.name) ?? - (input.name.includes("/") - ? undefined - : options.registry.byName.get(`agent/${input.name}`)); + const result = await executeTask(input, resolvedTask, options); - if (!resolvedTask) { - throw new EvalsError(`Task "${input.name}" not found in registry.`); - } + options.onProgress?.({ + type: result._success ? "passed" : "failed", + taskName: input.name, + modelName: input.modelName, + error: result._success + ? undefined + : formatProgressError(result.error), + }); - options.onProgress?.({ - type: "started", - taskName: input.name, - modelName: input.modelName, - }); + return result; + }, + scores: scores as unknown as never, + maxConcurrency: concurrency, + trialCount: trials, + }, + { + progress: silentBraintrustProgress, + reporter: silentBraintrustReporter, + ...(sendLogs ? {} : { noSendLogs: true }), + }, + ); - const result = await executeTask(input, resolvedTask, options); + if (sendLogs) { + await flush(); + } - options.onProgress?.({ - type: result._success ? "passed" : "failed", - taskName: input.name, - modelName: input.modelName, - error: result._success - ? undefined - : formatProgressError(result.error), - }); + const summaryResults = evalResult.results.map((result) => { + const output = + typeof result.output === "boolean" + ? { _success: result.output } + : result.output; + const categories = Array.isArray(result.metadata?.categories) + ? result.metadata.categories.filter( + (category): category is string => typeof category === "string", + ) + : undefined; + + return { + input: result.input, + output, + name: result.input.name, + score: output._success ? 1 : 0, + ...(categories && { categories }), + }; + }); - return result; + const resolvedExperimentName = + evalResult.summary?.experimentName ?? experimentName; + const resolvedExperimentUrl = evalResult.summary?.experimentUrl; + + // Cross-link local trajectories to the resolved Braintrust experiment. The + // hashed name (e.g. `agent/onlineMind2Web-92918006`) is only known now, after + // Eval() resolves — so write it once at the group-dir root of the group this + // run recorded into. + await writeExperimentLink( + resolveTrajectoryRoot(), + trajectoryGroup, + { + braintrustExperiment: resolvedExperimentName, + braintrustExperimentId: evalResult.summary?.experimentId ?? null, + braintrustExperimentUrl: resolvedExperimentUrl ?? null, + braintrustProject: + evalResult.summary?.projectName ?? braintrustProjectName, + braintrustProjectUrl: evalResult.summary?.projectUrl ?? null, + requestedExperimentName: experimentName, }, - scores: scores as unknown as never, - maxConcurrency: concurrency, - trialCount: trials, - }, - { - progress: silentBraintrustProgress, - reporter: silentBraintrustReporter, - ...(sendLogs ? {} : { noSendLogs: true }), - }, - ); + { persist: !hasCoreOnly && shouldPersistTrajectory(undefined) }, + ); - if (sendLogs) { - await flush(); - } + await generateSummary( + summaryResults, + resolvedExperimentName, + resolvedExperimentUrl, + evalResult.summary?.scores, + ); - const summaryResults = evalResult.results.map((result) => { - const output = - typeof result.output === "boolean" - ? { _success: result.output } - : result.output; - const categories = Array.isArray(result.metadata?.categories) - ? result.metadata.categories.filter( - (category): category is string => typeof category === "string", - ) - : undefined; + const passed = summaryResults.filter((r) => r.output._success).length; + const failed = summaryResults.filter((r) => !r.output._success).length; return { - input: result.input, - output, - name: result.input.name, - score: output._success ? 1 : 0, - ...(categories && { categories }), + experimentName: resolvedExperimentName, + summary: { passed, failed, total: summaryResults.length }, + results: summaryResults, }; - }); - - const resolvedExperimentName = - evalResult.summary?.experimentName ?? experimentName; - const resolvedExperimentUrl = evalResult.summary?.experimentUrl; - - // Cross-link local trajectories to the resolved Braintrust experiment. The - // hashed name (e.g. `agent/onlineMind2Web-92918006`) is only known now, after - // Eval() resolves — so write it once at the group-dir root of the group this - // run recorded into. - await writeExperimentLink( - resolveTrajectoryRoot(), - trajectoryGroup, - { - braintrustExperiment: resolvedExperimentName, - braintrustExperimentId: evalResult.summary?.experimentId ?? null, - braintrustExperimentUrl: resolvedExperimentUrl ?? null, - braintrustProject: - evalResult.summary?.projectName ?? braintrustProjectName, - braintrustProjectUrl: evalResult.summary?.projectUrl ?? null, - requestedExperimentName: experimentName, - }, - { persist: !hasCoreOnly && shouldPersistTrajectory(undefined) }, - ); - - await generateSummary( - summaryResults, - resolvedExperimentName, - resolvedExperimentUrl, - evalResult.summary?.scores, - ); - - const passed = summaryResults.filter((r) => r.output._success).length; - const failed = summaryResults.filter((r) => !r.output._success).length; - - return { - experimentName: resolvedExperimentName, - summary: { passed, failed, total: summaryResults.length }, - results: summaryResults, - }; + } finally { + if (traceTransport === "otel") { + try { + await shutdownTracing(); + } catch { + // Tracing shutdown must not mask the eval result or its exception. + } + } + } } diff --git a/packages/evals/lib/AISdkClientWrapped.ts b/packages/evals/lib/AISdkClientWrapped.ts index 1d4ab7df5f..07acd409be 100644 --- a/packages/evals/lib/AISdkClientWrapped.ts +++ b/packages/evals/lib/AISdkClientWrapped.ts @@ -36,6 +36,13 @@ async function loadWrappedAISDK(): Promise { return wrappedAiPromise; } +async function loadAISDK(): Promise { + if (process.env.EVAL_TRACE_TRANSPORT === "otel") { + return ai as unknown as WrappedAI; + } + return loadWrappedAISDK(); +} + export class AISdkClientWrapped extends LLMClient { public type = "aisdk" as const; private model: LanguageModelV2; @@ -149,7 +156,7 @@ export class AISdkClientWrapped extends LLMClient { }, ); - const { generateObject, generateText } = await loadWrappedAISDK(); + const { generateObject, generateText } = await loadAISDK(); let objectResponse: Awaited>; const isGPT5 = this.model.modelId.includes("gpt-5"); const isCodex = this.model.modelId.includes("codex"); @@ -182,6 +189,17 @@ You must respond in JSON format. respond WITH JSON. Do not include any other tex messages: formattedMessages, schema: options.response_model.schema, temperature, + ...(process.env.EVAL_TRACE_TRANSPORT === "otel" && { + experimental_telemetry: { + isEnabled: true, + functionId: "AISdkClientWrapped.createChatCompletion", + metadata: { + phase: "generateObject", + model: this.model.modelId, + task: options.requestId, + }, + }, + }), providerOptions: resolvedReasoningEffort ? { openai: { @@ -281,6 +299,17 @@ You must respond in JSON format. respond WITH JSON. Do not include any other tex await generateText({ model: this.model, messages: formattedMessages, + ...(process.env.EVAL_TRACE_TRANSPORT === "otel" && { + experimental_telemetry: { + isEnabled: true, + functionId: "AISdkClientWrapped.createChatCompletion", + metadata: { + phase: "generateText", + model: this.model.modelId, + task: options.requestId, + }, + }, + }), tools: Object.keys(tools).length > 0 ? tools : undefined, toolChoice: Object.keys(tools).length > 0 diff --git a/packages/evals/tests/framework/tracedspan-otel.test.ts b/packages/evals/tests/framework/tracedspan-otel.test.ts new file mode 100644 index 0000000000..ee78adb176 --- /dev/null +++ b/packages/evals/tests/framework/tracedspan-otel.test.ts @@ -0,0 +1,125 @@ +import { SpanStatusCode } from "@opentelemetry/api"; +import { afterAll, beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => { + const span = { + setAttribute: vi.fn(), + addEvent: vi.fn(), + setStatus: vi.fn(), + recordException: vi.fn(), + end: vi.fn(), + }; + const startActiveSpan = vi.fn( + async (_name: string, callback: (activeSpan: typeof span) => unknown) => + callback(span), + ); + const getTracer = vi.fn(() => ({ startActiveSpan })); + + return { getTracer, span, startActiveSpan }; +}); + +vi.mock("../../framework/otel.js", () => ({ + getTracer: mocks.getTracer, +})); + +const originalEnv = { ...process.env }; + +describe("tracedSpan OTEL transport", () => { + beforeEach(() => { + process.env = { ...originalEnv }; + delete process.env.BRAINTRUST_API_KEY; + delete process.env.EVAL_TRACE_TRANSPORT; + vi.clearAllMocks(); + vi.resetModules(); + }); + + afterAll(() => { + process.env = originalEnv; + }); + + it("maps logged fields to OTEL attributes and events", async () => { + process.env.EVAL_TRACE_TRANSPORT = "otel"; + const { tracedSpan } = await import("../../framework/braintrust.js"); + const callback = vi.fn(async (span) => { + span.log({ + output: { answer: "done" }, + scores: { accuracy: 1 }, + metrics: { duration_ms: 25 }, + metadata: { task: "example" }, + }); + return "result"; + }); + + await expect(tracedSpan(callback, { name: "test-span" })).resolves.toBe( + "result", + ); + + expect(callback).toHaveBeenCalledOnce(); + expect(mocks.startActiveSpan).toHaveBeenCalledWith( + "test-span", + expect.any(Function), + ); + // metadata rides the `langsmith.metadata.*` namespace — the only metadata + // carrier LangSmith honors for OTLP-native ingestion. + expect(mocks.span.setAttribute).toHaveBeenCalledWith( + "langsmith.metadata.task", + "example", + ); + expect(mocks.span.setAttribute).toHaveBeenCalledWith( + "metrics.duration_ms", + 25, + ); + expect(mocks.span.addEvent).toHaveBeenCalledWith("scores", { + "scores.accuracy": 1, + }); + // Output must be an ATTRIBUTE, not an event: verified against the live + // LangSmith API — `output.value` populates a run's outputs, whereas a + // span event named "output" is ignored (that was the "No outputs" bug). + expect(mocks.span.setAttribute).toHaveBeenCalledWith( + "output.value", + JSON.stringify({ answer: "done" }), + ); + expect(mocks.span.setAttribute).toHaveBeenCalledWith( + "braintrust.output_json", + JSON.stringify({ answer: "done" }), + ); + expect(mocks.span.end).toHaveBeenCalledOnce(); + }); + + it("records a throwing callback and still ends the span", async () => { + process.env.EVAL_TRACE_TRANSPORT = "otel"; + const { tracedSpan } = await import("../../framework/braintrust.js"); + const error = new Error("callback failed"); + + await expect( + tracedSpan( + async () => { + throw error; + }, + { name: "failing-span" }, + ), + ).rejects.toBe(error); + + expect(mocks.span.setStatus).toHaveBeenCalledWith({ + code: SpanStatusCode.ERROR, + message: "callback failed", + }); + expect(mocks.span.recordException).toHaveBeenCalledWith(error); + expect(mocks.span.end).toHaveBeenCalledOnce(); + }); + + it("uses the no-op span in native mode", async () => { + const { tracedSpan } = await import("../../framework/braintrust.js"); + const callback = vi.fn(async (span) => { + span.log({ output: "ignored" }); + return 42; + }); + + await expect(tracedSpan(callback, { name: "native-span" })).resolves.toBe( + 42, + ); + expect(callback).toHaveBeenCalledOnce(); + expect(mocks.getTracer).not.toHaveBeenCalled(); + expect(mocks.startActiveSpan).not.toHaveBeenCalled(); + }); +});