feat(evals): add LangSmith gating helpers + config (no behavior change)

.env.example

@@ -9,4 +9,18 @@ ENABLE_CACHING=false
 EVAL_MODELS="gpt-4o,claude-sonnet-4-6"
 EXPERIMENTAL_EVAL_MODELS="gpt-4o,claude-sonnet-4-6,o1-mini,o1-preview"
 EVAL_CATEGORIES="observe,act,combination,extract,experimental"
-AGENT_EVAL_MAX_STEPS=50
+AGENT_EVAL_MAX_STEPS=50
+
+# --- Experiment tracing / backends ------------------------------------------
+# Evals can push results to an experiment backend. Configure at least one of
+# Braintrust (BRAINTRUST_API_KEY, above) or LangSmith below; the doctor warns
+# when neither is set.
+#
+# LangSmith API key. Configure this and set LANGSMITH_TRACING="true" to emit traces.
+LANGSMITH_API_KEY=""
+# Master switch for LangSmith tracing ("true" to emit traces).
+LANGSMITH_TRACING="false"
+# Trace transport: how spans are shipped ("native" | "otel").
+EVAL_TRACE_TRANSPORT="native"
+# Primary trace sink when multiple are configured ("braintrust" | "langsmith").
+EVAL_TRACE_PRIMARY="braintrust"

packages/evals/framework/langsmith.ts

@@ -0,0 +1,31 @@
+let langSmithPromise: Promise<typeof import("langsmith")> | undefined;
+
+export function hasLangSmithApiKey(): boolean {
+  return Boolean(process.env.LANGSMITH_API_KEY);
+}
+
+export function langSmithTracingEnabled(): boolean {
+  return hasLangSmithApiKey() && process.env.LANGSMITH_TRACING === "true";
+}
+
+export function resolveTraceTransport(): "native" | "otel" {
+  return process.env.EVAL_TRACE_TRANSPORT === "otel" ? "otel" : "native";
+}
+
+export function loadLangSmith(): Promise<typeof import("langsmith")> {
+  langSmithPromise ??= import("langsmith");
+  return langSmithPromise;
+}
+
+export function assertLangSmithReady(): void {
+  if (!hasLangSmithApiKey()) {
+    throw new Error(
+      "LangSmith tracing was selected, but LANGSMITH_API_KEY is not set.",
+    );
+  }
+  if (!langSmithTracingEnabled()) {
+    throw new Error(
+      'LangSmith tracing was selected, but LANGSMITH_TRACING is not set to "true".',
+    );
+  }
+}

packages/evals/package.json

@@ -28,6 +28,7 @@
     "@openai/codex-sdk": "0.125.0",
     "ai": "^5.0.133",
     "dotenv": "^17.3.1",
+    "langsmith": "^0.5.16",
     "openai": "^4.104.0",
     "playwright": ">=1.55.1 <1.57.0",
     "sharp": "^0.34.5",

packages/evals/tests/framework/langsmith.test.ts

@@ -0,0 +1,107 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+const LANGSMITH_ENV_KEYS = [
+  "LANGSMITH_API_KEY",
+  "LANGSMITH_TRACING",
+  "EVAL_TRACE_TRANSPORT",
+] as const;
+
+const savedEnv = new Map<string, string | undefined>();
+
+beforeEach(() => {
+  for (const key of LANGSMITH_ENV_KEYS) {
+    savedEnv.set(key, process.env[key]);
+    delete process.env[key];
+  }
+  vi.resetModules();
+});
+
+afterEach(() => {
+  for (const key of LANGSMITH_ENV_KEYS) {
+    const value = savedEnv.get(key);
+    if (value === undefined) delete process.env[key];
+    else process.env[key] = value;
+  }
+  savedEnv.clear();
+  vi.resetModules();
+});
+
+async function loadLangSmithHelpers() {
+  return import("../../framework/langsmith.js");
+}
+
+describe("hasLangSmithApiKey", () => {
+  it("returns false without LANGSMITH_API_KEY", async () => {
+    const { hasLangSmithApiKey } = await loadLangSmithHelpers();
+    expect(hasLangSmithApiKey()).toBe(false);
+  });
+
+  it("returns true with LANGSMITH_API_KEY", async () => {
+    process.env.LANGSMITH_API_KEY = "ls-test";
+    const { hasLangSmithApiKey } = await loadLangSmithHelpers();
+    expect(hasLangSmithApiKey()).toBe(true);
+  });
+});
+
+describe("langSmithTracingEnabled", () => {
+  it.each([
+    { key: true, flag: true, expected: true },
+    { key: true, flag: false, expected: false },
+    { key: false, flag: true, expected: false },
+    { key: false, flag: false, expected: false },
+  ])(
+    "returns $expected when key=$key and flag=$flag",
+    async ({ key, flag, expected }) => {
+      if (key) process.env.LANGSMITH_API_KEY = "ls-test";
+      if (flag) process.env.LANGSMITH_TRACING = "true";
+      const { langSmithTracingEnabled } = await loadLangSmithHelpers();
+      expect(langSmithTracingEnabled()).toBe(expected);
+    },
+  );
+
+  it("reflects environment changes made after module import", async () => {
+    const { langSmithTracingEnabled } = await loadLangSmithHelpers();
+    expect(langSmithTracingEnabled()).toBe(false);
+
+    process.env.LANGSMITH_API_KEY = "ls-test";
+    process.env.LANGSMITH_TRACING = "true";
+    expect(langSmithTracingEnabled()).toBe(true);
+
+    delete process.env.LANGSMITH_TRACING;
+    expect(langSmithTracingEnabled()).toBe(false);
+  });
+});
+
+describe("resolveTraceTransport", () => {
+  it("defaults to native", async () => {
+    const { resolveTraceTransport } = await loadLangSmithHelpers();
+    expect(resolveTraceTransport()).toBe("native");
+  });
+
+  it('returns otel for the literal "otel" value', async () => {
+    process.env.EVAL_TRACE_TRANSPORT = "otel";
+    const { resolveTraceTransport } = await loadLangSmithHelpers();
+    expect(resolveTraceTransport()).toBe("otel");
+  });
+});
+
+describe("assertLangSmithReady", () => {
+  it("throws when tracing is enabled without an API key", async () => {
+    process.env.LANGSMITH_TRACING = "true";
+    const { assertLangSmithReady } = await loadLangSmithHelpers();
+    expect(() => assertLangSmithReady()).toThrow(/LANGSMITH_API_KEY/);
+  });
+
+  it("throws when the API key is set but tracing is disabled", async () => {
+    process.env.LANGSMITH_API_KEY = "ls-test";
+    const { assertLangSmithReady } = await loadLangSmithHelpers();
+    expect(() => assertLangSmithReady()).toThrow(/LANGSMITH_TRACING/);
+  });
+
+  it("does not throw when the API key and tracing flag are set", async () => {
+    process.env.LANGSMITH_API_KEY = "ls-test";
+    process.env.LANGSMITH_TRACING = "true";
+    const { assertLangSmithReady } = await loadLangSmithHelpers();
+    expect(() => assertLangSmithReady()).not.toThrow();
+  });
+});

packages/evals/tests/tui/doctor.test.ts

@@ -1,7 +1,15 @@
 import fs from "node:fs";
 import os from "node:os";
 import path from "node:path";
-import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import {
+  afterAll,
+  afterEach,
+  beforeEach,
+  describe,
+  expect,
+  it,
+  vi,
+} from "vitest";
 import { handleDoctor } from "../../tui/commands/doctor.js";
 import { __resetPackageEnvCacheForTests } from "../../tui/welcomeStatus.js";
 
@@ -15,11 +23,15 @@ const PROVIDER_KEYS = [
   "BB_API_KEY",
   "BB_PROJECT_ID",
   "BRAINTRUST_API_KEY",
+  "LANGSMITH_API_KEY",
+  "LANGSMITH_TRACING",
+  "EVAL_TRACE_PRIMARY",
+  "EVALS_DISABLE_PACKAGE_ENV",
 ];
 
 const savedEnv: Record<string, string | undefined> = {};
 const tempDirs: string[] = [];
-let savedDisablePkgEnv: string | undefined;
+let envSnapshotted = false;
 
 type DoctorJsonReport = {
   verdict: string;
@@ -28,13 +40,17 @@ type DoctorJsonReport = {
 };
 
 function clearProviderKeys(): void {
+  if (!envSnapshotted) {
+    for (const key of PROVIDER_KEYS) {
+      savedEnv[key] = process.env[key];
+    }
+    envSnapshotted = true;
+  }
   for (const key of PROVIDER_KEYS) {
-    savedEnv[key] = process.env[key];
     delete process.env[key];
   }
   // Neutralize the package-local .env loader so tests don't depend on
   // whatever real keys the developer happens to have at packages/evals/.env.
-  savedDisablePkgEnv = process.env.EVALS_DISABLE_PACKAGE_ENV;
   process.env.EVALS_DISABLE_PACKAGE_ENV = "1";
   __resetPackageEnvCacheForTests();
 }
@@ -44,11 +60,6 @@ function restoreProviderKeys(): void {
     if (savedEnv[key] === undefined) delete process.env[key];
     else process.env[key] = savedEnv[key];
   }
-  if (savedDisablePkgEnv === undefined) {
-    delete process.env.EVALS_DISABLE_PACKAGE_ENV;
-  } else {
-    process.env.EVALS_DISABLE_PACKAGE_ENV = savedDisablePkgEnv;
-  }
   __resetPackageEnvCacheForTests();
 }
 
@@ -89,13 +100,13 @@ async function runDoctorJson(
 
 beforeEach(() => clearProviderKeys());
 afterEach(() => {
-  restoreProviderKeys();
   while (tempDirs.length > 0) {
     const dir = tempDirs.pop();
     if (dir) fs.rmSync(dir, { recursive: true, force: true });
   }
   vi.restoreAllMocks();
 });
+afterAll(() => restoreProviderKeys());
 
 describe("handleDoctor --json", () => {
   it("always exits 0 even on fail verdict", async () => {
@@ -146,6 +157,70 @@ describe("handleDoctor verdicts", () => {
     expect(report.reasons.join(" ")).toMatch(/BRAINTRUST_API_KEY missing/);
   });
 
+  it("warn — LangSmith configured but experiments still require Braintrust", async () => {
+    process.env.OPENAI_API_KEY = "sk-test";
+    process.env.LANGSMITH_API_KEY = "ls-test";
+    process.env.LANGSMITH_TRACING = "true";
+    process.env.EVAL_TRACE_PRIMARY = "langsmith";
+    __resetPackageEnvCacheForTests();
+    const entryDir = makeTempEntryDir({ env: "local", trials: 3 });
+    const { report } = await runDoctorJson(entryDir);
+    expect(report.verdict).toBe("warn");
+    expect(report.reasons.join(" ")).toMatch(
+      /BRAINTRUST_API_KEY missing.*experiments.*require Braintrust/,
+    );
+  });
+
+  it("warn — LangSmith primary tracing is disabled without its flag", async () => {
+    process.env.OPENAI_API_KEY = "sk-test";
+    process.env.BRAINTRUST_API_KEY = "bt-test";
+    process.env.LANGSMITH_API_KEY = "ls-test";
+    process.env.EVAL_TRACE_PRIMARY = "langsmith";
+    __resetPackageEnvCacheForTests();
+    const entryDir = makeTempEntryDir({ env: "local", trials: 3 });
+    const { report } = await runDoctorJson(entryDir);
+    expect(report.verdict).toBe("warn");
+    expect(report.reasons.join(" ")).toMatch(/LANGSMITH_TRACING.*not "true"/);
+  });
+
+  it("warn — unrecognized trace primary", async () => {
+    process.env.OPENAI_API_KEY = "sk-test";
+    process.env.BRAINTRUST_API_KEY = "bt-test";
+    process.env.EVAL_TRACE_PRIMARY = "foo";
+    const entryDir = makeTempEntryDir({ env: "local", trials: 3 });
+    const { report } = await runDoctorJson(entryDir);
+    expect(report.verdict).toBe("warn");
+    expect(report.reasons.join(" ")).toMatch(
+      /EVAL_TRACE_PRIMARY="foo" is not recognized/,
+    );
+  });
+
+  it("does not warn when LangSmith primary tracing is enabled in process.env", async () => {
+    process.env.OPENAI_API_KEY = "sk-test";
+    process.env.BRAINTRUST_API_KEY = "bt-test";
+    process.env.LANGSMITH_API_KEY = "ls-test";
+    process.env.LANGSMITH_TRACING = "true";
+    process.env.EVAL_TRACE_PRIMARY = "langsmith";
+    const entryDir = makeTempEntryDir({ env: "local", trials: 3 });
+    const { report } = await runDoctorJson(entryDir);
+    expect(report.verdict).toBe("ok");
+    expect(report.reasons.join(" ")).not.toMatch(/LANGSMITH_TRACING/);
+  });
+
+  it("warns when LangSmith tracing is absent from process.env", async () => {
+    process.env.OPENAI_API_KEY = "sk-test";
+    process.env.BRAINTRUST_API_KEY = "bt-test";
+    process.env.LANGSMITH_API_KEY = "ls-test";
+    process.env.EVAL_TRACE_PRIMARY = "langsmith";
+    delete process.env.LANGSMITH_TRACING;
+    delete process.env.EVALS_DISABLE_PACKAGE_ENV;
+    __resetPackageEnvCacheForTests();
+    const entryDir = makeTempEntryDir({ env: "local", trials: 3 });
+    const { report } = await runDoctorJson(entryDir);
+    expect(report.verdict).toBe("warn");
+    expect(report.reasons.join(" ")).toMatch(/LANGSMITH_TRACING.*not "true"/);
+  });
+
   it("warn — Browserbase partially configured", async () => {
     process.env.OPENAI_API_KEY = "sk-test";
     process.env.BRAINTRUST_API_KEY = "bt-test";