From 53043c6df40b0cabb17a70254120d912597ddea9 Mon Sep 17 00:00:00 2001 From: miguel Date: Thu, 30 Jul 2026 16:05:57 -0700 Subject: [PATCH 1/2] evals: add LangSmith gating helpers + config (no behavior change) --- .env.example | 16 ++- packages/evals/framework/langsmith.ts | 31 +++++ packages/evals/package.json | 1 + .../evals/tests/framework/langsmith.test.ts | 107 ++++++++++++++++++ packages/evals/tests/tui/doctor.test.ts | 95 ++++++++++++++-- packages/evals/tui/commands/doctor.ts | 71 +++++++++++- packages/evals/tui/welcomeStatus.ts | 2 + pnpm-lock.yaml | 61 +++++++++- 8 files changed, 366 insertions(+), 18 deletions(-) create mode 100644 packages/evals/framework/langsmith.ts create mode 100644 packages/evals/tests/framework/langsmith.test.ts diff --git a/.env.example b/.env.example index eb08f00ee9..d7471ce3aa 100644 --- a/.env.example +++ b/.env.example @@ -9,4 +9,18 @@ ENABLE_CACHING=false EVAL_MODELS="gpt-4o,claude-sonnet-4-6" EXPERIMENTAL_EVAL_MODELS="gpt-4o,claude-sonnet-4-6,o1-mini,o1-preview" EVAL_CATEGORIES="observe,act,combination,extract,experimental" -AGENT_EVAL_MAX_STEPS=50 \ No newline at end of file +AGENT_EVAL_MAX_STEPS=50 + +# --- Experiment tracing / backends ------------------------------------------ +# Evals can push results to an experiment backend. Configure at least one of +# Braintrust (BRAINTRUST_API_KEY, above) or LangSmith below; the doctor warns +# when neither is set. +# +# LangSmith API key. Configure this and set LANGSMITH_TRACING="true" to emit traces. +LANGSMITH_API_KEY="" +# Master switch for LangSmith tracing ("true" to emit traces). +LANGSMITH_TRACING="false" +# Trace transport: how spans are shipped ("native" | "otel"). +EVAL_TRACE_TRANSPORT="native" +# Primary trace sink when multiple are configured ("braintrust" | "langsmith"). +EVAL_TRACE_PRIMARY="braintrust" diff --git a/packages/evals/framework/langsmith.ts b/packages/evals/framework/langsmith.ts new file mode 100644 index 0000000000..8bf9423a02 --- /dev/null +++ b/packages/evals/framework/langsmith.ts @@ -0,0 +1,31 @@ +let langSmithPromise: Promise | undefined; + +export function hasLangSmithApiKey(): boolean { + return Boolean(process.env.LANGSMITH_API_KEY); +} + +export function langSmithTracingEnabled(): boolean { + return hasLangSmithApiKey() && process.env.LANGSMITH_TRACING === "true"; +} + +export function resolveTraceTransport(): "native" | "otel" { + return process.env.EVAL_TRACE_TRANSPORT === "otel" ? "otel" : "native"; +} + +export function loadLangSmith(): Promise { + langSmithPromise ??= import("langsmith"); + return langSmithPromise; +} + +export function assertLangSmithReady(): void { + if (!hasLangSmithApiKey()) { + throw new Error( + "LangSmith tracing was selected, but LANGSMITH_API_KEY is not set.", + ); + } + if (!langSmithTracingEnabled()) { + throw new Error( + 'LangSmith tracing was selected, but LANGSMITH_TRACING is not set to "true".', + ); + } +} diff --git a/packages/evals/package.json b/packages/evals/package.json index a09d1c0da0..fcc15adfcb 100644 --- a/packages/evals/package.json +++ b/packages/evals/package.json @@ -28,6 +28,7 @@ "@openai/codex-sdk": "0.125.0", "ai": "^5.0.133", "dotenv": "^17.3.1", + "langsmith": "^0.5.16", "openai": "^4.104.0", "playwright": ">=1.55.1 <1.57.0", "sharp": "^0.34.5", diff --git a/packages/evals/tests/framework/langsmith.test.ts b/packages/evals/tests/framework/langsmith.test.ts new file mode 100644 index 0000000000..5a28c96461 --- /dev/null +++ b/packages/evals/tests/framework/langsmith.test.ts @@ -0,0 +1,107 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const LANGSMITH_ENV_KEYS = [ + "LANGSMITH_API_KEY", + "LANGSMITH_TRACING", + "EVAL_TRACE_TRANSPORT", +] as const; + +const savedEnv = new Map(); + +beforeEach(() => { + for (const key of LANGSMITH_ENV_KEYS) { + savedEnv.set(key, process.env[key]); + delete process.env[key]; + } + vi.resetModules(); +}); + +afterEach(() => { + for (const key of LANGSMITH_ENV_KEYS) { + const value = savedEnv.get(key); + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + savedEnv.clear(); + vi.resetModules(); +}); + +async function loadLangSmithHelpers() { + return import("../../framework/langsmith.js"); +} + +describe("hasLangSmithApiKey", () => { + it("returns false without LANGSMITH_API_KEY", async () => { + const { hasLangSmithApiKey } = await loadLangSmithHelpers(); + expect(hasLangSmithApiKey()).toBe(false); + }); + + it("returns true with LANGSMITH_API_KEY", async () => { + process.env.LANGSMITH_API_KEY = "ls-test"; + const { hasLangSmithApiKey } = await loadLangSmithHelpers(); + expect(hasLangSmithApiKey()).toBe(true); + }); +}); + +describe("langSmithTracingEnabled", () => { + it.each([ + { key: true, flag: true, expected: true }, + { key: true, flag: false, expected: false }, + { key: false, flag: true, expected: false }, + { key: false, flag: false, expected: false }, + ])( + "returns $expected when key=$key and flag=$flag", + async ({ key, flag, expected }) => { + if (key) process.env.LANGSMITH_API_KEY = "ls-test"; + if (flag) process.env.LANGSMITH_TRACING = "true"; + const { langSmithTracingEnabled } = await loadLangSmithHelpers(); + expect(langSmithTracingEnabled()).toBe(expected); + }, + ); + + it("reflects environment changes made after module import", async () => { + const { langSmithTracingEnabled } = await loadLangSmithHelpers(); + expect(langSmithTracingEnabled()).toBe(false); + + process.env.LANGSMITH_API_KEY = "ls-test"; + process.env.LANGSMITH_TRACING = "true"; + expect(langSmithTracingEnabled()).toBe(true); + + delete process.env.LANGSMITH_TRACING; + expect(langSmithTracingEnabled()).toBe(false); + }); +}); + +describe("resolveTraceTransport", () => { + it("defaults to native", async () => { + const { resolveTraceTransport } = await loadLangSmithHelpers(); + expect(resolveTraceTransport()).toBe("native"); + }); + + it('returns otel for the literal "otel" value', async () => { + process.env.EVAL_TRACE_TRANSPORT = "otel"; + const { resolveTraceTransport } = await loadLangSmithHelpers(); + expect(resolveTraceTransport()).toBe("otel"); + }); +}); + +describe("assertLangSmithReady", () => { + it("throws when tracing is enabled without an API key", async () => { + process.env.LANGSMITH_TRACING = "true"; + const { assertLangSmithReady } = await loadLangSmithHelpers(); + expect(() => assertLangSmithReady()).toThrow(/LANGSMITH_API_KEY/); + }); + + it("throws when the API key is set but tracing is disabled", async () => { + process.env.LANGSMITH_API_KEY = "ls-test"; + const { assertLangSmithReady } = await loadLangSmithHelpers(); + expect(() => assertLangSmithReady()).toThrow(/LANGSMITH_TRACING/); + }); + + it("does not throw when the API key and tracing flag are set", async () => { + process.env.LANGSMITH_API_KEY = "ls-test"; + process.env.LANGSMITH_TRACING = "true"; + const { assertLangSmithReady } = await loadLangSmithHelpers(); + expect(() => assertLangSmithReady()).not.toThrow(); + }); +}); diff --git a/packages/evals/tests/tui/doctor.test.ts b/packages/evals/tests/tui/doctor.test.ts index 7b44bdf3d1..9e46f5fc7b 100644 --- a/packages/evals/tests/tui/doctor.test.ts +++ b/packages/evals/tests/tui/doctor.test.ts @@ -1,7 +1,15 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + afterAll, + afterEach, + beforeEach, + describe, + expect, + it, + vi, +} from "vitest"; import { handleDoctor } from "../../tui/commands/doctor.js"; import { __resetPackageEnvCacheForTests } from "../../tui/welcomeStatus.js"; @@ -15,11 +23,15 @@ const PROVIDER_KEYS = [ "BB_API_KEY", "BB_PROJECT_ID", "BRAINTRUST_API_KEY", + "LANGSMITH_API_KEY", + "LANGSMITH_TRACING", + "EVAL_TRACE_PRIMARY", + "EVALS_DISABLE_PACKAGE_ENV", ]; const savedEnv: Record = {}; const tempDirs: string[] = []; -let savedDisablePkgEnv: string | undefined; +let envSnapshotted = false; type DoctorJsonReport = { verdict: string; @@ -28,13 +40,17 @@ type DoctorJsonReport = { }; function clearProviderKeys(): void { + if (!envSnapshotted) { + for (const key of PROVIDER_KEYS) { + savedEnv[key] = process.env[key]; + } + envSnapshotted = true; + } for (const key of PROVIDER_KEYS) { - savedEnv[key] = process.env[key]; delete process.env[key]; } // Neutralize the package-local .env loader so tests don't depend on // whatever real keys the developer happens to have at packages/evals/.env. - savedDisablePkgEnv = process.env.EVALS_DISABLE_PACKAGE_ENV; process.env.EVALS_DISABLE_PACKAGE_ENV = "1"; __resetPackageEnvCacheForTests(); } @@ -44,11 +60,6 @@ function restoreProviderKeys(): void { if (savedEnv[key] === undefined) delete process.env[key]; else process.env[key] = savedEnv[key]; } - if (savedDisablePkgEnv === undefined) { - delete process.env.EVALS_DISABLE_PACKAGE_ENV; - } else { - process.env.EVALS_DISABLE_PACKAGE_ENV = savedDisablePkgEnv; - } __resetPackageEnvCacheForTests(); } @@ -89,13 +100,13 @@ async function runDoctorJson( beforeEach(() => clearProviderKeys()); afterEach(() => { - restoreProviderKeys(); while (tempDirs.length > 0) { const dir = tempDirs.pop(); if (dir) fs.rmSync(dir, { recursive: true, force: true }); } vi.restoreAllMocks(); }); +afterAll(() => restoreProviderKeys()); describe("handleDoctor --json", () => { it("always exits 0 even on fail verdict", async () => { @@ -146,6 +157,70 @@ describe("handleDoctor verdicts", () => { expect(report.reasons.join(" ")).toMatch(/BRAINTRUST_API_KEY missing/); }); + it("warn — LangSmith configured but experiments still require Braintrust", async () => { + process.env.OPENAI_API_KEY = "sk-test"; + process.env.LANGSMITH_API_KEY = "ls-test"; + process.env.LANGSMITH_TRACING = "true"; + process.env.EVAL_TRACE_PRIMARY = "langsmith"; + __resetPackageEnvCacheForTests(); + const entryDir = makeTempEntryDir({ env: "local", trials: 3 }); + const { report } = await runDoctorJson(entryDir); + expect(report.verdict).toBe("warn"); + expect(report.reasons.join(" ")).toMatch( + /BRAINTRUST_API_KEY missing.*experiments.*require Braintrust/, + ); + }); + + it("warn — LangSmith primary tracing is disabled without its flag", async () => { + process.env.OPENAI_API_KEY = "sk-test"; + process.env.BRAINTRUST_API_KEY = "bt-test"; + process.env.LANGSMITH_API_KEY = "ls-test"; + process.env.EVAL_TRACE_PRIMARY = "langsmith"; + __resetPackageEnvCacheForTests(); + const entryDir = makeTempEntryDir({ env: "local", trials: 3 }); + const { report } = await runDoctorJson(entryDir); + expect(report.verdict).toBe("warn"); + expect(report.reasons.join(" ")).toMatch(/LANGSMITH_TRACING.*not "true"/); + }); + + it("warn — unrecognized trace primary", async () => { + process.env.OPENAI_API_KEY = "sk-test"; + process.env.BRAINTRUST_API_KEY = "bt-test"; + process.env.EVAL_TRACE_PRIMARY = "foo"; + const entryDir = makeTempEntryDir({ env: "local", trials: 3 }); + const { report } = await runDoctorJson(entryDir); + expect(report.verdict).toBe("warn"); + expect(report.reasons.join(" ")).toMatch( + /EVAL_TRACE_PRIMARY="foo" is not recognized/, + ); + }); + + it("does not warn when LangSmith primary tracing is enabled in process.env", async () => { + process.env.OPENAI_API_KEY = "sk-test"; + process.env.BRAINTRUST_API_KEY = "bt-test"; + process.env.LANGSMITH_API_KEY = "ls-test"; + process.env.LANGSMITH_TRACING = "true"; + process.env.EVAL_TRACE_PRIMARY = "langsmith"; + const entryDir = makeTempEntryDir({ env: "local", trials: 3 }); + const { report } = await runDoctorJson(entryDir); + expect(report.verdict).toBe("ok"); + expect(report.reasons.join(" ")).not.toMatch(/LANGSMITH_TRACING/); + }); + + it("warns when LangSmith tracing is absent from process.env", async () => { + process.env.OPENAI_API_KEY = "sk-test"; + process.env.BRAINTRUST_API_KEY = "bt-test"; + process.env.LANGSMITH_API_KEY = "ls-test"; + process.env.EVAL_TRACE_PRIMARY = "langsmith"; + delete process.env.LANGSMITH_TRACING; + delete process.env.EVALS_DISABLE_PACKAGE_ENV; + __resetPackageEnvCacheForTests(); + const entryDir = makeTempEntryDir({ env: "local", trials: 3 }); + const { report } = await runDoctorJson(entryDir); + expect(report.verdict).toBe("warn"); + expect(report.reasons.join(" ")).toMatch(/LANGSMITH_TRACING.*not "true"/); + }); + it("warn — Browserbase partially configured", async () => { process.env.OPENAI_API_KEY = "sk-test"; process.env.BRAINTRUST_API_KEY = "bt-test"; diff --git a/packages/evals/tui/commands/doctor.ts b/packages/evals/tui/commands/doctor.ts index 8271f7bd52..208993f86e 100644 --- a/packages/evals/tui/commands/doctor.ts +++ b/packages/evals/tui/commands/doctor.ts @@ -175,14 +175,18 @@ async function summarizeDiscovery(): Promise { * Verdict rules: * fail — zero provider keys, OR defaults.env=browserbase with both BB * vars missing, OR discovery threw. - * warn — at least one provider key present, but Braintrust missing or - * BB partial (only one of two BB vars set). + * warn — at least one provider key present, but no experiment backend + * (both Braintrust and LangSmith missing), the selected primary + * backend's key is missing, or BB is partial (only one of two BB + * vars set). * ok — otherwise. */ function computeVerdict( keys: EnvSnapshot, config: ConfigSummary, discovery: DiscoverySummary, + tracePrimary: string, + langSmithTracing: string, ): { verdict: Verdict; reasons: string[] } { const reasons: string[] = []; @@ -224,13 +228,55 @@ function computeVerdict( "Browserbase is partially configured (one of API key / project ID is missing).", ); } - if (keys.braintrust.state === "missing") { + const noExperimentBackend = + keys.braintrust.state === "missing" && keys.langsmith.state === "missing"; + if (noExperimentBackend) { reasons.push( - "BRAINTRUST_API_KEY missing — `experiments` commands will fail.", + "No experiment backend configured (BRAINTRUST_API_KEY and LANGSMITH_API_KEY both missing) — `experiments` commands will fail.", + ); + } else if (keys.braintrust.state === "missing") { + reasons.push( + "BRAINTRUST_API_KEY missing — `experiments` commands require Braintrust (LangSmith is not yet wired into them).", + ); + } + + const primaryBackendMissing = + (tracePrimary === "braintrust" && keys.braintrust.state === "missing") || + (tracePrimary === "langsmith" && keys.langsmith.state === "missing"); + const tracePrimaryUnrecognized = + tracePrimary !== "braintrust" && tracePrimary !== "langsmith"; + if (tracePrimaryUnrecognized) { + reasons.push( + `EVAL_TRACE_PRIMARY="${tracePrimary}" is not recognized (expected "braintrust" or "langsmith").`, + ); + } + if (primaryBackendMissing) { + const requiredKey = + tracePrimary === "braintrust" + ? "BRAINTRUST_API_KEY" + : "LANGSMITH_API_KEY"; + reasons.push( + `EVAL_TRACE_PRIMARY=${tracePrimary} but ${requiredKey} missing.`, + ); + } + + const langSmithTracingDisabled = + tracePrimary === "langsmith" && + keys.langsmith.state === "set" && + langSmithTracing !== "true"; + if (langSmithTracingDisabled) { + reasons.push( + 'EVAL_TRACE_PRIMARY=langsmith but LANGSMITH_TRACING is not "true" — tracing is disabled.', ); } - if (partialBB || keys.braintrust.state === "missing") { + if ( + partialBB || + keys.braintrust.state === "missing" || + tracePrimaryUnrecognized || + primaryBackendMissing || + langSmithTracingDisabled + ) { return { verdict: "warn", reasons }; } @@ -246,7 +292,17 @@ async function buildReport(entryDir: string): Promise { const config = summarizeConfig(entryDir); const discovery = await summarizeDiscovery(); const keys = snapshotEnv(); - const { verdict, reasons } = computeVerdict(keys, config, discovery); + const tracePrimary = resolveKey("EVAL_TRACE_PRIMARY").value || "braintrust"; + // This is a runtime behavior gate, so it must mirror langSmithTracingEnabled + // and deliberately bypass resolveKey's package-.env fallback. + const langSmithTracing = process.env.LANGSMITH_TRACING ?? ""; + const { verdict, reasons } = computeVerdict( + keys, + config, + discovery, + tracePrimary, + langSmithTracing, + ); return { verdict, runtime, config, discovery, keys, reasons }; } @@ -343,6 +399,9 @@ function renderHuman(report: DoctorReport): void { "(needed for `experiments`)", ), ); + console.log( + keyRow("LANGSMITH_API_KEY", r.keys.langsmith, "(needed for `experiments`)"), + ); console.log(""); console.log(` ${bold("Status")}`); diff --git a/packages/evals/tui/welcomeStatus.ts b/packages/evals/tui/welcomeStatus.ts index 6f2e8ad57d..4e7b4aaf55 100644 --- a/packages/evals/tui/welcomeStatus.ts +++ b/packages/evals/tui/welcomeStatus.ts @@ -45,6 +45,7 @@ export type EnvSnapshot = { google: GoogleKeyEntry; browserbase: BrowserbaseKeyEntry; braintrust: ProviderKeyEntry; + langsmith: ProviderKeyEntry; }; // --------------------------------------------------------------------------- @@ -167,6 +168,7 @@ export function snapshotEnv(): EnvSnapshot { google: googleEntry(), browserbase: browserbaseEntry(), braintrust: providerEntry("BRAINTRUST_API_KEY"), + langsmith: providerEntry("LANGSMITH_API_KEY"), }; } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 434749d4dc..49eee60df7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -350,6 +350,9 @@ importers: dotenv: specifier: ^17.3.1 version: 17.3.1 + langsmith: + specifier: ^0.5.16 + version: 0.5.26(@opentelemetry/api@1.9.0)(openai@4.104.0(ws@8.21.0(bufferutil@4.0.9))(zod@4.2.1))(ws@8.21.0(bufferutil@4.0.9)) openai: specifier: ^4.104.0 version: 4.104.0(ws@8.21.0(bufferutil@4.0.9))(zod@4.2.1) @@ -4060,6 +4063,9 @@ packages: resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==} engines: {node: '>=6'} + eventemitter3@4.0.7: + resolution: {integrity: sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==} + eventemitter3@5.0.4: resolution: {integrity: sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==} @@ -5165,6 +5171,26 @@ packages: resolution: {integrity: sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==} engines: {node: '>=0.10.0'} + langsmith@0.5.26: + resolution: {integrity: sha512-HmmFgeQR2n9x1Kq8NiVaNL/j72ta71qN11hYjbyePJ/QuYEnOMhQjbNv9KeyKB3bOetpIzNalQbhHm+RyKoPRQ==} + peerDependencies: + '@opentelemetry/api': '*' + '@opentelemetry/exporter-trace-otlp-proto': '*' + '@opentelemetry/sdk-trace-base': '*' + openai: '*' + ws: ^8.20.1 + peerDependenciesMeta: + '@opentelemetry/api': + optional: true + '@opentelemetry/exporter-trace-otlp-proto': + optional: true + '@opentelemetry/sdk-trace-base': + optional: true + openai: + optional: true + ws: + optional: true + lazystream@1.0.1: resolution: {integrity: sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw==} engines: {node: '>= 0.6.3'} @@ -5807,6 +5833,10 @@ packages: resolution: {integrity: sha512-ZBxxZ5sL2HghephhpGAQdoskxplTwr7ICaehZwLIlfL6acuVgZPm8yBNuRAFBGEqtD/hmUeq9eqLg2ys9Xr/yw==} engines: {node: '>=8'} + p-finally@1.0.0: + resolution: {integrity: sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==} + engines: {node: '>=4'} + p-limit@2.3.0: resolution: {integrity: sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==} engines: {node: '>=6'} @@ -5827,10 +5857,18 @@ packages: resolution: {integrity: sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==} engines: {node: '>=6'} + p-queue@6.6.2: + resolution: {integrity: sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==} + engines: {node: '>=8'} + p-some@6.0.0: resolution: {integrity: sha512-CJbQCKdfSX3fIh8/QKgS+9rjm7OBNUTmwWswAFQAhc8j1NR1dsEDETUEuVUtQHZpV+J03LqWBEwvu0g1Yn+TYg==} engines: {node: '>=12.20'} + p-timeout@3.2.0: + resolution: {integrity: sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==} + engines: {node: '>=8'} + p-timeout@5.1.0: resolution: {integrity: sha512-auFDyzzzGZZZdHz3BtET9VEz0SE/uMEAx7uWfGPucfzEwwe/xH0iVeZibQmANYE/hp9T2+UUZT5m+BKyrDp3Ew==} engines: {node: '>=12'} @@ -10682,7 +10720,7 @@ snapshots: obug: 2.1.1 std-env: 4.1.0 tinyrainbow: 3.1.0 - vitest: 4.1.9(@opentelemetry/api@1.9.0)(@types/node@25.6.2)(@vitest/coverage-v8@4.1.9)(jsdom@29.1.1)(vite@7.3.5(@types/node@25.6.2)(jiti@2.6.1)(tsx@4.19.4)(yaml@2.9.0)) + vitest: 4.1.9(@opentelemetry/api@1.9.0)(@types/node@20.17.32)(@vitest/coverage-v8@4.1.9)(jsdom@29.1.1)(vite@7.3.5(@types/node@20.17.32)(jiti@2.6.1)(tsx@4.22.4)(yaml@2.9.0)) '@vitest/expect@4.1.9': dependencies: @@ -12113,6 +12151,8 @@ snapshots: event-target-shim@5.0.1: {} + eventemitter3@4.0.7: {} + eventemitter3@5.0.4: {} events-universal@1.0.1: @@ -13568,6 +13608,14 @@ snapshots: kind-of@6.0.3: {} + langsmith@0.5.26(@opentelemetry/api@1.9.0)(openai@4.104.0(ws@8.21.0(bufferutil@4.0.9))(zod@4.2.1))(ws@8.21.0(bufferutil@4.0.9)): + dependencies: + p-queue: 6.6.2 + optionalDependencies: + '@opentelemetry/api': 1.9.0 + openai: 4.104.0(ws@8.21.0(bufferutil@4.0.9))(zod@4.2.1) + ws: 8.21.0(bufferutil@4.0.9) + lazystream@1.0.1: dependencies: readable-stream: 2.3.8 @@ -14557,6 +14605,8 @@ snapshots: dependencies: p-map: 2.1.0 + p-finally@1.0.0: {} + p-limit@2.3.0: dependencies: p-try: 2.2.0 @@ -14575,11 +14625,20 @@ snapshots: p-map@2.1.0: {} + p-queue@6.6.2: + dependencies: + eventemitter3: 4.0.7 + p-timeout: 3.2.0 + p-some@6.0.0: dependencies: aggregate-error: 4.0.1 p-cancelable: 3.0.0 + p-timeout@3.2.0: + dependencies: + p-finally: 1.0.0 + p-timeout@5.1.0: {} p-try@2.2.0: {} From a384a6e21ebd98f00acdb359e4f7ddadb335a461 Mon Sep 17 00:00:00 2001 From: miguel Date: Thu, 30 Jul 2026 16:57:57 -0700 Subject: [PATCH 2/2] chore(cli): format reduce-logs.ts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unrelated to this stack — main's #2373 landed this file unformatted, so the repo-wide `turbo run lint` job fails on every PR rebased onto current main. Formatting it here (stack base) unblocks CI for the whole stack. --- packages/cli/src/lib/cloud/reduce-logs.ts | 94 +++++++++++++++++++---- 1 file changed, 80 insertions(+), 14 deletions(-) diff --git a/packages/cli/src/lib/cloud/reduce-logs.ts b/packages/cli/src/lib/cloud/reduce-logs.ts index 90f0b84380..102f567316 100644 --- a/packages/cli/src/lib/cloud/reduce-logs.ts +++ b/packages/cli/src/lib/cloud/reduce-logs.ts @@ -3,6 +3,12 @@ // console errors/warnings/asserts, uncaught exceptions, HTTP 4xx/5xx responses, and net-level load // failures. No LLM — pure allowlist + severity/status filter + field projection + dedupe + stack trim. +/* eslint-disable @typescript-eslint/no-explicit-any -- + * This reducer parses the raw, untyped CDP event firehose; `any` on the + * dynamic JSON params is intentional and load-bearing (narrowing to typed + * shapes would only re-introduce optional-chaining noise for the same + * runtime behavior). */ + export interface ReduceLogsOptions { /** Only failed / error-status network requests (4xx/5xx + load failures). */ failedRequests?: boolean; @@ -15,7 +21,10 @@ interface RawLog { function paramsOf(e: RawLog): Record { try { - return (JSON.parse(e.request?.rawBody ?? "{}").params as Record) ?? {}; + return ( + (JSON.parse(e.request?.rawBody ?? "{}").params as Record) ?? + {} + ); } catch { return {}; } @@ -25,13 +34,24 @@ function paramsOf(e: RawLog): Record { function trimStack(s: string): string { return (s || "") .split("\n") - .filter((l, i) => i === 0 || (/\/src\//.test(l) && !/node_modules|\.vite/.test(l))) + .filter( + (l, i) => + i === 0 || (/\/src\//.test(l) && !/node_modules|\.vite/.test(l)), + ) .slice(0, 4) - .map((l) => l.replace(/https?:\/\/[^/)]+/g, "").replace(/\?[^):]*/, "").trim()) + .map((l) => + l + .replace(/https?:\/\/[^/)]+/g, "") + .replace(/\?[^):]*/, "") + .trim(), + ) .join("\n"); } -export function reduceLogs(raw: RawLog[], opts: ReduceLogsOptions = {}): unknown[] { +export function reduceLogs( + raw: RawLog[], + opts: ReduceLogsOptions = {}, +): unknown[] { const out: Record[] = []; const seen = new Set(); const push = (rec: Record) => { @@ -47,17 +67,63 @@ export function reduceLogs(raw: RawLog[], opts: ReduceLogsOptions = {}): unknown const m = e.method; let rec: Record | null = null; - if (m === "Runtime.consoleAPICalled" && ["error", "warning", "assert"].includes(p.type)) { - const text = (p.args ?? []).map((a: any) => a.description || a.value || "").join(" "); - if (text && !/^%[os]/.test(text)) rec = { kind: `console.${p.type}`, domain: "Runtime", severity: p.type, text: trimStack(text) }; + if ( + m === "Runtime.consoleAPICalled" && + ["error", "warning", "assert"].includes(p.type) + ) { + const text = (p.args ?? []) + .map((a: any) => a.description || a.value || "") + .join(" "); + if (text && !/^%[os]/.test(text)) + rec = { + kind: `console.${p.type}`, + domain: "Runtime", + severity: p.type, + text: trimStack(text), + }; } else if (m === "Runtime.exceptionThrown") { - rec = { kind: "exception", domain: "Runtime", severity: "error", text: trimStack(p.exceptionDetails?.exception?.description ?? p.exceptionDetails?.text ?? "") }; - } else if (m === "Log.entryAdded" && ["error", "warning"].includes(p.entry?.level)) { - rec = { kind: `log.${p.entry.level}`, domain: "Log", severity: p.entry.level, text: p.entry.text, url: p.entry.url }; - } else if (m === "Network.responseReceived" && (p.response?.status ?? 0) >= 400) { - rec = { kind: "network", domain: "Network", status: p.response.status, url: p.response.url, type: p.type }; - } else if (m === "Network.loadingFailed" && p.errorText !== "net::ERR_ABORTED") { - rec = { kind: "network.failed", domain: "Network", error: p.errorText, type: p.type }; + rec = { + kind: "exception", + domain: "Runtime", + severity: "error", + text: trimStack( + p.exceptionDetails?.exception?.description ?? + p.exceptionDetails?.text ?? + "", + ), + }; + } else if ( + m === "Log.entryAdded" && + ["error", "warning"].includes(p.entry?.level) + ) { + rec = { + kind: `log.${p.entry.level}`, + domain: "Log", + severity: p.entry.level, + text: p.entry.text, + url: p.entry.url, + }; + } else if ( + m === "Network.responseReceived" && + (p.response?.status ?? 0) >= 400 + ) { + rec = { + kind: "network", + domain: "Network", + status: p.response.status, + url: p.response.url, + type: p.type, + }; + } else if ( + m === "Network.loadingFailed" && + p.errorText !== "net::ERR_ABORTED" + ) { + rec = { + kind: "network.failed", + domain: "Network", + error: p.errorText, + type: p.type, + }; } else { continue; // everything else (byte-chunk / lifecycle events) is noise }