diff --git a/packages/evals/framework/runner.ts b/packages/evals/framework/runner.ts index 71ddca4f29..796ace1ed0 100644 --- a/packages/evals/framework/runner.ts +++ b/packages/evals/framework/runner.ts @@ -36,6 +36,7 @@ import { onceAsync, registerActiveRunCleanup } from "./activeRunCleanup.js"; import { loadTaskModuleFromPath } from "./taskLoader.js"; import { resolveTraceTransport } from "./langsmith.js"; import { buildTracerProvider, shutdownTracing } from "./otel.js"; +import { randomUUID } from "node:crypto"; export { discoverTasks, resolveTarget } from "./discovery.js"; export { @@ -279,6 +280,87 @@ export interface RunEvalsResult { }>; } +/** + * Upper bound on the JSON size of a span payload. Measured against the live + * LangSmith OTLP endpoint: 300KB/700KB/1MB/3MB payloads all round-trip intact + * (it offloads large values to S3), and real agent runs land around 600KB. 2MB + * keeps ~3x headroom over observed runs while staying under the proven ceiling + * — a single oversized span can fail its whole OTLP export batch and silently + * take every other span in that batch with it. + */ +const MAX_SPAN_PAYLOAD_BYTES = 2_000_000; + +/** UTF-8 byte length of a value's JSON, or undefined if it can't serialize. */ +function jsonByteSize(value: unknown): number | undefined { + try { + const json = JSON.stringify(value); + return json === undefined ? undefined : Buffer.byteLength(json, "utf8"); + } catch { + return undefined; + } +} + +/** + * Keep `value` whole when it serializes small enough. Otherwise progressively + * drop the oldest `logs` entries (the only unbounded field) until it fits, so a + * huge run still yields a useful tail rather than nothing, and record what was + * dropped. If the non-`logs` fields alone already exceed the cap, `logs` is + * dropped entirely — the return value is always within the cap, never a payload + * that would fail its OTLP batch. `.trajectories` remains the complete record + * either way. + * + * Exported for unit testing; not part of the module's public surface. + */ +export function capForSpan( + value: Record, +): Record { + const size = jsonByteSize(value); + if (size === undefined) return { _truncated: "payload not serializable" }; + if (size <= MAX_SPAN_PAYLOAD_BYTES) return value; + + const logs = value.logs; + const rest = { ...value }; + delete rest.logs; + const baseSize = jsonByteSize(rest); + + // Non-`logs` fields already over the cap (or `logs` isn't a trimmable array, + // or `rest` won't serialize): trimming logs can't bring us under. Keep only + // the fields that are individually small so the return is guaranteed within + // the cap rather than shipping an oversized payload that fails its batch. + if ( + !Array.isArray(logs) || + baseSize === undefined || + baseSize > MAX_SPAN_PAYLOAD_BYTES + ) { + const PER_FIELD_LIMIT = 64_000; + const compact: Record = {}; + for (const [key, fieldValue] of Object.entries(rest)) { + const fieldSize = jsonByteSize(fieldValue); + if (fieldSize !== undefined && fieldSize <= PER_FIELD_LIMIT) { + compact[key] = fieldValue; + } + } + return { + ...compact, + _truncated: `payload ${size}B exceeded cap ${MAX_SPAN_PAYLOAD_BYTES}B; oversized fields omitted — see .trajectories for the complete record`, + }; + } + + // Halve the retained tail (dropping the oldest entries) until it fits. The + // baseSize check above guarantees this terminates within the cap. + let kept = logs; + while (kept.length > 0) { + kept = kept.slice(Math.ceil(kept.length / 2)); + const probe = jsonByteSize({ ...rest, logs: kept }); + if (probe !== undefined && probe <= MAX_SPAN_PAYLOAD_BYTES) break; + } + return { + ...rest, + logs: kept, + _truncated: `kept the last ${kept.length} of ${logs.length} log entries: full payload was ${size}B (cap ${MAX_SPAN_PAYLOAD_BYTES}B) — see .trajectories for the complete record`, + }; +} + function formatProgressError(error: unknown): string | undefined { if (error === undefined || error === null) return undefined; if (typeof error === "string") return error; @@ -338,6 +420,19 @@ export async function runEvals( testcases.map((testcase) => testcase.input?.modelName), ); + // LangSmith addresses threads by URL path segment (/v2/threads//...), + // and its gateway rejects percent-encoded slashes with a 403 HTML page — + // which the browser reports as a CORS preflight failure, breaking the whole + // thread/peek view. Experiment names contain "/" (e.g. "agent/onlineMind2Web"), + // so the id must be sanitized to a path-safe form. The timestamp + random + // suffix scopes the thread to this invocation (so one eval run groups as one + // thread) while the random component avoids collisions between runs that + // start within the same millisecond. + const traceThreadId = `${experimentName.replace( + /[^A-Za-z0-9._-]+/g, + "__", + )}-${Date.now().toString(36)}-${randomUUID().slice(0, 8)}`; + // Stamp the run-scoped trajectory group; the token is generated once here and // reused for the completion-time experiment link. Local persistence only. const trajectoryGroup = buildTrajectoryGroupSlug({ @@ -443,7 +538,59 @@ export async function runEvals( modelName: input.modelName, }); - const result = await executeTask(input, resolvedTask, options); + // Task-root span for the OTEL transport only: it gives LangSmith a + // per-task trace root (agent/verifier spans nest beneath) and carries + // the thread_id that groups one eval run in the Threads view. Gated + // off in native mode so the Braintrust span tree stays byte-identical. + const result = + traceTransport === "otel" + ? await tracedSpan( + async (span) => { + const taskResult = await executeTask( + input, + resolvedTask, + options, + ); + // Mirror the whole TaskResult onto the root span so the + // trace carries the agent's trajectory. Braintrust gets this + // for free: its Eval() framework hands the TaskResult to the + // scorers, so the logs show up in the scorer spans (~400KB). + // OTEL has no equivalent, so we attach it explicitly here. + // `logs` is the Stagehand logger output — i.e. what the + // agent actually did — and is already screenshot-redacted by + // EvalLogger. Capped so a pathological run can't blow up the + // OTLP payload. + span?.log({ + output: capForSpan({ + _success: taskResult._success, + ...(taskResult.error !== undefined && { + error: taskResult.error, + }), + ...(taskResult.metrics !== undefined && { + metrics: taskResult.metrics, + }), + ...(taskResult.logs !== undefined && { + logs: taskResult.logs, + }), + }), + metadata: { + experiment_name: experimentName, + thread_id: traceThreadId, + model: input.modelName, + task: input.name, + }, + }); + return taskResult; + }, + { + name: input.name, + type: "task", + event: { + input: { task: input.name, model: input.modelName }, + }, + }, + ) + : await executeTask(input, resolvedTask, options); options.onProgress?.({ type: result._success ? "passed" : "failed", diff --git a/packages/evals/tests/framework/capforspan.test.ts b/packages/evals/tests/framework/capforspan.test.ts new file mode 100644 index 0000000000..7ca7c32126 --- /dev/null +++ b/packages/evals/tests/framework/capforspan.test.ts @@ -0,0 +1,77 @@ +import { describe, expect, it } from "vitest"; +import { capForSpan } from "../../framework/runner.js"; + +const MAX_SPAN_PAYLOAD_BYTES = 2_000_000; + +function jsonBytes(value: unknown): number { + return Buffer.byteLength(JSON.stringify(value), "utf8"); +} + +describe("capForSpan", () => { + it("returns small payloads unchanged", () => { + const value = { _success: true, metrics: { steps: 3 }, logs: ["a", "b"] }; + expect(capForSpan(value)).toBe(value); + }); + + it("flags payloads that cannot be serialized", () => { + const circular: Record = { _success: true }; + circular.self = circular; + expect(capForSpan(circular)).toEqual({ + _truncated: "payload not serializable", + }); + }); + + it("keeps a tail of logs when the payload exceeds the cap", () => { + // ~4MB of logs: 400 entries × ~10KB each. Entries are distinguishable so + // we can assert the *tail* (newest) is what survives. + const logs = Array.from( + { length: 400 }, + (_, i) => `entry-${i}-${"x".repeat(10_000)}`, + ); + const value = { _success: false, logs }; + + const capped = capForSpan(value); + + expect(jsonBytes(capped)).toBeLessThanOrEqual(MAX_SPAN_PAYLOAD_BYTES); + const keptLogs = capped.logs as string[]; + expect(keptLogs.length).toBeGreaterThan(0); + expect(keptLogs.length).toBeLessThan(logs.length); + // Tail-preserving: the kept entries are a contiguous suffix of the input. + expect(keptLogs).toEqual(logs.slice(logs.length - keptLogs.length)); + expect(keptLogs[keptLogs.length - 1]).toBe(logs[logs.length - 1]); + expect(capped._truncated).toMatch(/kept the last \d+ of 400 log entries/); + }); + + it("drops oversized non-log fields so the result is always within the cap (P1)", () => { + // A single non-`logs` field alone blows the cap; trimming logs can't help. + const value = { + _success: false, + error: "e".repeat(2_500_000), + metrics: { steps: 2 }, + logs: ["a", "b"], + }; + + const capped = capForSpan(value); + + expect(jsonBytes(capped)).toBeLessThanOrEqual(MAX_SPAN_PAYLOAD_BYTES); + expect(capped.error).toBeUndefined(); + expect(capped._success).toBe(false); // small scalar survives + expect(capped.metrics).toEqual({ steps: 2 }); // small object survives + expect(capped._truncated).toMatch(/oversized fields omitted/); + }); + + it("measures bytes, not UTF-16 code units (P2)", () => { + // "€" is 1 UTF-16 unit but 3 UTF-8 bytes. This string's JSON char length + // is under the cap, but its byte length is over — a char-counting cap + // would wrongly pass it through whole. + const value = { note: "€".repeat(700_000) }; + expect(JSON.stringify(value).length).toBeLessThan(MAX_SPAN_PAYLOAD_BYTES); + expect(jsonBytes(value)).toBeGreaterThan(MAX_SPAN_PAYLOAD_BYTES); + + const capped = capForSpan(value); + + expect(jsonBytes(capped)).toBeLessThanOrEqual(MAX_SPAN_PAYLOAD_BYTES); + expect(capped.note).toBeUndefined(); + expect(capped._truncated).toBeDefined(); + }); +});