[STG-2284] feat(evals): richer OTEL traces — task root, thread grouping, trajectory

packages/evals/framework/runner.ts

@@ -36,6 +36,7 @@ import { onceAsync, registerActiveRunCleanup } from "./activeRunCleanup.js";
 import { loadTaskModuleFromPath } from "./taskLoader.js";
 import { resolveTraceTransport } from "./langsmith.js";
 import { buildTracerProvider, shutdownTracing } from "./otel.js";
+import { randomUUID } from "node:crypto";
 
 export { discoverTasks, resolveTarget } from "./discovery.js";
 export {
@@ -279,6 +280,87 @@ export interface RunEvalsResult {
   }>;
 }
 
+/**
+ * Upper bound on the JSON size of a span payload. Measured against the live
+ * LangSmith OTLP endpoint: 300KB/700KB/1MB/3MB payloads all round-trip intact
+ * (it offloads large values to S3), and real agent runs land around 600KB. 2MB
+ * keeps ~3x headroom over observed runs while staying under the proven ceiling
+ * — a single oversized span can fail its whole OTLP export batch and silently
+ * take every other span in that batch with it.
+ */
+const MAX_SPAN_PAYLOAD_BYTES = 2_000_000;
+
+/** UTF-8 byte length of a value's JSON, or undefined if it can't serialize. */
+function jsonByteSize(value: unknown): number | undefined {
+  try {
+    const json = JSON.stringify(value);
+    return json === undefined ? undefined : Buffer.byteLength(json, "utf8");
+  } catch {
+    return undefined;
+  }
+}
+
+/**
+ * Keep `value` whole when it serializes small enough. Otherwise progressively
+ * drop the oldest `logs` entries (the only unbounded field) until it fits, so a
+ * huge run still yields a useful tail rather than nothing, and record what was
+ * dropped. If the non-`logs` fields alone already exceed the cap, `logs` is
+ * dropped entirely — the return value is always within the cap, never a payload
+ * that would fail its OTLP batch. `.trajectories` remains the complete record
+ * either way.
+ *
+ * Exported for unit testing; not part of the module's public surface.
+ */
+export function capForSpan(
+  value: Record<string, unknown>,
+): Record<string, unknown> {
+  const size = jsonByteSize(value);
+  if (size === undefined) return { _truncated: "payload not serializable" };
+  if (size <= MAX_SPAN_PAYLOAD_BYTES) return value;
+
+  const logs = value.logs;
+  const rest = { ...value };
+  delete rest.logs;
+  const baseSize = jsonByteSize(rest);
+
+  // Non-`logs` fields already over the cap (or `logs` isn't a trimmable array,
+  // or `rest` won't serialize): trimming logs can't bring us under. Keep only
+  // the fields that are individually small so the return is guaranteed within
+  // the cap rather than shipping an oversized payload that fails its batch.
+  if (
+    !Array.isArray(logs) ||
+    baseSize === undefined ||
+    baseSize > MAX_SPAN_PAYLOAD_BYTES
+  ) {
+    const PER_FIELD_LIMIT = 64_000;
+    const compact: Record<string, unknown> = {};
+    for (const [key, fieldValue] of Object.entries(rest)) {
+      const fieldSize = jsonByteSize(fieldValue);
+      if (fieldSize !== undefined && fieldSize <= PER_FIELD_LIMIT) {
+        compact[key] = fieldValue;
+      }
+    }
+    return {
+      ...compact,
+      _truncated: `payload ${size}B exceeded cap ${MAX_SPAN_PAYLOAD_BYTES}B; oversized fields omitted — see .trajectories for the complete record`,
+    };
+  }
+
+  // Halve the retained tail (dropping the oldest entries) until it fits. The
+  // baseSize check above guarantees this terminates within the cap.
+  let kept = logs;
+  while (kept.length > 0) {
+    kept = kept.slice(Math.ceil(kept.length / 2));
+    const probe = jsonByteSize({ ...rest, logs: kept });
+    if (probe !== undefined && probe <= MAX_SPAN_PAYLOAD_BYTES) break;
+  }
+  return {
+    ...rest,
+    logs: kept,
+    _truncated: `kept the last ${kept.length} of ${logs.length} log entries: full payload was ${size}B (cap ${MAX_SPAN_PAYLOAD_BYTES}B) — see .trajectories for the complete record`,
+  };
+}
+
 function formatProgressError(error: unknown): string | undefined {
   if (error === undefined || error === null) return undefined;
   if (typeof error === "string") return error;
@@ -338,6 +420,19 @@ export async function runEvals(
     testcases.map((testcase) => testcase.input?.modelName),
   );
 
+  // LangSmith addresses threads by URL path segment (/v2/threads/<id>/...),
+  // and its gateway rejects percent-encoded slashes with a 403 HTML page —
+  // which the browser reports as a CORS preflight failure, breaking the whole
+  // thread/peek view. Experiment names contain "/" (e.g. "agent/onlineMind2Web"),
+  // so the id must be sanitized to a path-safe form. The timestamp + random
+  // suffix scopes the thread to this invocation (so one eval run groups as one
+  // thread) while the random component avoids collisions between runs that
+  // start within the same millisecond.
+  const traceThreadId = `${experimentName.replace(
+    /[^A-Za-z0-9._-]+/g,
+    "__",
+  )}-${Date.now().toString(36)}-${randomUUID().slice(0, 8)}`;
+
   // Stamp the run-scoped trajectory group; the token is generated once here and
   // reused for the completion-time experiment link. Local persistence only.
   const trajectoryGroup = buildTrajectoryGroupSlug({
@@ -443,7 +538,59 @@ export async function runEvals(
           modelName: input.modelName,
         });
 
-        const result = await executeTask(input, resolvedTask, options);
+        // Task-root span for the OTEL transport only: it gives LangSmith a
+        // per-task trace root (agent/verifier spans nest beneath) and carries
+        // the thread_id that groups one eval run in the Threads view. Gated
+        // off in native mode so the Braintrust span tree stays byte-identical.
+        const result =
+          traceTransport === "otel"
+            ? await tracedSpan(
+                async (span) => {
+                  const taskResult = await executeTask(
+                    input,
+                    resolvedTask,
+                    options,
+                  );
+                  // Mirror the whole TaskResult onto the root span so the
+                  // trace carries the agent's trajectory. Braintrust gets this
+                  // for free: its Eval() framework hands the TaskResult to the
+                  // scorers, so the logs show up in the scorer spans (~400KB).
+                  // OTEL has no equivalent, so we attach it explicitly here.
+                  // `logs` is the Stagehand logger output — i.e. what the
+                  // agent actually did — and is already screenshot-redacted by
+                  // EvalLogger. Capped so a pathological run can't blow up the
+                  // OTLP payload.
+                  span?.log({
+                    output: capForSpan({
+                      _success: taskResult._success,
+                      ...(taskResult.error !== undefined && {
+                        error: taskResult.error,
+                      }),
+                      ...(taskResult.metrics !== undefined && {
+                        metrics: taskResult.metrics,
+                      }),
+                      ...(taskResult.logs !== undefined && {
+                        logs: taskResult.logs,
+                      }),
+                    }),
+                    metadata: {
+                      experiment_name: experimentName,
+                      thread_id: traceThreadId,
+                      model: input.modelName,
+                      task: input.name,
+                    },
+                  });
+                  return taskResult;
+                },
+                {
+                  name: input.name,
+                  type: "task",
+                  event: {
+                    input: { task: input.name, model: input.modelName },
+                  },
+                },
+              )
+            : await executeTask(input, resolvedTask, options);
 
         options.onProgress?.({
           type: result._success ? "passed" : "failed",

packages/evals/tests/framework/capforspan.test.ts

@@ -0,0 +1,77 @@
+import { describe, expect, it } from "vitest";
+import { capForSpan } from "../../framework/runner.js";
+
+const MAX_SPAN_PAYLOAD_BYTES = 2_000_000;
+
+function jsonBytes(value: unknown): number {
+  return Buffer.byteLength(JSON.stringify(value), "utf8");
+}
+
+describe("capForSpan", () => {
+  it("returns small payloads unchanged", () => {
+    const value = { _success: true, metrics: { steps: 3 }, logs: ["a", "b"] };
+    expect(capForSpan(value)).toBe(value);
+  });
+
+  it("flags payloads that cannot be serialized", () => {
+    const circular: Record<string, unknown> = { _success: true };
+    circular.self = circular;
+    expect(capForSpan(circular)).toEqual({
+      _truncated: "payload not serializable",
+    });
+  });
+
+  it("keeps a tail of logs when the payload exceeds the cap", () => {
+    // ~4MB of logs: 400 entries × ~10KB each. Entries are distinguishable so
+    // we can assert the *tail* (newest) is what survives.
+    const logs = Array.from(
+      { length: 400 },
+      (_, i) => `entry-${i}-${"x".repeat(10_000)}`,
+    );
+    const value = { _success: false, logs };
+
+    const capped = capForSpan(value);
+
+    expect(jsonBytes(capped)).toBeLessThanOrEqual(MAX_SPAN_PAYLOAD_BYTES);
+    const keptLogs = capped.logs as string[];
+    expect(keptLogs.length).toBeGreaterThan(0);
+    expect(keptLogs.length).toBeLessThan(logs.length);
+    // Tail-preserving: the kept entries are a contiguous suffix of the input.
+    expect(keptLogs).toEqual(logs.slice(logs.length - keptLogs.length));
+    expect(keptLogs[keptLogs.length - 1]).toBe(logs[logs.length - 1]);
+    expect(capped._truncated).toMatch(/kept the last \d+ of 400 log entries/);
+  });
+
+  it("drops oversized non-log fields so the result is always within the cap (P1)", () => {
+    // A single non-`logs` field alone blows the cap; trimming logs can't help.
+    const value = {
+      _success: false,
+      error: "e".repeat(2_500_000),
+      metrics: { steps: 2 },
+      logs: ["a", "b"],
+    };
+
+    const capped = capForSpan(value);
+
+    expect(jsonBytes(capped)).toBeLessThanOrEqual(MAX_SPAN_PAYLOAD_BYTES);
+    expect(capped.error).toBeUndefined();
+    expect(capped._success).toBe(false); // small scalar survives
+    expect(capped.metrics).toEqual({ steps: 2 }); // small object survives
+    expect(capped._truncated).toMatch(/oversized fields omitted/);
+  });
+
+  it("measures bytes, not UTF-16 code units (P2)", () => {
+    // "€" is 1 UTF-16 unit but 3 UTF-8 bytes. This string's JSON char length
+    // is under the cap, but its byte length is over — a char-counting cap
+    // would wrongly pass it through whole.
+    const value = { note: "€".repeat(700_000) };
+    expect(JSON.stringify(value).length).toBeLessThan(MAX_SPAN_PAYLOAD_BYTES);
+    expect(jsonBytes(value)).toBeGreaterThan(MAX_SPAN_PAYLOAD_BYTES);
+
+    const capped = capForSpan(value);
+
+    expect(jsonBytes(capped)).toBeLessThanOrEqual(MAX_SPAN_PAYLOAD_BYTES);
+    expect(capped.note).toBeUndefined();
+    expect(capped._truncated).toBeDefined();
+  });
+});