[STG-2284] fix(evals): default experimental to true on the SDK path

packages/evals/initV3.ts

@@ -24,6 +24,24 @@ import {
 import { getEnv } from "./env.js";
 import { EvalLogger } from "./logger.js";
 
+/**
+ * Resolve the V3 `experimental` flag for eval runs.
+ *
+ * Experimental features (hybrid agent mode, verifier agent callbacks) require
+ * it on the SDK path, so it defaults ON there. But core forbids `experimental`
+ * together with API mode (`ExperimentalNotConfiguredError`), so it is forced
+ * OFF whenever USE_API=true — even when the caller explicitly asks for it.
+ *
+ * Exported for unit testing.
+ */
+export function resolveExperimental(
+  requested: boolean | undefined,
+  useApi: boolean = process.env.USE_API === "true",
+): boolean {
+  if (useApi) return false;
+  return typeof requested === "boolean" ? requested : true;
+}
+
 type InitV3Args = {
   llmClient?: LLMClient;
   modelClientOptions?: ClientOptions;
@@ -108,10 +126,7 @@ export async function initV3({
         configOverrides?.chromeFlags,
     },
     model: resolvedModelConfig,
-    experimental:
-      typeof configOverrides?.experimental === "boolean"
-        ? configOverrides.experimental && process.env.USE_API !== "true" // experimental only when not using API
-        : false,
+    experimental: resolveExperimental(configOverrides?.experimental),
     verbose: verbose ? 2 : 0,
     browserbaseSessionCreateParams:
       configOverrides?.browserbaseSessionCreateParams,

packages/evals/tests/initv3-experimental.test.ts

@@ -0,0 +1,36 @@
+import { describe, expect, it } from "vitest";
+import { resolveExperimental } from "../initV3.js";
+
+describe("resolveExperimental", () => {
+  it("defaults experimental ON for SDK-path runs", () => {
+    expect(resolveExperimental(undefined, false)).toBe(true);
+  });
+
+  it("forces experimental OFF under API mode, even when requested", () => {
+    // Core rejects `experimental` together with API mode, so the API guard
+    // wins over both the default and an explicit opt-in.
+    expect(resolveExperimental(undefined, true)).toBe(false);
+    expect(resolveExperimental(true, true)).toBe(false);
+    expect(resolveExperimental(false, true)).toBe(false);
+  });
+
+  it("honors an explicit boolean on the SDK path", () => {
+    expect(resolveExperimental(true, false)).toBe(true);
+    expect(resolveExperimental(false, false)).toBe(false);
+  });
+
+  it("reads USE_API from the environment when not passed explicitly", () => {
+    const prev = process.env.USE_API;
+    try {
+      process.env.USE_API = "true";
+      expect(resolveExperimental(undefined)).toBe(false);
+      process.env.USE_API = "false";
+      expect(resolveExperimental(undefined)).toBe(true);
+      delete process.env.USE_API;
+      expect(resolveExperimental(undefined)).toBe(true);
+    } finally {
+      if (prev === undefined) delete process.env.USE_API;
+      else process.env.USE_API = prev;
+    }
+  });
+});