From 3248fc3d434ea2569f99b4ecae1b690f6b92bca1 Mon Sep 17 00:00:00 2001
From: miguel <miguelg71921@gmail.com>
Date: Thu, 30 Jul 2026 16:17:10 -0700
Subject: [PATCH] fix(evals): default experimental to true on the SDK path

Extract resolveExperimental() so the SDK-default-ON / API-forced-OFF
logic is unit-tested (addresses review: add regression test).
---
 packages/evals/initV3.ts                      | 23 +++++++++---
 .../evals/tests/initv3-experimental.test.ts   | 36 +++++++++++++++++++
 2 files changed, 55 insertions(+), 4 deletions(-)
 create mode 100644 packages/evals/tests/initv3-experimental.test.ts

diff --git a/packages/evals/initV3.ts b/packages/evals/initV3.ts
index 850b12126f..c5543f93a5 100644
--- a/packages/evals/initV3.ts
+++ b/packages/evals/initV3.ts
@@ -24,6 +24,24 @@ import {
 import { getEnv } from "./env.js";
 import { EvalLogger } from "./logger.js";
 
+/**
+ * Resolve the V3 `experimental` flag for eval runs.
+ *
+ * Experimental features (hybrid agent mode, verifier agent callbacks) require
+ * it on the SDK path, so it defaults ON there. But core forbids `experimental`
+ * together with API mode (`ExperimentalNotConfiguredError`), so it is forced
+ * OFF whenever USE_API=true — even when the caller explicitly asks for it.
+ *
+ * Exported for unit testing.
+ */
+export function resolveExperimental(
+  requested: boolean | undefined,
+  useApi: boolean = process.env.USE_API === "true",
+): boolean {
+  if (useApi) return false;
+  return typeof requested === "boolean" ? requested : true;
+}
+
 type InitV3Args = {
   llmClient?: LLMClient;
   modelClientOptions?: ClientOptions;
@@ -108,10 +126,7 @@ export async function initV3({
         configOverrides?.chromeFlags,
     },
     model: resolvedModelConfig,
-    experimental:
-      typeof configOverrides?.experimental === "boolean"
-        ? configOverrides.experimental && process.env.USE_API !== "true" // experimental only when not using API
-        : false,
+    experimental: resolveExperimental(configOverrides?.experimental),
     verbose: verbose ? 2 : 0,
     browserbaseSessionCreateParams:
       configOverrides?.browserbaseSessionCreateParams,
diff --git a/packages/evals/tests/initv3-experimental.test.ts b/packages/evals/tests/initv3-experimental.test.ts
new file mode 100644
index 0000000000..8d265385b5
--- /dev/null
+++ b/packages/evals/tests/initv3-experimental.test.ts
@@ -0,0 +1,36 @@
+import { describe, expect, it } from "vitest";
+import { resolveExperimental } from "../initV3.js";
+
+describe("resolveExperimental", () => {
+  it("defaults experimental ON for SDK-path runs", () => {
+    expect(resolveExperimental(undefined, false)).toBe(true);
+  });
+
+  it("forces experimental OFF under API mode, even when requested", () => {
+    // Core rejects `experimental` together with API mode, so the API guard
+    // wins over both the default and an explicit opt-in.
+    expect(resolveExperimental(undefined, true)).toBe(false);
+    expect(resolveExperimental(true, true)).toBe(false);
+    expect(resolveExperimental(false, true)).toBe(false);
+  });
+
+  it("honors an explicit boolean on the SDK path", () => {
+    expect(resolveExperimental(true, false)).toBe(true);
+    expect(resolveExperimental(false, false)).toBe(false);
+  });
+
+  it("reads USE_API from the environment when not passed explicitly", () => {
+    const prev = process.env.USE_API;
+    try {
+      process.env.USE_API = "true";
+      expect(resolveExperimental(undefined)).toBe(false);
+      process.env.USE_API = "false";
+      expect(resolveExperimental(undefined)).toBe(true);
+      delete process.env.USE_API;
+      expect(resolveExperimental(undefined)).toBe(true);
+    } finally {
+      if (prev === undefined) delete process.env.USE_API;
+      else process.env.USE_API = prev;
+    }
+  });
+});