diff --git a/src/benchmarks/benchmark-config.test.ts b/src/benchmarks/benchmark-config.test.ts index 4cba901a..5aa74440 100644 --- a/src/benchmarks/benchmark-config.test.ts +++ b/src/benchmarks/benchmark-config.test.ts @@ -38,13 +38,14 @@ describe("benchmark config", () => { assertLeft(result); }); - it("requires reasoningEffort in model benchmark configs", () => { + it("accepts model benchmark configs without reasoningEffort", () => { const result = parseSchema(BenchmarkRunConfigSchema, { benchmarkId: "gpqa_diamond", model: "openai/gpt-5", }); - assertLeft(result); + assertRight(result); + expect(result.right).not.toHaveProperty("reasoningEffort"); }); it("parses injected benchmark configs with opaque options", () => { diff --git a/src/benchmarks/benchmark-config.ts b/src/benchmarks/benchmark-config.ts index 24f309ea..668676f3 100644 --- a/src/benchmarks/benchmark-config.ts +++ b/src/benchmarks/benchmark-config.ts @@ -44,7 +44,7 @@ export type GeminiMediaResolution = ValueOf; export const InferenceOverrideSchema = z.object({ temperature: z.number().optional(), maxTokens: z.number().optional(), - reasoningEffort: z.enum(REASONING_EFFORTS), + reasoningEffort: z.enum(REASONING_EFFORTS).optional(), costTier: z.enum(COST_TIERS).optional(), timeoutMs: z.number().optional(), sort: z.nativeEnum(ProviderSort).optional(), diff --git a/src/benchmarks/search/core/request.test.ts b/src/benchmarks/search/core/request.test.ts index da276ff8..60b349b5 100644 --- a/src/benchmarks/search/core/request.test.ts +++ b/src/benchmarks/search/core/request.test.ts @@ -196,6 +196,10 @@ describe("buildSearchRequestBody", () => { expect(body.temperature).toBe(0); expect(body.reasoning).toEqual({ effort: "high" }); }); + it("omits reasoning when no effort is requested", () => { + const body = buildSearchRequestBody({ ...BASE, lane: lane({}) }); + expect(body).not.toHaveProperty("reasoning"); + }); it("threads the candidate models list and omits it by default", () => { const routed = buildSearchRequestBody({ ...BASE, diff --git a/src/benchmarks/search/core/request.ts b/src/benchmarks/search/core/request.ts index c2cf6588..aec4730b 100644 --- a/src/benchmarks/search/core/request.ts +++ b/src/benchmarks/search/core/request.ts @@ -28,7 +28,7 @@ export interface SearchRequestOptions { readonly lane: SearchLaneConfig; readonly maxOutputTokens?: number; readonly temperature?: number; - readonly reasoningEffort: ReasoningEffort; + readonly reasoningEffort?: ReasoningEffort; readonly sort?: ProviderSort; readonly providerOrder?: readonly string[]; readonly providerOnly?: readonly string[]; @@ -127,7 +127,10 @@ export function buildSearchRequestBody( models: opts.models === undefined ? undefined : [...opts.models], maxOutputTokens: opts.maxOutputTokens, temperature: opts.temperature, - reasoning: { effort: opts.reasoningEffort }, + reasoning: + opts.reasoningEffort !== undefined + ? { effort: opts.reasoningEffort } + : undefined, provider: opts.sort !== undefined || opts.providerOrder !== undefined || diff --git a/src/benchmarks/search/core/solver.ts b/src/benchmarks/search/core/solver.ts index f9aae09a..2adfbd28 100644 --- a/src/benchmarks/search/core/solver.ts +++ b/src/benchmarks/search/core/solver.ts @@ -50,7 +50,7 @@ export interface SearchSolverOptions { readonly timeoutMs?: number; readonly maxOutputTokens?: number; readonly temperature?: number; - readonly reasoningEffort: ReasoningEffort; + readonly reasoningEffort?: ReasoningEffort; readonly endpointId?: string; readonly sort?: ProviderSort; readonly providerOrder?: readonly string[]; diff --git a/src/cli/index.test.ts b/src/cli/index.test.ts index 326bff7b..5d20a922 100644 --- a/src/cli/index.test.ts +++ b/src/cli/index.test.ts @@ -12,6 +12,28 @@ describe("bench-harness CLI", () => { ); }); + it("leaves reasoning effort unset for --reasoning-effort default", () => { + const args = parseArgs([ + "--benchmark", + "gpqa_diamond", + "--model", + "openrouter/jev", + "--reasoning-effort", + "default", + ]); + expect(args.reasoningEffort).toBeUndefined(); + const config = buildBenchmarkConfig({ + benchmarkId: "gpqa_diamond", + model: "openrouter/jev", + panelConfig: undefined, + artifactDir: undefined, + endpointId: undefined, + imageDetail: undefined, + reasoningEffort: args.reasoningEffort, + }); + expect(config).not.toHaveProperty("reasoningEffort"); + }); + it("rejects an invalid --reasoning-effort value", () => { expect(() => parseArgs(["--reasoning-effort", "invalid"])).toThrow( "--reasoning-effort must be one of" diff --git a/src/cli/index.ts b/src/cli/index.ts index 4c9aea8d..f04cec47 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -46,7 +46,7 @@ interface CliArgs { readonly resumeId?: string; readonly imageDetail?: ImageDetail; readonly costTier?: CostTier; - readonly reasoningEffort: ReasoningEffort; + readonly reasoningEffort: ReasoningEffort | undefined; } export function parseArgs(argv: readonly string[]): CliArgs { @@ -190,7 +190,7 @@ function main(): Promise { reasoningEffort: args.reasoningEffort, }); process.stderr.write( - `Running ${args.benchmark}${args.model !== undefined ? ` on ${args.model}` : ""}${args.solverConfig !== undefined ? ` (solver-config=${args.solverConfig})` : ""}${artifactDir !== undefined ? ` (artifact-dir=${artifactDir})` : ""} (epochs=${epochs}, concurrency=${args.concurrency}, reasoning-effort=${args.reasoningEffort}${range !== undefined ? `, range=${range.start ?? 0}..${range.end ?? "end"}` : ""}, session=${sessionId})...\n` + `Running ${args.benchmark}${args.model !== undefined ? ` on ${args.model}` : ""}${args.solverConfig !== undefined ? ` (solver-config=${args.solverConfig})` : ""}${artifactDir !== undefined ? ` (artifact-dir=${artifactDir})` : ""} (epochs=${epochs}, concurrency=${args.concurrency}, reasoning-effort=${args.reasoningEffort ?? MODEL_DEFAULT_REASONING_EFFORT}${range !== undefined ? `, range=${range.start ?? 0}..${range.end ?? "end"}` : ""}, session=${sessionId})...\n` ); const total = yield* promise(() => resolveTotalEvaluations(args.benchmark, range, epochs) @@ -304,13 +304,20 @@ function validateCostTier(raw: string | undefined): CostTier | undefined { return raw; } -function validateReasoningEffort(raw: string | undefined): ReasoningEffort { +const MODEL_DEFAULT_REASONING_EFFORT = "default"; + +function validateReasoningEffort( + raw: string | undefined +): ReasoningEffort | undefined { if (raw === undefined) { return DEFAULT_REASONING_EFFORT; } + if (raw === MODEL_DEFAULT_REASONING_EFFORT) { + return undefined; + } if (!isMember(raw, REASONING_EFFORTS)) { throw new Error( - `--reasoning-effort must be one of: ${REASONING_EFFORTS.join(", ")} (got "${raw}")` + `--reasoning-effort must be one of: ${[...REASONING_EFFORTS, MODEL_DEFAULT_REASONING_EFFORT].join(", ")} (got "${raw}")` ); } return raw; @@ -330,7 +337,7 @@ function buildSchemaValidatedConfig(opts: { panelConfig: unknown; imageDetail?: ImageDetail; costTier?: CostTier; - reasoningEffort: ReasoningEffort; + reasoningEffort: ReasoningEffort | undefined; }): BenchmarkRunConfig { const { benchmarkId, @@ -372,6 +379,7 @@ function buildSchemaValidatedConfig(opts: { : undefined; if ( optionsSchema !== undefined && + reasoningEffort !== undefined && Object.hasOwn(optionsSchema.shape, "agentReasoningEffort") && !( typeof panelConfig === "object" && @@ -410,7 +418,7 @@ export function buildBenchmarkConfig(opts: { endpointId: string | undefined; imageDetail: ImageDetail | undefined; costTier?: CostTier; - reasoningEffort: ReasoningEffort; + reasoningEffort: ReasoningEffort | undefined; }): BenchmarkRunConfig { const { benchmarkId, diff --git a/src/harness/model.ts b/src/harness/model.ts index 0e9d4b5f..0f85c282 100644 --- a/src/harness/model.ts +++ b/src/harness/model.ts @@ -20,7 +20,7 @@ export interface GenerateConfig { readonly maxTokens?: number; readonly endpointId?: string; readonly tools?: readonly ToolDefinition[]; - readonly reasoningEffort: ReasoningEffort; + readonly reasoningEffort?: ReasoningEffort; readonly costTier?: CostTier; readonly timeoutMs?: number; readonly sort?: ProviderSort; diff --git a/src/providers/responses-model.test.ts b/src/providers/responses-model.test.ts index e3dd04e2..ec840278 100644 --- a/src/providers/responses-model.test.ts +++ b/src/providers/responses-model.test.ts @@ -313,6 +313,7 @@ describe("responses-model", () => { }, ]); expect(sentOptions?.extraBody).toBeUndefined(); + expect(sentBody).not.toHaveProperty("reasoning"); expect(exit.value.outputItems).toEqual([ { type: "function_call", diff --git a/src/providers/responses-model.ts b/src/providers/responses-model.ts index b558d63d..81293e59 100644 --- a/src/providers/responses-model.ts +++ b/src/providers/responses-model.ts @@ -185,8 +185,11 @@ export function generate( ? [...genConfig.tools] : undefined, }), - reasoning: { effort: genConfig.reasoningEffort }, ...definedValues({ + reasoning: + genConfig.reasoningEffort !== undefined + ? { effort: genConfig.reasoningEffort } + : undefined, provider: sendProvider ? providerPreferences : undefined, }), ...definedValues({