diff --git a/packages/junior-evals/src/guardian-harness.ts b/packages/junior-evals/src/guardian-harness.ts index d0e4bd20e8..a28e196841 100644 --- a/packages/junior-evals/src/guardian-harness.ts +++ b/packages/junior-evals/src/guardian-harness.ts @@ -41,7 +41,7 @@ function resolveGuardianModelId(): string { if (configured) { return configured; } - return "openai/gpt-5.6-luna"; + return "typesafe-ai/jev"; } /** Run one Guardian proposal through the production reviewer boundary. */ diff --git a/packages/junior-evals/vitest.evals.guardian.config.ts b/packages/junior-evals/vitest.evals.guardian.config.ts index 2550a313e2..89668b5129 100644 --- a/packages/junior-evals/vitest.evals.guardian.config.ts +++ b/packages/junior-evals/vitest.evals.guardian.config.ts @@ -29,7 +29,7 @@ process.env.JUNIOR_STATE_ADAPTER = "redis"; process.env.JUNIOR_STATE_KEY_PREFIX ??= `junior:eval-guardian:${randomUUID()}`; process.env.REDIS_URL = process.env.JUNIOR_EVAL_REDIS_URL?.trim() || "redis://127.0.0.1:6382"; -process.env.AI_GUARDIAN_MODEL ??= "openai/gpt-5.6-luna"; +process.env.AI_GUARDIAN_MODEL ??= "typesafe-ai/jev"; export default defineConfig({ resolve: { diff --git a/packages/junior/package.json b/packages/junior/package.json index 99901fd91a..f2a8783d54 100644 --- a/packages/junior/package.json +++ b/packages/junior/package.json @@ -70,6 +70,7 @@ "dependencies": { "@agentclientprotocol/sdk": "1.3.0", "@ai-sdk/gateway": "^3.0.119", + "@ai-sdk/gateway-v4": "npm:@ai-sdk/gateway@4.0.85", "@chat-adapter/slack": "4.29.0", "@chat-adapter/state-memory": "4.29.0", "@chat-adapter/state-redis": "4.29.0", @@ -91,6 +92,7 @@ "@vercel/queue": "^0.2.0", "@vercel/sandbox": "2.8.0", "ai": "^6.0.190", + "ai-v7": "npm:ai@7.0.105", "chat": "4.29.0", "commander": "^14.0.3", "drizzle-orm": "catalog:", diff --git a/packages/junior/src/chat/config.ts b/packages/junior/src/chat/config.ts index 7a360f10e1..1f4018ae86 100644 --- a/packages/junior/src/chat/config.ts +++ b/packages/junior/src/chat/config.ts @@ -242,10 +242,7 @@ const DEFAULT_FAST_MODEL_ID = getModel( "vercel-ai-gateway", "openai/gpt-5.6-luna", ).id; -const DEFAULT_GUARDIAN_MODEL_ID = getModel( - "vercel-ai-gateway", - "openai/gpt-5.6-luna", -).id; +const DEFAULT_GUARDIAN_MODEL_ID = "typesafe-ai/jev"; const DEFAULT_HANDOFF_MODEL_ID = getModel( "vercel-ai-gateway", "openai/gpt-5.6-sol", diff --git a/packages/junior/src/chat/pi/evaluate.ts b/packages/junior/src/chat/pi/evaluate.ts new file mode 100644 index 0000000000..b0fce7fed0 --- /dev/null +++ b/packages/junior/src/chat/pi/evaluate.ts @@ -0,0 +1,27 @@ +import { createGatewayProvider } from "@ai-sdk/gateway-v4"; +import { + experimental_evaluate as evaluate, + type Experimental_EvaluationQuestion as EvaluationQuestion, +} from "ai-v7"; +import { resolveGatewayCredential } from "@/chat/pi/gateway-auth"; + +/** Evaluate typed questions with an AI Gateway evaluation model. */ +export async function evaluateQuestions< + const Questions extends Record, +>(args: { + modelId: string; + state: string; + questions: Questions; + signal?: AbortSignal; +}) { + const credential = await resolveGatewayCredential(); + const gateway = createGatewayProvider( + credential ? { apiKey: credential.token } : {}, + ); + return evaluate({ + model: gateway.evaluationModel(args.modelId), + state: args.state, + questions: args.questions, + abortSignal: args.signal, + }); +} diff --git a/packages/junior/src/chat/services/guardian-action-review.ts b/packages/junior/src/chat/services/guardian-action-review.ts index cf0fb79be5..ad3db6aae6 100644 --- a/packages/junior/src/chat/services/guardian-action-review.ts +++ b/packages/junior/src/chat/services/guardian-action-review.ts @@ -1,6 +1,7 @@ import { z } from "zod"; import { logWarn } from "@/chat/logging"; import type { completeObject } from "@/chat/pi/client"; +import { evaluateQuestions } from "@/chat/pi/evaluate"; import { GUARDIAN_ACTION_POLICY } from "@/chat/services/guardian-action-policy"; import { ProviderError } from "@/chat/services/provider-error"; import type { @@ -20,6 +21,7 @@ const guardianDecisionSchema = z type CompleteObject = typeof completeObject; const GUARDIAN_REVIEW_TIMEOUT_MS = 60_000; const GUARDIAN_REVIEW_MAX_TOKENS = 2_000; +const JEV_MODEL_ID = "typesafe-ai/jev"; const MAX_PROPOSAL_CHARS = 192_000; /** Serialize one bounded proposal while keeping its contents untrusted. */ @@ -51,6 +53,54 @@ export function createGuardianActionReviewer(options: { const signal = reviewOptions?.signal ? AbortSignal.any([reviewOptions.signal, timeoutSignal]) : timeoutSignal; + if (options.modelId === JEV_MODEL_ID) { + const result = await evaluateQuestions({ + modelId: options.modelId, + state: [GUARDIAN_ACTION_POLICY, guardianPrompt(proposal)].join( + "\n\n", + ), + questions: { + decision: { + type: "choice", + instructions: "Choose the required Guardian decision.", + criteria: { + allow: "The policy allows the action without confirmation.", + ask: "The policy requires explicit user confirmation.", + deny: "The policy prohibits the action.", + }, + }, + riskLevel: { + type: "choice", + instructions: "Choose the risk level of the planned action.", + criteria: { + low: "Low impact and easy to reverse.", + medium: "Meaningful but limited impact.", + high: "Substantial impact or hard to reverse.", + critical: "Severe, broad, or irreversible impact.", + }, + }, + userAuthorization: { + type: "choice", + instructions: + "Choose how clearly the user authorized the planned action.", + criteria: { + high: "The user explicitly authorized this exact action.", + medium: "The user intent supports the action but is not exact.", + low: "The action is only weakly implied.", + unknown: "The proposal shows no user authorization.", + }, + }, + }, + signal, + }); + return guardianDecisionSchema.parse({ + decision: result.answers.decision.choice, + reason: "jev_evaluation", + riskLevel: result.answers.riskLevel.choice, + userAuthorization: result.answers.userAuthorization.choice, + }); + } + const completeReview = () => options.completeObject({ modelId: options.modelId, @@ -87,7 +137,9 @@ export function createGuardianActionReviewer(options: { } return { ...guardianDecisionSchema.parse(result.object), - ...(result.costUsd !== undefined ? { costUsd: result.costUsd } : undefined), + ...(result.costUsd !== undefined + ? { costUsd: result.costUsd } + : undefined), }; }, }; diff --git a/packages/junior/tests/component/briefs/conversation-brief-task.test.ts b/packages/junior/tests/component/briefs/conversation-brief-task.test.ts index 94f21a086f..af1163c687 100644 --- a/packages/junior/tests/component/briefs/conversation-brief-task.test.ts +++ b/packages/junior/tests/component/briefs/conversation-brief-task.test.ts @@ -79,6 +79,7 @@ vi.mock("@/chat/pi/client", () => ({ }; }), embedTexts: vi.fn(), + resolveGatewayModel: vi.fn((modelId: string) => ({ id: modelId })), })); function piMessages(instruction: string, turnId: string): PiMessage[] { diff --git a/packages/junior/tests/component/config/chat-config.test.ts b/packages/junior/tests/component/config/chat-config.test.ts index bcb34ec1e7..440b3be6d5 100644 --- a/packages/junior/tests/component/config/chat-config.test.ts +++ b/packages/junior/tests/component/config/chat-config.test.ts @@ -76,13 +76,13 @@ describe("chat config", () => { expect(botConfig.fastModelId).toBe("openai/gpt-5.6-luna"); }); - it("uses Luna for Guardian when no override is configured", async () => { + it("uses Jev for Guardian when no override is configured", async () => { process.env.AI_MODEL = "anthropic/claude-opus-4.6"; process.env.AI_FAST_MODEL = "anthropic/claude-haiku-4.5"; delete process.env.AI_GUARDIAN_MODEL; const { botConfig } = await loadConfig(); - expect(botConfig.guardianModelId).toBe("openai/gpt-5.6-luna"); + expect(botConfig.guardianModelId).toBe("typesafe-ai/jev"); }); it("uses the configured Guardian model override", async () => { diff --git a/packages/junior/vitest.config.ts b/packages/junior/vitest.config.ts index e73b041641..31e33fede9 100644 --- a/packages/junior/vitest.config.ts +++ b/packages/junior/vitest.config.ts @@ -8,6 +8,9 @@ const packageRoot = process.cwd(); loadJuniorTestEnvFiles({ workspaceRoot, packageRoots: [packageRoot] }); process.env.AI_GATEWAY_API_KEY = "test-gateway-key"; +// The shared fake LLM implements chat completions, not evaluation models. +// Guardian evals cover the Jev path through the real AI Gateway protocol. +process.env.AI_GUARDIAN_MODEL = "openai/gpt-5.6-luna"; process.env.JUNIOR_SECRET = "junior-test-secret"; process.env.JUNIOR_STATE_ADAPTER = "memory"; process.env.JUNIOR_STATE_KEY_PREFIX ??= `junior:test:${process.pid}`; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bd7115ba00..cea11f18b3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -182,6 +182,9 @@ importers: '@ai-sdk/gateway': specifier: ^3.0.119 version: 3.0.119(zod@4.5.4) + '@ai-sdk/gateway-v4': + specifier: npm:@ai-sdk/gateway@4.0.85 + version: '@ai-sdk/gateway@4.0.85(zod@4.5.4)' '@chat-adapter/slack': specifier: 4.29.0 version: 4.29.0(ai@6.0.190(zod@4.5.4))(zod@4.5.4) @@ -245,6 +248,9 @@ importers: ai: specifier: 6.0.190 version: 6.0.190(zod@4.5.4) + ai-v7: + specifier: npm:ai@7.0.105 + version: ai@7.0.105(zod@4.5.4) chat: specifier: 4.29.0 version: 4.29.0(ai@6.0.190(zod@4.5.4))(zod@4.5.4) @@ -744,16 +750,32 @@ packages: peerDependencies: zod: ^3.25.76 || ^4.1.8 + '@ai-sdk/gateway@4.0.85': + resolution: {integrity: sha512-c8ztST/CslupqO3p7izsAuG3TlKg9mhE6mGJ42iUADsSMJIV5IOKd9AsLG33QCAY7SWs6NDO2giM/rq7bUTNlQ==} + engines: {node: '>=22'} + peerDependencies: + zod: ^3.25.76 || ^4.1.8 + '@ai-sdk/provider-utils@4.0.27': resolution: {integrity: sha512-ubkAJ+xODouwtmN1tYlvTPphH1hPOBfZaEQe8U7skGvFAnIRs9PPpsq57bC2+Ky/MB4yzhd6YOsxTAx9sGpazw==} engines: {node: '>=18'} peerDependencies: zod: ^3.25.76 || ^4.1.8 + '@ai-sdk/provider-utils@5.0.43': + resolution: {integrity: sha512-gw/bcNseOGSs59TMtV4H1KwqXXe24NHgx+uBYr98pa4Fg6Uvp8hPPxjuckVnfFFyIxHCDew8so0ujpvcSuyMZA==} + engines: {node: '>=22'} + peerDependencies: + zod: ^3.25.76 || ^4.1.8 + '@ai-sdk/provider@3.0.10': resolution: {integrity: sha512-Q3BZ27qfpYqnCYGvE3vt+Qi6LGOF9R5Nmzn+9JoM1lCRsD9mYaIhfJLkSunN48nfGXJ6n+XNV0J/XVpqGQl7Dw==} engines: {node: '>=18'} + '@ai-sdk/provider@4.0.17': + resolution: {integrity: sha512-VYMBxIQdcHqbIf1j+YZlI9Ati6LZ4wJe0GGd4z4a5H/KxTggjeOiyaVYTnfF7LHZK5jMQ+rofmzz4QPqf++NUw==} + engines: {node: '>=22'} + '@andrewbranch/untar.js@1.0.4': resolution: {integrity: sha512-pVXSwPsLuw8IGLo2Di0EaOfsk+ntVvpkk942J/sHYIkwvtKUakEcPh7HBgZ6tuimgzKSEHgCvO4XgQ05DEbwDw==} @@ -4462,6 +4484,9 @@ packages: '@vscode/l10n@0.0.18': resolution: {integrity: sha512-KYSIHVmslkaCDyw013pphY+d7x1qV8IZupYfeIfzNA+nsaWHbn5uPuQRvdRFsa9zFzGeudPuoGoZ1Op4jrJXIQ==} + '@workflow/serde@4.1.0': + resolution: {integrity: sha512-pav4F2BoirECWR7Nf1TKt+2eETcBj7jj4cBefQ8VXQCA6NPkaKeLfj/zMgi+3zYV5ZIBT4GuUiphsj0/b9hPQQ==} + '@workflow/serde@4.1.0-beta.2': resolution: {integrity: sha512-8kkeoQKLDaKXefjV5dbhBj2aErfKp1Mc4pb6tj8144cF+Em5SPbyMbyLCHp+BVrFfFVCBluCtMx+jjvaFVZGww==} @@ -4650,6 +4675,12 @@ packages: peerDependencies: zod: ^3.25.76 || ^4.1.8 + ai@7.0.105: + resolution: {integrity: sha512-gkV3W+tTtWwQ41KNV/KjV8wabBrk+bnbnZ3f0AWp/T9LLyljose8uhFViKleVy7tIdEiR2PiS4BpbNdcExf7rQ==} + engines: {node: '>=22'} + peerDependencies: + zod: ^3.25.76 || ^4.1.8 + ajv-draft-04@1.0.0: resolution: {integrity: sha512-mv00Te6nmYbRp5DCwclxtt7yV/joXJPGS7nM+97GdxvuttCOfgI3K4U25zboyeX0O+myI8ERluxQe5wljMmVIw==} peerDependencies: @@ -9023,6 +9054,13 @@ snapshots: '@vercel/oidc': 3.2.0 zod: 4.5.4 + '@ai-sdk/gateway@4.0.85(zod@4.5.4)': + dependencies: + '@ai-sdk/provider': 4.0.17 + '@ai-sdk/provider-utils': 5.0.43(zod@4.5.4) + '@vercel/oidc': 3.2.0 + zod: 4.5.4 + '@ai-sdk/provider-utils@4.0.27(zod@4.5.4)': dependencies: '@ai-sdk/provider': 3.0.10 @@ -9030,10 +9068,23 @@ snapshots: eventsource-parser: 3.0.8 zod: 4.5.4 + '@ai-sdk/provider-utils@5.0.43(zod@4.5.4)': + dependencies: + '@ai-sdk/provider': 4.0.17 + '@standard-schema/spec': 1.1.0 + '@workflow/serde': 4.1.0 + eventsource-parser: 3.0.8 + undici: 7.29.0 + zod: 4.5.4 + '@ai-sdk/provider@3.0.10': dependencies: json-schema: 0.4.0 + '@ai-sdk/provider@4.0.17': + dependencies: + json-schema: 0.4.0 + '@andrewbranch/untar.js@1.0.4': {} '@anthropic-ai/sdk@0.123.0(zod@4.5.4)': @@ -13201,6 +13252,8 @@ snapshots: '@vscode/l10n@0.0.18': {} + '@workflow/serde@4.1.0': {} + '@workflow/serde@4.1.0-beta.2': {} '@yuku-codegen/binding-android-arm64@0.9.4': @@ -13322,6 +13375,13 @@ snapshots: '@opentelemetry/api': 1.9.1 zod: 4.5.4 + ai@7.0.105(zod@4.5.4): + dependencies: + '@ai-sdk/gateway': 4.0.85(zod@4.5.4) + '@ai-sdk/provider': 4.0.17 + '@ai-sdk/provider-utils': 5.0.43(zod@4.5.4) + zod: 4.5.4 + ajv-draft-04@1.0.0(ajv@8.20.0): optionalDependencies: ajv: 8.20.0 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 0112f40071..615da02a2a 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -34,6 +34,11 @@ syncInjectedDepsAfterScripts: - build minimumReleaseAge: 1440 minimumReleaseAgeExclude: + # Temporary exception for the Jev evaluation model spike. + - "ai@7.0.105" + - "@ai-sdk/gateway@4.0.85" + - "@ai-sdk/provider@4.0.17" + - "@ai-sdk/provider-utils@5.0.43" - "@sentry/starlight-theme" - "@vitest-evals/core@0.16.1" - "@vitest-evals/report-ui@0.16.1"