diff --git a/app/api/vapi/generate/route.ts b/app/api/vapi/generate/route.ts index 7305425..19760bf 100644 --- a/app/api/vapi/generate/route.ts +++ b/app/api/vapi/generate/route.ts @@ -10,6 +10,7 @@ import { generateText } from "ai"; import { z } from "zod"; import { env } from "@/config/env"; +import { GEMINI_FLASH_MODEL } from "@/config/llm"; import { opLogger } from "@/server/observability/logger"; @@ -57,7 +58,7 @@ export async function POST(request: Request) { try { const { text } = await generateText({ - model: google("gemini-2.0-flash-001"), + model: google(GEMINI_FLASH_MODEL), prompt: `Prepare questions for a job interview. The job role is ${role}. The job experience level is ${level}. diff --git a/config/llm.ts b/config/llm.ts new file mode 100644 index 0000000..263def6 --- /dev/null +++ b/config/llm.ts @@ -0,0 +1,9 @@ +// Model ids in one place. Google retires Gemini ids without notice (gemini-2.0-flash-001 was +// rejected on 2026-09-20 with a pointer to gemini-3.6-flash), so nothing hard-codes a model. +// P2.3 replaces this with config/providers.ts (per-task routing + Groq fallback) and a contract +// test that fails when a configured model stops being served. +// +// Newer Flash models spend output tokens on reasoning before the answer: do not set a small +// maxOutputTokens on structured-output calls or the JSON gets truncated. + +export const GEMINI_FLASH_MODEL = "gemini-3.6-flash"; diff --git a/constants/index.ts b/constants/index.ts index 3b98d37..23bb83a 100644 --- a/constants/index.ts +++ b/constants/index.ts @@ -59,35 +59,28 @@ End the conversation on a polite and positive note. }, }; +export const FEEDBACK_CATEGORIES = [ + "Communication Skills", + "Technical Knowledge", + "Problem Solving", + "Cultural Fit", + "Confidence and Clarity", +] as const; + +// Structured-output schema. Deliberately a bounded array with an enum `name` rather than a tuple: +// tuples compile to JSON Schema `items: [...]`, which Google's structured-output API rejects. +// Moves to server/llm/schemas/feedback.ts in P2.3 (v2 with per-question scoring in P4.4). export const feedbackSchema = z.object({ - totalScore: z.number(), - categoryScores: z.tuple([ - z.object({ - name: z.literal("Communication Skills"), - score: z.number(), - comment: z.string(), - }), - z.object({ - name: z.literal("Technical Knowledge"), - score: z.number(), - comment: z.string(), - }), - z.object({ - name: z.literal("Problem Solving"), - score: z.number(), - comment: z.string(), - }), - z.object({ - name: z.literal("Cultural Fit"), - score: z.number(), - comment: z.string(), - }), - z.object({ - name: z.literal("Confidence and Clarity"), - score: z.number(), - comment: z.string(), - }), - ]), + totalScore: z.number().min(0).max(100), + categoryScores: z + .array( + z.object({ + name: z.enum(FEEDBACK_CATEGORIES), + score: z.number().min(0).max(100), + comment: z.string(), + }), + ) + .length(FEEDBACK_CATEGORIES.length), strengths: z.array(z.string()), areasForImprovement: z.array(z.string()), finalAssessment: z.string(), diff --git a/docs/MILESTONES.md b/docs/MILESTONES.md index 0ca511d..a981e94 100644 --- a/docs/MILESTONES.md +++ b/docs/MILESTONES.md @@ -323,7 +323,7 @@ Read these rows before starting the phase; link them in the PR that closes them. | P1.3 | R17 (`!=` + `orderBy` query) | | P1.4 | **R2 (open endpoints)**, R12 | | P1.5 | R3 (verify Upstash limits) | -| P2.3 | R3 (Gemini/Groq limits), R24 (global daily caps), **R26 (valid Gemini key required)** | +| P2.3 | R3 (Gemini/Groq limits), R24 (global daily caps), **R27 (model-availability contract test, 429 fallback)** | | P2.5 | **R8 (prod migration procedure)** | | P2.6 | R4 (R2 card check → Supabase fallback), R5 (never Firebase Storage), R9 (CSP domains) | | P3 start | **R10 (do not start before P2 exit)** | diff --git a/docs/RISKS.md b/docs/RISKS.md index a9a2e27..a1d40fa 100644 --- a/docs/RISKS.md +++ b/docs/RISKS.md @@ -31,7 +31,8 @@ Severity: **S1** = security or data loss · **S2** = outage / quota burn / broke | R23 | S3 | Vercel Hobby is licensed for non-commercial use. | If monetisation is ever revisited | Any ADR superseding ADR-0002 must also cover hosting. | Standing | | R24 | S2 | Groq/Gemini free tiers rate-limit by requests per minute and per day; a burst of users at launch can exhaust the global daily cap early. | P2.3, P4.1 | Generation cache by input hash (P4.1), global daily counters in Redis with provider fallback (P2.3), and per-user daily caps. Alert in PostHog when 80 % of a daily cap is reached. | Open | | R25 | S3 | Stray uncommitted edits exist in the working tree (`app/api/auth/signout/route.ts` whitespace, `components/SettingsMenu.tsx` CRLF) alongside the handbook. | First commit | Commit the handbook separately (`docs:`); revert the two stray edits (`git checkout -- `) unless intentional. | Closed (`e11dd62`) | -| R26 | S2 | The stored `GEMINI_API_KEY` is rejected by Google ("API key not valid"); it predates the rebuild. Every LLM path (question generation, feedback) fails until it is replaced, and the free-tier limits (R3) cannot be verified. | Now; P2.3 contract tests need it | Generate a new key in Google AI Studio (free), update `.env.local` and the Vercel project env. Verify with the structured-output smoke call from PR 0.4 before P2.3. | Open | +| R26 | S2 | The stored `GEMINI_API_KEY` is rejected by Google ("API key not valid"); it predates the rebuild. Every LLM path (question generation, feedback) fails until it is replaced, and the free-tier limits (R3) cannot be verified. | Now; P2.3 contract tests need it | Generate a new key in Google AI Studio (free), update `.env.local` and the Vercel project env. Verify with the structured-output smoke call from PR 0.4 before P2.3. | Closed (2026-09-20): new key verified with a live structured-output call | +| R27 | S2 | Google retires Gemini model ids without notice (`gemini-2.0-flash-001` rejected 2026-09-20 → `gemini-3.6-flash`), and structured output now rejects tuple schemas — both broke feedback generation silently. Newer Flash models also spend output tokens on reasoning (a small `maxOutputTokens` truncates JSON) and free-tier RPM limits produce transient `RetryError`s on back-to-back calls. | Now; P2.3; every phase | Model ids only in `config/llm.ts` (→ `config/providers.ts` in P2.3) with a contract test that fails when a configured model is not served; schemas stay JSON-Schema-plain (test in `feedback.schema.test.ts`); no small output caps on structured calls; P2.3 router retries with Groq on 429/503; R24 caps. | Open (mitigated for feedback; contract test → P2.3) | ## Closing a row diff --git a/server/services/feedback.schema.test.ts b/server/services/feedback.schema.test.ts new file mode 100644 index 0000000..95d66c2 --- /dev/null +++ b/server/services/feedback.schema.test.ts @@ -0,0 +1,36 @@ +import { zodSchema } from "ai"; +import { describe, expect, it } from "vitest"; + +import { FEEDBACK_CATEGORIES, feedbackSchema } from "@/constants"; + +// Regression guard: Google's structured-output API rejects JSON Schema tuples (`items` as an +// array). The schema must compile to plain object/array shapes. +describe("feedbackSchema JSON Schema compatibility", () => { + it("compiles categoryScores to an array with an object `items`", () => { + const json = zodSchema(feedbackSchema).jsonSchema as { + properties: { categoryScores: { type: string; items: unknown; minItems?: number } }; + }; + const categoryScores = json.properties.categoryScores; + expect(categoryScores.type).toBe("array"); + expect(Array.isArray(categoryScores.items)).toBe(false); + expect(typeof categoryScores.items).toBe("object"); + expect(categoryScores.minItems).toBe(FEEDBACK_CATEGORIES.length); + }); + + it("accepts a well-formed model response and rejects unknown categories", () => { + const valid = { + totalScore: 72, + categoryScores: FEEDBACK_CATEGORIES.map((name) => ({ name, score: 70, comment: "ok" })), + strengths: ["clear"], + areasForImprovement: ["depth"], + finalAssessment: "Solid.", + }; + expect(feedbackSchema.safeParse(valid).success).toBe(true); + expect( + feedbackSchema.safeParse({ + ...valid, + categoryScores: [{ name: "Vibes", score: 1, comment: "" }], + }).success, + ).toBe(false); + }); +}); diff --git a/server/services/feedback.service.ts b/server/services/feedback.service.ts index bf1fddf..7b57711 100644 --- a/server/services/feedback.service.ts +++ b/server/services/feedback.service.ts @@ -5,6 +5,7 @@ import { createGoogle } from "@ai-sdk/google"; import { generateText, Output } from "ai"; import { env } from "@/config/env"; +import { GEMINI_FLASH_MODEL } from "@/config/llm"; import { AppError } from "@/server/errors"; import { opLogger } from "@/server/observability/logger"; @@ -56,7 +57,7 @@ export async function generateAndStoreFeedback(params: { const started = Date.now(); const { output } = await generateText({ - model: google("gemini-2.0-flash-001"), + model: google(GEMINI_FLASH_MODEL), output: Output.object({ schema: feedbackSchema }), system: "You are a professional interviewer analyzing a mock interview. Evaluate the candidate on structured categories.",