diff --git a/e2e-live/live-analysis.pw.ts b/e2e-live/live-analysis.pw.ts index 7fe2888..861f643 100644 --- a/e2e-live/live-analysis.pw.ts +++ b/e2e-live/live-analysis.pw.ts @@ -19,39 +19,29 @@ test.describe("opt-in live browser analysis", () => { page, }) => { test.setTimeout(10 * 60_000); + const startedAt = performance.now(); + const timings: Array<{ stage: string; elapsedMs: number }> = []; + const mark = (stage: string) => + timings.push({ stage, elapsedMs: Math.round(performance.now() - startedAt) }); const browserErrors: string[] = []; page.on("console", (message) => { if (message.type() === "error") browserErrors.push(message.text()); }); page.on("pageerror", (error) => browserErrors.push(error.message)); - await page.goto("/"); + await page.goto("/analyze"); if (source?.kind === "upload") { await page.locator('input[type="file"]').setInputFiles(source.path); } else if (source?.kind === "url") { await page.getByLabel("Video URL").fill(source.url); } await page.getByRole("button", { name: "Check it" }).click(); + mark("submitted"); await expect( - page.getByText(/Downloading the source video|Staging the uploaded video/), - ).toBeVisible({ timeout: 2 * 60_000 }); - await expect(page.getByText("Preparing the video with Gemini")).toBeVisible({ - timeout: 3 * 60_000, - }); - await expect( - page.getByText("Extracting transcript and financial claims"), - ).toBeVisible({ timeout: 3 * 60_000 }); - await expect(page.getByText(/Verifying \d+ checkable claims?/)).toBeVisible({ - timeout: 5 * 60_000, - }); - await expect(page.getByText("Building the CapCheck scorecard")).toBeVisible({ - timeout: 5 * 60_000, - }); - - await expect(page.getByRole("img", { name: /Cap Score \d+ out of 100/ })).toBeVisible({ - timeout: 10 * 60_000, - }); + page.locator('[aria-roledescription="Cap Score"]'), + ).toBeVisible({ timeout: 10 * 60_000 }); + mark("complete"); await expect(page.getByRole("tab", { name: /Claims reviewed/ })).toBeVisible(); const claimCards = page.locator("details.claim"); @@ -64,5 +54,9 @@ test.describe("opt-in live browser analysis", () => { page.getByRole("link", { name: /Open source:.*opens in new tab/ }).first(), ).toBeVisible(); expect(browserErrors).toEqual([]); + await test.info().attach("live-stage-timings.json", { + body: Buffer.from(JSON.stringify(timings, null, 2)), + contentType: "application/json", + }); }); }); diff --git a/playwright.live.config.ts b/playwright.live.config.ts index 1a14b9f..47c8dca 100644 --- a/playwright.live.config.ts +++ b/playwright.live.config.ts @@ -8,6 +8,8 @@ const liveEnvironment = Object.fromEntries( // Prevent a local dotenv file from silently turning this live smoke into the // deterministic fixture path. liveEnvironment.CAPCHECK_ANALYSIS_MODE = "live"; +const livePort = process.env.CAPCHECK_LIVE_PORT ?? "3317"; +const liveBaseUrl = `http://127.0.0.1:${livePort}`; export default defineConfig({ testDir: "./e2e-live", @@ -17,9 +19,11 @@ export default defineConfig({ retries: 0, reporter: "list", use: { - baseURL: "http://127.0.0.1:3100", + baseURL: liveBaseUrl, screenshot: "only-on-failure", - trace: "retain-on-failure", + trace: + process.env.CAPCHECK_LIVE_TRACE === "1" ? "on" : "retain-on-failure", + video: process.env.CAPCHECK_LIVE_VIDEO === "1" ? "on" : "off", }, projects: [ { @@ -28,8 +32,8 @@ export default defineConfig({ }, ], webServer: { - command: "npm run dev -- --hostname 127.0.0.1 --port 3100", - url: "http://127.0.0.1:3100", + command: `npm run dev -- --hostname 127.0.0.1 --port ${livePort}`, + url: `${liveBaseUrl}/analyze`, env: liveEnvironment, reuseExistingServer: false, timeout: 120_000, diff --git a/src/server/analysis/claim-extraction.ts b/src/server/analysis/claim-extraction.ts index 1026cb5..3181206 100644 --- a/src/server/analysis/claim-extraction.ts +++ b/src/server/analysis/claim-extraction.ts @@ -2,10 +2,7 @@ import { z } from "zod"; import { ClaimSchema } from "@/domain/analysis"; import type { ProgressEvent } from "@/domain/analysis"; -import type { - ActiveGeminiFile, - VideoIngestionSource, -} from "@/server/ingestion/video-ingestion"; +import type { VideoIngestionSource } from "@/server/ingestion/video-ingestion"; export const QuantitativeClaimSchema = z .object({ @@ -53,17 +50,23 @@ type ExtractionOptions = { onProgress(event: ProgressEvent): void; }; +type GeminiVideoReference = { + name?: string; + uri: string; + mimeType?: string; +}; + type ClaimExtractionIngestor = { withActiveFile( source: VideoIngestionSource, options: ExtractionOptions, - consume: (file: ActiveGeminiFile) => Promise, + consume: (file: GeminiVideoReference) => Promise, ): Promise; }; type GeminiClaimGenerator = { generate(input: { - file: ActiveGeminiFile; + file: GeminiVideoReference; signal: AbortSignal; }): Promise; }; diff --git a/src/server/analysis/gemini-claim-generator.test.ts b/src/server/analysis/gemini-claim-generator.test.ts index c9e91c6..4b4040a 100644 --- a/src/server/analysis/gemini-claim-generator.test.ts +++ b/src/server/analysis/gemini-claim-generator.test.ts @@ -92,6 +92,7 @@ describe("Gemini claim generator", () => { "Every claim containing a number", ); expect(body.generationConfig).toMatchObject({ + thinkingConfig: { thinkingLevel: "low" }, responseMimeType: "application/json", responseJsonSchema: { type: "object", diff --git a/src/server/analysis/gemini-claim-generator.ts b/src/server/analysis/gemini-claim-generator.ts index 338c310..0d44050 100644 --- a/src/server/analysis/gemini-claim-generator.ts +++ b/src/server/analysis/gemini-claim-generator.ts @@ -1,5 +1,3 @@ -import type { ActiveGeminiFile } from "@/server/ingestion/video-ingestion"; - const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com"; const DEFAULT_MODEL = "gemini-3.5-flash"; const transientStatuses = new Set([408, 429, 500, 502, 503, 504]); @@ -105,7 +103,11 @@ type GeminiClaimGeneratorOptions = { }; type GenerateInput = { - file: ActiveGeminiFile; + file: { + name?: string; + uri: string; + mimeType?: string; + }; signal: AbortSignal; }; @@ -196,7 +198,7 @@ export function createGeminiClaimGenerator({ { fileData: { fileUri: file.uri, - mimeType: file.mimeType, + ...(file.mimeType ? { mimeType: file.mimeType } : {}), }, }, { text: extractionPrompt }, @@ -204,6 +206,7 @@ export function createGeminiClaimGenerator({ }, ], generationConfig: { + thinkingConfig: { thinkingLevel: "low" }, responseMimeType: "application/json", responseJsonSchema, }, diff --git a/src/server/analysis/gemini-claim-verifier.test.ts b/src/server/analysis/gemini-claim-verifier.test.ts index e945ff6..2e19ce6 100644 --- a/src/server/analysis/gemini-claim-verifier.test.ts +++ b/src/server/analysis/gemini-claim-verifier.test.ts @@ -180,7 +180,9 @@ describe("Gemini claim verifier", () => { "The company increased revenue by five percent.", ); expect(body.contents[0].parts[0].text).toContain("Use Google Search"); - expect(body.generationConfig).toBeUndefined(); + expect(body.generationConfig).toEqual({ + thinkingConfig: { thinkingLevel: "low" }, + }); const classificationBody = JSON.parse(String(fetch.mock.calls[1][1]?.body)); expect(classificationBody.contents[1]).toEqual(signedSearchTurn); expect(classificationBody.generationConfig.responseJsonSchema.required).toEqual([ @@ -188,6 +190,9 @@ describe("Gemini claim verifier", () => { "confidence", "explanation", ]); + expect(classificationBody.generationConfig.thinkingConfig).toEqual({ + thinkingLevel: "low", + }); }); it("does not grant authority from hostile page titles or substring hostnames", async () => { diff --git a/src/server/analysis/gemini-claim-verifier.ts b/src/server/analysis/gemini-claim-verifier.ts index 502822d..ab0fca4 100644 --- a/src/server/analysis/gemini-claim-verifier.ts +++ b/src/server/analysis/gemini-claim-verifier.ts @@ -321,6 +321,16 @@ export function createGeminiClaimVerifier({ } const request = async (body: unknown, signal: AbortSignal) => { + const requestBody = + body && typeof body === "object" && !Array.isArray(body) + ? { + ...body, + generationConfig: { + ...((body as { generationConfig?: object }).generationConfig ?? {}), + thinkingConfig: { thinkingLevel: "low" }, + }, + } + : body; let response: Response | undefined; for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { const timeout = new AbortController(); @@ -337,7 +347,7 @@ export function createGeminiClaimVerifier({ "Content-Type": "application/json", "X-Goog-Api-Key": apiKey, }, - body: JSON.stringify(body), + body: JSON.stringify(requestBody), signal: AbortSignal.any([signal, timeout.signal]), }, ); diff --git a/src/server/analysis/gemini-scorecard-synthesizer.test.ts b/src/server/analysis/gemini-scorecard-synthesizer.test.ts index 8bcb328..8a6d490 100644 --- a/src/server/analysis/gemini-scorecard-synthesizer.test.ts +++ b/src/server/analysis/gemini-scorecard-synthesizer.test.ts @@ -90,6 +90,7 @@ describe("Gemini scorecard synthesizer", () => { }); const request = JSON.parse(fetch.mock.calls[0][1].body as string); expect(request.generationConfig).toMatchObject({ + thinkingConfig: { thinkingLevel: "low" }, responseMimeType: "application/json", responseJsonSchema: { properties: { diff --git a/src/server/analysis/gemini-scorecard-synthesizer.ts b/src/server/analysis/gemini-scorecard-synthesizer.ts index 432027f..ebd2aad 100644 --- a/src/server/analysis/gemini-scorecard-synthesizer.ts +++ b/src/server/analysis/gemini-scorecard-synthesizer.ts @@ -163,6 +163,7 @@ export function createGeminiScorecardSynthesizer({ }, ], generationConfig: { + thinkingConfig: { thinkingLevel: "low" }, responseMimeType: "application/json", responseJsonSchema, }, diff --git a/src/server/analysis/live-analysis.live.test.ts b/src/server/analysis/live-analysis.live.test.ts index 3eec2ce..738e0ec 100644 --- a/src/server/analysis/live-analysis.live.test.ts +++ b/src/server/analysis/live-analysis.live.test.ts @@ -46,10 +46,28 @@ describe.skipIf(!enabled)("live analysis smoke", () => { finnhubApiKey: process.env.FINNHUB_KEY!, }); const events = []; + const startedAt = performance.now(); + const timings: Array<{ + stage: string; + elapsedMs: number; + detail?: string; + }> = []; for await (const event of stream(source, new AbortController().signal)) { events.push(AnalysisEventSchema.parse(event)); + timings.push({ + stage: event.type === "progress" ? event.stage : event.type, + elapsedMs: Math.round(performance.now() - startedAt), + detail: + event.type === "progress" + ? event.message + : event.type === "complete" + ? `${event.scorecard.verifications.length} verified, ${event.scorecard.skippedClaims?.length ?? 0} skipped, Cap Score ${event.scorecard.capScore}` + : event.error.code, + }); } + console.info(`[live-analysis-timing] ${JSON.stringify(timings)}`); + expect( events.flatMap((event) => event.type === "progress" ? [event.stage] : [], diff --git a/src/server/analysis/node-claim-extraction-pipeline.test.ts b/src/server/analysis/node-claim-extraction-pipeline.test.ts index d3eebb8..4ce93e0 100644 --- a/src/server/analysis/node-claim-extraction-pipeline.test.ts +++ b/src/server/analysis/node-claim-extraction-pipeline.test.ts @@ -9,6 +9,98 @@ const MP4_BYTES = Uint8Array.from([ ]); describe("createNodeClaimExtractionPipeline", () => { + it("sends a public YouTube URL directly to Gemini without downloading or uploading it", async () => { + const extraction = { + transcript: [{ timestampSeconds: 1, text: "Index funds reduce risk." }], + claims: [ + { + id: "claim-1", + text: "Index funds reduce risk.", + timestampSeconds: 1, + kind: "factual", + checkable: true, + }, + ], + }; + const fetch = vi.fn().mockResolvedValue( + Response.json({ + candidates: [ + { content: { parts: [{ text: JSON.stringify(extraction) }] } }, + ], + }), + ); + const ytDlpRun = vi.fn(); + const pipeline = createNodeClaimExtractionPipeline({ + apiKey: "test-api-key", + fetch, + ytDlpRun, + }); + const progress: unknown[] = []; + + await expect( + pipeline.extract( + { kind: "url", url: "https://www.youtube.com/shorts/demo" }, + { + signal: new AbortController().signal, + onProgress: (event) => progress.push(event), + }, + ), + ).resolves.toEqual(extraction); + + expect(ytDlpRun).not.toHaveBeenCalled(); + expect(fetch).toHaveBeenCalledOnce(); + const body = JSON.parse(String(fetch.mock.calls[0][1]?.body)); + expect(body.contents[0].parts[0]).toEqual({ + fileData: { + fileUri: "https://www.youtube.com/watch?v=demo", + }, + }); + expect(progress).toEqual([ + { + type: "progress", + stage: "fetching", + message: "Sending the public YouTube video to Gemini", + }, + { + type: "progress", + stage: "processing", + message: "Preparing the video with Gemini", + }, + { + type: "progress", + stage: "extracting", + message: "Extracting transcript and financial claims", + }, + ]); + }); + + it("falls back to yt-dlp when Gemini rejects the direct YouTube URL", async () => { + const fetch = vi.fn().mockResolvedValue(new Response(null, { status: 503 })); + const ytDlpRun = vi.fn().mockResolvedValue({ + exitCode: 1, + stdout: "", + stderr: "source unavailable", + }); + const pipeline = createNodeClaimExtractionPipeline({ + apiKey: "test-api-key", + fetch, + ytDlpRun, + }); + + await expect( + pipeline.extract( + { kind: "url", url: "https://youtu.be/demo" }, + { + signal: new AbortController().signal, + onProgress: () => undefined, + }, + ), + ).rejects.toMatchObject({ code: "SOURCE_VIDEO_UNAVAILABLE" }); + + expect(fetch).toHaveBeenCalledOnce(); + expect(ytDlpRun).toHaveBeenCalledOnce(); + }); + it("extracts claims inside the ACTIVE-file lease and leaves cleanup to ingestion", async () => { const extraction = { transcript: [ diff --git a/src/server/analysis/node-claim-extraction-pipeline.ts b/src/server/analysis/node-claim-extraction-pipeline.ts index 8ade98e..95382be 100644 --- a/src/server/analysis/node-claim-extraction-pipeline.ts +++ b/src/server/analysis/node-claim-extraction-pipeline.ts @@ -2,6 +2,7 @@ import { createNodeVideoIngestor } from "@/server/ingestion/node-video-ingestor" import type { VideoIngestionPolicy, } from "@/server/ingestion/video-ingestion"; +import { normalizeYouTubeVideoUrl } from "@/server/ingestion/video-ingestion"; import type { ProcessRunner } from "@/server/ingestion/yt-dlp"; import { createClaimExtractionPipeline } from "./claim-extraction"; @@ -26,14 +27,44 @@ export function createNodeClaimExtractionPipeline({ ytDlpExecutable, ytDlpRun, }: NodeClaimExtractionPipelineOptions) { + const fileIngestor = createNodeVideoIngestor({ + apiKey, + fetch, + policy: ingestionPolicy, + ytDlpExecutable, + ytDlpRun, + }); + return createClaimExtractionPipeline({ - ingestor: createNodeVideoIngestor({ - apiKey, - fetch, - policy: ingestionPolicy, - ytDlpExecutable, - ytDlpRun, - }), + ingestor: { + async withActiveFile(source, options, consume) { + const directUrl = + source.kind === "url" + ? normalizeYouTubeVideoUrl(source.url) + : undefined; + if (!directUrl) { + return fileIngestor.withActiveFile(source, options, consume); + } + + options.onProgress({ + type: "progress", + stage: "fetching", + message: "Sending the public YouTube video to Gemini", + }); + options.onProgress({ + type: "progress", + stage: "processing", + message: "Preparing the video with Gemini", + }); + + try { + return await consume({ uri: directUrl }); + } catch (cause) { + if (options.signal.aborted) throw cause; + return fileIngestor.withActiveFile(source, options, consume); + } + }, + }, gemini: createGeminiClaimGenerator({ apiKey, fetch, diff --git a/src/server/ingestion/video-ingestion.ts b/src/server/ingestion/video-ingestion.ts index 5528bca..beb22a7 100644 --- a/src/server/ingestion/video-ingestion.ts +++ b/src/server/ingestion/video-ingestion.ts @@ -185,7 +185,7 @@ const tikTokHosts = new Set([ "vt.tiktok.com", ]); -const isSupportedVideoUrl = (value: string) => { +export const normalizeYouTubeVideoUrl = (value: string) => { try { const url = new URL(value); if ( @@ -193,15 +193,45 @@ const isSupportedVideoUrl = (value: string) => { url.username || url.password ) { - return false; + return undefined; } const parts = url.pathname.split("/").filter(Boolean); + let videoId: string | undefined; if (youtubeHosts.has(url.hostname)) { - if (parts[0] === "shorts") return Boolean(parts[1]); - return parts[0] === "watch" && Boolean(url.searchParams.get("v")); + videoId = + parts[0] === "shorts" && parts[1] + ? parts[1] + : parts[0] === "watch" + ? url.searchParams.get("v") ?? undefined + : undefined; + } else if (url.hostname === "youtu.be") { + videoId = parts[0]; + } + return videoId + ? `https://www.youtube.com/watch?v=${encodeURIComponent(videoId)}` + : undefined; + } catch { + return undefined; + } +}; + +export const isYouTubeVideoUrl = (value: string) => + normalizeYouTubeVideoUrl(value) !== undefined; + +const isSupportedVideoUrl = (value: string) => { + if (isYouTubeVideoUrl(value)) return true; + try { + const url = new URL(value); + if ( + !["http:", "https:"].includes(url.protocol) || + url.username || + url.password + ) { + return false; } - if (url.hostname === "youtu.be") return Boolean(parts[0]); + + const parts = url.pathname.split("/").filter(Boolean); if (tikTokHosts.has(url.hostname)) return parts.length > 0; return false; } catch {