diff --git a/apps/api/src/__tests__/snips/v2/audio-routing.test.ts b/apps/api/src/__tests__/snips/v2/audio-routing.test.ts index 5c2d1cac3db..1c4cf040777 100644 --- a/apps/api/src/__tests__/snips/v2/audio-routing.test.ts +++ b/apps/api/src/__tests__/snips/v2/audio-routing.test.ts @@ -6,12 +6,16 @@ describe("Audio format engine routing (buildFallbackList)", () => { let buildFallbackList: typeof import("../../../scraper/scrapeURL/engines/index.js").buildFallbackList; + let clearExchangeProvidersForTest: typeof import("../../../lib/exchange.js").clearExchangeProvidersForTest; + let setExchangeProvidersForTest: typeof import("../../../lib/exchange.js").setExchangeProvidersForTest; const originalFireEngineUrl = process.env.FIRE_ENGINE_BETA_URL; const originalIndexUrl = process.env.INDEX_DATABASE_URL; + const originalExchangeUrl = process.env.FIRE_EXCHANGE_URL; beforeAll(async () => { process.env.FIRE_ENGINE_BETA_URL = "http://test-fire-engine"; + process.env.FIRE_EXCHANGE_URL = "http://test-exchange"; process.env.INDEX_DATABASE_URL = "postgresql://postgres:postgres@localhost:5432/postgres"; @@ -20,6 +24,14 @@ describe("Audio format engine routing (buildFallbackList)", () => { ({ buildFallbackList } = await import( "../../../scraper/scrapeURL/engines/index.js" )); + ({ + clearExchangeProvidersForTest, + setExchangeProvidersForTest, + } = await import("../../../lib/exchange.js")); + }); + + afterEach(() => { + clearExchangeProvidersForTest(); }); afterAll(() => { @@ -33,6 +45,11 @@ describe("Audio format engine routing (buildFallbackList)", () => { } else { process.env.INDEX_DATABASE_URL = originalIndexUrl; } + if (originalExchangeUrl === undefined) { + delete process.env.FIRE_EXCHANGE_URL; + } else { + process.env.FIRE_EXCHANGE_URL = originalExchangeUrl; + } }); const buildStubMeta = (featureFlags: string[]) => @@ -110,4 +127,20 @@ describe("Audio format engine routing (buildFallbackList)", () => { expect(engines).toContain("fire-engine;chrome-cdp"); }); + it("does not route agent index-only requests through the Exchange", async () => { + setExchangeProvidersForTest([ + { id: "acme", routes: [{ domains: ["profiles.example"] }] }, + ]); + + const meta = buildStubMeta([]); + meta.url = "https://profiles.example/person/example-person"; + meta.options.formats = [{ type: "markdown" }]; + meta.internalOptions.agentIndexOnly = true; + meta.internalOptions.teamFlags = { professionalProfileCompanyDataBeta: true }; + + const fallback = await buildFallbackList(meta); + const engines = fallback.map(f => f.engine); + + expect(engines).toEqual(["index", "index;documents"]); + }); }); diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts index a372c24e772..9168f7f605d 100644 --- a/apps/api/src/config.ts +++ b/apps/api/src/config.ts @@ -165,6 +165,9 @@ const configSchema = z.object({ HIGHLIGHT_MODEL_TOKEN: z.string().optional(), HIGHLIGHT_SHADOW_RATE: z.coerce.number().min(0).max(1).default(0), + // Exchange (routed data sources service) + FIRE_EXCHANGE_URL: z.url().optional(), + // Fire Engine FIRE_ENGINE_BETA_URL: z.string().optional(), FIRE_ENGINE_STAGING_URL: z.string().optional(), diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 650a4a0f979..a163c46fc6c 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -34,6 +34,7 @@ import { v2Router } from "./routes/v2"; import { nuqShutdown } from "./services/worker/nuq"; import { getErrorContactMessage } from "./lib/deployment"; import { initializeBlocklist } from "./scraper/WebScraper/utils/blocklist"; +import { warmExchangeCatalog } from "./lib/exchange"; import { initializeEngineForcing } from "./scraper/WebScraper/utils/engine-forcing"; import responseTime from "response-time"; import { shutdownWebhookQueue } from "./services/webhook"; @@ -132,6 +133,7 @@ async function startServer(port = DEFAULT_PORT) { try { await initializeBlocklist(); initializeEngineForcing(); + warmExchangeCatalog(); } catch (error) { logger.error("Failed to initialize API startup dependencies", { error, diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.ts index 935ef612b79..aaa59db8e6b 100644 --- a/apps/api/src/lib/error-serde.ts +++ b/apps/api/src/lib/error-serde.ts @@ -39,6 +39,8 @@ import { UnsafeDomainBlockedError } from "./threat-protection/error"; // TODO: figure out correct typing for this const errorMap: Record = { + // Terms responses are API-level, never transported through workers. + THIRD_PARTY_DATA_TERMS_REQUIRED: null, SCRAPE_TIMEOUT: ScrapeJobTimeoutError, MAP_TIMEOUT: MapTimeoutError, UNKNOWN_ERROR: UnknownError, diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.ts index 62df2572bfe..d77a878f860 100644 --- a/apps/api/src/lib/error.ts +++ b/apps/api/src/lib/error.ts @@ -1,4 +1,5 @@ export type ErrorCodes = + | "THIRD_PARTY_DATA_TERMS_REQUIRED" | "SCRAPE_TIMEOUT" | "MAP_TIMEOUT" | "UNKNOWN_ERROR" diff --git a/apps/api/src/lib/exchange-request.ts b/apps/api/src/lib/exchange-request.ts new file mode 100644 index 00000000000..17238155b4d --- /dev/null +++ b/apps/api/src/lib/exchange-request.ts @@ -0,0 +1,47 @@ +import { + getExchangeAccessForRequest, + type ExchangeAccess, +} from "./exchange"; + +type ExchangeFlags = Parameters< + typeof getExchangeAccessForRequest +>[0]["flags"]; + +/** + * Run the Exchange access gate against an API request body, sourcing the + * effective per-page options from a nested scrapeOptions object (crawl + * bodies) or the body itself (scrape and batch scrape bodies), with v1 + * pageOptions fallbacks. + */ +export function getExchangeAccessForRequestBody(input: { + body: Record; + flags: ExchangeFlags; + url: string; + zeroDataRetention: boolean; +}): Promise { + const body = input.body ?? {}; + const scrapeOptions = + typeof body.scrapeOptions === "object" && body.scrapeOptions !== null + ? body.scrapeOptions + : body; + + return getExchangeAccessForRequest({ + url: input.url, + formats: scrapeOptions.formats, + actions: scrapeOptions.actions, + headers: scrapeOptions.headers, + waitFor: scrapeOptions.waitFor, + mobile: scrapeOptions.mobile, + location: scrapeOptions.location, + proxy: scrapeOptions.proxy, + blockAds: scrapeOptions.blockAds, + profile: scrapeOptions.profile, + atsv: scrapeOptions.atsv ?? body.pageOptions?.atsv, + minAge: scrapeOptions.minAge, + includeTags: scrapeOptions.includeTags ?? body.pageOptions?.includeTags, + excludeTags: scrapeOptions.excludeTags ?? body.pageOptions?.excludeTags, + zeroDataRetention: input.zeroDataRetention, + lockdown: scrapeOptions.lockdown ?? body.lockdown, + flags: input.flags, + }); +} diff --git a/apps/api/src/lib/exchange.test.ts b/apps/api/src/lib/exchange.test.ts new file mode 100644 index 00000000000..416e2b627ad --- /dev/null +++ b/apps/api/src/lib/exchange.test.ts @@ -0,0 +1,589 @@ +import { fetch } from "undici"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { config } from "../config"; +import { + canUseExchangeForRequest, + clearExchangeProvidersForTest, + reportExchangeBilling, + getExchangeAccessForRequest, + getExchangeRequestLogContext, + getExchangeResponseLogContext, + getExchangeSuccessCredits, + getThirdPartyDataTermsRequiredResponse, + isSuccessfulExchangeStatusCode, + isSupportedExchangeFormatRequest, + resolveExchangeProvider, + setExchangeProvidersForTest, +} from "./exchange"; + +vi.mock("undici", () => ({ + fetch: vi.fn(), +})); + +const originalConfig = { + FIRE_EXCHANGE_URL: config.FIRE_EXCHANGE_URL, + USE_DB_AUTHENTICATION: config.USE_DB_AUTHENTICATION, +}; + +const ACME_TERMS = { key: "acme", version: "2026-01-01" }; + +const TEST_PROVIDERS = [ + { + id: "acme", + creditsCost: 12, + terms: ACME_TERMS, + routes: [ + { + domains: ["profiles.example", "www.profiles.example"], + pathPrefixes: ["/person/", "/company/"], + }, + ], + }, + { + id: "openfacts", + creditsCost: 0, + routes: [{ domains: ["facts.example"] }], + }, +]; + +const ENABLED_EXCHANGE_FLAGS = { + professionalProfileCompanyDataBeta: true, + organizationDataSourceAccess: { + acme: { + status: "enabled", + termsKey: "acme", + termsVersion: "2026-01-01", + termsAcceptedAt: "2026-01-01T00:00:00.000Z", + enabledAt: "2026-01-01T00:00:00.000Z", + disabledAt: null, + disabledReason: null, + }, + }, +}; + +describe("Exchange routing", () => { + beforeEach(() => { + vi.mocked(fetch).mockReset(); + config.FIRE_EXCHANGE_URL = "https://exchange.example"; + config.USE_DB_AUTHENTICATION = true; + setExchangeProvidersForTest(TEST_PROVIDERS); + }); + + afterEach(() => { + config.FIRE_EXCHANGE_URL = originalConfig.FIRE_EXCHANGE_URL; + config.USE_DB_AUTHENTICATION = originalConfig.USE_DB_AUTHENTICATION; + clearExchangeProvidersForTest(); + }); + + it("resolves URLs to providers using catalog routes", async () => { + await expect( + resolveExchangeProvider( + "https://profiles.example/person/example-person/details/?trk=foo", + ), + ).resolves.toMatchObject({ id: "acme", creditsCost: 12 }); + await expect( + resolveExchangeProvider("https://www.profiles.example/company/example"), + ).resolves.toMatchObject({ id: "acme" }); + await expect( + resolveExchangeProvider("https://facts.example/any/path"), + ).resolves.toMatchObject({ id: "openfacts" }); + await expect( + resolveExchangeProvider("https://profiles.example/jobs/example"), + ).resolves.toBeNull(); + await expect( + resolveExchangeProvider("https://other.example/person/example"), + ).resolves.toBeNull(); + await expect(resolveExchangeProvider("not a url")).resolves.toBeNull(); + + await expect( + resolveExchangeProvider("https://profiles.example/person/example-person"), + ).resolves.not.toBeNull(); + }); + + it("respects path segment boundaries for prefixes without trailing slashes", async () => { + setExchangeProvidersForTest([ + { + id: "acme", + creditsCost: 12, + routes: [{ domains: ["profiles.example"], pathPrefixes: ["/person"] }], + }, + ]); + + await expect( + resolveExchangeProvider("https://profiles.example/person"), + ).resolves.toMatchObject({ id: "acme" }); + await expect( + resolveExchangeProvider("https://profiles.example/person/example"), + ).resolves.toMatchObject({ id: "acme" }); + await expect( + resolveExchangeProvider("https://profiles.example/personality/details"), + ).resolves.toBeNull(); + await expect( + resolveExchangeProvider("https://profiles.example/personnel"), + ).resolves.toBeNull(); + }); + + it("serves the stale catalog while refreshing in the background", async () => { + setExchangeProvidersForTest(TEST_PROVIDERS, -1); + let resolveFetch: (value: unknown) => void = () => {}; + vi.mocked(fetch).mockReturnValue( + new Promise(resolve => { + resolveFetch = resolve; + }) as ReturnType, + ); + + // The catalog is expired and the refresh never resolves during the + // request - the stale value must be served without waiting. + await expect( + resolveExchangeProvider("https://profiles.example/person/example"), + ).resolves.toMatchObject({ id: "acme" }); + expect(fetch).toHaveBeenCalledTimes(1); + + resolveFetch({ ok: false, status: 503 }); + }); + + it("keeps serving the last good catalog when a refresh fails", async () => { + setExchangeProvidersForTest(TEST_PROVIDERS, -1); + vi.mocked(fetch).mockResolvedValue({ + ok: false, + status: 503, + } as Awaited>); + + await expect( + resolveExchangeProvider("https://profiles.example/person/example"), + ).resolves.toMatchObject({ id: "acme" }); + + // Let the background refresh settle - the failure must not clobber + // the catalog, and the failure TTL must throttle the next attempt. + await new Promise(resolve => setTimeout(resolve, 0)); + await expect( + resolveExchangeProvider("https://profiles.example/person/example"), + ).resolves.toMatchObject({ id: "acme" }); + expect(fetch).toHaveBeenCalledTimes(1); + }); + + it("degrades to ineligible when the access check itself throws", async () => { + const explodingFlags = new Proxy( + {}, + { + get() { + throw new Error("flags backend exploded"); + }, + }, + ) as { professionalProfileCompanyDataBeta?: boolean }; + + await expect( + getExchangeAccessForRequest({ + url: "https://profiles.example/person/example-person", + formats: [{ type: "markdown" }], + flags: explodingFlags, + }), + ).resolves.toEqual({ allowed: false, termsRequired: false }); + }); + + it("caches failed provider catalog lookups briefly", async () => { + clearExchangeProvidersForTest(); + vi.mocked(fetch).mockResolvedValue({ + ok: false, + status: 503, + } as Awaited>); + + await expect( + resolveExchangeProvider("https://profiles.example/person/example-person"), + ).resolves.toBeNull(); + await expect( + resolveExchangeProvider("https://profiles.example/person/example-person"), + ).resolves.toBeNull(); + + expect(fetch).toHaveBeenCalledTimes(1); + }); + + it("parses the provider catalog response", async () => { + clearExchangeProvidersForTest(); + vi.mocked(fetch).mockResolvedValue({ + ok: true, + status: 200, + json: async () => ({ + success: true, + data: [ + { + id: "acme", + name: "Acme", + description: "Structured records.", + creditsCost: 12, + terms: ACME_TERMS, + capabilities: { + scrape: { + resourceTypes: ["records"], + includes: [], + urlRoutes: [ + { + domains: ["Profiles.Example"], + pathPrefixes: ["person/"], + }, + ], + }, + }, + }, + { + id: "searchonly", + name: "Search Only", + description: "No scrape routes.", + creditsCost: 0, + capabilities: { search: { modes: ["semantic"] } }, + }, + ], + }), + } as unknown as Awaited>); + + await expect( + resolveExchangeProvider("https://profiles.example/person/example"), + ).resolves.toMatchObject({ id: "acme", terms: ACME_TERMS }); + await expect( + resolveExchangeProvider("https://searchonly.example/anything"), + ).resolves.toBeNull(); + expect(fetch).toHaveBeenCalledTimes(1); + }); + + it("builds a compact request log context", () => { + expect( + getExchangeRequestLogContext( + "https://profiles.example/person/example-person/details/?trk=foo", + ), + ).toEqual({ + url: "https://profiles.example/person/example-person/details/?trk=foo", + host: "profiles.example", + pathPrefix: "person", + }); + + expect(getExchangeRequestLogContext("not a url")).toBeUndefined(); + + // Embedded credentials must never reach the logs. + expect( + getExchangeRequestLogContext( + "https://user:secret@profiles.example/person/example", + ), + ).toEqual({ + url: "https://profiles.example/person/example", + host: "profiles.example", + pathPrefix: "person", + }); + }); + + it("extracts response cache metadata for logs", () => { + expect( + getExchangeResponseLogContext({ + cacheState: "hit", + cachedAt: "2026-06-21T10:00:00.000Z", + cacheAgeMs: 1000, + request_id: "req_123", + extra: "ignored", + }), + ).toEqual({ + cacheState: "hit", + cachedAt: "2026-06-21T10:00:00.000Z", + cacheAgeMs: 1000, + providerRequestId: "req_123", + }); + + expect(getExchangeResponseLogContext(null)).toEqual({}); + }); + + it("accepts only formats the Exchange can return directly", () => { + expect(isSupportedExchangeFormatRequest(undefined)).toBe(true); + expect(isSupportedExchangeFormatRequest([{ type: "markdown" }])).toBe( + true, + ); + expect(isSupportedExchangeFormatRequest(["json"])).toBe(true); + expect( + isSupportedExchangeFormatRequest([ + { type: "markdown" }, + { type: "json" }, + ]), + ).toBe(true); + expect(isSupportedExchangeFormatRequest([{ type: "html" }])).toBe(false); + // deterministicJson extractors run against page HTML, which Exchange + // responses do not carry. + expect(isSupportedExchangeFormatRequest([{ type: "deterministicJson" }])).toBe( + false, + ); + expect(isSupportedExchangeFormatRequest([])).toBe(false); + }); + + it("allows eligible requests when access and terms are current", async () => { + await expect( + getExchangeAccessForRequest({ + url: "https://profiles.example/person/example-person", + formats: [{ type: "markdown" }], + flags: ENABLED_EXCHANGE_FLAGS, + }), + ).resolves.toMatchObject({ + allowed: true, + provider: { id: "acme", creditsCost: 12 }, + }); + + await expect( + canUseExchangeForRequest({ + url: "https://profiles.example/person/example-person", + formats: [{ type: "json" }], + actions: [{ type: "wait" }], + flags: ENABLED_EXCHANGE_FLAGS, + }), + ).resolves.toBe(false); + + await expect( + canUseExchangeForRequest({ + url: "https://profiles.example/person/example-person", + formats: [{ type: "json" }], + zeroDataRetention: true, + flags: ENABLED_EXCHANGE_FLAGS, + }), + ).resolves.toBe(false); + + // Profile-backed scrapes expect session-specific content. + await expect( + canUseExchangeForRequest({ + url: "https://profiles.example/person/example-person", + formats: [{ type: "markdown" }], + profile: { id: "profile-1" }, + flags: ENABLED_EXCHANGE_FLAGS, + }), + ).resolves.toBe(false); + + // atsv requests stay on engines that support the flag. + await expect( + canUseExchangeForRequest({ + url: "https://profiles.example/person/example-person", + formats: [{ type: "markdown" }], + atsv: true, + flags: ENABLED_EXCHANGE_FLAGS, + }), + ).resolves.toBe(false); + + // minAge asks for Firecrawl-cached data, which the Exchange never has. + await expect( + canUseExchangeForRequest({ + url: "https://profiles.example/person/example-person", + formats: [{ type: "markdown" }], + minAge: 3_600_000, + flags: ENABLED_EXCHANGE_FLAGS, + }), + ).resolves.toBe(false); + + // Selector-based filtering does not apply to provider records. + await expect( + canUseExchangeForRequest({ + url: "https://profiles.example/person/example-person", + formats: [{ type: "markdown" }], + includeTags: ["article"], + flags: ENABLED_EXCHANGE_FLAGS, + }), + ).resolves.toBe(false); + await expect( + canUseExchangeForRequest({ + url: "https://profiles.example/person/example-person", + formats: [{ type: "markdown" }], + excludeTags: ["nav"], + flags: ENABLED_EXCHANGE_FLAGS, + }), + ).resolves.toBe(false); + }); + + it("requires the provider's terms before routing", async () => { + const access = await getExchangeAccessForRequest({ + url: "https://profiles.example/person/example-person", + formats: [{ type: "markdown" }], + flags: { professionalProfileCompanyDataBeta: true }, + }); + + expect(access).toEqual({ + allowed: false, + termsRequired: true, + terms: ACME_TERMS, + }); + + expect(getThirdPartyDataTermsRequiredResponse(ACME_TERMS)).toMatchObject({ + success: false, + code: "THIRD_PARTY_DATA_TERMS_REQUIRED", + requiresAction: { + type: "accept_terms", + terms: "acme", + version: "2026-01-01", + }, + }); + }); + + it("requires current terms when the accepted version is stale", async () => { + await expect( + getExchangeAccessForRequest({ + url: "https://profiles.example/person/example-person", + formats: [{ type: "markdown" }], + flags: { + professionalProfileCompanyDataBeta: true, + organizationDataSourceAccess: { + acme: { + status: "enabled", + termsKey: "acme", + termsVersion: "2025-12-01", + }, + }, + }, + }), + ).resolves.toEqual({ + allowed: false, + termsRequired: true, + terms: ACME_TERMS, + }); + }); + + it("does not route or prompt for terms when access is disabled", async () => { + await expect( + getExchangeAccessForRequest({ + url: "https://profiles.example/person/example-person", + formats: [{ type: "markdown" }], + flags: { + professionalProfileCompanyDataBeta: true, + organizationDataSourceAccess: { + acme: { + status: "disabled", + termsKey: "acme", + termsVersion: "2026-01-01", + disabledAt: "2026-01-02T00:00:00.000Z", + disabledReason: "customer_disabled", + }, + }, + }, + }), + ).resolves.toEqual({ allowed: false, termsRequired: false }); + }); + + it("allows providers with no declared terms without acceptance", async () => { + await expect( + getExchangeAccessForRequest({ + url: "https://facts.example/records/1", + formats: [{ type: "markdown" }], + flags: { professionalProfileCompanyDataBeta: true }, + }), + ).resolves.toMatchObject({ allowed: true, provider: { id: "openfacts" } }); + + await expect( + getExchangeAccessForRequest({ + url: "https://facts.example/records/1", + formats: [{ type: "markdown" }], + flags: { + professionalProfileCompanyDataBeta: true, + organizationDataSourceAccess: { + openfacts: { status: "disabled" }, + }, + }, + }), + ).resolves.toEqual({ allowed: false, termsRequired: false }); + }); + + it("does not route unless the beta flag is enabled", async () => { + await expect( + canUseExchangeForRequest({ + url: "https://profiles.example/person/example-person", + formats: [{ type: "markdown" }], + }), + ).resolves.toBe(false); + + await expect( + canUseExchangeForRequest({ + url: "https://profiles.example/person/example-person", + formats: [{ type: "markdown" }], + flags: { professionalProfileCompanyDataBeta: false }, + }), + ).resolves.toBe(false); + }); + + it("does not route unless the Exchange is configured", async () => { + config.FIRE_EXCHANGE_URL = undefined; + + await expect( + canUseExchangeForRequest({ + url: "https://profiles.example/person/example-person", + formats: [{ type: "markdown" }], + flags: ENABLED_EXCHANGE_FLAGS, + }), + ).resolves.toBe(false); + }); + + it("bills the reported credit cost only for successful handled responses", () => { + expect(isSuccessfulExchangeStatusCode(200)).toBe(true); + expect(isSuccessfulExchangeStatusCode(204)).toBe(true); + expect(isSuccessfulExchangeStatusCode(304)).toBe(true); + expect(isSuccessfulExchangeStatusCode(404)).toBe(false); + + expect( + getExchangeSuccessCredits({ + exchange: { handled: true, creditsCost: 12 }, + statusCode: 200, + }), + ).toBe(12); + + expect( + getExchangeSuccessCredits({ + exchange: { handled: true, creditsCost: 0 }, + statusCode: 304, + }), + ).toBe(0); + + expect( + getExchangeSuccessCredits({ + exchange: { handled: true, creditsCost: 12 }, + statusCode: 404, + }), + ).toBeNull(); + + expect( + getExchangeSuccessCredits({ + statusCode: 200, + }), + ).toBeNull(); + }); + + it("reports billing outcomes without throwing on service failures", async () => { + vi.mocked(fetch).mockResolvedValue({ + ok: true, + status: 200, + } as Awaited>); + + await reportExchangeBilling({ + accessEventId: "6f1f5aab-3f78-4d0a-8a3d-2b1d3c4e5f60", + status: "confirmed", + billingReference: "bill-1", + }); + + expect(fetch).toHaveBeenCalledWith( + "https://exchange.example/v1/access-events/6f1f5aab-3f78-4d0a-8a3d-2b1d3c4e5f60/billing", + expect.objectContaining({ + method: "POST", + body: JSON.stringify({ status: "confirmed", billingReference: "bill-1" }), + }), + ); + + await reportExchangeBilling({ + accessEventId: "6f1f5aab-3f78-4d0a-8a3d-2b1d3c4e5f60", + status: "void", + }); + + expect(fetch).toHaveBeenLastCalledWith( + "https://exchange.example/v1/access-events/6f1f5aab-3f78-4d0a-8a3d-2b1d3c4e5f60/billing", + expect.objectContaining({ + method: "POST", + body: JSON.stringify({ status: "void" }), + }), + ); + + vi.mocked(fetch).mockRejectedValue(new Error("connect timeout")); + await expect( + reportExchangeBilling({ + accessEventId: "6f1f5aab-3f78-4d0a-8a3d-2b1d3c4e5f60", + status: "confirmed", + billingReference: "bill-1", + }), + ).resolves.toBeUndefined(); + }); +}); diff --git a/apps/api/src/lib/exchange.ts b/apps/api/src/lib/exchange.ts new file mode 100644 index 00000000000..9de05a4733e --- /dev/null +++ b/apps/api/src/lib/exchange.ts @@ -0,0 +1,639 @@ +import { fetch } from "undici"; +import { z } from "zod"; + +import { config } from "../config"; +import type { FormatObject } from "../controllers/v2/types"; +import { logger as rootLogger } from "./logger"; + +type OrganizationDataSourceAccessRecord = { + status?: string | null; + termsKey?: string | null; + termsVersion?: string | null; + termsAcceptedAt?: string | null; + enabledAt?: string | null; + disabledAt?: string | null; + disabledReason?: string | null; +}; + +type OrganizationDataSourceAccess = Record< + string, + OrganizationDataSourceAccessRecord | null | undefined +>; + +type RouteInput = { + url: string; + formats?: FormatObject[] | unknown[]; + actions?: unknown[]; + headers?: Record; + waitFor?: number; + mobile?: boolean; + location?: unknown; + proxy?: unknown; + blockAds?: boolean; + profile?: unknown; + atsv?: boolean; + minAge?: number; + includeTags?: unknown[]; + excludeTags?: unknown[]; + zeroDataRetention?: boolean; + lockdown?: boolean; + flags?: { + professionalProfileCompanyDataBeta?: boolean; + organizationDataSourceAccess?: OrganizationDataSourceAccess | null; + } | null; +}; + +export type ExchangeScrapeMetadata = { + handled: true; + creditsCost: number; + accessEventId?: string; + integrationId?: string; +}; + +export type ExchangeTerms = { + key: string; + version: string; +}; + +export type ExchangeProvider = { + id: string; + creditsCost: number; + terms?: ExchangeTerms; + routes: { + domains: Set; + pathPrefixes: string[]; + }[]; +}; + +// deterministicJson is deliberately unsupported: its extractor scripts run +// against page HTML, which Exchange responses do not carry. +const SUPPORTED_FORMATS = new Set(["markdown", "json"]); +const EXCHANGE_BETA_FLAG = "professionalProfileCompanyDataBeta"; +const THIRD_PARTY_DATA_TERMS_REQUIRED_CODE = "THIRD_PARTY_DATA_TERMS_REQUIRED"; +const THIRD_PARTY_DATA_TERMS_REQUIRED_MESSAGE = + "An organization admin must accept this data source's terms before this URL can be processed."; + +const EXCHANGE_PROVIDERS_PATH = "/v1/providers"; +const EXCHANGE_PROVIDERS_TIMEOUT_MS = 2_000; +const EXCHANGE_PROVIDERS_TTL_MS = 60_000; +const EXCHANGE_PROVIDERS_FAILURE_TTL_MS = 30_000; + +const exchangeProvidersSchema = z.object({ + success: z.literal(true), + data: z.array( + z + .object({ + id: z.string(), + // No .catch() here: a malformed credit cost must reject the catalog + // (keeping the last good one) rather than silently billing 0. + creditsCost: z.number().int().nonnegative(), + terms: z + .object({ + key: z.string(), + version: z.string(), + }) + .optional(), + capabilities: z + .object({ + scrape: z + .object({ + urlRoutes: z + .array( + z + .object({ + domains: z.string().array(), + pathPrefixes: z.string().array(), + }) + .passthrough(), + ) + .optional(), + }) + .passthrough() + .optional(), + }) + .passthrough(), + }) + .passthrough(), + ), +}); + +let cachedProviders: + | { + expiresAt: number; + value: ExchangeProvider[] | null; + } + | undefined; +let providersRequest: Promise | undefined; + +function normalizeHost(host: string): string { + return host.trim().toLowerCase().replace(/\.$/, ""); +} + +function normalizePathPrefix(prefix: string): string { + const trimmed = prefix.trim(); + if (!trimmed) { + return "/"; + } + + return trimmed.startsWith("/") ? trimmed : `/${trimmed}`; +} + +function getExchangeBaseUrl(): string | null { + if (!config.FIRE_EXCHANGE_URL) { + return null; + } + + return config.FIRE_EXCHANGE_URL.replace(/\/+$/, ""); +} + +function normalizeProviders( + raw: z.infer, +): ExchangeProvider[] { + return raw.data + .map(provider => ({ + id: provider.id, + creditsCost: provider.creditsCost, + ...(provider.terms === undefined ? {} : { terms: provider.terms }), + routes: (provider.capabilities.scrape?.urlRoutes ?? []).map(route => ({ + domains: new Set(route.domains.map(normalizeHost)), + pathPrefixes: route.pathPrefixes.map(normalizePathPrefix), + })), + })) + .filter(provider => provider.routes.length > 0); +} + +async function fetchExchangeProviders(): Promise { + const baseUrl = getExchangeBaseUrl(); + if (!baseUrl) { + return null; + } + + try { + const response = await fetch(`${baseUrl}${EXCHANGE_PROVIDERS_PATH}`, { + method: "GET", + signal: AbortSignal.timeout(EXCHANGE_PROVIDERS_TIMEOUT_MS), + }); + + if (!response.ok) { + rootLogger.warn("Exchange providers request failed", { + statusCode: response.status, + }); + return null; + } + + const parsed = exchangeProvidersSchema.parse(await response.json()); + return normalizeProviders(parsed); + } catch (error) { + rootLogger.warn("Exchange providers request errored", { error }); + return null; + } +} + +async function getExchangeProviders(): Promise { + if (cachedProviders && cachedProviders.expiresAt > Date.now()) { + return cachedProviders.value; + } + + if (!providersRequest) { + providersRequest = fetchExchangeProviders() + .then(providers => { + if (providers === null) { + // Keep serving the last good catalog through transient outages; + // the failure TTL only delays the next refresh attempt. + cachedProviders = { + value: cachedProviders?.value ?? null, + expiresAt: Date.now() + EXCHANGE_PROVIDERS_FAILURE_TTL_MS, + }; + } else { + cachedProviders = { + value: providers, + expiresAt: Date.now() + EXCHANGE_PROVIDERS_TTL_MS, + }; + } + return cachedProviders.value; + }) + .finally(() => { + providersRequest = undefined; + }); + } + + // Serve the stale catalog while the refresh runs in the background so + // request latency never depends on the catalog endpoint; only the very + // first lookup after boot has nothing to serve and waits. + if (cachedProviders) { + return cachedProviders.value; + } + + return providersRequest; +} + +function providerMatchesUrl(provider: ExchangeProvider, inputUrl: string): boolean { + let parsed: URL; + try { + parsed = new URL(inputUrl); + } catch { + return false; + } + + const host = normalizeHost(parsed.hostname); + const pathname = parsed.pathname || "/"; + + return provider.routes.some(route => { + if (!route.domains.has(host)) { + return false; + } + + if (route.pathPrefixes.length === 0) { + return true; + } + + // Prefix matches respect path segment boundaries: "/person" matches + // "/person" and "/person/x" but never "/personality". + return route.pathPrefixes.some(prefix => + prefix.endsWith("/") + ? pathname.startsWith(prefix) + : pathname === prefix || pathname.startsWith(`${prefix}/`), + ); + }); +} + +export async function resolveExchangeProvider( + inputUrl: string, +): Promise { + const providers = await getExchangeProviders(); + if (providers === null) { + return null; + } + + return providers.find(provider => providerMatchesUrl(provider, inputUrl)) ?? null; +} + +export function getExchangeRequestLogContext(inputUrl: string): + | { + url: string; + host: string; + pathPrefix: string | null; + } + | undefined { + let parsed: URL; + try { + parsed = new URL(inputUrl); + } catch { + return undefined; + } + + // Never log embedded credentials from user-submitted URLs. + parsed.username = ""; + parsed.password = ""; + + return { + url: parsed.href, + host: parsed.hostname.toLowerCase(), + pathPrefix: + parsed.pathname + .split("/") + .map(part => part.trim()) + .filter(part => part.length > 0)[0] ?? null, + }; +} + +export function getExchangeResponseLogContext(meta: unknown): { + cacheState?: string; + cachedAt?: string; + cacheAgeMs?: number; + providerRequestId?: string; +} { + if (typeof meta !== "object" || meta === null) { + return {}; + } + + const record = meta as Record; + const requestId = record.request_id ?? record.requestId; + + return { + ...(typeof record.cacheState === "string" + ? { cacheState: record.cacheState } + : {}), + ...(typeof record.cachedAt === "string" + ? { cachedAt: record.cachedAt } + : {}), + ...(typeof record.cacheAgeMs === "number" + ? { cacheAgeMs: record.cacheAgeMs } + : {}), + ...(typeof requestId === "string" ? { providerRequestId: requestId } : {}), + }; +} + +export function isSuccessfulExchangeStatusCode(statusCode: number): boolean { + return (statusCode >= 200 && statusCode < 300) || statusCode === 304; +} + +export function isSupportedExchangeFormatRequest( + formats?: FormatObject[] | unknown[], +): boolean { + if (formats === undefined) { + return true; + } + + if (!Array.isArray(formats) || formats.length === 0) { + return false; + } + + return formats.every(format => { + const type = + typeof format === "string" + ? format + : typeof format === "object" && format !== null && "type" in format + ? (format as { type?: unknown }).type + : undefined; + + return typeof type === "string" && SUPPORTED_FORMATS.has(type); + }); +} + +type DataSourceAccessDecision = "allowed" | "terms_required" | "not_enabled"; + +function getProviderAccessDecision( + provider: ExchangeProvider, + flags: RouteInput["flags"], +): DataSourceAccessDecision { + if (config.USE_DB_AUTHENTICATION !== true) { + return "allowed"; + } + + const access = flags?.organizationDataSourceAccess?.[provider.id]; + const entry = typeof access === "object" && access !== null ? access : null; + + if (provider.terms === undefined) { + return entry !== null && entry.status !== "enabled" ? "not_enabled" : "allowed"; + } + + if (entry === null) { + return "terms_required"; + } + + if (entry.status !== "enabled") { + return "not_enabled"; + } + + return entry.termsKey === provider.terms.key && + entry.termsVersion === provider.terms.version + ? "allowed" + : "terms_required"; +} + +function isExchangeEligibleRequest(input: RouteInput): boolean { + if (input.flags?.[EXCHANGE_BETA_FLAG] !== true) { + return false; + } + + if (!config.FIRE_EXCHANGE_URL) { + return false; + } + + if (!input.url) { + return false; + } + + if (input.zeroDataRetention || input.lockdown) { + return false; + } + + if (Array.isArray(input.actions) && input.actions.length > 0) { + return false; + } + + if (input.headers && Object.keys(input.headers).length > 0) { + return false; + } + + if (input.waitFor !== undefined && input.waitFor !== 0) { + return false; + } + + if (input.mobile || input.location || input.blockAds === false) { + return false; + } + + // Profile-backed scrapes expect session-specific content, which the + // Exchange cannot serve. + if (input.profile !== undefined) { + return false; + } + + // atsv is only supported by browser engines; requests that set it keep an + // engine that can honor it instead of routing to the Exchange. + if (input.atsv === true) { + return false; + } + + // minAge requests ask for Firecrawl-cached data; the Exchange serves + // provider data and Firecrawl never caches it, so the semantics cannot + // be honored here. + if (input.minAge !== undefined) { + return false; + } + + // Selector-based content filtering does not apply to provider records. + if ( + (Array.isArray(input.includeTags) && input.includeTags.length > 0) || + (Array.isArray(input.excludeTags) && input.excludeTags.length > 0) + ) { + return false; + } + + if (input.proxy === "stealth" || input.proxy === "enhanced") { + return false; + } + + if (!isSupportedExchangeFormatRequest(input.formats)) { + return false; + } + + return true; +} + +export type ExchangeAccess = + | { + allowed: true; + termsRequired: false; + provider: ExchangeProvider; + } + | { + allowed: false; + termsRequired: true; + terms: ExchangeTerms; + } + | { + allowed: false; + termsRequired: false; + }; + +export async function getExchangeAccessForRequest( + input: RouteInput, +): Promise { + // The Exchange gate sits on the hot path of every scrape request; no + // failure inside it may ever fail the request itself. Anything unexpected + // degrades to "not eligible" and the request continues on the normal path. + try { + if (!isExchangeEligibleRequest(input)) { + return { allowed: false, termsRequired: false }; + } + + const provider = await resolveExchangeProvider(input.url); + if (provider === null) { + return { allowed: false, termsRequired: false }; + } + + const decision = getProviderAccessDecision(provider, input.flags); + if (decision === "terms_required" && provider.terms !== undefined) { + return { allowed: false, termsRequired: true, terms: provider.terms }; + } + if (decision !== "allowed") { + return { allowed: false, termsRequired: false }; + } + + return { allowed: true, termsRequired: false, provider }; + } catch (error) { + rootLogger.warn("Exchange access check errored; treating as ineligible", { + error, + }); + return { allowed: false, termsRequired: false }; + } +} + +export async function canUseExchangeForRequest( + input: RouteInput, +): Promise { + return (await getExchangeAccessForRequest(input)).allowed; +} + +function getThirdPartyDataTermsSettingsUrl(): string { + return `${config.FIRECRAWL_DASHBOARD_URL.replace(/\/+$/, "")}/app/settings?tab=data-sources`; +} + +export function getThirdPartyDataTermsRequiredResponse(terms: ExchangeTerms) { + return { + success: false as const, + code: THIRD_PARTY_DATA_TERMS_REQUIRED_CODE as "THIRD_PARTY_DATA_TERMS_REQUIRED", + error: THIRD_PARTY_DATA_TERMS_REQUIRED_MESSAGE, + requiresAction: { + type: "accept_terms", + terms: terms.key, + version: terms.version, + url: getThirdPartyDataTermsSettingsUrl(), + }, + }; +} + +export function getExchangeSuccessCredits(input: { + exchange?: ExchangeScrapeMetadata; + statusCode?: number | null; +}): number | null { + if (input.exchange?.handled !== true) { + return null; + } + + const statusCode = input.statusCode; + if ( + statusCode === undefined || + statusCode === null || + !isSuccessfulExchangeStatusCode(statusCode) + ) { + return null; + } + + return input.exchange.creditsCost; +} + +const EXCHANGE_BILLING_TIMEOUT_MS = 5_000; + +/** + * Report the billing outcome of a delivered Exchange access so the service + * can reconcile its ledger: "confirmed" once the customer was billed, "void" + * when the delivered access was ultimately discarded and never billed. + * Failures only log - the service flags unresolved events for follow-up. + */ +export async function reportExchangeBilling(input: { + accessEventId: string; + status: "confirmed" | "void"; + billingReference?: string; +}): Promise { + const baseUrl = getExchangeBaseUrl(); + if (!baseUrl) { + return; + } + + try { + const response = await fetch( + `${baseUrl}/v1/access-events/${encodeURIComponent(input.accessEventId)}/billing`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + status: input.status, + ...(input.billingReference === undefined + ? {} + : { billingReference: input.billingReference }), + }), + signal: AbortSignal.timeout(EXCHANGE_BILLING_TIMEOUT_MS), + }, + ); + + if (!response.ok) { + rootLogger.warn("Exchange billing report failed", { + accessEventId: input.accessEventId, + status: input.status, + statusCode: response.status, + }); + } + } catch (error) { + rootLogger.warn("Exchange billing report errored", { + accessEventId: input.accessEventId, + status: input.status, + error, + }); + } +} + +/** + * Warm the provider catalog at process startup so the first flagged-org + * request never waits on the fetch; after this, stale-while-revalidate + * keeps every lookup in-memory. No-op when the Exchange is not configured; + * never throws. + */ +export function warmExchangeCatalog(): void { + if (!config.FIRE_EXCHANGE_URL) { + return; + } + + void getExchangeProviders(); +} + +export function setExchangeProvidersForTest( + providers: { + id: string; + creditsCost?: number; + terms?: ExchangeTerms; + routes: { domains: string[]; pathPrefixes?: string[] }[]; + }[], + ttlMs = 300_000, +) { + cachedProviders = { + value: providers.map(provider => ({ + id: provider.id, + creditsCost: provider.creditsCost ?? 0, + ...(provider.terms === undefined ? {} : { terms: provider.terms }), + routes: provider.routes.map(route => ({ + domains: new Set(route.domains.map(normalizeHost)), + pathPrefixes: (route.pathPrefixes ?? []).map(normalizePathPrefix), + })), + })), + expiresAt: Date.now() + ttlMs, + }; +} + +export function clearExchangeProvidersForTest() { + cachedProviders = undefined; + providersRequest = undefined; +} diff --git a/apps/api/src/lib/scrape-billing.test.ts b/apps/api/src/lib/scrape-billing.test.ts index a6c67caee87..9c1f9e7680f 100644 --- a/apps/api/src/lib/scrape-billing.test.ts +++ b/apps/api/src/lib/scrape-billing.test.ts @@ -6,6 +6,33 @@ import { UnsafeDomainBlockedError } from "./threat-protection/error"; import type { ThreatDecision } from "./threat-protection/types"; describe("calculateCreditsToBeBilled", () => { + it("bills handled Exchange successes at the reported credit cost", async () => { + const credits = await calculateCreditsToBeBilled( + { + formats: [{ type: "markdown" }], + } as any, + { + teamId: "team-id", + }, + { + metadata: { + statusCode: 200, + url: "https://profiles.example/person/example-person", + proxyUsed: "basic", + }, + } as any, + { + totalCost: 0, + } as any, + {} as any, + undefined, + undefined, + { handled: true, creditsCost: 12 }, + ); + + expect(credits).toBe(12); + }); + it("bills X/Twitter scrapes at 30 credits", async () => { const credits = await calculateCreditsToBeBilled( { @@ -142,6 +169,7 @@ const billWithDecisions = (args: { {} as any, args.error, undefined, + undefined, args.threatDecisions, ); diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.ts index b393646d739..58347aec371 100644 --- a/apps/api/src/lib/scrape-billing.ts +++ b/apps/api/src/lib/scrape-billing.ts @@ -10,6 +10,10 @@ import { hasFormatOfType } from "./format-utils"; import { TransportableError } from "./error"; import { FeatureFlag } from "../scraper/scrapeURL/engines"; import { isUrlBlocked } from "../scraper/WebScraper/utils/blocklist"; +import { + ExchangeScrapeMetadata, + getExchangeSuccessCredits, +} from "./exchange"; import type { ThreatDecision } from "./threat-protection/types"; import { UnsafeDomainBlockedError } from "./threat-protection/error"; @@ -64,6 +68,7 @@ export async function calculateCreditsToBeBilled( flags: TeamFlags, error?: Error | null, unsupportedFeatures?: Set, + exchange?: ExchangeScrapeMetadata, // Threat protection decisions for this scrape (initial + redirect checks, // in order). Each decision with `providerConsulted` bills a scan fee (+2 // per unique scanned URL) on top of the scrape's own cost — on both success @@ -112,6 +117,14 @@ export async function calculateCreditsToBeBilled( return creditsToBeBilled + threatScanCredits; } + const exchangeCredits = getExchangeSuccessCredits({ + exchange, + statusCode: document.metadata?.statusCode, + }); + if (exchangeCredits !== null) { + return exchangeCredits + threatScanCredits; + } + let creditsToBeBilled = 1; // Assuming 1 credit per document if (options.lockdown) { diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.ts index 5528a9aa852..73f553e3b53 100644 --- a/apps/api/src/routes/shared.ts +++ b/apps/api/src/routes/shared.ts @@ -29,6 +29,9 @@ import { CREDITS_FEATURE_ID, } from "../services/autumn/autumn.service"; import { getTeamBalance } from "../services/autumn/usage"; +import { getThirdPartyDataTermsRequiredResponse } from "../lib/exchange"; +import { getExchangeAccessForRequestBody } from "../lib/exchange-request"; +import { getScrapeZDR } from "../lib/zdr-helpers"; export function checkCreditsMiddleware( _minimum?: number, @@ -284,10 +287,58 @@ export function blocklistMiddleware( req: RequestWithMaybeACUC, res: Response, next: NextFunction, +) { + return blocklistGate(req, res, next, { exchange: false }); +} + +/** + * Blocklist gate for single-URL scrape-shaped routes (scrape, crawl), where + * an Exchange-eligible URL may bypass the blocklist because the exchange + * engine can serve it. Everything else (map, search, batch scrape, monitors) + * keeps plain blocklist behavior - batch stays out until its jobs carry the + * access flags the worker-side recheck needs. + */ +export function scrapeBlocklistMiddleware( + req: RequestWithMaybeACUC, + res: Response, + next: NextFunction, +) { + return blocklistGate(req, res, next, { exchange: true }); +} + +function blocklistGate( + req: RequestWithMaybeACUC, + res: Response, + next: NextFunction, + options: { exchange: boolean }, ) { (async () => { + const zeroDataRetention = + getScrapeZDR(req.acuc?.flags) === "forced" || + req.body?.zeroDataRetention === true; + const exchangeAccess = + options.exchange && + typeof req.body.url === "string" && + (await getExchangeAccessForRequestBody({ + body: req.body, + flags: req.acuc?.flags ?? null, + url: req.body.url, + zeroDataRetention, + })); + const canUseExchange = + typeof exchangeAccess === "object" && exchangeAccess.allowed; + + if (typeof exchangeAccess === "object" && exchangeAccess.termsRequired) { + if (!res.headersSent) { + return res + .status(403) + .json(getThirdPartyDataTermsRequiredResponse(exchangeAccess.terms)); + } + } + if ( typeof req.body.url === "string" && + !canUseExchange && isUrlBlocked(req.body.url, req.acuc?.flags ?? null, { team_id: req.acuc?.team_id ?? null, origin: typeof req.body.origin === "string" ? req.body.origin : null, diff --git a/apps/api/src/routes/v1.ts b/apps/api/src/routes/v1.ts index bc92811d1eb..d2cdf6dff3d 100644 --- a/apps/api/src/routes/v1.ts +++ b/apps/api/src/routes/v1.ts @@ -28,6 +28,7 @@ import { authMiddleware, checkCreditsMiddleware, blocklistMiddleware, + scrapeBlocklistMiddleware, countryCheck, idempotencyMiddleware, requestTimingMiddleware, @@ -51,7 +52,7 @@ v1Router.post( authMiddleware(RateLimiterMode.Scrape, { allowKeyless: true }), countryCheck, checkCreditsMiddleware(1), - blocklistMiddleware, + scrapeBlocklistMiddleware, wrap(scrapeController), ); @@ -60,7 +61,7 @@ v1Router.post( authMiddleware(RateLimiterMode.Crawl), countryCheck, checkCreditsMiddleware(), - blocklistMiddleware, + scrapeBlocklistMiddleware, idempotencyMiddleware, wrap(crawlController), ); diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.ts index e3a0991dd27..31c9b39975b 100644 --- a/apps/api/src/routes/v2.ts +++ b/apps/api/src/routes/v2.ts @@ -38,6 +38,7 @@ import { authMiddleware, checkCreditsMiddleware, blocklistMiddleware, + scrapeBlocklistMiddleware, countryCheck, idempotencyMiddleware, requestTimingMiddleware, @@ -210,7 +211,7 @@ v2Router.post( authMiddleware(RateLimiterMode.Scrape, { allowKeyless: true }), countryCheck, checkCreditsMiddleware(1), - blocklistMiddleware, + scrapeBlocklistMiddleware, wrap(scrapeController), ); @@ -257,7 +258,7 @@ v2Router.post( authMiddleware(RateLimiterMode.Crawl), countryCheck, checkCreditsMiddleware(), - blocklistMiddleware, + scrapeBlocklistMiddleware, idempotencyMiddleware, wrap(crawlController), ); diff --git a/apps/api/src/scraper/scrapeURL/engines/exchange.ts b/apps/api/src/scraper/scrapeURL/engines/exchange.ts new file mode 100644 index 00000000000..c6c41de2dc0 --- /dev/null +++ b/apps/api/src/scraper/scrapeURL/engines/exchange.ts @@ -0,0 +1,186 @@ +import { z } from "zod"; + +import { Meta } from ".."; +import { EngineScrapeResult } from "."; +import { config } from "../../../config"; +import { + getExchangeRequestLogContext, + getExchangeResponseLogContext, +} from "../../../lib/exchange"; +import { setSpanAttributes, withSpan } from "../../../lib/otel-tracer"; +import { robustFetch } from "../lib/fetch"; +import { EngineError } from "../error"; + +const exchangeScrapeResponseSchema = z.union([ + z + .object({ + success: z.literal(true), + accessEventId: z.string().optional(), + // No .catch() here: a malformed credit cost must fail the scrape + // loudly rather than silently billing 0 for a delivered access. + creditsCost: z.number().int().nonnegative(), + data: z + .object({ + url: z.string().optional(), + title: z.string().optional(), + description: z.string().optional(), + source: z + .object({ + provider: z.string().optional(), + }) + .passthrough() + .optional(), + metadata: z.record(z.string(), z.unknown()).optional(), + markdown: z.string().optional(), + json: z.unknown().optional(), + }) + .passthrough(), + }) + .passthrough(), + z + .object({ + success: z.literal(false), + error: z + .object({ + code: z.string().optional(), + message: z.string().optional(), + }) + .passthrough() + .optional(), + }) + .passthrough(), +]); + +export function exchangeMaxReasonableTime(meta: Meta): number { + return meta.options.timeout ?? 60_000; +} + +function escapeHtml(value: string): string { + return value + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """); +} + +// Exchange responses carry no page HTML; synthesize a minimal head so the +// metadata transformer can populate the document's title and description. +function buildMetadataHtml(title?: string, description?: string): string { + const titleTag = title === undefined ? "" : `${escapeHtml(title)}`; + const descriptionTag = + description === undefined + ? "" + : ``; + return `${titleTag}${descriptionTag}`; +} + +export async function scrapeURLWithExchange( + meta: Meta, +): Promise { + return withSpan("engine.exchange.scrape", async span => { + const startTime = Date.now(); + const url = meta.rewrittenUrl ?? meta.url; + const requestLogContext = getExchangeRequestLogContext(url); + const logger = meta.logger.child({ method: "scrapeURLWithExchange" }); + + setSpanAttributes(span, { + "engine.type": "exchange", + // Follow the same credential redaction as the log context. + "engine.url": requestLogContext?.url ?? "", + "engine.team_id": meta.internalOptions.teamId, + }); + + logger.info("Exchange scrape started", { + ...requestLogContext, + scrapeId: meta.id, + teamId: meta.internalOptions.teamId, + maxAge: meta.options.maxAge, + }); + + try { + const response = await robustFetch({ + url: `${config.FIRE_EXCHANGE_URL!.replace(/\/+$/, "")}/v1/scrape`, + method: "POST", + body: { + requestId: meta.id, + teamId: meta.internalOptions.teamId, + url, + formats: ["markdown", "json"], + ...(meta.options.maxAge === undefined + ? {} + : { maxAge: meta.options.maxAge }), + }, + logger: logger.child({ method: "exchangeScrape/robustFetch" }), + tryCount: 2, + ignoreFailureStatus: true, + mock: meta.mock, + abort: meta.abort.asSignal(), + schema: exchangeScrapeResponseSchema, + }); + + if (!response.success) { + logger.warn("Exchange scrape failed", { + ...requestLogContext, + scrapeId: meta.id, + teamId: meta.internalOptions.teamId, + errorCode: response.error?.code, + durationMs: Date.now() - startTime, + }); + throw new EngineError("Exchange request failed"); + } + + const responseLogContext = getExchangeResponseLogContext( + response.data.metadata, + ); + + logger.info("Exchange scrape completed", { + ...requestLogContext, + ...responseLogContext, + scrapeId: meta.id, + teamId: meta.internalOptions.teamId, + integrationId: response.data.source?.provider, + accessEventId: response.accessEventId, + creditsCost: response.creditsCost, + durationMs: Date.now() - startTime, + }); + + setSpanAttributes(span, { + "exchange.integration_id": response.data.source?.provider, + "exchange.credits_cost": response.creditsCost, + "exchange.cache_state": responseLogContext.cacheState, + "exchange.cache_age_ms": responseLogContext.cacheAgeMs, + "exchange.duration_ms": Date.now() - startTime, + }); + + return { + url: response.data.url ?? url, + html: buildMetadataHtml(response.data.title, response.data.description), + markdown: response.data.markdown, + json: response.data.json, + statusCode: 200, + contentType: "text/markdown", + proxyUsed: "basic", + exchange: { + handled: true, + creditsCost: response.creditsCost, + ...(response.accessEventId === undefined + ? {} + : { accessEventId: response.accessEventId }), + ...(response.data.source?.provider === undefined + ? {} + : { integrationId: response.data.source.provider }), + }, + }; + } catch (error) { + logger.warn("Exchange scrape errored", { + ...requestLogContext, + scrapeId: meta.id, + teamId: meta.internalOptions.teamId, + durationMs: Date.now() - startTime, + errorMessage: error instanceof Error ? error.message : String(error), + error, + }); + throw error; + } + }); +} diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts index 8a9b2f7275c..60b986bc721 100644 --- a/apps/api/src/scraper/scrapeURL/engines/index.ts +++ b/apps/api/src/scraper/scrapeURL/engines/index.ts @@ -7,6 +7,10 @@ import { scrapeURLWithFireEngineChromeCDP, scrapeURLWithFireEngineTLSClient, } from "./fire-engine"; +import { + exchangeMaxReasonableTime, + scrapeURLWithExchange, +} from "./exchange"; import { pdfMaxReasonableTime, scrapePDF } from "./pdf"; import { fetchMaxReasonableTime, scrapeURLWithFetch } from "./fetch"; import { @@ -30,8 +34,14 @@ import { getPDFMaxPages } from "../../../controllers/v2/types"; import type { PdfMetadata } from "./pdf/types"; import { BrandingProfile } from "../../../types/branding"; import { BrandingNotSupportedError } from "../error"; +import { isUrlBlocked } from "../../WebScraper/utils/blocklist"; +import { + canUseExchangeForRequest, + type ExchangeScrapeMetadata, +} from "../../../lib/exchange"; export type Engine = + | "exchange" | "fire-engine;chrome-cdp" | "fire-engine(retry);chrome-cdp" | "fire-engine;chrome-cdp;stealth" @@ -162,11 +172,13 @@ export type EngineScrapeResult = { proxyUsed: "basic" | "stealth"; timezone?: string; + exchange?: ExchangeScrapeMetadata; }; const engineHandlers: { [E in Engine]: (meta: Meta) => Promise; } = { + "exchange": scrapeURLWithExchange, index: scrapeURLWithIndex, "index;documents": scrapeURLWithIndex, "fire-engine;chrome-cdp": scrapeURLWithFireEngineChromeCDP, @@ -186,6 +198,7 @@ const engineHandlers: { const engineMRTs: { [E in Engine]: (meta: Meta) => number; } = { + "exchange": exchangeMaxReasonableTime, index: indexMaxReasonableTime, "index;documents": indexMaxReasonableTime, "fire-engine;chrome-cdp": meta => @@ -218,6 +231,27 @@ const engineOptions: { quality: number; }; } = { + "exchange": { + features: { + actions: false, + waitFor: false, + screenshot: false, + "screenshot@fullScreen": false, + pdf: false, + document: false, + audio: false, + video: false, + atsv: false, + location: false, + mobile: false, + skipTlsVerification: true, + useFastMode: true, + stealthProxy: false, + branding: false, + disableAdblock: false, + }, + quality: 2000, + }, index: { features: { actions: false, @@ -547,6 +581,73 @@ export async function buildFallbackList(meta: Meta): Promise< unsupportedFeatures: Set; }[] > { + if ( + !meta.internalOptions.agentIndexOnly && + meta.internalOptions.forceEngine === undefined + ) { + if ( + await canUseExchangeForRequest({ + url: meta.rewrittenUrl ?? meta.url, + formats: meta.options.formats, + actions: meta.options.actions, + headers: meta.options.headers, + waitFor: meta.options.waitFor, + mobile: meta.options.mobile, + location: meta.options.location, + proxy: meta.options.proxy, + blockAds: meta.options.blockAds, + profile: meta.options.profile, + atsv: meta.internalOptions.atsv, + minAge: meta.options.minAge, + includeTags: meta.options.includeTags, + excludeTags: meta.options.excludeTags, + zeroDataRetention: meta.internalOptions.zeroDataRetention, + lockdown: meta.options.lockdown, + flags: meta.internalOptions.teamFlags ?? null, + }) + ) { + return [ + { + engine: "exchange", + unsupportedFeatures: new Set(), + }, + ]; + } + + // A blocked URL can only have been admitted by the Exchange bypass in + // blocklistMiddleware, which only applies to flagged orgs; if the + // Exchange is no longer usable by execution time (catalog changed, + // service down), fail closed rather than letting normal engines scrape + // a blocklisted site. An error here also fails closed: this branch only + // runs for flagged orgs, and a retryable scrape failure is preferable + // to scraping a potentially blocklisted site with normal engines. + if ( + meta.internalOptions.teamFlags?.professionalProfileCompanyDataBeta === + true + ) { + try { + if ( + isUrlBlocked( + meta.rewrittenUrl ?? meta.url, + meta.internalOptions.teamFlags ?? null, + { + team_id: meta.internalOptions.teamId ?? null, + origin: null, + }, + ) + ) { + return []; + } + } catch (error) { + meta.logger.warn( + "Exchange blocklist re-check failed; failing closed", + { error }, + ); + return []; + } + } + } + const shouldPrioritizeTlsClient = meta.options.__experimental_engpicker ? (await queryEngpickerVerdict( meta.options.__experimental_omceDomain ?? new URL(meta.url).hostname, diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts index 53012521b4c..3863f972e25 100644 --- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts +++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts @@ -52,6 +52,9 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) { !meta.internalOptions.zeroDataRetention && meta.winnerEngine !== "index" && meta.winnerEngine !== "index;documents" && + // Exchange-delivered content is never stored on the Firecrawl side: + // every access must go through the Exchange and its ledger. + meta.winnerEngine !== "exchange" && !(meta.winnerEngine === "pdf" && !shouldParsePDF(meta.options.parsers)) && !meta.options.parsers?.some(parser => { if ( diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts index fb1e738f685..5357e616745 100644 --- a/apps/api/src/scraper/scrapeURL/index.ts +++ b/apps/api/src/scraper/scrapeURL/index.ts @@ -87,6 +87,7 @@ import { writeFile } from "node:fs/promises"; import path from "node:path"; import { tmpdir } from "node:os"; import { randomUUID } from "node:crypto"; +import type { ExchangeScrapeMetadata } from "../../lib/exchange"; import { checkUrl, type ThreatCheckDedup, @@ -101,6 +102,7 @@ export type ScrapeUrlResponse = success: true; document: Document; unsupportedFeatures?: Set; + exchange?: ExchangeScrapeMetadata; /** * Threat protection decisions made for this scrape (initial domain * check + any redirect re-checks, in order). Read by the billing layer @@ -1071,6 +1073,7 @@ async function scrapeURLLoop(meta: Meta): Promise { success: true, document, unsupportedFeatures: result.unsupportedFeatures, + exchange: engineResult.exchange, }; }); } diff --git a/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts b/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts index 7b350ea4222..41a626e68b7 100644 --- a/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts +++ b/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts @@ -27,6 +27,11 @@ function shouldIndexForSearch(meta: Meta, document: Document): boolean { return false; } + // Exchange-delivered content is never stored on the Firecrawl side. + if (meta.winnerEngine === "exchange") { + return false; + } + // Lockdown must not forward the target URL to any external service. if (meta.options.lockdown) { return false; diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts index f3d968e0209..41d0b7b2062 100644 --- a/apps/api/src/services/worker/scrape-worker.ts +++ b/apps/api/src/services/worker/scrape-worker.ts @@ -99,6 +99,11 @@ import { recordMonitorScrapeFailure, recordMonitorScrapeSuccess, } from "../monitoring/results"; +import { + reportExchangeBilling, + warmExchangeCatalog, + type ExchangeScrapeMetadata, +} from "../../lib/exchange"; configDotenv(); @@ -108,6 +113,7 @@ const jobLockExtensionTime = config.JOB_LOCK_EXTENSION_TIME; if (require.main === module) { cacheableLookup.install(http.globalAgent); cacheableLookup.install(https.globalAgent); + warmExchangeCatalog(); } async function billScrapeJob( @@ -118,6 +124,7 @@ async function billScrapeJob( flags: TeamFlags, error?: Error | null, unsupportedFeatures?: Set, + exchange?: ExchangeScrapeMetadata, threatDecisions?: ThreatDecision[], ) { let creditsToBeBilled: number | null = null; @@ -146,6 +153,7 @@ async function billScrapeJob( flags, error, unsupportedFeatures, + exchange, threatDecisions, ); @@ -196,6 +204,18 @@ async function billScrapeJob( }, ); + // Reconcile the Exchange ledger. Awaited so the confirmation lands + // before any later failure path could attempt to void this access - + // the service rejects confirmed->void, making the order decisive. + // reportExchangeBilling never throws and is bounded by its timeout. + if (exchange?.accessEventId !== undefined) { + await reportExchangeBilling({ + accessEventId: exchange.accessEventId, + status: "confirmed", + billingReference: billingJobId, + }); + } + return creditsToBeBilled; } catch (error) { logger.error( @@ -663,6 +683,7 @@ async function processJob(job: NuQJob) { (await getACUCTeam(job.data.team_id))?.flags ?? null, undefined, pipeline.unsupportedFeatures, + pipeline.exchange, pipeline.threatDecisions, ); @@ -754,6 +775,7 @@ async function processJob(job: NuQJob) { (await getACUCTeam(job.data.team_id))?.flags ?? null, undefined, pipeline.unsupportedFeatures, + pipeline.exchange, pipeline.threatDecisions, ); @@ -954,9 +976,23 @@ async function processJob(job: NuQJob) { (await getACUCTeam(job.data.team_id))?.flags ?? null, error instanceof Error ? error : null, undefined, + undefined, pipeline?.threatDecisions, ); + // The Exchange delivered this access but the scrape ultimately failed, + // so the customer was never billed for it - void the access event so + // the Exchange ledger reconciles. Fire-and-forget. If billing did + // happen before a later step failed, its confirmation was awaited + // above and the service rejects confirmed->void, so a paid access can + // never be marked void. + if (pipeline?.success && pipeline.exchange?.accessEventId !== undefined) { + void reportExchangeBilling({ + accessEventId: pipeline.exchange.accessEventId, + status: "void", + }); + } + logger.debug("Logging job to DB..."); await logScrape( { diff --git a/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs b/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs index 799039f11a0..3ab23a0697b 100644 --- a/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs +++ b/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs @@ -87,6 +87,7 @@ public void SearchOptions_SerializesCorrectly() Limit = 5, Location = "US", Tbs = "qdr:w", + Highlights = false, IncludeDomains = new() { "firecrawl.dev" }, ExcludeDomains = new() { "example.com" } }; @@ -95,6 +96,7 @@ public void SearchOptions_SerializesCorrectly() Assert.Contains("\"limit\":5", json); Assert.Contains("\"location\":\"US\"", json); Assert.Contains("\"tbs\":\"qdr:w\"", json); + Assert.Contains("\"highlights\":false", json); Assert.Contains("\"includeDomains\":[\"firecrawl.dev\"]", json); Assert.Contains("\"excludeDomains\":[\"example.com\"]", json); } diff --git a/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj b/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj index ad364cc6204..b84ed222ad4 100644 --- a/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj +++ b/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj @@ -8,7 +8,7 @@ firecrawl-sdk - 1.9.0 + 1.9.1 Firecrawl Firecrawl .NET SDK for the Firecrawl API - web scraping, crawling, and data extraction diff --git a/apps/dot-net-sdk/Firecrawl/Models/SearchOptions.cs b/apps/dot-net-sdk/Firecrawl/Models/SearchOptions.cs index f917a20e85e..709e7cf044b 100644 --- a/apps/dot-net-sdk/Firecrawl/Models/SearchOptions.cs +++ b/apps/dot-net-sdk/Firecrawl/Models/SearchOptions.cs @@ -43,6 +43,11 @@ public class SearchOptions [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public int? Timeout { get; set; } + /// Generate query-relevant highlights for search results. Defaults to true. + [JsonPropertyName("highlights")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? Highlights { get; set; } + [JsonPropertyName("scrapeOptions")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public ScrapeOptions? ScrapeOptions { get; set; } diff --git a/apps/elixir-sdk/lib/firecrawl.ex b/apps/elixir-sdk/lib/firecrawl.ex index c75cd467478..760957ce58f 100644 --- a/apps/elixir-sdk/lib/firecrawl.ex +++ b/apps/elixir-sdk/lib/firecrawl.ex @@ -1150,6 +1150,7 @@ defmodule Firecrawl do enterprise: [type: {:list, :string}, doc: "Enterprise search options for Zero Data Retention (ZDR). Use `[\"zdr\"]` for end-to-end ZDR (10 credits / 10 results) or `[\"anon\"]` for anonymized ZDR (2 credits / 10 results). Must be enabled for your team."], exclude_domains: [type: {:list, :string}, doc: "Domains to exclude from search results."], ignore_invalid_urls: [type: :boolean, doc: "Excludes URLs from the search results that are invalid for other Firecrawl endpoints. This helps reduce errors if you are piping data from search into other Firecrawl API endpoints."], + highlights: [type: :boolean, doc: "Generate query-relevant highlights for search results. Defaults to true."], include_domains: [type: {:list, :string}, doc: "Domains to include in search results."], limit: [type: :integer, doc: "Maximum number of results to return"], location: [type: :string, doc: "Location parameter for search results (e.g. `San Francisco,California,United States`). For best results, set both this and the `country` parameter."], @@ -1160,7 +1161,7 @@ defmodule Firecrawl do timeout: [type: :integer, doc: "Timeout in milliseconds"] ]) - @search_and_scrape_key_mapping %{categories: "categories", country: "country", enterprise: "enterprise", exclude_domains: "excludeDomains", ignore_invalid_urls: "ignoreInvalidURLs", include_domains: "includeDomains", limit: "limit", location: "location", query: "query", scrape_options: "scrapeOptions", sources: "sources", tbs: "tbs", timeout: "timeout"} + @search_and_scrape_key_mapping %{categories: "categories", country: "country", enterprise: "enterprise", exclude_domains: "excludeDomains", highlights: "highlights", ignore_invalid_urls: "ignoreInvalidURLs", include_domains: "includeDomains", limit: "limit", location: "location", query: "query", scrape_options: "scrapeOptions", sources: "sources", tbs: "tbs", timeout: "timeout"} @doc """ Search and optionally scrape search results diff --git a/apps/elixir-sdk/mix.exs b/apps/elixir-sdk/mix.exs index 446c7055366..163203b4b42 100644 --- a/apps/elixir-sdk/mix.exs +++ b/apps/elixir-sdk/mix.exs @@ -1,7 +1,7 @@ defmodule Firecrawl.MixProject do use Mix.Project - @version "1.8.0" + @version "1.8.1" @source_url "https://github.com/firecrawl/firecrawl/tree/main/apps/elixir-sdk" def project do diff --git a/apps/elixir-sdk/test/firecrawl_test.exs b/apps/elixir-sdk/test/firecrawl_test.exs index 9da76abd86b..be8c4400ec7 100644 --- a/apps/elixir-sdk/test/firecrawl_test.exs +++ b/apps/elixir-sdk/test/firecrawl_test.exs @@ -317,6 +317,40 @@ defmodule FirecrawlTest do assert body["urls"] == ["https://example.com"] end + test "search maps highlights to highlights" do + parent = self() + + adapter = fn request -> + send(parent, {:request, request}) + + resp = Req.Response.new( + status: 200, + headers: %{"content-type" => ["application/json"]}, + body: Jason.encode!(%{"success" => true, "data" => %{}}) + ) + + {request, resp} + end + + assert {:ok, %Req.Response{status: 200}} = + Firecrawl.search_and_scrape( + [query: "firecrawl", highlights: false], + api_key: "test-key", + adapter: adapter + ) + + assert_receive {:request, request} + + body = + cond do + is_binary(request.body) -> Jason.decode!(request.body) + is_map(request.body) -> request.body + true -> request.options[:json] + end + + assert body["highlights"] == false + end + test "all expected API functions are defined with bang variants" do functions = Firecrawl.__info__(:functions) diff --git a/apps/go-sdk/options.go b/apps/go-sdk/options.go index f38f296dfde..c7022f57e86 100644 --- a/apps/go-sdk/options.go +++ b/apps/go-sdk/options.go @@ -171,6 +171,7 @@ type SearchOptions struct { Location *string `json:"location,omitempty"` IgnoreInvalidURLs *bool `json:"ignoreInvalidURLs,omitempty"` Timeout *int `json:"timeout,omitempty"` + Highlights *bool `json:"highlights,omitempty"` ScrapeOptions *ScrapeOptions `json:"scrapeOptions,omitempty"` Integration *string `json:"integration,omitempty"` } diff --git a/apps/go-sdk/options_test.go b/apps/go-sdk/options_test.go index 35603d42138..d41057c2d7a 100644 --- a/apps/go-sdk/options_test.go +++ b/apps/go-sdk/options_test.go @@ -76,3 +76,14 @@ func TestScrapeOptionsSerializesRedactPII(t *testing.T) { t.Fatalf("serialized redactPII = %s", payload) } } + +func TestSearchOptionsSerializesHighlights(t *testing.T) { + payload, err := json.Marshal(SearchOptions{Highlights: Bool(false)}) + if err != nil { + t.Fatalf("Marshal SearchOptions: %v", err) + } + + if !strings.Contains(string(payload), `"highlights":false`) { + t.Fatalf("serialized search options = %s", payload) + } +} diff --git a/apps/go-sdk/version.go b/apps/go-sdk/version.go index 4d99375f14c..79775e6d872 100644 --- a/apps/go-sdk/version.go +++ b/apps/go-sdk/version.go @@ -9,4 +9,4 @@ package firecrawl // Bump this when preparing a new release. The publish-go-sdk GitHub workflow // reads this value and creates the corresponding monorepo-prefixed tag on // merge to main. -const Version = "1.8.0" +const Version = "1.8.1" diff --git a/apps/java-sdk/build.gradle.kts b/apps/java-sdk/build.gradle.kts index c74af612ef8..0310669b89f 100644 --- a/apps/java-sdk/build.gradle.kts +++ b/apps/java-sdk/build.gradle.kts @@ -4,7 +4,7 @@ plugins { } group = "com.firecrawl" -version = "1.11.0" +version = "1.11.1" java { sourceCompatibility = JavaVersion.VERSION_11 diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/SearchOptions.java b/apps/java-sdk/src/main/java/com/firecrawl/models/SearchOptions.java index 8b36091f89b..1553d4f65b7 100644 --- a/apps/java-sdk/src/main/java/com/firecrawl/models/SearchOptions.java +++ b/apps/java-sdk/src/main/java/com/firecrawl/models/SearchOptions.java @@ -18,6 +18,7 @@ public class SearchOptions { private String location; private Boolean ignoreInvalidURLs; private Integer timeout; + private Boolean highlights; private ScrapeOptions scrapeOptions; private String integration; @@ -32,6 +33,7 @@ private SearchOptions() {} public String getLocation() { return location; } public Boolean getIgnoreInvalidURLs() { return ignoreInvalidURLs; } public Integer getTimeout() { return timeout; } + public Boolean getHighlights() { return highlights; } public ScrapeOptions getScrapeOptions() { return scrapeOptions; } public String getIntegration() { return integration; } @@ -47,6 +49,7 @@ public static final class Builder { private String location; private Boolean ignoreInvalidURLs; private Integer timeout; + private Boolean highlights; private ScrapeOptions scrapeOptions; private String integration; @@ -70,6 +73,8 @@ private Builder() {} public Builder ignoreInvalidURLs(Boolean ignoreInvalidURLs) { this.ignoreInvalidURLs = ignoreInvalidURLs; return this; } /** Timeout in milliseconds. */ public Builder timeout(Integer timeout) { this.timeout = timeout; return this; } + /** Generate query-relevant highlights for search results. Defaults to true. */ + public Builder highlights(Boolean highlights) { this.highlights = highlights; return this; } /** Scrape options applied to search result pages. */ public Builder scrapeOptions(ScrapeOptions scrapeOptions) { this.scrapeOptions = scrapeOptions; return this; } /** Integration identifier. */ @@ -86,6 +91,7 @@ public SearchOptions build() { o.location = this.location; o.ignoreInvalidURLs = this.ignoreInvalidURLs; o.timeout = this.timeout; + o.highlights = this.highlights; o.scrapeOptions = this.scrapeOptions; o.integration = this.integration; return o; diff --git a/apps/java-sdk/src/test/java/com/firecrawl/SearchOptionsTest.java b/apps/java-sdk/src/test/java/com/firecrawl/SearchOptionsTest.java new file mode 100644 index 00000000000..7b8f1dc547e --- /dev/null +++ b/apps/java-sdk/src/test/java/com/firecrawl/SearchOptionsTest.java @@ -0,0 +1,25 @@ +package com.firecrawl; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.firecrawl.models.SearchOptions; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class SearchOptionsTest { + @Test + void serializesHighlightsOption() { + ObjectMapper mapper = new ObjectMapper(); + JsonNode enabled = mapper.valueToTree( + SearchOptions.builder().highlights(true).build()); + JsonNode disabled = mapper.valueToTree( + SearchOptions.builder().highlights(false).build()); + JsonNode omitted = mapper.valueToTree(SearchOptions.builder().build()); + + assertTrue(enabled.get("highlights").asBoolean()); + assertFalse(disabled.get("highlights").asBoolean()); + assertFalse(omitted.has("highlights")); + } +} diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json index 0442e591a11..611d79bdc9a 100644 --- a/apps/js-sdk/firecrawl/package.json +++ b/apps/js-sdk/firecrawl/package.json @@ -1,6 +1,6 @@ { "name": "@mendable/firecrawl-js", - "version": "4.30.0", + "version": "4.30.1", "description": "JavaScript SDK for Firecrawl API", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/search-highlights.unit.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/search-highlights.unit.test.ts new file mode 100644 index 00000000000..88265b2bde8 --- /dev/null +++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/search-highlights.unit.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, jest, test } from "@jest/globals"; +import { search } from "../../../v2/methods/search"; + +describe("v2 search highlights", () => { + test("forwards the highlights option", async () => { + const http = { + post: jest.fn(async () => ({ status: 200, data: { success: true } })), + } as any; + + await search(http, { query: "firecrawl", highlights: false }); + + expect(http.post).toHaveBeenCalledWith( + "/v2/search", + { query: "firecrawl", highlights: false }, + {}, + ); + }); +}); diff --git a/apps/js-sdk/firecrawl/src/v2/methods/search.ts b/apps/js-sdk/firecrawl/src/v2/methods/search.ts index d1610959d96..00332394e3f 100644 --- a/apps/js-sdk/firecrawl/src/v2/methods/search.ts +++ b/apps/js-sdk/firecrawl/src/v2/methods/search.ts @@ -37,6 +37,7 @@ function prepareSearchPayload(req: SearchRequest): Record { if (req.ignoreInvalidURLs != null) payload.ignoreInvalidURLs = req.ignoreInvalidURLs; if (req.timeout != null) payload.timeout = req.timeout; + if (req.highlights != null) payload.highlights = req.highlights; if (req.integration && req.integration.trim()) payload.integration = req.integration.trim(); if (req.origin) payload.origin = req.origin; diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.ts index c75b174c967..da2ea320837 100644 --- a/apps/js-sdk/firecrawl/src/v2/types.ts +++ b/apps/js-sdk/firecrawl/src/v2/types.ts @@ -697,6 +697,8 @@ export interface SearchRequest { location?: string; ignoreInvalidURLs?: boolean; timeout?: number; // ms + /** Generate query-relevant highlights for search results. Defaults to true. */ + highlights?: boolean; scrapeOptions?: ScrapeOptions; /** * Enterprise search options. Use `["zdr"]` for end-to-end Zero Data diff --git a/apps/php-sdk/src/Models/SearchOptions.php b/apps/php-sdk/src/Models/SearchOptions.php index 30dd484fd8f..bc413f3a7c4 100644 --- a/apps/php-sdk/src/Models/SearchOptions.php +++ b/apps/php-sdk/src/Models/SearchOptions.php @@ -24,6 +24,7 @@ private function __construct( private readonly ?string $integration = null, private readonly ?array $includeDomains = null, private readonly ?array $excludeDomains = null, + private readonly ?bool $highlights = null, ) {} /** @@ -44,10 +45,12 @@ public static function with( ?string $integration = null, ?array $includeDomains = null, ?array $excludeDomains = null, + ?bool $highlights = null, ): self { return new self( $sources, $categories, $limit, $tbs, $location, $ignoreInvalidURLs, $timeout, $scrapeOptions, $integration, $includeDomains, $excludeDomains, + $highlights, ); } @@ -64,6 +67,7 @@ public function toArray(): array 'location' => $this->location, 'ignoreInvalidURLs' => $this->ignoreInvalidURLs, 'timeout' => $this->timeout, + 'highlights' => $this->highlights, 'scrapeOptions' => $this->scrapeOptions?->toArray(), 'integration' => $this->integration, ]; diff --git a/apps/php-sdk/src/Version.php b/apps/php-sdk/src/Version.php index 14feb905343..4e52a3fcfc1 100644 --- a/apps/php-sdk/src/Version.php +++ b/apps/php-sdk/src/Version.php @@ -6,5 +6,5 @@ final class Version { - public const SDK_VERSION = '1.9.0'; + public const SDK_VERSION = '1.9.1'; } diff --git a/apps/php-sdk/tests/Unit/ModelsTest.php b/apps/php-sdk/tests/Unit/ModelsTest.php index 2b77c63005a..e55ac7a9b3d 100644 --- a/apps/php-sdk/tests/Unit/ModelsTest.php +++ b/apps/php-sdk/tests/Unit/ModelsTest.php @@ -13,6 +13,7 @@ use Firecrawl\Models\QueryFormat; use Firecrawl\Models\QuestionFormat; use Firecrawl\Models\ScrapeOptions; +use Firecrawl\Models\SearchOptions; use Firecrawl\Models\Monitor; use Firecrawl\Models\MonitorCheck; @@ -259,6 +260,12 @@ expect($doc->getMenu())->toBeNull(); }); +it('serializes the search highlights option', function (): void { + $options = SearchOptions::with(highlights: false); + + expect($options->toArray())->toBe(['highlights' => false]); +}); + it('coerces non-string scalar identity fields without a TypeError under strict_types', function (): void { // Defensive: upstream data could carry a non-string scalar (e.g. a numeric // brand). Under declare(strict_types=1) these must be cast, not passed raw. diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py index 64a6d45f861..fc8b578bbed 100644 --- a/apps/python-sdk/firecrawl/__init__.py +++ b/apps/python-sdk/firecrawl/__init__.py @@ -17,7 +17,7 @@ V1ChangeTrackingOptions, ) -__version__ = "4.32.0" +__version__ = "4.32.1" # Define the logger for the Firecrawl project logger: logging.Logger = logging.getLogger("firecrawl") diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_search_request_preparation.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_search_request_preparation.py index f63219591e2..221f2e2d0d3 100644 --- a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_search_request_preparation.py +++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_search_request_preparation.py @@ -44,6 +44,7 @@ def test_all_fields_conversion(self): location="US", ignore_invalid_urls=False, timeout=30000, + highlights=False, scrape_options=scrape_opts, integration=" _e2e-test ", ) @@ -56,6 +57,7 @@ def test_all_fields_conversion(self): assert data["tbs"] == "qdr:w" assert data["location"] == "US" assert data["timeout"] == 30000 + assert data["highlights"] is False # Check snake_case to camelCase conversions assert "ignoreInvalidURLs" in data @@ -181,4 +183,4 @@ def test_scrape_options_shared_function_integration(self): assert "only_main_content" not in scrape_data assert "wait_for" not in scrape_data assert "skip_tls_verification" not in scrape_data - assert "remove_base64_images" not in scrape_data \ No newline at end of file + assert "remove_base64_images" not in scrape_data diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.py index b04810290c9..a8236e57302 100644 --- a/apps/python-sdk/firecrawl/v2/client.py +++ b/apps/python-sdk/firecrawl/v2/client.py @@ -395,6 +395,7 @@ def search( location: Optional[str] = None, ignore_invalid_urls: Optional[bool] = None, timeout: Optional[int] = None, + highlights: Optional[bool] = None, scrape_options: Optional[ScrapeOptions] = None, integration: Optional[str] = None, enterprise: Optional[List[str]] = None, @@ -409,6 +410,8 @@ def search( tbs: Time-based search filter location: Location string for search timeout: Request timeout in milliseconds (default: 300000) + highlights: Generate query-relevant highlights for search results + (default: true) page_options: Options for scraping individual pages enterprise: Enterprise search options. Use ["zdr"] for end-to-end Zero Data Retention or ["anon"] for anonymized search. Must be @@ -430,6 +433,7 @@ def search( location=location, ignore_invalid_urls=ignore_invalid_urls, timeout=timeout, + highlights=highlights, scrape_options=scrape_options, integration=integration, enterprise=enterprise, diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.py index 7b3e043af5e..a769e843a4b 100644 --- a/apps/python-sdk/firecrawl/v2/types.py +++ b/apps/python-sdk/firecrawl/v2/types.py @@ -1602,6 +1602,7 @@ class SearchRequest(BaseModel): location: Optional[str] = None ignore_invalid_urls: Optional[bool] = None timeout: Optional[int] = 300000 + highlights: Optional[bool] = None scrape_options: Optional[ScrapeOptions] = None # Enterprise search options. Use ["zdr"] for end-to-end Zero Data # Retention or ["anon"] for anonymized search. Must be enabled for your team. diff --git a/apps/ruby-sdk/lib/firecrawl/models/search_options.rb b/apps/ruby-sdk/lib/firecrawl/models/search_options.rb index b0e8d96ed11..4df9bb4ad4d 100644 --- a/apps/ruby-sdk/lib/firecrawl/models/search_options.rb +++ b/apps/ruby-sdk/lib/firecrawl/models/search_options.rb @@ -6,7 +6,7 @@ module Models class SearchOptions FIELDS = %i[ sources categories include_domains exclude_domains limit tbs location - ignore_invalid_urls timeout scrape_options integration enterprise + ignore_invalid_urls timeout highlights scrape_options integration enterprise ].freeze attr_reader(*FIELDS) @@ -26,6 +26,7 @@ def to_h "location" => location, "ignoreInvalidURLs" => ignore_invalid_urls, "timeout" => timeout, + "highlights" => highlights, "scrapeOptions" => scrape_options&.to_h, "integration" => integration, # Enterprise search options. Use ["zdr"] for end-to-end Zero Data diff --git a/apps/ruby-sdk/lib/firecrawl/version.rb b/apps/ruby-sdk/lib/firecrawl/version.rb index ce4e0ce3890..fd14285d4f8 100644 --- a/apps/ruby-sdk/lib/firecrawl/version.rb +++ b/apps/ruby-sdk/lib/firecrawl/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module Firecrawl - VERSION = "1.10.0" + VERSION = "1.10.1" end diff --git a/apps/ruby-sdk/test/firecrawl/client_test.rb b/apps/ruby-sdk/test/firecrawl/client_test.rb index a620506e86e..8019993f981 100644 --- a/apps/ruby-sdk/test/firecrawl/client_test.rb +++ b/apps/ruby-sdk/test/firecrawl/client_test.rb @@ -742,6 +742,7 @@ def test_search_options_to_h limit: 10, location: "US", tbs: "qdr:w", + highlights: false, include_domains: ["firecrawl.dev"], exclude_domains: ["example.com"] ) @@ -749,6 +750,7 @@ def test_search_options_to_h assert_equal 10, h["limit"] assert_equal "US", h["location"] assert_equal "qdr:w", h["tbs"] + assert_equal false, h["highlights"] assert_equal ["firecrawl.dev"], h["includeDomains"] assert_equal ["example.com"], h["excludeDomains"] end diff --git a/apps/rust-sdk/Cargo.lock b/apps/rust-sdk/Cargo.lock index 2ac74062ac5..c5fda5043e5 100644 --- a/apps/rust-sdk/Cargo.lock +++ b/apps/rust-sdk/Cargo.lock @@ -250,7 +250,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "firecrawl" -version = "2.11.0" +version = "2.11.1" dependencies = [ "mockito", "reqwest", diff --git a/apps/rust-sdk/Cargo.toml b/apps/rust-sdk/Cargo.toml index dd989220b66..40611dcad68 100644 --- a/apps/rust-sdk/Cargo.toml +++ b/apps/rust-sdk/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "firecrawl" -version = "2.11.0" +version = "2.11.1" edition = "2021" license = "MIT" homepage = "https://www.firecrawl.dev/" diff --git a/apps/rust-sdk/src/search.rs b/apps/rust-sdk/src/search.rs index 945f724ea85..9a7a328b961 100644 --- a/apps/rust-sdk/src/search.rs +++ b/apps/rust-sdk/src/search.rs @@ -41,6 +41,9 @@ pub struct SearchOptions { /// Timeout in milliseconds. pub timeout: Option, + /// Whether to generate query-relevant highlights. Defaults to true. + pub highlights: Option, + /// Scrape options to apply to each search result. pub scrape_options: Option, @@ -265,6 +268,19 @@ mod tests { use super::*; use serde_json::json; + #[test] + fn serializes_highlights_option() { + let options = SearchOptions { + highlights: Some(false), + ..Default::default() + }; + + assert_eq!( + serde_json::to_value(options).unwrap(), + json!({ "highlights": false }) + ); + } + #[tokio::test] async fn test_search_with_mock() { let mut server = mockito::Server::new_async().await;