[pull] main from firecrawl:main

apps/api/src/__tests__/snips/v2/audio-routing.test.ts

@@ -6,12 +6,16 @@
 
 describe("Audio format engine routing (buildFallbackList)", () => {
   let buildFallbackList: typeof import("../../../scraper/scrapeURL/engines/index.js").buildFallbackList;
+  let clearExchangeProvidersForTest: typeof import("../../../lib/exchange.js").clearExchangeProvidersForTest;
+  let setExchangeProvidersForTest: typeof import("../../../lib/exchange.js").setExchangeProvidersForTest;
 
   const originalFireEngineUrl = process.env.FIRE_ENGINE_BETA_URL;
   const originalIndexUrl = process.env.INDEX_DATABASE_URL;
+  const originalExchangeUrl = process.env.FIRE_EXCHANGE_URL;
 
   beforeAll(async () => {
     process.env.FIRE_ENGINE_BETA_URL = "http://test-fire-engine";
+    process.env.FIRE_EXCHANGE_URL = "http://test-exchange";
     process.env.INDEX_DATABASE_URL =
       "postgresql://postgres:postgres@localhost:5432/postgres";
 
@@ -20,6 +24,14 @@ describe("Audio format engine routing (buildFallbackList)", () => {
     ({ buildFallbackList } = await import(
       "../../../scraper/scrapeURL/engines/index.js"
     ));
+    ({
+      clearExchangeProvidersForTest,
+      setExchangeProvidersForTest,
+    } = await import("../../../lib/exchange.js"));
+  });
+
+  afterEach(() => {
+    clearExchangeProvidersForTest();
   });
 
   afterAll(() => {
@@ -33,6 +45,11 @@ describe("Audio format engine routing (buildFallbackList)", () => {
     } else {
       process.env.INDEX_DATABASE_URL = originalIndexUrl;
     }
+    if (originalExchangeUrl === undefined) {
+      delete process.env.FIRE_EXCHANGE_URL;
+    } else {
+      process.env.FIRE_EXCHANGE_URL = originalExchangeUrl;
+    }
   });
 
   const buildStubMeta = (featureFlags: string[]) =>
@@ -110,4 +127,20 @@ describe("Audio format engine routing (buildFallbackList)", () => {
     expect(engines).toContain("fire-engine;chrome-cdp");
   });
 
+  it("does not route agent index-only requests through the Exchange", async () => {
+    setExchangeProvidersForTest([
+      { id: "acme", routes: [{ domains: ["profiles.example"] }] },
+    ]);
+
+    const meta = buildStubMeta([]);
+    meta.url = "https://profiles.example/person/example-person";
+    meta.options.formats = [{ type: "markdown" }];
+    meta.internalOptions.agentIndexOnly = true;
+    meta.internalOptions.teamFlags = { professionalProfileCompanyDataBeta: true };
+
+    const fallback = await buildFallbackList(meta);
+    const engines = fallback.map(f => f.engine);
+
+    expect(engines).toEqual(["index", "index;documents"]);
+  });
 });

apps/api/src/config.ts

@@ -165,6 +165,9 @@ const configSchema = z.object({
   HIGHLIGHT_MODEL_TOKEN: z.string().optional(),
   HIGHLIGHT_SHADOW_RATE: z.coerce.number().min(0).max(1).default(0),
 
+  // Exchange (routed data sources service)
+  FIRE_EXCHANGE_URL: z.url().optional(),
+
   // Fire Engine
   FIRE_ENGINE_BETA_URL: z.string().optional(),
   FIRE_ENGINE_STAGING_URL: z.string().optional(),

apps/api/src/index.ts

@@ -34,6 +34,7 @@ import { v2Router } from "./routes/v2";
 import { nuqShutdown } from "./services/worker/nuq";
 import { getErrorContactMessage } from "./lib/deployment";
 import { initializeBlocklist } from "./scraper/WebScraper/utils/blocklist";
+import { warmExchangeCatalog } from "./lib/exchange";
 import { initializeEngineForcing } from "./scraper/WebScraper/utils/engine-forcing";
 import responseTime from "response-time";
 import { shutdownWebhookQueue } from "./services/webhook";
@@ -132,6 +133,7 @@ async function startServer(port = DEFAULT_PORT) {
   try {
     await initializeBlocklist();
     initializeEngineForcing();
+    warmExchangeCatalog();
   } catch (error) {
     logger.error("Failed to initialize API startup dependencies", {
       error,

apps/api/src/lib/error-serde.ts

@@ -39,6 +39,8 @@ import { UnsafeDomainBlockedError } from "./threat-protection/error";
 
 // TODO: figure out correct typing for this
 const errorMap: Record<ErrorCodes, any> = {
+  // Terms responses are API-level, never transported through workers.
+  THIRD_PARTY_DATA_TERMS_REQUIRED: null,
   SCRAPE_TIMEOUT: ScrapeJobTimeoutError,
   MAP_TIMEOUT: MapTimeoutError,
   UNKNOWN_ERROR: UnknownError,

apps/api/src/lib/error.ts

@@ -1,4 +1,5 @@
 export type ErrorCodes =
+  | "THIRD_PARTY_DATA_TERMS_REQUIRED"
   | "SCRAPE_TIMEOUT"
   | "MAP_TIMEOUT"
   | "UNKNOWN_ERROR"

apps/api/src/lib/exchange-request.ts

@@ -0,0 +1,47 @@
+import {
+  getExchangeAccessForRequest,
+  type ExchangeAccess,
+} from "./exchange";
+
+type ExchangeFlags = Parameters<
+  typeof getExchangeAccessForRequest
+>[0]["flags"];
+
+/**
+ * Run the Exchange access gate against an API request body, sourcing the
+ * effective per-page options from a nested scrapeOptions object (crawl
+ * bodies) or the body itself (scrape and batch scrape bodies), with v1
+ * pageOptions fallbacks.
+ */
+export function getExchangeAccessForRequestBody(input: {
+  body: Record<string, any>;
+  flags: ExchangeFlags;
+  url: string;
+  zeroDataRetention: boolean;
+}): Promise<ExchangeAccess> {
+  const body = input.body ?? {};
+  const scrapeOptions =
+    typeof body.scrapeOptions === "object" && body.scrapeOptions !== null
+      ? body.scrapeOptions
+      : body;
+
+  return getExchangeAccessForRequest({
+    url: input.url,
+    formats: scrapeOptions.formats,
+    actions: scrapeOptions.actions,
+    headers: scrapeOptions.headers,
+    waitFor: scrapeOptions.waitFor,
+    mobile: scrapeOptions.mobile,
+    location: scrapeOptions.location,
+    proxy: scrapeOptions.proxy,
+    blockAds: scrapeOptions.blockAds,
+    profile: scrapeOptions.profile,
+    atsv: scrapeOptions.atsv ?? body.pageOptions?.atsv,
+    minAge: scrapeOptions.minAge,
+    includeTags: scrapeOptions.includeTags ?? body.pageOptions?.includeTags,
+    excludeTags: scrapeOptions.excludeTags ?? body.pageOptions?.excludeTags,
+    zeroDataRetention: input.zeroDataRetention,
+    lockdown: scrapeOptions.lockdown ?? body.lockdown,
+    flags: input.flags,
+  });
+}