From eb5b5ca0d641f8beb77212cf07aba1037cd26a3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= Date: Thu, 16 Jul 2026 16:09:30 +0200 Subject: [PATCH 1/2] feat(sdks): add search highlights option (#4042) --- .../Firecrawl.Tests/ModelsTests.cs | 2 ++ apps/dot-net-sdk/Firecrawl/Firecrawl.csproj | 2 +- .../Firecrawl/Models/SearchOptions.cs | 5 +++ apps/elixir-sdk/lib/firecrawl.ex | 3 +- apps/elixir-sdk/mix.exs | 2 +- apps/elixir-sdk/test/firecrawl_test.exs | 34 +++++++++++++++++++ apps/go-sdk/options.go | 1 + apps/go-sdk/options_test.go | 11 ++++++ apps/go-sdk/version.go | 2 +- apps/java-sdk/build.gradle.kts | 2 +- .../com/firecrawl/models/SearchOptions.java | 6 ++++ .../java/com/firecrawl/SearchOptionsTest.java | 25 ++++++++++++++ apps/js-sdk/firecrawl/package.json | 2 +- .../unit/v2/search-highlights.unit.test.ts | 18 ++++++++++ .../js-sdk/firecrawl/src/v2/methods/search.ts | 1 + apps/js-sdk/firecrawl/src/v2/types.ts | 2 ++ apps/php-sdk/src/Models/SearchOptions.php | 4 +++ apps/php-sdk/src/Version.php | 2 +- apps/php-sdk/tests/Unit/ModelsTest.php | 7 ++++ apps/python-sdk/firecrawl/__init__.py | 2 +- .../test_search_request_preparation.py | 4 ++- apps/python-sdk/firecrawl/v2/client.py | 4 +++ apps/python-sdk/firecrawl/v2/types.py | 1 + .../lib/firecrawl/models/search_options.rb | 3 +- apps/ruby-sdk/lib/firecrawl/version.rb | 2 +- apps/ruby-sdk/test/firecrawl/client_test.rb | 2 ++ apps/rust-sdk/Cargo.lock | 2 +- apps/rust-sdk/Cargo.toml | 2 +- apps/rust-sdk/src/search.rs | 16 +++++++++ 29 files changed, 156 insertions(+), 13 deletions(-) create mode 100644 apps/java-sdk/src/test/java/com/firecrawl/SearchOptionsTest.java create mode 100644 apps/js-sdk/firecrawl/src/__tests__/unit/v2/search-highlights.unit.test.ts diff --git a/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs b/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs index 799039f11a0..3ab23a0697b 100644 --- a/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs +++ b/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs @@ -87,6 +87,7 @@ public void SearchOptions_SerializesCorrectly() Limit = 5, Location = "US", Tbs = "qdr:w", + Highlights = false, IncludeDomains = new() { "firecrawl.dev" }, ExcludeDomains = new() { "example.com" } }; @@ -95,6 +96,7 @@ public void SearchOptions_SerializesCorrectly() Assert.Contains("\"limit\":5", json); Assert.Contains("\"location\":\"US\"", json); Assert.Contains("\"tbs\":\"qdr:w\"", json); + Assert.Contains("\"highlights\":false", json); Assert.Contains("\"includeDomains\":[\"firecrawl.dev\"]", json); Assert.Contains("\"excludeDomains\":[\"example.com\"]", json); } diff --git a/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj b/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj index ad364cc6204..b84ed222ad4 100644 --- a/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj +++ b/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj @@ -8,7 +8,7 @@ firecrawl-sdk - 1.9.0 + 1.9.1 Firecrawl Firecrawl .NET SDK for the Firecrawl API - web scraping, crawling, and data extraction diff --git a/apps/dot-net-sdk/Firecrawl/Models/SearchOptions.cs b/apps/dot-net-sdk/Firecrawl/Models/SearchOptions.cs index f917a20e85e..709e7cf044b 100644 --- a/apps/dot-net-sdk/Firecrawl/Models/SearchOptions.cs +++ b/apps/dot-net-sdk/Firecrawl/Models/SearchOptions.cs @@ -43,6 +43,11 @@ public class SearchOptions [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public int? Timeout { get; set; } + /// Generate query-relevant highlights for search results. Defaults to true. + [JsonPropertyName("highlights")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? Highlights { get; set; } + [JsonPropertyName("scrapeOptions")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public ScrapeOptions? ScrapeOptions { get; set; } diff --git a/apps/elixir-sdk/lib/firecrawl.ex b/apps/elixir-sdk/lib/firecrawl.ex index c75cd467478..760957ce58f 100644 --- a/apps/elixir-sdk/lib/firecrawl.ex +++ b/apps/elixir-sdk/lib/firecrawl.ex @@ -1150,6 +1150,7 @@ defmodule Firecrawl do enterprise: [type: {:list, :string}, doc: "Enterprise search options for Zero Data Retention (ZDR). Use `[\"zdr\"]` for end-to-end ZDR (10 credits / 10 results) or `[\"anon\"]` for anonymized ZDR (2 credits / 10 results). Must be enabled for your team."], exclude_domains: [type: {:list, :string}, doc: "Domains to exclude from search results."], ignore_invalid_urls: [type: :boolean, doc: "Excludes URLs from the search results that are invalid for other Firecrawl endpoints. This helps reduce errors if you are piping data from search into other Firecrawl API endpoints."], + highlights: [type: :boolean, doc: "Generate query-relevant highlights for search results. Defaults to true."], include_domains: [type: {:list, :string}, doc: "Domains to include in search results."], limit: [type: :integer, doc: "Maximum number of results to return"], location: [type: :string, doc: "Location parameter for search results (e.g. `San Francisco,California,United States`). For best results, set both this and the `country` parameter."], @@ -1160,7 +1161,7 @@ defmodule Firecrawl do timeout: [type: :integer, doc: "Timeout in milliseconds"] ]) - @search_and_scrape_key_mapping %{categories: "categories", country: "country", enterprise: "enterprise", exclude_domains: "excludeDomains", ignore_invalid_urls: "ignoreInvalidURLs", include_domains: "includeDomains", limit: "limit", location: "location", query: "query", scrape_options: "scrapeOptions", sources: "sources", tbs: "tbs", timeout: "timeout"} + @search_and_scrape_key_mapping %{categories: "categories", country: "country", enterprise: "enterprise", exclude_domains: "excludeDomains", highlights: "highlights", ignore_invalid_urls: "ignoreInvalidURLs", include_domains: "includeDomains", limit: "limit", location: "location", query: "query", scrape_options: "scrapeOptions", sources: "sources", tbs: "tbs", timeout: "timeout"} @doc """ Search and optionally scrape search results diff --git a/apps/elixir-sdk/mix.exs b/apps/elixir-sdk/mix.exs index 446c7055366..163203b4b42 100644 --- a/apps/elixir-sdk/mix.exs +++ b/apps/elixir-sdk/mix.exs @@ -1,7 +1,7 @@ defmodule Firecrawl.MixProject do use Mix.Project - @version "1.8.0" + @version "1.8.1" @source_url "https://github.com/firecrawl/firecrawl/tree/main/apps/elixir-sdk" def project do diff --git a/apps/elixir-sdk/test/firecrawl_test.exs b/apps/elixir-sdk/test/firecrawl_test.exs index 9da76abd86b..be8c4400ec7 100644 --- a/apps/elixir-sdk/test/firecrawl_test.exs +++ b/apps/elixir-sdk/test/firecrawl_test.exs @@ -317,6 +317,40 @@ defmodule FirecrawlTest do assert body["urls"] == ["https://example.com"] end + test "search maps highlights to highlights" do + parent = self() + + adapter = fn request -> + send(parent, {:request, request}) + + resp = Req.Response.new( + status: 200, + headers: %{"content-type" => ["application/json"]}, + body: Jason.encode!(%{"success" => true, "data" => %{}}) + ) + + {request, resp} + end + + assert {:ok, %Req.Response{status: 200}} = + Firecrawl.search_and_scrape( + [query: "firecrawl", highlights: false], + api_key: "test-key", + adapter: adapter + ) + + assert_receive {:request, request} + + body = + cond do + is_binary(request.body) -> Jason.decode!(request.body) + is_map(request.body) -> request.body + true -> request.options[:json] + end + + assert body["highlights"] == false + end + test "all expected API functions are defined with bang variants" do functions = Firecrawl.__info__(:functions) diff --git a/apps/go-sdk/options.go b/apps/go-sdk/options.go index f38f296dfde..c7022f57e86 100644 --- a/apps/go-sdk/options.go +++ b/apps/go-sdk/options.go @@ -171,6 +171,7 @@ type SearchOptions struct { Location *string `json:"location,omitempty"` IgnoreInvalidURLs *bool `json:"ignoreInvalidURLs,omitempty"` Timeout *int `json:"timeout,omitempty"` + Highlights *bool `json:"highlights,omitempty"` ScrapeOptions *ScrapeOptions `json:"scrapeOptions,omitempty"` Integration *string `json:"integration,omitempty"` } diff --git a/apps/go-sdk/options_test.go b/apps/go-sdk/options_test.go index 35603d42138..d41057c2d7a 100644 --- a/apps/go-sdk/options_test.go +++ b/apps/go-sdk/options_test.go @@ -76,3 +76,14 @@ func TestScrapeOptionsSerializesRedactPII(t *testing.T) { t.Fatalf("serialized redactPII = %s", payload) } } + +func TestSearchOptionsSerializesHighlights(t *testing.T) { + payload, err := json.Marshal(SearchOptions{Highlights: Bool(false)}) + if err != nil { + t.Fatalf("Marshal SearchOptions: %v", err) + } + + if !strings.Contains(string(payload), `"highlights":false`) { + t.Fatalf("serialized search options = %s", payload) + } +} diff --git a/apps/go-sdk/version.go b/apps/go-sdk/version.go index 4d99375f14c..79775e6d872 100644 --- a/apps/go-sdk/version.go +++ b/apps/go-sdk/version.go @@ -9,4 +9,4 @@ package firecrawl // Bump this when preparing a new release. The publish-go-sdk GitHub workflow // reads this value and creates the corresponding monorepo-prefixed tag on // merge to main. -const Version = "1.8.0" +const Version = "1.8.1" diff --git a/apps/java-sdk/build.gradle.kts b/apps/java-sdk/build.gradle.kts index c74af612ef8..0310669b89f 100644 --- a/apps/java-sdk/build.gradle.kts +++ b/apps/java-sdk/build.gradle.kts @@ -4,7 +4,7 @@ plugins { } group = "com.firecrawl" -version = "1.11.0" +version = "1.11.1" java { sourceCompatibility = JavaVersion.VERSION_11 diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/SearchOptions.java b/apps/java-sdk/src/main/java/com/firecrawl/models/SearchOptions.java index 8b36091f89b..1553d4f65b7 100644 --- a/apps/java-sdk/src/main/java/com/firecrawl/models/SearchOptions.java +++ b/apps/java-sdk/src/main/java/com/firecrawl/models/SearchOptions.java @@ -18,6 +18,7 @@ public class SearchOptions { private String location; private Boolean ignoreInvalidURLs; private Integer timeout; + private Boolean highlights; private ScrapeOptions scrapeOptions; private String integration; @@ -32,6 +33,7 @@ private SearchOptions() {} public String getLocation() { return location; } public Boolean getIgnoreInvalidURLs() { return ignoreInvalidURLs; } public Integer getTimeout() { return timeout; } + public Boolean getHighlights() { return highlights; } public ScrapeOptions getScrapeOptions() { return scrapeOptions; } public String getIntegration() { return integration; } @@ -47,6 +49,7 @@ public static final class Builder { private String location; private Boolean ignoreInvalidURLs; private Integer timeout; + private Boolean highlights; private ScrapeOptions scrapeOptions; private String integration; @@ -70,6 +73,8 @@ private Builder() {} public Builder ignoreInvalidURLs(Boolean ignoreInvalidURLs) { this.ignoreInvalidURLs = ignoreInvalidURLs; return this; } /** Timeout in milliseconds. */ public Builder timeout(Integer timeout) { this.timeout = timeout; return this; } + /** Generate query-relevant highlights for search results. Defaults to true. */ + public Builder highlights(Boolean highlights) { this.highlights = highlights; return this; } /** Scrape options applied to search result pages. */ public Builder scrapeOptions(ScrapeOptions scrapeOptions) { this.scrapeOptions = scrapeOptions; return this; } /** Integration identifier. */ @@ -86,6 +91,7 @@ public SearchOptions build() { o.location = this.location; o.ignoreInvalidURLs = this.ignoreInvalidURLs; o.timeout = this.timeout; + o.highlights = this.highlights; o.scrapeOptions = this.scrapeOptions; o.integration = this.integration; return o; diff --git a/apps/java-sdk/src/test/java/com/firecrawl/SearchOptionsTest.java b/apps/java-sdk/src/test/java/com/firecrawl/SearchOptionsTest.java new file mode 100644 index 00000000000..7b8f1dc547e --- /dev/null +++ b/apps/java-sdk/src/test/java/com/firecrawl/SearchOptionsTest.java @@ -0,0 +1,25 @@ +package com.firecrawl; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.firecrawl.models.SearchOptions; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class SearchOptionsTest { + @Test + void serializesHighlightsOption() { + ObjectMapper mapper = new ObjectMapper(); + JsonNode enabled = mapper.valueToTree( + SearchOptions.builder().highlights(true).build()); + JsonNode disabled = mapper.valueToTree( + SearchOptions.builder().highlights(false).build()); + JsonNode omitted = mapper.valueToTree(SearchOptions.builder().build()); + + assertTrue(enabled.get("highlights").asBoolean()); + assertFalse(disabled.get("highlights").asBoolean()); + assertFalse(omitted.has("highlights")); + } +} diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json index 0442e591a11..611d79bdc9a 100644 --- a/apps/js-sdk/firecrawl/package.json +++ b/apps/js-sdk/firecrawl/package.json @@ -1,6 +1,6 @@ { "name": "@mendable/firecrawl-js", - "version": "4.30.0", + "version": "4.30.1", "description": "JavaScript SDK for Firecrawl API", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/search-highlights.unit.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/search-highlights.unit.test.ts new file mode 100644 index 00000000000..88265b2bde8 --- /dev/null +++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/search-highlights.unit.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, jest, test } from "@jest/globals"; +import { search } from "../../../v2/methods/search"; + +describe("v2 search highlights", () => { + test("forwards the highlights option", async () => { + const http = { + post: jest.fn(async () => ({ status: 200, data: { success: true } })), + } as any; + + await search(http, { query: "firecrawl", highlights: false }); + + expect(http.post).toHaveBeenCalledWith( + "/v2/search", + { query: "firecrawl", highlights: false }, + {}, + ); + }); +}); diff --git a/apps/js-sdk/firecrawl/src/v2/methods/search.ts b/apps/js-sdk/firecrawl/src/v2/methods/search.ts index d1610959d96..00332394e3f 100644 --- a/apps/js-sdk/firecrawl/src/v2/methods/search.ts +++ b/apps/js-sdk/firecrawl/src/v2/methods/search.ts @@ -37,6 +37,7 @@ function prepareSearchPayload(req: SearchRequest): Record { if (req.ignoreInvalidURLs != null) payload.ignoreInvalidURLs = req.ignoreInvalidURLs; if (req.timeout != null) payload.timeout = req.timeout; + if (req.highlights != null) payload.highlights = req.highlights; if (req.integration && req.integration.trim()) payload.integration = req.integration.trim(); if (req.origin) payload.origin = req.origin; diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.ts index c75b174c967..da2ea320837 100644 --- a/apps/js-sdk/firecrawl/src/v2/types.ts +++ b/apps/js-sdk/firecrawl/src/v2/types.ts @@ -697,6 +697,8 @@ export interface SearchRequest { location?: string; ignoreInvalidURLs?: boolean; timeout?: number; // ms + /** Generate query-relevant highlights for search results. Defaults to true. */ + highlights?: boolean; scrapeOptions?: ScrapeOptions; /** * Enterprise search options. Use `["zdr"]` for end-to-end Zero Data diff --git a/apps/php-sdk/src/Models/SearchOptions.php b/apps/php-sdk/src/Models/SearchOptions.php index 30dd484fd8f..bc413f3a7c4 100644 --- a/apps/php-sdk/src/Models/SearchOptions.php +++ b/apps/php-sdk/src/Models/SearchOptions.php @@ -24,6 +24,7 @@ private function __construct( private readonly ?string $integration = null, private readonly ?array $includeDomains = null, private readonly ?array $excludeDomains = null, + private readonly ?bool $highlights = null, ) {} /** @@ -44,10 +45,12 @@ public static function with( ?string $integration = null, ?array $includeDomains = null, ?array $excludeDomains = null, + ?bool $highlights = null, ): self { return new self( $sources, $categories, $limit, $tbs, $location, $ignoreInvalidURLs, $timeout, $scrapeOptions, $integration, $includeDomains, $excludeDomains, + $highlights, ); } @@ -64,6 +67,7 @@ public function toArray(): array 'location' => $this->location, 'ignoreInvalidURLs' => $this->ignoreInvalidURLs, 'timeout' => $this->timeout, + 'highlights' => $this->highlights, 'scrapeOptions' => $this->scrapeOptions?->toArray(), 'integration' => $this->integration, ]; diff --git a/apps/php-sdk/src/Version.php b/apps/php-sdk/src/Version.php index 14feb905343..4e52a3fcfc1 100644 --- a/apps/php-sdk/src/Version.php +++ b/apps/php-sdk/src/Version.php @@ -6,5 +6,5 @@ final class Version { - public const SDK_VERSION = '1.9.0'; + public const SDK_VERSION = '1.9.1'; } diff --git a/apps/php-sdk/tests/Unit/ModelsTest.php b/apps/php-sdk/tests/Unit/ModelsTest.php index 2b77c63005a..e55ac7a9b3d 100644 --- a/apps/php-sdk/tests/Unit/ModelsTest.php +++ b/apps/php-sdk/tests/Unit/ModelsTest.php @@ -13,6 +13,7 @@ use Firecrawl\Models\QueryFormat; use Firecrawl\Models\QuestionFormat; use Firecrawl\Models\ScrapeOptions; +use Firecrawl\Models\SearchOptions; use Firecrawl\Models\Monitor; use Firecrawl\Models\MonitorCheck; @@ -259,6 +260,12 @@ expect($doc->getMenu())->toBeNull(); }); +it('serializes the search highlights option', function (): void { + $options = SearchOptions::with(highlights: false); + + expect($options->toArray())->toBe(['highlights' => false]); +}); + it('coerces non-string scalar identity fields without a TypeError under strict_types', function (): void { // Defensive: upstream data could carry a non-string scalar (e.g. a numeric // brand). Under declare(strict_types=1) these must be cast, not passed raw. diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py index 64a6d45f861..fc8b578bbed 100644 --- a/apps/python-sdk/firecrawl/__init__.py +++ b/apps/python-sdk/firecrawl/__init__.py @@ -17,7 +17,7 @@ V1ChangeTrackingOptions, ) -__version__ = "4.32.0" +__version__ = "4.32.1" # Define the logger for the Firecrawl project logger: logging.Logger = logging.getLogger("firecrawl") diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_search_request_preparation.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_search_request_preparation.py index f63219591e2..221f2e2d0d3 100644 --- a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_search_request_preparation.py +++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_search_request_preparation.py @@ -44,6 +44,7 @@ def test_all_fields_conversion(self): location="US", ignore_invalid_urls=False, timeout=30000, + highlights=False, scrape_options=scrape_opts, integration=" _e2e-test ", ) @@ -56,6 +57,7 @@ def test_all_fields_conversion(self): assert data["tbs"] == "qdr:w" assert data["location"] == "US" assert data["timeout"] == 30000 + assert data["highlights"] is False # Check snake_case to camelCase conversions assert "ignoreInvalidURLs" in data @@ -181,4 +183,4 @@ def test_scrape_options_shared_function_integration(self): assert "only_main_content" not in scrape_data assert "wait_for" not in scrape_data assert "skip_tls_verification" not in scrape_data - assert "remove_base64_images" not in scrape_data \ No newline at end of file + assert "remove_base64_images" not in scrape_data diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.py index b04810290c9..a8236e57302 100644 --- a/apps/python-sdk/firecrawl/v2/client.py +++ b/apps/python-sdk/firecrawl/v2/client.py @@ -395,6 +395,7 @@ def search( location: Optional[str] = None, ignore_invalid_urls: Optional[bool] = None, timeout: Optional[int] = None, + highlights: Optional[bool] = None, scrape_options: Optional[ScrapeOptions] = None, integration: Optional[str] = None, enterprise: Optional[List[str]] = None, @@ -409,6 +410,8 @@ def search( tbs: Time-based search filter location: Location string for search timeout: Request timeout in milliseconds (default: 300000) + highlights: Generate query-relevant highlights for search results + (default: true) page_options: Options for scraping individual pages enterprise: Enterprise search options. Use ["zdr"] for end-to-end Zero Data Retention or ["anon"] for anonymized search. Must be @@ -430,6 +433,7 @@ def search( location=location, ignore_invalid_urls=ignore_invalid_urls, timeout=timeout, + highlights=highlights, scrape_options=scrape_options, integration=integration, enterprise=enterprise, diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.py index 7b3e043af5e..a769e843a4b 100644 --- a/apps/python-sdk/firecrawl/v2/types.py +++ b/apps/python-sdk/firecrawl/v2/types.py @@ -1602,6 +1602,7 @@ class SearchRequest(BaseModel): location: Optional[str] = None ignore_invalid_urls: Optional[bool] = None timeout: Optional[int] = 300000 + highlights: Optional[bool] = None scrape_options: Optional[ScrapeOptions] = None # Enterprise search options. Use ["zdr"] for end-to-end Zero Data # Retention or ["anon"] for anonymized search. Must be enabled for your team. diff --git a/apps/ruby-sdk/lib/firecrawl/models/search_options.rb b/apps/ruby-sdk/lib/firecrawl/models/search_options.rb index b0e8d96ed11..4df9bb4ad4d 100644 --- a/apps/ruby-sdk/lib/firecrawl/models/search_options.rb +++ b/apps/ruby-sdk/lib/firecrawl/models/search_options.rb @@ -6,7 +6,7 @@ module Models class SearchOptions FIELDS = %i[ sources categories include_domains exclude_domains limit tbs location - ignore_invalid_urls timeout scrape_options integration enterprise + ignore_invalid_urls timeout highlights scrape_options integration enterprise ].freeze attr_reader(*FIELDS) @@ -26,6 +26,7 @@ def to_h "location" => location, "ignoreInvalidURLs" => ignore_invalid_urls, "timeout" => timeout, + "highlights" => highlights, "scrapeOptions" => scrape_options&.to_h, "integration" => integration, # Enterprise search options. Use ["zdr"] for end-to-end Zero Data diff --git a/apps/ruby-sdk/lib/firecrawl/version.rb b/apps/ruby-sdk/lib/firecrawl/version.rb index ce4e0ce3890..fd14285d4f8 100644 --- a/apps/ruby-sdk/lib/firecrawl/version.rb +++ b/apps/ruby-sdk/lib/firecrawl/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module Firecrawl - VERSION = "1.10.0" + VERSION = "1.10.1" end diff --git a/apps/ruby-sdk/test/firecrawl/client_test.rb b/apps/ruby-sdk/test/firecrawl/client_test.rb index a620506e86e..8019993f981 100644 --- a/apps/ruby-sdk/test/firecrawl/client_test.rb +++ b/apps/ruby-sdk/test/firecrawl/client_test.rb @@ -742,6 +742,7 @@ def test_search_options_to_h limit: 10, location: "US", tbs: "qdr:w", + highlights: false, include_domains: ["firecrawl.dev"], exclude_domains: ["example.com"] ) @@ -749,6 +750,7 @@ def test_search_options_to_h assert_equal 10, h["limit"] assert_equal "US", h["location"] assert_equal "qdr:w", h["tbs"] + assert_equal false, h["highlights"] assert_equal ["firecrawl.dev"], h["includeDomains"] assert_equal ["example.com"], h["excludeDomains"] end diff --git a/apps/rust-sdk/Cargo.lock b/apps/rust-sdk/Cargo.lock index 2ac74062ac5..c5fda5043e5 100644 --- a/apps/rust-sdk/Cargo.lock +++ b/apps/rust-sdk/Cargo.lock @@ -250,7 +250,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "firecrawl" -version = "2.11.0" +version = "2.11.1" dependencies = [ "mockito", "reqwest", diff --git a/apps/rust-sdk/Cargo.toml b/apps/rust-sdk/Cargo.toml index dd989220b66..40611dcad68 100644 --- a/apps/rust-sdk/Cargo.toml +++ b/apps/rust-sdk/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "firecrawl" -version = "2.11.0" +version = "2.11.1" edition = "2021" license = "MIT" homepage = "https://www.firecrawl.dev/" diff --git a/apps/rust-sdk/src/search.rs b/apps/rust-sdk/src/search.rs index 945f724ea85..9a7a328b961 100644 --- a/apps/rust-sdk/src/search.rs +++ b/apps/rust-sdk/src/search.rs @@ -41,6 +41,9 @@ pub struct SearchOptions { /// Timeout in milliseconds. pub timeout: Option, + /// Whether to generate query-relevant highlights. Defaults to true. + pub highlights: Option, + /// Scrape options to apply to each search result. pub scrape_options: Option, @@ -265,6 +268,19 @@ mod tests { use super::*; use serde_json::json; + #[test] + fn serializes_highlights_option() { + let options = SearchOptions { + highlights: Some(false), + ..Default::default() + }; + + assert_eq!( + serde_json::to_value(options).unwrap(), + json!({ "highlights": false }) + ); + } + #[tokio::test] async fn test_search_with_mock() { let mut server = mockito::Server::new_async().await; From 692d6b7e10fe6b17fc7ba5cae64f56a12f05c75f Mon Sep 17 00:00:00 2001 From: Abimael Martell <1450169+abimaelmartell@users.noreply.github.com> Date: Thu, 16 Jul 2026 10:22:12 -0700 Subject: [PATCH 2/2] feat(api): route eligible scrapes through the Exchange (#4030) * feat(api): route eligible scrapes through the Exchange Reimplement the removed data-layer path against the Exchange service: - the provider catalog (GET /v1/providers) supplies URL routes, per-access credit cost, and required legal terms, cached with a short TTL; new providers need no changes in this repo - access gating is generic: the org's data-source access entry is looked up by provider id and compared against the terms the catalog declares - billing uses the creditsCost reported per response, and each billed access is confirmed back to the Exchange so its ledger reconciles - terms-required responses carry the provider's terms key and version from the catalog Configured via EXCHANGE_URL; the beta flag and the org data-source access flags are unchanged. Co-Authored-By: Claude Fable 5 * perf(api): serve the stale Exchange catalog while refreshing Catalog expiry no longer adds latency to flagged-org requests: the expired catalog is served immediately while the refresh runs in the background. Only the very first lookup after boot waits for the fetch. Orgs without the beta flag never reach the catalog at all - the flag check is the first, synchronous eligibility gate. Co-Authored-By: Claude Fable 5 * fix(api): degrade gracefully when the Exchange is unavailable Two hardening changes so an Exchange outage or a bug in its gate can never affect the normal scrape path: - a failed catalog refresh keeps serving the last good catalog instead of clobbering it with null; the failure TTL only throttles the next refresh attempt - the access gate wraps everything in a fail-safe: any unexpected error degrades the request to ineligible (normal blocklist behavior) with a warning log, instead of failing the customer's request Co-Authored-By: Claude Fable 5 * fix(api): address review findings on the Exchange path - void the Exchange access event when a delivered scrape ultimately fails (post-delivery crawl rejection), so the ledger reconciles a never-billed access instead of leaving it pending - fail closed in buildFallbackList when a blocklisted URL reaches the worker but the Exchange is no longer usable (bypass TOCTOU); scoped to flagged orgs and fail-safe so it can never break ordinary scrapes - respect internalOptions.forceEngine over Exchange routing - drop deterministicJson from supported formats - its extractors need page HTML, which Exchange responses do not carry - remove .catch(0) on credit costs in both schemas: malformed costs now reject loudly instead of silently billing 0 - synthesize a minimal HTML head from the provider's title/description so document metadata is populated - validate EXCHANGE_URL as a URL at config load - strip embedded credentials from logged URLs - drop the redundant lockdown term from the middleware ZDR computation Co-Authored-By: Claude Fable 5 * chore(api): rename the Exchange env var to FIRE_EXCHANGE_URL Matches the service's deployment naming (fire-exchange). Co-Authored-By: Claude Fable 5 * fix(api): fail closed when the Exchange blocklist re-check errors For flagged orgs, an error in the worker-side blocklist re-check now returns no engines instead of continuing: a retryable scrape failure is preferable to normal engines scraping a potentially blocklisted site that was admitted through the Exchange bypass. Co-Authored-By: Claude Fable 5 * fix(api): address second review round on the Exchange path - crawl bodies nest options under scrapeOptions; eligibility now reads the effective per-page options so ineligible crawls cannot bypass the blocklist - profile-backed scrapes are Exchange-ineligible (session-specific content cannot come from a provider) - the engine span uses the credential-redacted URL, matching the log context - the billing confirmation is awaited before billScrapeJob returns, so a later failure-path void can never race it; the service rejects confirmed->void, making the order decisive Co-Authored-By: Claude Fable 5 * perf(api): warm the Exchange catalog at process startup The API server and scrape worker prefetch the provider catalog at boot, so the first flagged-org request never waits on the catalog fetch; stale-while-revalidate keeps every later lookup in-memory. No-op when FIRE_EXCHANGE_URL is not configured. Co-Authored-By: Claude Fable 5 * fix(api): keep atsv requests off the Exchange path The exchange engine declares atsv unsupported, but the eligibility gate never checked it, so atsv-enabled scrapes of provider URLs were routed exclusively to the Exchange and the flag was silently ignored. atsv now makes a request Exchange-ineligible, checked in the middleware and in engine selection like every other engine-shaping option. Co-Authored-By: Claude Fable 5 * fix(api): never store Exchange content on the Firecrawl side Exchange-delivered documents are excluded from the index cache and the search index: every access to provider content must go through the Exchange and its ledger, and no copy may persist outside it. Also from review: - minAge requests are Exchange-ineligible - they ask for Firecrawl-cached data, which never exists for Exchange content - includeTags/excludeTags make a request Exchange-ineligible; selector filtering does not apply to provider records - provider route prefixes match on path segment boundaries, so /person no longer matches /personality Co-Authored-By: Claude Fable 5 * fix(api): scope the Exchange blocklist bypass to scrape-shaped routes The Exchange gate now runs only for scrape, crawl, and batch scrape - the flows whose jobs the exchange engine can actually serve. Map, search, monitors, and other routes keep plain blocklist behavior, so a provider URL can no longer bypass the blocklist on flows that never route through the Exchange, and terms errors no longer surface there. Also removes the unused isExchangeSupportedUrl wrapper. Co-Authored-By: Claude Fable 5 * fix(api): honor Exchange access in batch scrape URL validation Batch bodies carry urls[], so the middleware gate never applied and the controllers' per-URL blocklist checks rejected Exchange-eligible URLs as unsupported (or dropped them into invalidURLs). Both v1 and v2 batch controllers now consult the Exchange gate for blocked URLs only: allowed URLs are admitted, missing terms surface the structured terms response, and everything else keeps existing blocklist behavior. The gate-input mapping is shared with the middleware via lib/exchange-request. Co-Authored-By: Claude Fable 5 * revert(api): keep batch scrape off the Exchange path Batch is out of scope for the pilot: queued batch jobs would need the access flags threaded through internalOptions for the worker-side recheck to hold, so rather than half-supporting it, batch keeps plain blocklist behavior with no Exchange semantics. Only single scrape and crawl are Exchange-aware. The shared gate-input helper stays, used by the middleware. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- .../__tests__/snips/v2/audio-routing.test.ts | 33 + apps/api/src/config.ts | 3 + apps/api/src/index.ts | 2 + apps/api/src/lib/error-serde.ts | 2 + apps/api/src/lib/error.ts | 1 + apps/api/src/lib/exchange-request.ts | 47 ++ apps/api/src/lib/exchange.test.ts | 589 ++++++++++++++++ apps/api/src/lib/exchange.ts | 639 ++++++++++++++++++ apps/api/src/lib/scrape-billing.test.ts | 28 + apps/api/src/lib/scrape-billing.ts | 13 + apps/api/src/routes/shared.ts | 51 ++ apps/api/src/routes/v1.ts | 5 +- apps/api/src/routes/v2.ts | 5 +- .../src/scraper/scrapeURL/engines/exchange.ts | 186 +++++ .../src/scraper/scrapeURL/engines/index.ts | 101 +++ .../scraper/scrapeURL/engines/index/index.ts | 3 + apps/api/src/scraper/scrapeURL/index.ts | 3 + .../transformers/sendToSearchIndex.ts | 5 + apps/api/src/services/worker/scrape-worker.ts | 36 + 19 files changed, 1748 insertions(+), 4 deletions(-) create mode 100644 apps/api/src/lib/exchange-request.ts create mode 100644 apps/api/src/lib/exchange.test.ts create mode 100644 apps/api/src/lib/exchange.ts create mode 100644 apps/api/src/scraper/scrapeURL/engines/exchange.ts diff --git a/apps/api/src/__tests__/snips/v2/audio-routing.test.ts b/apps/api/src/__tests__/snips/v2/audio-routing.test.ts index 5c2d1cac3db..1c4cf040777 100644 --- a/apps/api/src/__tests__/snips/v2/audio-routing.test.ts +++ b/apps/api/src/__tests__/snips/v2/audio-routing.test.ts @@ -6,12 +6,16 @@ describe("Audio format engine routing (buildFallbackList)", () => { let buildFallbackList: typeof import("../../../scraper/scrapeURL/engines/index.js").buildFallbackList; + let clearExchangeProvidersForTest: typeof import("../../../lib/exchange.js").clearExchangeProvidersForTest; + let setExchangeProvidersForTest: typeof import("../../../lib/exchange.js").setExchangeProvidersForTest; const originalFireEngineUrl = process.env.FIRE_ENGINE_BETA_URL; const originalIndexUrl = process.env.INDEX_DATABASE_URL; + const originalExchangeUrl = process.env.FIRE_EXCHANGE_URL; beforeAll(async () => { process.env.FIRE_ENGINE_BETA_URL = "http://test-fire-engine"; + process.env.FIRE_EXCHANGE_URL = "http://test-exchange"; process.env.INDEX_DATABASE_URL = "postgresql://postgres:postgres@localhost:5432/postgres"; @@ -20,6 +24,14 @@ describe("Audio format engine routing (buildFallbackList)", () => { ({ buildFallbackList } = await import( "../../../scraper/scrapeURL/engines/index.js" )); + ({ + clearExchangeProvidersForTest, + setExchangeProvidersForTest, + } = await import("../../../lib/exchange.js")); + }); + + afterEach(() => { + clearExchangeProvidersForTest(); }); afterAll(() => { @@ -33,6 +45,11 @@ describe("Audio format engine routing (buildFallbackList)", () => { } else { process.env.INDEX_DATABASE_URL = originalIndexUrl; } + if (originalExchangeUrl === undefined) { + delete process.env.FIRE_EXCHANGE_URL; + } else { + process.env.FIRE_EXCHANGE_URL = originalExchangeUrl; + } }); const buildStubMeta = (featureFlags: string[]) => @@ -110,4 +127,20 @@ describe("Audio format engine routing (buildFallbackList)", () => { expect(engines).toContain("fire-engine;chrome-cdp"); }); + it("does not route agent index-only requests through the Exchange", async () => { + setExchangeProvidersForTest([ + { id: "acme", routes: [{ domains: ["profiles.example"] }] }, + ]); + + const meta = buildStubMeta([]); + meta.url = "https://profiles.example/person/example-person"; + meta.options.formats = [{ type: "markdown" }]; + meta.internalOptions.agentIndexOnly = true; + meta.internalOptions.teamFlags = { professionalProfileCompanyDataBeta: true }; + + const fallback = await buildFallbackList(meta); + const engines = fallback.map(f => f.engine); + + expect(engines).toEqual(["index", "index;documents"]); + }); }); diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts index a372c24e772..9168f7f605d 100644 --- a/apps/api/src/config.ts +++ b/apps/api/src/config.ts @@ -165,6 +165,9 @@ const configSchema = z.object({ HIGHLIGHT_MODEL_TOKEN: z.string().optional(), HIGHLIGHT_SHADOW_RATE: z.coerce.number().min(0).max(1).default(0), + // Exchange (routed data sources service) + FIRE_EXCHANGE_URL: z.url().optional(), + // Fire Engine FIRE_ENGINE_BETA_URL: z.string().optional(), FIRE_ENGINE_STAGING_URL: z.string().optional(), diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 650a4a0f979..a163c46fc6c 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -34,6 +34,7 @@ import { v2Router } from "./routes/v2"; import { nuqShutdown } from "./services/worker/nuq"; import { getErrorContactMessage } from "./lib/deployment"; import { initializeBlocklist } from "./scraper/WebScraper/utils/blocklist"; +import { warmExchangeCatalog } from "./lib/exchange"; import { initializeEngineForcing } from "./scraper/WebScraper/utils/engine-forcing"; import responseTime from "response-time"; import { shutdownWebhookQueue } from "./services/webhook"; @@ -132,6 +133,7 @@ async function startServer(port = DEFAULT_PORT) { try { await initializeBlocklist(); initializeEngineForcing(); + warmExchangeCatalog(); } catch (error) { logger.error("Failed to initialize API startup dependencies", { error, diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.ts index 935ef612b79..aaa59db8e6b 100644 --- a/apps/api/src/lib/error-serde.ts +++ b/apps/api/src/lib/error-serde.ts @@ -39,6 +39,8 @@ import { UnsafeDomainBlockedError } from "./threat-protection/error"; // TODO: figure out correct typing for this const errorMap: Record = { + // Terms responses are API-level, never transported through workers. + THIRD_PARTY_DATA_TERMS_REQUIRED: null, SCRAPE_TIMEOUT: ScrapeJobTimeoutError, MAP_TIMEOUT: MapTimeoutError, UNKNOWN_ERROR: UnknownError, diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.ts index 62df2572bfe..d77a878f860 100644 --- a/apps/api/src/lib/error.ts +++ b/apps/api/src/lib/error.ts @@ -1,4 +1,5 @@ export type ErrorCodes = + | "THIRD_PARTY_DATA_TERMS_REQUIRED" | "SCRAPE_TIMEOUT" | "MAP_TIMEOUT" | "UNKNOWN_ERROR" diff --git a/apps/api/src/lib/exchange-request.ts b/apps/api/src/lib/exchange-request.ts new file mode 100644 index 00000000000..17238155b4d --- /dev/null +++ b/apps/api/src/lib/exchange-request.ts @@ -0,0 +1,47 @@ +import { + getExchangeAccessForRequest, + type ExchangeAccess, +} from "./exchange"; + +type ExchangeFlags = Parameters< + typeof getExchangeAccessForRequest +>[0]["flags"]; + +/** + * Run the Exchange access gate against an API request body, sourcing the + * effective per-page options from a nested scrapeOptions object (crawl + * bodies) or the body itself (scrape and batch scrape bodies), with v1 + * pageOptions fallbacks. + */ +export function getExchangeAccessForRequestBody(input: { + body: Record; + flags: ExchangeFlags; + url: string; + zeroDataRetention: boolean; +}): Promise { + const body = input.body ?? {}; + const scrapeOptions = + typeof body.scrapeOptions === "object" && body.scrapeOptions !== null + ? body.scrapeOptions + : body; + + return getExchangeAccessForRequest({ + url: input.url, + formats: scrapeOptions.formats, + actions: scrapeOptions.actions, + headers: scrapeOptions.headers, + waitFor: scrapeOptions.waitFor, + mobile: scrapeOptions.mobile, + location: scrapeOptions.location, + proxy: scrapeOptions.proxy, + blockAds: scrapeOptions.blockAds, + profile: scrapeOptions.profile, + atsv: scrapeOptions.atsv ?? body.pageOptions?.atsv, + minAge: scrapeOptions.minAge, + includeTags: scrapeOptions.includeTags ?? body.pageOptions?.includeTags, + excludeTags: scrapeOptions.excludeTags ?? body.pageOptions?.excludeTags, + zeroDataRetention: input.zeroDataRetention, + lockdown: scrapeOptions.lockdown ?? body.lockdown, + flags: input.flags, + }); +} diff --git a/apps/api/src/lib/exchange.test.ts b/apps/api/src/lib/exchange.test.ts new file mode 100644 index 00000000000..416e2b627ad --- /dev/null +++ b/apps/api/src/lib/exchange.test.ts @@ -0,0 +1,589 @@ +import { fetch } from "undici"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { config } from "../config"; +import { + canUseExchangeForRequest, + clearExchangeProvidersForTest, + reportExchangeBilling, + getExchangeAccessForRequest, + getExchangeRequestLogContext, + getExchangeResponseLogContext, + getExchangeSuccessCredits, + getThirdPartyDataTermsRequiredResponse, + isSuccessfulExchangeStatusCode, + isSupportedExchangeFormatRequest, + resolveExchangeProvider, + setExchangeProvidersForTest, +} from "./exchange"; + +vi.mock("undici", () => ({ + fetch: vi.fn(), +})); + +const originalConfig = { + FIRE_EXCHANGE_URL: config.FIRE_EXCHANGE_URL, + USE_DB_AUTHENTICATION: config.USE_DB_AUTHENTICATION, +}; + +const ACME_TERMS = { key: "acme", version: "2026-01-01" }; + +const TEST_PROVIDERS = [ + { + id: "acme", + creditsCost: 12, + terms: ACME_TERMS, + routes: [ + { + domains: ["profiles.example", "www.profiles.example"], + pathPrefixes: ["/person/", "/company/"], + }, + ], + }, + { + id: "openfacts", + creditsCost: 0, + routes: [{ domains: ["facts.example"] }], + }, +]; + +const ENABLED_EXCHANGE_FLAGS = { + professionalProfileCompanyDataBeta: true, + organizationDataSourceAccess: { + acme: { + status: "enabled", + termsKey: "acme", + termsVersion: "2026-01-01", + termsAcceptedAt: "2026-01-01T00:00:00.000Z", + enabledAt: "2026-01-01T00:00:00.000Z", + disabledAt: null, + disabledReason: null, + }, + }, +}; + +describe("Exchange routing", () => { + beforeEach(() => { + vi.mocked(fetch).mockReset(); + config.FIRE_EXCHANGE_URL = "https://exchange.example"; + config.USE_DB_AUTHENTICATION = true; + setExchangeProvidersForTest(TEST_PROVIDERS); + }); + + afterEach(() => { + config.FIRE_EXCHANGE_URL = originalConfig.FIRE_EXCHANGE_URL; + config.USE_DB_AUTHENTICATION = originalConfig.USE_DB_AUTHENTICATION; + clearExchangeProvidersForTest(); + }); + + it("resolves URLs to providers using catalog routes", async () => { + await expect( + resolveExchangeProvider( + "https://profiles.example/person/example-person/details/?trk=foo", + ), + ).resolves.toMatchObject({ id: "acme", creditsCost: 12 }); + await expect( + resolveExchangeProvider("https://www.profiles.example/company/example"), + ).resolves.toMatchObject({ id: "acme" }); + await expect( + resolveExchangeProvider("https://facts.example/any/path"), + ).resolves.toMatchObject({ id: "openfacts" }); + await expect( + resolveExchangeProvider("https://profiles.example/jobs/example"), + ).resolves.toBeNull(); + await expect( + resolveExchangeProvider("https://other.example/person/example"), + ).resolves.toBeNull(); + await expect(resolveExchangeProvider("not a url")).resolves.toBeNull(); + + await expect( + resolveExchangeProvider("https://profiles.example/person/example-person"), + ).resolves.not.toBeNull(); + }); + + it("respects path segment boundaries for prefixes without trailing slashes", async () => { + setExchangeProvidersForTest([ + { + id: "acme", + creditsCost: 12, + routes: [{ domains: ["profiles.example"], pathPrefixes: ["/person"] }], + }, + ]); + + await expect( + resolveExchangeProvider("https://profiles.example/person"), + ).resolves.toMatchObject({ id: "acme" }); + await expect( + resolveExchangeProvider("https://profiles.example/person/example"), + ).resolves.toMatchObject({ id: "acme" }); + await expect( + resolveExchangeProvider("https://profiles.example/personality/details"), + ).resolves.toBeNull(); + await expect( + resolveExchangeProvider("https://profiles.example/personnel"), + ).resolves.toBeNull(); + }); + + it("serves the stale catalog while refreshing in the background", async () => { + setExchangeProvidersForTest(TEST_PROVIDERS, -1); + let resolveFetch: (value: unknown) => void = () => {}; + vi.mocked(fetch).mockReturnValue( + new Promise(resolve => { + resolveFetch = resolve; + }) as ReturnType, + ); + + // The catalog is expired and the refresh never resolves during the + // request - the stale value must be served without waiting. + await expect( + resolveExchangeProvider("https://profiles.example/person/example"), + ).resolves.toMatchObject({ id: "acme" }); + expect(fetch).toHaveBeenCalledTimes(1); + + resolveFetch({ ok: false, status: 503 }); + }); + + it("keeps serving the last good catalog when a refresh fails", async () => { + setExchangeProvidersForTest(TEST_PROVIDERS, -1); + vi.mocked(fetch).mockResolvedValue({ + ok: false, + status: 503, + } as Awaited>); + + await expect( + resolveExchangeProvider("https://profiles.example/person/example"), + ).resolves.toMatchObject({ id: "acme" }); + + // Let the background refresh settle - the failure must not clobber + // the catalog, and the failure TTL must throttle the next attempt. + await new Promise(resolve => setTimeout(resolve, 0)); + await expect( + resolveExchangeProvider("https://profiles.example/person/example"), + ).resolves.toMatchObject({ id: "acme" }); + expect(fetch).toHaveBeenCalledTimes(1); + }); + + it("degrades to ineligible when the access check itself throws", async () => { + const explodingFlags = new Proxy( + {}, + { + get() { + throw new Error("flags backend exploded"); + }, + }, + ) as { professionalProfileCompanyDataBeta?: boolean }; + + await expect( + getExchangeAccessForRequest({ + url: "https://profiles.example/person/example-person", + formats: [{ type: "markdown" }], + flags: explodingFlags, + }), + ).resolves.toEqual({ allowed: false, termsRequired: false }); + }); + + it("caches failed provider catalog lookups briefly", async () => { + clearExchangeProvidersForTest(); + vi.mocked(fetch).mockResolvedValue({ + ok: false, + status: 503, + } as Awaited>); + + await expect( + resolveExchangeProvider("https://profiles.example/person/example-person"), + ).resolves.toBeNull(); + await expect( + resolveExchangeProvider("https://profiles.example/person/example-person"), + ).resolves.toBeNull(); + + expect(fetch).toHaveBeenCalledTimes(1); + }); + + it("parses the provider catalog response", async () => { + clearExchangeProvidersForTest(); + vi.mocked(fetch).mockResolvedValue({ + ok: true, + status: 200, + json: async () => ({ + success: true, + data: [ + { + id: "acme", + name: "Acme", + description: "Structured records.", + creditsCost: 12, + terms: ACME_TERMS, + capabilities: { + scrape: { + resourceTypes: ["records"], + includes: [], + urlRoutes: [ + { + domains: ["Profiles.Example"], + pathPrefixes: ["person/"], + }, + ], + }, + }, + }, + { + id: "searchonly", + name: "Search Only", + description: "No scrape routes.", + creditsCost: 0, + capabilities: { search: { modes: ["semantic"] } }, + }, + ], + }), + } as unknown as Awaited>); + + await expect( + resolveExchangeProvider("https://profiles.example/person/example"), + ).resolves.toMatchObject({ id: "acme", terms: ACME_TERMS }); + await expect( + resolveExchangeProvider("https://searchonly.example/anything"), + ).resolves.toBeNull(); + expect(fetch).toHaveBeenCalledTimes(1); + }); + + it("builds a compact request log context", () => { + expect( + getExchangeRequestLogContext( + "https://profiles.example/person/example-person/details/?trk=foo", + ), + ).toEqual({ + url: "https://profiles.example/person/example-person/details/?trk=foo", + host: "profiles.example", + pathPrefix: "person", + }); + + expect(getExchangeRequestLogContext("not a url")).toBeUndefined(); + + // Embedded credentials must never reach the logs. + expect( + getExchangeRequestLogContext( + "https://user:secret@profiles.example/person/example", + ), + ).toEqual({ + url: "https://profiles.example/person/example", + host: "profiles.example", + pathPrefix: "person", + }); + }); + + it("extracts response cache metadata for logs", () => { + expect( + getExchangeResponseLogContext({ + cacheState: "hit", + cachedAt: "2026-06-21T10:00:00.000Z", + cacheAgeMs: 1000, + request_id: "req_123", + extra: "ignored", + }), + ).toEqual({ + cacheState: "hit", + cachedAt: "2026-06-21T10:00:00.000Z", + cacheAgeMs: 1000, + providerRequestId: "req_123", + }); + + expect(getExchangeResponseLogContext(null)).toEqual({}); + }); + + it("accepts only formats the Exchange can return directly", () => { + expect(isSupportedExchangeFormatRequest(undefined)).toBe(true); + expect(isSupportedExchangeFormatRequest([{ type: "markdown" }])).toBe( + true, + ); + expect(isSupportedExchangeFormatRequest(["json"])).toBe(true); + expect( + isSupportedExchangeFormatRequest([ + { type: "markdown" }, + { type: "json" }, + ]), + ).toBe(true); + expect(isSupportedExchangeFormatRequest([{ type: "html" }])).toBe(false); + // deterministicJson extractors run against page HTML, which Exchange + // responses do not carry. + expect(isSupportedExchangeFormatRequest([{ type: "deterministicJson" }])).toBe( + false, + ); + expect(isSupportedExchangeFormatRequest([])).toBe(false); + }); + + it("allows eligible requests when access and terms are current", async () => { + await expect( + getExchangeAccessForRequest({ + url: "https://profiles.example/person/example-person", + formats: [{ type: "markdown" }], + flags: ENABLED_EXCHANGE_FLAGS, + }), + ).resolves.toMatchObject({ + allowed: true, + provider: { id: "acme", creditsCost: 12 }, + }); + + await expect( + canUseExchangeForRequest({ + url: "https://profiles.example/person/example-person", + formats: [{ type: "json" }], + actions: [{ type: "wait" }], + flags: ENABLED_EXCHANGE_FLAGS, + }), + ).resolves.toBe(false); + + await expect( + canUseExchangeForRequest({ + url: "https://profiles.example/person/example-person", + formats: [{ type: "json" }], + zeroDataRetention: true, + flags: ENABLED_EXCHANGE_FLAGS, + }), + ).resolves.toBe(false); + + // Profile-backed scrapes expect session-specific content. + await expect( + canUseExchangeForRequest({ + url: "https://profiles.example/person/example-person", + formats: [{ type: "markdown" }], + profile: { id: "profile-1" }, + flags: ENABLED_EXCHANGE_FLAGS, + }), + ).resolves.toBe(false); + + // atsv requests stay on engines that support the flag. + await expect( + canUseExchangeForRequest({ + url: "https://profiles.example/person/example-person", + formats: [{ type: "markdown" }], + atsv: true, + flags: ENABLED_EXCHANGE_FLAGS, + }), + ).resolves.toBe(false); + + // minAge asks for Firecrawl-cached data, which the Exchange never has. + await expect( + canUseExchangeForRequest({ + url: "https://profiles.example/person/example-person", + formats: [{ type: "markdown" }], + minAge: 3_600_000, + flags: ENABLED_EXCHANGE_FLAGS, + }), + ).resolves.toBe(false); + + // Selector-based filtering does not apply to provider records. + await expect( + canUseExchangeForRequest({ + url: "https://profiles.example/person/example-person", + formats: [{ type: "markdown" }], + includeTags: ["article"], + flags: ENABLED_EXCHANGE_FLAGS, + }), + ).resolves.toBe(false); + await expect( + canUseExchangeForRequest({ + url: "https://profiles.example/person/example-person", + formats: [{ type: "markdown" }], + excludeTags: ["nav"], + flags: ENABLED_EXCHANGE_FLAGS, + }), + ).resolves.toBe(false); + }); + + it("requires the provider's terms before routing", async () => { + const access = await getExchangeAccessForRequest({ + url: "https://profiles.example/person/example-person", + formats: [{ type: "markdown" }], + flags: { professionalProfileCompanyDataBeta: true }, + }); + + expect(access).toEqual({ + allowed: false, + termsRequired: true, + terms: ACME_TERMS, + }); + + expect(getThirdPartyDataTermsRequiredResponse(ACME_TERMS)).toMatchObject({ + success: false, + code: "THIRD_PARTY_DATA_TERMS_REQUIRED", + requiresAction: { + type: "accept_terms", + terms: "acme", + version: "2026-01-01", + }, + }); + }); + + it("requires current terms when the accepted version is stale", async () => { + await expect( + getExchangeAccessForRequest({ + url: "https://profiles.example/person/example-person", + formats: [{ type: "markdown" }], + flags: { + professionalProfileCompanyDataBeta: true, + organizationDataSourceAccess: { + acme: { + status: "enabled", + termsKey: "acme", + termsVersion: "2025-12-01", + }, + }, + }, + }), + ).resolves.toEqual({ + allowed: false, + termsRequired: true, + terms: ACME_TERMS, + }); + }); + + it("does not route or prompt for terms when access is disabled", async () => { + await expect( + getExchangeAccessForRequest({ + url: "https://profiles.example/person/example-person", + formats: [{ type: "markdown" }], + flags: { + professionalProfileCompanyDataBeta: true, + organizationDataSourceAccess: { + acme: { + status: "disabled", + termsKey: "acme", + termsVersion: "2026-01-01", + disabledAt: "2026-01-02T00:00:00.000Z", + disabledReason: "customer_disabled", + }, + }, + }, + }), + ).resolves.toEqual({ allowed: false, termsRequired: false }); + }); + + it("allows providers with no declared terms without acceptance", async () => { + await expect( + getExchangeAccessForRequest({ + url: "https://facts.example/records/1", + formats: [{ type: "markdown" }], + flags: { professionalProfileCompanyDataBeta: true }, + }), + ).resolves.toMatchObject({ allowed: true, provider: { id: "openfacts" } }); + + await expect( + getExchangeAccessForRequest({ + url: "https://facts.example/records/1", + formats: [{ type: "markdown" }], + flags: { + professionalProfileCompanyDataBeta: true, + organizationDataSourceAccess: { + openfacts: { status: "disabled" }, + }, + }, + }), + ).resolves.toEqual({ allowed: false, termsRequired: false }); + }); + + it("does not route unless the beta flag is enabled", async () => { + await expect( + canUseExchangeForRequest({ + url: "https://profiles.example/person/example-person", + formats: [{ type: "markdown" }], + }), + ).resolves.toBe(false); + + await expect( + canUseExchangeForRequest({ + url: "https://profiles.example/person/example-person", + formats: [{ type: "markdown" }], + flags: { professionalProfileCompanyDataBeta: false }, + }), + ).resolves.toBe(false); + }); + + it("does not route unless the Exchange is configured", async () => { + config.FIRE_EXCHANGE_URL = undefined; + + await expect( + canUseExchangeForRequest({ + url: "https://profiles.example/person/example-person", + formats: [{ type: "markdown" }], + flags: ENABLED_EXCHANGE_FLAGS, + }), + ).resolves.toBe(false); + }); + + it("bills the reported credit cost only for successful handled responses", () => { + expect(isSuccessfulExchangeStatusCode(200)).toBe(true); + expect(isSuccessfulExchangeStatusCode(204)).toBe(true); + expect(isSuccessfulExchangeStatusCode(304)).toBe(true); + expect(isSuccessfulExchangeStatusCode(404)).toBe(false); + + expect( + getExchangeSuccessCredits({ + exchange: { handled: true, creditsCost: 12 }, + statusCode: 200, + }), + ).toBe(12); + + expect( + getExchangeSuccessCredits({ + exchange: { handled: true, creditsCost: 0 }, + statusCode: 304, + }), + ).toBe(0); + + expect( + getExchangeSuccessCredits({ + exchange: { handled: true, creditsCost: 12 }, + statusCode: 404, + }), + ).toBeNull(); + + expect( + getExchangeSuccessCredits({ + statusCode: 200, + }), + ).toBeNull(); + }); + + it("reports billing outcomes without throwing on service failures", async () => { + vi.mocked(fetch).mockResolvedValue({ + ok: true, + status: 200, + } as Awaited>); + + await reportExchangeBilling({ + accessEventId: "6f1f5aab-3f78-4d0a-8a3d-2b1d3c4e5f60", + status: "confirmed", + billingReference: "bill-1", + }); + + expect(fetch).toHaveBeenCalledWith( + "https://exchange.example/v1/access-events/6f1f5aab-3f78-4d0a-8a3d-2b1d3c4e5f60/billing", + expect.objectContaining({ + method: "POST", + body: JSON.stringify({ status: "confirmed", billingReference: "bill-1" }), + }), + ); + + await reportExchangeBilling({ + accessEventId: "6f1f5aab-3f78-4d0a-8a3d-2b1d3c4e5f60", + status: "void", + }); + + expect(fetch).toHaveBeenLastCalledWith( + "https://exchange.example/v1/access-events/6f1f5aab-3f78-4d0a-8a3d-2b1d3c4e5f60/billing", + expect.objectContaining({ + method: "POST", + body: JSON.stringify({ status: "void" }), + }), + ); + + vi.mocked(fetch).mockRejectedValue(new Error("connect timeout")); + await expect( + reportExchangeBilling({ + accessEventId: "6f1f5aab-3f78-4d0a-8a3d-2b1d3c4e5f60", + status: "confirmed", + billingReference: "bill-1", + }), + ).resolves.toBeUndefined(); + }); +}); diff --git a/apps/api/src/lib/exchange.ts b/apps/api/src/lib/exchange.ts new file mode 100644 index 00000000000..9de05a4733e --- /dev/null +++ b/apps/api/src/lib/exchange.ts @@ -0,0 +1,639 @@ +import { fetch } from "undici"; +import { z } from "zod"; + +import { config } from "../config"; +import type { FormatObject } from "../controllers/v2/types"; +import { logger as rootLogger } from "./logger"; + +type OrganizationDataSourceAccessRecord = { + status?: string | null; + termsKey?: string | null; + termsVersion?: string | null; + termsAcceptedAt?: string | null; + enabledAt?: string | null; + disabledAt?: string | null; + disabledReason?: string | null; +}; + +type OrganizationDataSourceAccess = Record< + string, + OrganizationDataSourceAccessRecord | null | undefined +>; + +type RouteInput = { + url: string; + formats?: FormatObject[] | unknown[]; + actions?: unknown[]; + headers?: Record; + waitFor?: number; + mobile?: boolean; + location?: unknown; + proxy?: unknown; + blockAds?: boolean; + profile?: unknown; + atsv?: boolean; + minAge?: number; + includeTags?: unknown[]; + excludeTags?: unknown[]; + zeroDataRetention?: boolean; + lockdown?: boolean; + flags?: { + professionalProfileCompanyDataBeta?: boolean; + organizationDataSourceAccess?: OrganizationDataSourceAccess | null; + } | null; +}; + +export type ExchangeScrapeMetadata = { + handled: true; + creditsCost: number; + accessEventId?: string; + integrationId?: string; +}; + +export type ExchangeTerms = { + key: string; + version: string; +}; + +export type ExchangeProvider = { + id: string; + creditsCost: number; + terms?: ExchangeTerms; + routes: { + domains: Set; + pathPrefixes: string[]; + }[]; +}; + +// deterministicJson is deliberately unsupported: its extractor scripts run +// against page HTML, which Exchange responses do not carry. +const SUPPORTED_FORMATS = new Set(["markdown", "json"]); +const EXCHANGE_BETA_FLAG = "professionalProfileCompanyDataBeta"; +const THIRD_PARTY_DATA_TERMS_REQUIRED_CODE = "THIRD_PARTY_DATA_TERMS_REQUIRED"; +const THIRD_PARTY_DATA_TERMS_REQUIRED_MESSAGE = + "An organization admin must accept this data source's terms before this URL can be processed."; + +const EXCHANGE_PROVIDERS_PATH = "/v1/providers"; +const EXCHANGE_PROVIDERS_TIMEOUT_MS = 2_000; +const EXCHANGE_PROVIDERS_TTL_MS = 60_000; +const EXCHANGE_PROVIDERS_FAILURE_TTL_MS = 30_000; + +const exchangeProvidersSchema = z.object({ + success: z.literal(true), + data: z.array( + z + .object({ + id: z.string(), + // No .catch() here: a malformed credit cost must reject the catalog + // (keeping the last good one) rather than silently billing 0. + creditsCost: z.number().int().nonnegative(), + terms: z + .object({ + key: z.string(), + version: z.string(), + }) + .optional(), + capabilities: z + .object({ + scrape: z + .object({ + urlRoutes: z + .array( + z + .object({ + domains: z.string().array(), + pathPrefixes: z.string().array(), + }) + .passthrough(), + ) + .optional(), + }) + .passthrough() + .optional(), + }) + .passthrough(), + }) + .passthrough(), + ), +}); + +let cachedProviders: + | { + expiresAt: number; + value: ExchangeProvider[] | null; + } + | undefined; +let providersRequest: Promise | undefined; + +function normalizeHost(host: string): string { + return host.trim().toLowerCase().replace(/\.$/, ""); +} + +function normalizePathPrefix(prefix: string): string { + const trimmed = prefix.trim(); + if (!trimmed) { + return "/"; + } + + return trimmed.startsWith("/") ? trimmed : `/${trimmed}`; +} + +function getExchangeBaseUrl(): string | null { + if (!config.FIRE_EXCHANGE_URL) { + return null; + } + + return config.FIRE_EXCHANGE_URL.replace(/\/+$/, ""); +} + +function normalizeProviders( + raw: z.infer, +): ExchangeProvider[] { + return raw.data + .map(provider => ({ + id: provider.id, + creditsCost: provider.creditsCost, + ...(provider.terms === undefined ? {} : { terms: provider.terms }), + routes: (provider.capabilities.scrape?.urlRoutes ?? []).map(route => ({ + domains: new Set(route.domains.map(normalizeHost)), + pathPrefixes: route.pathPrefixes.map(normalizePathPrefix), + })), + })) + .filter(provider => provider.routes.length > 0); +} + +async function fetchExchangeProviders(): Promise { + const baseUrl = getExchangeBaseUrl(); + if (!baseUrl) { + return null; + } + + try { + const response = await fetch(`${baseUrl}${EXCHANGE_PROVIDERS_PATH}`, { + method: "GET", + signal: AbortSignal.timeout(EXCHANGE_PROVIDERS_TIMEOUT_MS), + }); + + if (!response.ok) { + rootLogger.warn("Exchange providers request failed", { + statusCode: response.status, + }); + return null; + } + + const parsed = exchangeProvidersSchema.parse(await response.json()); + return normalizeProviders(parsed); + } catch (error) { + rootLogger.warn("Exchange providers request errored", { error }); + return null; + } +} + +async function getExchangeProviders(): Promise { + if (cachedProviders && cachedProviders.expiresAt > Date.now()) { + return cachedProviders.value; + } + + if (!providersRequest) { + providersRequest = fetchExchangeProviders() + .then(providers => { + if (providers === null) { + // Keep serving the last good catalog through transient outages; + // the failure TTL only delays the next refresh attempt. + cachedProviders = { + value: cachedProviders?.value ?? null, + expiresAt: Date.now() + EXCHANGE_PROVIDERS_FAILURE_TTL_MS, + }; + } else { + cachedProviders = { + value: providers, + expiresAt: Date.now() + EXCHANGE_PROVIDERS_TTL_MS, + }; + } + return cachedProviders.value; + }) + .finally(() => { + providersRequest = undefined; + }); + } + + // Serve the stale catalog while the refresh runs in the background so + // request latency never depends on the catalog endpoint; only the very + // first lookup after boot has nothing to serve and waits. + if (cachedProviders) { + return cachedProviders.value; + } + + return providersRequest; +} + +function providerMatchesUrl(provider: ExchangeProvider, inputUrl: string): boolean { + let parsed: URL; + try { + parsed = new URL(inputUrl); + } catch { + return false; + } + + const host = normalizeHost(parsed.hostname); + const pathname = parsed.pathname || "/"; + + return provider.routes.some(route => { + if (!route.domains.has(host)) { + return false; + } + + if (route.pathPrefixes.length === 0) { + return true; + } + + // Prefix matches respect path segment boundaries: "/person" matches + // "/person" and "/person/x" but never "/personality". + return route.pathPrefixes.some(prefix => + prefix.endsWith("/") + ? pathname.startsWith(prefix) + : pathname === prefix || pathname.startsWith(`${prefix}/`), + ); + }); +} + +export async function resolveExchangeProvider( + inputUrl: string, +): Promise { + const providers = await getExchangeProviders(); + if (providers === null) { + return null; + } + + return providers.find(provider => providerMatchesUrl(provider, inputUrl)) ?? null; +} + +export function getExchangeRequestLogContext(inputUrl: string): + | { + url: string; + host: string; + pathPrefix: string | null; + } + | undefined { + let parsed: URL; + try { + parsed = new URL(inputUrl); + } catch { + return undefined; + } + + // Never log embedded credentials from user-submitted URLs. + parsed.username = ""; + parsed.password = ""; + + return { + url: parsed.href, + host: parsed.hostname.toLowerCase(), + pathPrefix: + parsed.pathname + .split("/") + .map(part => part.trim()) + .filter(part => part.length > 0)[0] ?? null, + }; +} + +export function getExchangeResponseLogContext(meta: unknown): { + cacheState?: string; + cachedAt?: string; + cacheAgeMs?: number; + providerRequestId?: string; +} { + if (typeof meta !== "object" || meta === null) { + return {}; + } + + const record = meta as Record; + const requestId = record.request_id ?? record.requestId; + + return { + ...(typeof record.cacheState === "string" + ? { cacheState: record.cacheState } + : {}), + ...(typeof record.cachedAt === "string" + ? { cachedAt: record.cachedAt } + : {}), + ...(typeof record.cacheAgeMs === "number" + ? { cacheAgeMs: record.cacheAgeMs } + : {}), + ...(typeof requestId === "string" ? { providerRequestId: requestId } : {}), + }; +} + +export function isSuccessfulExchangeStatusCode(statusCode: number): boolean { + return (statusCode >= 200 && statusCode < 300) || statusCode === 304; +} + +export function isSupportedExchangeFormatRequest( + formats?: FormatObject[] | unknown[], +): boolean { + if (formats === undefined) { + return true; + } + + if (!Array.isArray(formats) || formats.length === 0) { + return false; + } + + return formats.every(format => { + const type = + typeof format === "string" + ? format + : typeof format === "object" && format !== null && "type" in format + ? (format as { type?: unknown }).type + : undefined; + + return typeof type === "string" && SUPPORTED_FORMATS.has(type); + }); +} + +type DataSourceAccessDecision = "allowed" | "terms_required" | "not_enabled"; + +function getProviderAccessDecision( + provider: ExchangeProvider, + flags: RouteInput["flags"], +): DataSourceAccessDecision { + if (config.USE_DB_AUTHENTICATION !== true) { + return "allowed"; + } + + const access = flags?.organizationDataSourceAccess?.[provider.id]; + const entry = typeof access === "object" && access !== null ? access : null; + + if (provider.terms === undefined) { + return entry !== null && entry.status !== "enabled" ? "not_enabled" : "allowed"; + } + + if (entry === null) { + return "terms_required"; + } + + if (entry.status !== "enabled") { + return "not_enabled"; + } + + return entry.termsKey === provider.terms.key && + entry.termsVersion === provider.terms.version + ? "allowed" + : "terms_required"; +} + +function isExchangeEligibleRequest(input: RouteInput): boolean { + if (input.flags?.[EXCHANGE_BETA_FLAG] !== true) { + return false; + } + + if (!config.FIRE_EXCHANGE_URL) { + return false; + } + + if (!input.url) { + return false; + } + + if (input.zeroDataRetention || input.lockdown) { + return false; + } + + if (Array.isArray(input.actions) && input.actions.length > 0) { + return false; + } + + if (input.headers && Object.keys(input.headers).length > 0) { + return false; + } + + if (input.waitFor !== undefined && input.waitFor !== 0) { + return false; + } + + if (input.mobile || input.location || input.blockAds === false) { + return false; + } + + // Profile-backed scrapes expect session-specific content, which the + // Exchange cannot serve. + if (input.profile !== undefined) { + return false; + } + + // atsv is only supported by browser engines; requests that set it keep an + // engine that can honor it instead of routing to the Exchange. + if (input.atsv === true) { + return false; + } + + // minAge requests ask for Firecrawl-cached data; the Exchange serves + // provider data and Firecrawl never caches it, so the semantics cannot + // be honored here. + if (input.minAge !== undefined) { + return false; + } + + // Selector-based content filtering does not apply to provider records. + if ( + (Array.isArray(input.includeTags) && input.includeTags.length > 0) || + (Array.isArray(input.excludeTags) && input.excludeTags.length > 0) + ) { + return false; + } + + if (input.proxy === "stealth" || input.proxy === "enhanced") { + return false; + } + + if (!isSupportedExchangeFormatRequest(input.formats)) { + return false; + } + + return true; +} + +export type ExchangeAccess = + | { + allowed: true; + termsRequired: false; + provider: ExchangeProvider; + } + | { + allowed: false; + termsRequired: true; + terms: ExchangeTerms; + } + | { + allowed: false; + termsRequired: false; + }; + +export async function getExchangeAccessForRequest( + input: RouteInput, +): Promise { + // The Exchange gate sits on the hot path of every scrape request; no + // failure inside it may ever fail the request itself. Anything unexpected + // degrades to "not eligible" and the request continues on the normal path. + try { + if (!isExchangeEligibleRequest(input)) { + return { allowed: false, termsRequired: false }; + } + + const provider = await resolveExchangeProvider(input.url); + if (provider === null) { + return { allowed: false, termsRequired: false }; + } + + const decision = getProviderAccessDecision(provider, input.flags); + if (decision === "terms_required" && provider.terms !== undefined) { + return { allowed: false, termsRequired: true, terms: provider.terms }; + } + if (decision !== "allowed") { + return { allowed: false, termsRequired: false }; + } + + return { allowed: true, termsRequired: false, provider }; + } catch (error) { + rootLogger.warn("Exchange access check errored; treating as ineligible", { + error, + }); + return { allowed: false, termsRequired: false }; + } +} + +export async function canUseExchangeForRequest( + input: RouteInput, +): Promise { + return (await getExchangeAccessForRequest(input)).allowed; +} + +function getThirdPartyDataTermsSettingsUrl(): string { + return `${config.FIRECRAWL_DASHBOARD_URL.replace(/\/+$/, "")}/app/settings?tab=data-sources`; +} + +export function getThirdPartyDataTermsRequiredResponse(terms: ExchangeTerms) { + return { + success: false as const, + code: THIRD_PARTY_DATA_TERMS_REQUIRED_CODE as "THIRD_PARTY_DATA_TERMS_REQUIRED", + error: THIRD_PARTY_DATA_TERMS_REQUIRED_MESSAGE, + requiresAction: { + type: "accept_terms", + terms: terms.key, + version: terms.version, + url: getThirdPartyDataTermsSettingsUrl(), + }, + }; +} + +export function getExchangeSuccessCredits(input: { + exchange?: ExchangeScrapeMetadata; + statusCode?: number | null; +}): number | null { + if (input.exchange?.handled !== true) { + return null; + } + + const statusCode = input.statusCode; + if ( + statusCode === undefined || + statusCode === null || + !isSuccessfulExchangeStatusCode(statusCode) + ) { + return null; + } + + return input.exchange.creditsCost; +} + +const EXCHANGE_BILLING_TIMEOUT_MS = 5_000; + +/** + * Report the billing outcome of a delivered Exchange access so the service + * can reconcile its ledger: "confirmed" once the customer was billed, "void" + * when the delivered access was ultimately discarded and never billed. + * Failures only log - the service flags unresolved events for follow-up. + */ +export async function reportExchangeBilling(input: { + accessEventId: string; + status: "confirmed" | "void"; + billingReference?: string; +}): Promise { + const baseUrl = getExchangeBaseUrl(); + if (!baseUrl) { + return; + } + + try { + const response = await fetch( + `${baseUrl}/v1/access-events/${encodeURIComponent(input.accessEventId)}/billing`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + status: input.status, + ...(input.billingReference === undefined + ? {} + : { billingReference: input.billingReference }), + }), + signal: AbortSignal.timeout(EXCHANGE_BILLING_TIMEOUT_MS), + }, + ); + + if (!response.ok) { + rootLogger.warn("Exchange billing report failed", { + accessEventId: input.accessEventId, + status: input.status, + statusCode: response.status, + }); + } + } catch (error) { + rootLogger.warn("Exchange billing report errored", { + accessEventId: input.accessEventId, + status: input.status, + error, + }); + } +} + +/** + * Warm the provider catalog at process startup so the first flagged-org + * request never waits on the fetch; after this, stale-while-revalidate + * keeps every lookup in-memory. No-op when the Exchange is not configured; + * never throws. + */ +export function warmExchangeCatalog(): void { + if (!config.FIRE_EXCHANGE_URL) { + return; + } + + void getExchangeProviders(); +} + +export function setExchangeProvidersForTest( + providers: { + id: string; + creditsCost?: number; + terms?: ExchangeTerms; + routes: { domains: string[]; pathPrefixes?: string[] }[]; + }[], + ttlMs = 300_000, +) { + cachedProviders = { + value: providers.map(provider => ({ + id: provider.id, + creditsCost: provider.creditsCost ?? 0, + ...(provider.terms === undefined ? {} : { terms: provider.terms }), + routes: provider.routes.map(route => ({ + domains: new Set(route.domains.map(normalizeHost)), + pathPrefixes: (route.pathPrefixes ?? []).map(normalizePathPrefix), + })), + })), + expiresAt: Date.now() + ttlMs, + }; +} + +export function clearExchangeProvidersForTest() { + cachedProviders = undefined; + providersRequest = undefined; +} diff --git a/apps/api/src/lib/scrape-billing.test.ts b/apps/api/src/lib/scrape-billing.test.ts index a6c67caee87..9c1f9e7680f 100644 --- a/apps/api/src/lib/scrape-billing.test.ts +++ b/apps/api/src/lib/scrape-billing.test.ts @@ -6,6 +6,33 @@ import { UnsafeDomainBlockedError } from "./threat-protection/error"; import type { ThreatDecision } from "./threat-protection/types"; describe("calculateCreditsToBeBilled", () => { + it("bills handled Exchange successes at the reported credit cost", async () => { + const credits = await calculateCreditsToBeBilled( + { + formats: [{ type: "markdown" }], + } as any, + { + teamId: "team-id", + }, + { + metadata: { + statusCode: 200, + url: "https://profiles.example/person/example-person", + proxyUsed: "basic", + }, + } as any, + { + totalCost: 0, + } as any, + {} as any, + undefined, + undefined, + { handled: true, creditsCost: 12 }, + ); + + expect(credits).toBe(12); + }); + it("bills X/Twitter scrapes at 30 credits", async () => { const credits = await calculateCreditsToBeBilled( { @@ -142,6 +169,7 @@ const billWithDecisions = (args: { {} as any, args.error, undefined, + undefined, args.threatDecisions, ); diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.ts index b393646d739..58347aec371 100644 --- a/apps/api/src/lib/scrape-billing.ts +++ b/apps/api/src/lib/scrape-billing.ts @@ -10,6 +10,10 @@ import { hasFormatOfType } from "./format-utils"; import { TransportableError } from "./error"; import { FeatureFlag } from "../scraper/scrapeURL/engines"; import { isUrlBlocked } from "../scraper/WebScraper/utils/blocklist"; +import { + ExchangeScrapeMetadata, + getExchangeSuccessCredits, +} from "./exchange"; import type { ThreatDecision } from "./threat-protection/types"; import { UnsafeDomainBlockedError } from "./threat-protection/error"; @@ -64,6 +68,7 @@ export async function calculateCreditsToBeBilled( flags: TeamFlags, error?: Error | null, unsupportedFeatures?: Set, + exchange?: ExchangeScrapeMetadata, // Threat protection decisions for this scrape (initial + redirect checks, // in order). Each decision with `providerConsulted` bills a scan fee (+2 // per unique scanned URL) on top of the scrape's own cost — on both success @@ -112,6 +117,14 @@ export async function calculateCreditsToBeBilled( return creditsToBeBilled + threatScanCredits; } + const exchangeCredits = getExchangeSuccessCredits({ + exchange, + statusCode: document.metadata?.statusCode, + }); + if (exchangeCredits !== null) { + return exchangeCredits + threatScanCredits; + } + let creditsToBeBilled = 1; // Assuming 1 credit per document if (options.lockdown) { diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.ts index 5528a9aa852..73f553e3b53 100644 --- a/apps/api/src/routes/shared.ts +++ b/apps/api/src/routes/shared.ts @@ -29,6 +29,9 @@ import { CREDITS_FEATURE_ID, } from "../services/autumn/autumn.service"; import { getTeamBalance } from "../services/autumn/usage"; +import { getThirdPartyDataTermsRequiredResponse } from "../lib/exchange"; +import { getExchangeAccessForRequestBody } from "../lib/exchange-request"; +import { getScrapeZDR } from "../lib/zdr-helpers"; export function checkCreditsMiddleware( _minimum?: number, @@ -284,10 +287,58 @@ export function blocklistMiddleware( req: RequestWithMaybeACUC, res: Response, next: NextFunction, +) { + return blocklistGate(req, res, next, { exchange: false }); +} + +/** + * Blocklist gate for single-URL scrape-shaped routes (scrape, crawl), where + * an Exchange-eligible URL may bypass the blocklist because the exchange + * engine can serve it. Everything else (map, search, batch scrape, monitors) + * keeps plain blocklist behavior - batch stays out until its jobs carry the + * access flags the worker-side recheck needs. + */ +export function scrapeBlocklistMiddleware( + req: RequestWithMaybeACUC, + res: Response, + next: NextFunction, +) { + return blocklistGate(req, res, next, { exchange: true }); +} + +function blocklistGate( + req: RequestWithMaybeACUC, + res: Response, + next: NextFunction, + options: { exchange: boolean }, ) { (async () => { + const zeroDataRetention = + getScrapeZDR(req.acuc?.flags) === "forced" || + req.body?.zeroDataRetention === true; + const exchangeAccess = + options.exchange && + typeof req.body.url === "string" && + (await getExchangeAccessForRequestBody({ + body: req.body, + flags: req.acuc?.flags ?? null, + url: req.body.url, + zeroDataRetention, + })); + const canUseExchange = + typeof exchangeAccess === "object" && exchangeAccess.allowed; + + if (typeof exchangeAccess === "object" && exchangeAccess.termsRequired) { + if (!res.headersSent) { + return res + .status(403) + .json(getThirdPartyDataTermsRequiredResponse(exchangeAccess.terms)); + } + } + if ( typeof req.body.url === "string" && + !canUseExchange && isUrlBlocked(req.body.url, req.acuc?.flags ?? null, { team_id: req.acuc?.team_id ?? null, origin: typeof req.body.origin === "string" ? req.body.origin : null, diff --git a/apps/api/src/routes/v1.ts b/apps/api/src/routes/v1.ts index bc92811d1eb..d2cdf6dff3d 100644 --- a/apps/api/src/routes/v1.ts +++ b/apps/api/src/routes/v1.ts @@ -28,6 +28,7 @@ import { authMiddleware, checkCreditsMiddleware, blocklistMiddleware, + scrapeBlocklistMiddleware, countryCheck, idempotencyMiddleware, requestTimingMiddleware, @@ -51,7 +52,7 @@ v1Router.post( authMiddleware(RateLimiterMode.Scrape, { allowKeyless: true }), countryCheck, checkCreditsMiddleware(1), - blocklistMiddleware, + scrapeBlocklistMiddleware, wrap(scrapeController), ); @@ -60,7 +61,7 @@ v1Router.post( authMiddleware(RateLimiterMode.Crawl), countryCheck, checkCreditsMiddleware(), - blocklistMiddleware, + scrapeBlocklistMiddleware, idempotencyMiddleware, wrap(crawlController), ); diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.ts index e3a0991dd27..31c9b39975b 100644 --- a/apps/api/src/routes/v2.ts +++ b/apps/api/src/routes/v2.ts @@ -38,6 +38,7 @@ import { authMiddleware, checkCreditsMiddleware, blocklistMiddleware, + scrapeBlocklistMiddleware, countryCheck, idempotencyMiddleware, requestTimingMiddleware, @@ -210,7 +211,7 @@ v2Router.post( authMiddleware(RateLimiterMode.Scrape, { allowKeyless: true }), countryCheck, checkCreditsMiddleware(1), - blocklistMiddleware, + scrapeBlocklistMiddleware, wrap(scrapeController), ); @@ -257,7 +258,7 @@ v2Router.post( authMiddleware(RateLimiterMode.Crawl), countryCheck, checkCreditsMiddleware(), - blocklistMiddleware, + scrapeBlocklistMiddleware, idempotencyMiddleware, wrap(crawlController), ); diff --git a/apps/api/src/scraper/scrapeURL/engines/exchange.ts b/apps/api/src/scraper/scrapeURL/engines/exchange.ts new file mode 100644 index 00000000000..c6c41de2dc0 --- /dev/null +++ b/apps/api/src/scraper/scrapeURL/engines/exchange.ts @@ -0,0 +1,186 @@ +import { z } from "zod"; + +import { Meta } from ".."; +import { EngineScrapeResult } from "."; +import { config } from "../../../config"; +import { + getExchangeRequestLogContext, + getExchangeResponseLogContext, +} from "../../../lib/exchange"; +import { setSpanAttributes, withSpan } from "../../../lib/otel-tracer"; +import { robustFetch } from "../lib/fetch"; +import { EngineError } from "../error"; + +const exchangeScrapeResponseSchema = z.union([ + z + .object({ + success: z.literal(true), + accessEventId: z.string().optional(), + // No .catch() here: a malformed credit cost must fail the scrape + // loudly rather than silently billing 0 for a delivered access. + creditsCost: z.number().int().nonnegative(), + data: z + .object({ + url: z.string().optional(), + title: z.string().optional(), + description: z.string().optional(), + source: z + .object({ + provider: z.string().optional(), + }) + .passthrough() + .optional(), + metadata: z.record(z.string(), z.unknown()).optional(), + markdown: z.string().optional(), + json: z.unknown().optional(), + }) + .passthrough(), + }) + .passthrough(), + z + .object({ + success: z.literal(false), + error: z + .object({ + code: z.string().optional(), + message: z.string().optional(), + }) + .passthrough() + .optional(), + }) + .passthrough(), +]); + +export function exchangeMaxReasonableTime(meta: Meta): number { + return meta.options.timeout ?? 60_000; +} + +function escapeHtml(value: string): string { + return value + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """); +} + +// Exchange responses carry no page HTML; synthesize a minimal head so the +// metadata transformer can populate the document's title and description. +function buildMetadataHtml(title?: string, description?: string): string { + const titleTag = title === undefined ? "" : `${escapeHtml(title)}`; + const descriptionTag = + description === undefined + ? "" + : ``; + return `${titleTag}${descriptionTag}`; +} + +export async function scrapeURLWithExchange( + meta: Meta, +): Promise { + return withSpan("engine.exchange.scrape", async span => { + const startTime = Date.now(); + const url = meta.rewrittenUrl ?? meta.url; + const requestLogContext = getExchangeRequestLogContext(url); + const logger = meta.logger.child({ method: "scrapeURLWithExchange" }); + + setSpanAttributes(span, { + "engine.type": "exchange", + // Follow the same credential redaction as the log context. + "engine.url": requestLogContext?.url ?? "", + "engine.team_id": meta.internalOptions.teamId, + }); + + logger.info("Exchange scrape started", { + ...requestLogContext, + scrapeId: meta.id, + teamId: meta.internalOptions.teamId, + maxAge: meta.options.maxAge, + }); + + try { + const response = await robustFetch({ + url: `${config.FIRE_EXCHANGE_URL!.replace(/\/+$/, "")}/v1/scrape`, + method: "POST", + body: { + requestId: meta.id, + teamId: meta.internalOptions.teamId, + url, + formats: ["markdown", "json"], + ...(meta.options.maxAge === undefined + ? {} + : { maxAge: meta.options.maxAge }), + }, + logger: logger.child({ method: "exchangeScrape/robustFetch" }), + tryCount: 2, + ignoreFailureStatus: true, + mock: meta.mock, + abort: meta.abort.asSignal(), + schema: exchangeScrapeResponseSchema, + }); + + if (!response.success) { + logger.warn("Exchange scrape failed", { + ...requestLogContext, + scrapeId: meta.id, + teamId: meta.internalOptions.teamId, + errorCode: response.error?.code, + durationMs: Date.now() - startTime, + }); + throw new EngineError("Exchange request failed"); + } + + const responseLogContext = getExchangeResponseLogContext( + response.data.metadata, + ); + + logger.info("Exchange scrape completed", { + ...requestLogContext, + ...responseLogContext, + scrapeId: meta.id, + teamId: meta.internalOptions.teamId, + integrationId: response.data.source?.provider, + accessEventId: response.accessEventId, + creditsCost: response.creditsCost, + durationMs: Date.now() - startTime, + }); + + setSpanAttributes(span, { + "exchange.integration_id": response.data.source?.provider, + "exchange.credits_cost": response.creditsCost, + "exchange.cache_state": responseLogContext.cacheState, + "exchange.cache_age_ms": responseLogContext.cacheAgeMs, + "exchange.duration_ms": Date.now() - startTime, + }); + + return { + url: response.data.url ?? url, + html: buildMetadataHtml(response.data.title, response.data.description), + markdown: response.data.markdown, + json: response.data.json, + statusCode: 200, + contentType: "text/markdown", + proxyUsed: "basic", + exchange: { + handled: true, + creditsCost: response.creditsCost, + ...(response.accessEventId === undefined + ? {} + : { accessEventId: response.accessEventId }), + ...(response.data.source?.provider === undefined + ? {} + : { integrationId: response.data.source.provider }), + }, + }; + } catch (error) { + logger.warn("Exchange scrape errored", { + ...requestLogContext, + scrapeId: meta.id, + teamId: meta.internalOptions.teamId, + durationMs: Date.now() - startTime, + errorMessage: error instanceof Error ? error.message : String(error), + error, + }); + throw error; + } + }); +} diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts index 8a9b2f7275c..60b986bc721 100644 --- a/apps/api/src/scraper/scrapeURL/engines/index.ts +++ b/apps/api/src/scraper/scrapeURL/engines/index.ts @@ -7,6 +7,10 @@ import { scrapeURLWithFireEngineChromeCDP, scrapeURLWithFireEngineTLSClient, } from "./fire-engine"; +import { + exchangeMaxReasonableTime, + scrapeURLWithExchange, +} from "./exchange"; import { pdfMaxReasonableTime, scrapePDF } from "./pdf"; import { fetchMaxReasonableTime, scrapeURLWithFetch } from "./fetch"; import { @@ -30,8 +34,14 @@ import { getPDFMaxPages } from "../../../controllers/v2/types"; import type { PdfMetadata } from "./pdf/types"; import { BrandingProfile } from "../../../types/branding"; import { BrandingNotSupportedError } from "../error"; +import { isUrlBlocked } from "../../WebScraper/utils/blocklist"; +import { + canUseExchangeForRequest, + type ExchangeScrapeMetadata, +} from "../../../lib/exchange"; export type Engine = + | "exchange" | "fire-engine;chrome-cdp" | "fire-engine(retry);chrome-cdp" | "fire-engine;chrome-cdp;stealth" @@ -162,11 +172,13 @@ export type EngineScrapeResult = { proxyUsed: "basic" | "stealth"; timezone?: string; + exchange?: ExchangeScrapeMetadata; }; const engineHandlers: { [E in Engine]: (meta: Meta) => Promise; } = { + "exchange": scrapeURLWithExchange, index: scrapeURLWithIndex, "index;documents": scrapeURLWithIndex, "fire-engine;chrome-cdp": scrapeURLWithFireEngineChromeCDP, @@ -186,6 +198,7 @@ const engineHandlers: { const engineMRTs: { [E in Engine]: (meta: Meta) => number; } = { + "exchange": exchangeMaxReasonableTime, index: indexMaxReasonableTime, "index;documents": indexMaxReasonableTime, "fire-engine;chrome-cdp": meta => @@ -218,6 +231,27 @@ const engineOptions: { quality: number; }; } = { + "exchange": { + features: { + actions: false, + waitFor: false, + screenshot: false, + "screenshot@fullScreen": false, + pdf: false, + document: false, + audio: false, + video: false, + atsv: false, + location: false, + mobile: false, + skipTlsVerification: true, + useFastMode: true, + stealthProxy: false, + branding: false, + disableAdblock: false, + }, + quality: 2000, + }, index: { features: { actions: false, @@ -547,6 +581,73 @@ export async function buildFallbackList(meta: Meta): Promise< unsupportedFeatures: Set; }[] > { + if ( + !meta.internalOptions.agentIndexOnly && + meta.internalOptions.forceEngine === undefined + ) { + if ( + await canUseExchangeForRequest({ + url: meta.rewrittenUrl ?? meta.url, + formats: meta.options.formats, + actions: meta.options.actions, + headers: meta.options.headers, + waitFor: meta.options.waitFor, + mobile: meta.options.mobile, + location: meta.options.location, + proxy: meta.options.proxy, + blockAds: meta.options.blockAds, + profile: meta.options.profile, + atsv: meta.internalOptions.atsv, + minAge: meta.options.minAge, + includeTags: meta.options.includeTags, + excludeTags: meta.options.excludeTags, + zeroDataRetention: meta.internalOptions.zeroDataRetention, + lockdown: meta.options.lockdown, + flags: meta.internalOptions.teamFlags ?? null, + }) + ) { + return [ + { + engine: "exchange", + unsupportedFeatures: new Set(), + }, + ]; + } + + // A blocked URL can only have been admitted by the Exchange bypass in + // blocklistMiddleware, which only applies to flagged orgs; if the + // Exchange is no longer usable by execution time (catalog changed, + // service down), fail closed rather than letting normal engines scrape + // a blocklisted site. An error here also fails closed: this branch only + // runs for flagged orgs, and a retryable scrape failure is preferable + // to scraping a potentially blocklisted site with normal engines. + if ( + meta.internalOptions.teamFlags?.professionalProfileCompanyDataBeta === + true + ) { + try { + if ( + isUrlBlocked( + meta.rewrittenUrl ?? meta.url, + meta.internalOptions.teamFlags ?? null, + { + team_id: meta.internalOptions.teamId ?? null, + origin: null, + }, + ) + ) { + return []; + } + } catch (error) { + meta.logger.warn( + "Exchange blocklist re-check failed; failing closed", + { error }, + ); + return []; + } + } + } + const shouldPrioritizeTlsClient = meta.options.__experimental_engpicker ? (await queryEngpickerVerdict( meta.options.__experimental_omceDomain ?? new URL(meta.url).hostname, diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts index 53012521b4c..3863f972e25 100644 --- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts +++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts @@ -52,6 +52,9 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) { !meta.internalOptions.zeroDataRetention && meta.winnerEngine !== "index" && meta.winnerEngine !== "index;documents" && + // Exchange-delivered content is never stored on the Firecrawl side: + // every access must go through the Exchange and its ledger. + meta.winnerEngine !== "exchange" && !(meta.winnerEngine === "pdf" && !shouldParsePDF(meta.options.parsers)) && !meta.options.parsers?.some(parser => { if ( diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts index fb1e738f685..5357e616745 100644 --- a/apps/api/src/scraper/scrapeURL/index.ts +++ b/apps/api/src/scraper/scrapeURL/index.ts @@ -87,6 +87,7 @@ import { writeFile } from "node:fs/promises"; import path from "node:path"; import { tmpdir } from "node:os"; import { randomUUID } from "node:crypto"; +import type { ExchangeScrapeMetadata } from "../../lib/exchange"; import { checkUrl, type ThreatCheckDedup, @@ -101,6 +102,7 @@ export type ScrapeUrlResponse = success: true; document: Document; unsupportedFeatures?: Set; + exchange?: ExchangeScrapeMetadata; /** * Threat protection decisions made for this scrape (initial domain * check + any redirect re-checks, in order). Read by the billing layer @@ -1071,6 +1073,7 @@ async function scrapeURLLoop(meta: Meta): Promise { success: true, document, unsupportedFeatures: result.unsupportedFeatures, + exchange: engineResult.exchange, }; }); } diff --git a/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts b/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts index 7b350ea4222..41a626e68b7 100644 --- a/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts +++ b/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts @@ -27,6 +27,11 @@ function shouldIndexForSearch(meta: Meta, document: Document): boolean { return false; } + // Exchange-delivered content is never stored on the Firecrawl side. + if (meta.winnerEngine === "exchange") { + return false; + } + // Lockdown must not forward the target URL to any external service. if (meta.options.lockdown) { return false; diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts index f3d968e0209..41d0b7b2062 100644 --- a/apps/api/src/services/worker/scrape-worker.ts +++ b/apps/api/src/services/worker/scrape-worker.ts @@ -99,6 +99,11 @@ import { recordMonitorScrapeFailure, recordMonitorScrapeSuccess, } from "../monitoring/results"; +import { + reportExchangeBilling, + warmExchangeCatalog, + type ExchangeScrapeMetadata, +} from "../../lib/exchange"; configDotenv(); @@ -108,6 +113,7 @@ const jobLockExtensionTime = config.JOB_LOCK_EXTENSION_TIME; if (require.main === module) { cacheableLookup.install(http.globalAgent); cacheableLookup.install(https.globalAgent); + warmExchangeCatalog(); } async function billScrapeJob( @@ -118,6 +124,7 @@ async function billScrapeJob( flags: TeamFlags, error?: Error | null, unsupportedFeatures?: Set, + exchange?: ExchangeScrapeMetadata, threatDecisions?: ThreatDecision[], ) { let creditsToBeBilled: number | null = null; @@ -146,6 +153,7 @@ async function billScrapeJob( flags, error, unsupportedFeatures, + exchange, threatDecisions, ); @@ -196,6 +204,18 @@ async function billScrapeJob( }, ); + // Reconcile the Exchange ledger. Awaited so the confirmation lands + // before any later failure path could attempt to void this access - + // the service rejects confirmed->void, making the order decisive. + // reportExchangeBilling never throws and is bounded by its timeout. + if (exchange?.accessEventId !== undefined) { + await reportExchangeBilling({ + accessEventId: exchange.accessEventId, + status: "confirmed", + billingReference: billingJobId, + }); + } + return creditsToBeBilled; } catch (error) { logger.error( @@ -663,6 +683,7 @@ async function processJob(job: NuQJob) { (await getACUCTeam(job.data.team_id))?.flags ?? null, undefined, pipeline.unsupportedFeatures, + pipeline.exchange, pipeline.threatDecisions, ); @@ -754,6 +775,7 @@ async function processJob(job: NuQJob) { (await getACUCTeam(job.data.team_id))?.flags ?? null, undefined, pipeline.unsupportedFeatures, + pipeline.exchange, pipeline.threatDecisions, ); @@ -954,9 +976,23 @@ async function processJob(job: NuQJob) { (await getACUCTeam(job.data.team_id))?.flags ?? null, error instanceof Error ? error : null, undefined, + undefined, pipeline?.threatDecisions, ); + // The Exchange delivered this access but the scrape ultimately failed, + // so the customer was never billed for it - void the access event so + // the Exchange ledger reconciles. Fire-and-forget. If billing did + // happen before a later step failed, its confirmation was awaited + // above and the service rejects confirmed->void, so a paid access can + // never be marked void. + if (pipeline?.success && pipeline.exchange?.accessEventId !== undefined) { + void reportExchangeBilling({ + accessEventId: pipeline.exchange.accessEventId, + status: "void", + }); + } + logger.debug("Logging job to DB..."); await logScrape( {