From 79212157b16b2e7012a0e8fbdf3306c79b6968b7 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 08:26:14 +0000 Subject: [PATCH] fix(web-search): reduce domain entries on every route, not just site: MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `normalizeDomain` ran only on the way into a `site:` operator, so a provider declaring `domainFilter: "native"` received the caller's raw entry. `includeDomains: ["https://arxiv.org/"]` restricted the search on one route and reached Tavily, OpenAI, Anthropic or OpenRouter as a scheme-bearing value on the other — all of which document bare hosts. The caller then gets no results for a filter that should have matched, or, for a vendor that skips an entry it cannot parse, an unrestricted search reported as successful with nothing saying the filter was dropped. With `"auto"` the route is picked at run time, so it is not reproducible from the input either. `adaptRequest` now reduces both lists before either direction is decided, so the native list and the operator translation read the same value. An entry that names no domain is still refused up front, unchanged. `domainInput` is exported from the barrel so a host writing its own native adapter can apply the same reduction. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01T8DU24G9GWuRUJncc5TJFZ --- packages/web-search/src/WebSearchTask.ts | 21 ++++++--- .../src/__tests__/WebSearchTask.test.ts | 46 ++++++++++++++++++- packages/web-search/src/common.ts | 1 + packages/web-search/src/domainInput.ts | 15 ++++++ 4 files changed, 76 insertions(+), 7 deletions(-) diff --git a/packages/web-search/src/WebSearchTask.ts b/packages/web-search/src/WebSearchTask.ts index 30667ad80..46c351092 100644 --- a/packages/web-search/src/WebSearchTask.ts +++ b/packages/web-search/src/WebSearchTask.ts @@ -9,7 +9,7 @@ import { Task, TaskConfigurationError } from "@workglow/task-graph"; import { FetchUrlTask, fetchUrlEntitlementsFor } from "@workglow/tasks"; import type { DataPortSchema, FromSchema } from "@workglow/util/schema"; import { unhonorableOptions } from "./capabilityCheck"; -import { unusableDomainEntries } from "./domainInput"; +import { normalizeDomains, unusableDomainEntries } from "./domainInput"; import type { IWebSearchProvider, SearchResult, @@ -350,6 +350,13 @@ export class WebSearchTask extends Task * includes and `"query-operator"` excludes pass routing and then receive an * `excludeDomains` list it has no native way to send — a restriction silently * dropped on a request the task reported as honored. + * + * Domains are reduced to bare hosts here, before either direction is decided, + * so the list a native adapter forwards and the list the `site:` translation + * reads are the same value. Reducing them only on the way into an operator + * left `"https://arxiv.org/"` restricting one route and reaching a vendor API + * as a scheme-bearing entry on the other, where it matches nothing or is + * skipped — and with `"auto"` the caller cannot tell which route ran. */ private adaptRequest(provider: IWebSearchProvider, request: WebSearchRequest): WebSearchRequest { const capabilities = provider.capabilities; @@ -358,23 +365,25 @@ export class WebSearchTask extends Task cap !== undefined && request.maxResults !== undefined ? Math.min(request.maxResults, cap) : request.maxResults; + const includeDomains = normalizeDomains(request.includeDomains); + const excludeDomains = normalizeDomains(request.excludeDomains); const excludeSupport = capabilities.excludeDomainFilter ?? capabilities.domainFilter; const translateIncludes = capabilities.domainFilter === "query-operator"; const translateExcludes = excludeSupport === "query-operator"; if (!translateIncludes && !translateExcludes) { - return { ...request, maxResults }; + return { ...request, maxResults, includeDomains, excludeDomains }; } return { ...request, maxResults, query: applyDomainOperators( request.query, - translateIncludes ? request.includeDomains : undefined, - translateExcludes ? request.excludeDomains : undefined + translateIncludes ? includeDomains : undefined, + translateExcludes ? excludeDomains : undefined ), - includeDomains: translateIncludes ? undefined : request.includeDomains, - excludeDomains: translateExcludes ? undefined : request.excludeDomains, + includeDomains: translateIncludes ? undefined : includeDomains, + excludeDomains: translateExcludes ? undefined : excludeDomains, }; } } diff --git a/packages/web-search/src/__tests__/WebSearchTask.test.ts b/packages/web-search/src/__tests__/WebSearchTask.test.ts index 70e04f984..c78dce503 100644 --- a/packages/web-search/src/__tests__/WebSearchTask.test.ts +++ b/packages/web-search/src/__tests__/WebSearchTask.test.ts @@ -116,7 +116,7 @@ describe("WebSearchTask", () => { expect(request.excludeDomains).toBeUndefined(); }); - it("passes domain lists through untouched to a native provider", async () => { + it("passes domain lists to a native provider rather than into the query", async () => { const seen = vi.fn(); WebSearchProviderRegistry.register(fake("tavily", { domainFilter: "native" }, seen)); await new WebSearchTask().run({ @@ -129,6 +129,50 @@ describe("WebSearchTask", () => { expect(request.includeDomains).toEqual(["arxiv.org"]); }); + it("hands a native provider the same reduction the site: translation applies", async () => { + // The vendor APIs behind `domainFilter: "native"` all document bare hosts, + // so a scheme, a `www.` or a trailing slash is at best matched by nothing + // and at worst skipped as unparseable — an unrestricted search reported as + // successful with nothing saying the filter was dropped. Under `"auto"` the + // caller cannot even predict which route ran, so the reduction has to be the + // same on both. + const native = vi.fn(); + const operator = vi.fn(); + WebSearchProviderRegistry.register(fake("tavily", { domainFilter: "native" }, native)); + WebSearchProviderRegistry.register(fake("brave", { domainFilter: "query-operator" }, operator)); + + const domains = { + includeDomains: ["https://ArXiv.org/"], + excludeDomains: ["www.spam.example"], + }; + await new WebSearchTask().run({ query: "cats", provider: "tavily", ...domains }); + await new WebSearchTask().run({ query: "cats", provider: "brave", ...domains }); + + const nativeRequest = native.mock.calls[0][0] as WebSearchRequest; + expect(nativeRequest.includeDomains).toEqual(["arxiv.org"]); + expect(nativeRequest.excludeDomains).toEqual(["spam.example"]); + const operatorRequest = operator.mock.calls[0][0] as WebSearchRequest; + expect(operatorRequest.query).toBe("cats site:arxiv.org -site:spam.example"); + }); + + it("normalizes the direction a split provider filters natively", async () => { + // `excludeDomainFilter` is decided separately from `domainFilter`, so the + // list left for the adapter to send must be reduced whichever side it is on. + const seen = vi.fn(); + WebSearchProviderRegistry.register( + fake("split", { domainFilter: "native", excludeDomainFilter: "query-operator" }, seen) + ); + await new WebSearchTask().run({ + query: "cats", + provider: "split", + includeDomains: ["https://arxiv.org/"], + excludeDomains: ["https://spam.example/"], + }); + const request = seen.mock.calls[0][0] as WebSearchRequest; + expect(request.includeDomains).toEqual(["arxiv.org"]); + expect(request.query).toBe("cats -site:spam.example"); + }); + it("clamps maxResults to the provider cap instead of refusing", async () => { const seen = vi.fn(); WebSearchProviderRegistry.register(fake("brave", { maxResultsCap: 5 }, seen)); diff --git a/packages/web-search/src/common.ts b/packages/web-search/src/common.ts index 206593ee0..e1ca0058e 100644 --- a/packages/web-search/src/common.ts +++ b/packages/web-search/src/common.ts @@ -15,6 +15,7 @@ import { WebSearchProviderRegistry } from "./WebSearchProviderRegistry"; import { WebSearchTask } from "./WebSearchTask"; export * from "./capabilityCheck"; +export * from "./domainInput"; export * from "./IWebSearchProvider"; export * from "./limitResults"; export * from "./providers/BraveWebSearchProvider"; diff --git a/packages/web-search/src/domainInput.ts b/packages/web-search/src/domainInput.ts index 6b9a0c628..f666a4a05 100644 --- a/packages/web-search/src/domainInput.ts +++ b/packages/web-search/src/domainInput.ts @@ -31,6 +31,21 @@ export function normalizeDomain(domain: string): string { return value; } +/** + * {@link normalizeDomain} over a whole list, preserving an absent one. + * + * Applied to every route rather than only the `site:` translation: the vendor + * APIs behind a native domain filter document bare hosts, so an entry the + * operator route reduces and the native route forwards raw is the same request + * filtered two different ways — and a vendor that skips an entry it cannot parse + * runs the search unrestricted and reports it as honored. + */ +export function normalizeDomains( + domains: readonly string[] | undefined +): readonly string[] | undefined { + return domains?.map(normalizeDomain); +} + /** * Whether a domain survives normalization as something a filter can express. *