diff --git a/packages/web-search/src/WebSearchTask.ts b/packages/web-search/src/WebSearchTask.ts index 30667ad80..46c351092 100644 --- a/packages/web-search/src/WebSearchTask.ts +++ b/packages/web-search/src/WebSearchTask.ts @@ -9,7 +9,7 @@ import { Task, TaskConfigurationError } from "@workglow/task-graph"; import { FetchUrlTask, fetchUrlEntitlementsFor } from "@workglow/tasks"; import type { DataPortSchema, FromSchema } from "@workglow/util/schema"; import { unhonorableOptions } from "./capabilityCheck"; -import { unusableDomainEntries } from "./domainInput"; +import { normalizeDomains, unusableDomainEntries } from "./domainInput"; import type { IWebSearchProvider, SearchResult, @@ -350,6 +350,13 @@ export class WebSearchTask extends Task * includes and `"query-operator"` excludes pass routing and then receive an * `excludeDomains` list it has no native way to send — a restriction silently * dropped on a request the task reported as honored. + * + * Domains are reduced to bare hosts here, before either direction is decided, + * so the list a native adapter forwards and the list the `site:` translation + * reads are the same value. Reducing them only on the way into an operator + * left `"https://arxiv.org/"` restricting one route and reaching a vendor API + * as a scheme-bearing entry on the other, where it matches nothing or is + * skipped — and with `"auto"` the caller cannot tell which route ran. */ private adaptRequest(provider: IWebSearchProvider, request: WebSearchRequest): WebSearchRequest { const capabilities = provider.capabilities; @@ -358,23 +365,25 @@ export class WebSearchTask extends Task cap !== undefined && request.maxResults !== undefined ? Math.min(request.maxResults, cap) : request.maxResults; + const includeDomains = normalizeDomains(request.includeDomains); + const excludeDomains = normalizeDomains(request.excludeDomains); const excludeSupport = capabilities.excludeDomainFilter ?? capabilities.domainFilter; const translateIncludes = capabilities.domainFilter === "query-operator"; const translateExcludes = excludeSupport === "query-operator"; if (!translateIncludes && !translateExcludes) { - return { ...request, maxResults }; + return { ...request, maxResults, includeDomains, excludeDomains }; } return { ...request, maxResults, query: applyDomainOperators( request.query, - translateIncludes ? request.includeDomains : undefined, - translateExcludes ? request.excludeDomains : undefined + translateIncludes ? includeDomains : undefined, + translateExcludes ? excludeDomains : undefined ), - includeDomains: translateIncludes ? undefined : request.includeDomains, - excludeDomains: translateExcludes ? undefined : request.excludeDomains, + includeDomains: translateIncludes ? undefined : includeDomains, + excludeDomains: translateExcludes ? undefined : excludeDomains, }; } } diff --git a/packages/web-search/src/__tests__/WebSearchTask.test.ts b/packages/web-search/src/__tests__/WebSearchTask.test.ts index 70e04f984..c78dce503 100644 --- a/packages/web-search/src/__tests__/WebSearchTask.test.ts +++ b/packages/web-search/src/__tests__/WebSearchTask.test.ts @@ -116,7 +116,7 @@ describe("WebSearchTask", () => { expect(request.excludeDomains).toBeUndefined(); }); - it("passes domain lists through untouched to a native provider", async () => { + it("passes domain lists to a native provider rather than into the query", async () => { const seen = vi.fn(); WebSearchProviderRegistry.register(fake("tavily", { domainFilter: "native" }, seen)); await new WebSearchTask().run({ @@ -129,6 +129,50 @@ describe("WebSearchTask", () => { expect(request.includeDomains).toEqual(["arxiv.org"]); }); + it("hands a native provider the same reduction the site: translation applies", async () => { + // The vendor APIs behind `domainFilter: "native"` all document bare hosts, + // so a scheme, a `www.` or a trailing slash is at best matched by nothing + // and at worst skipped as unparseable — an unrestricted search reported as + // successful with nothing saying the filter was dropped. Under `"auto"` the + // caller cannot even predict which route ran, so the reduction has to be the + // same on both. + const native = vi.fn(); + const operator = vi.fn(); + WebSearchProviderRegistry.register(fake("tavily", { domainFilter: "native" }, native)); + WebSearchProviderRegistry.register(fake("brave", { domainFilter: "query-operator" }, operator)); + + const domains = { + includeDomains: ["https://ArXiv.org/"], + excludeDomains: ["www.spam.example"], + }; + await new WebSearchTask().run({ query: "cats", provider: "tavily", ...domains }); + await new WebSearchTask().run({ query: "cats", provider: "brave", ...domains }); + + const nativeRequest = native.mock.calls[0][0] as WebSearchRequest; + expect(nativeRequest.includeDomains).toEqual(["arxiv.org"]); + expect(nativeRequest.excludeDomains).toEqual(["spam.example"]); + const operatorRequest = operator.mock.calls[0][0] as WebSearchRequest; + expect(operatorRequest.query).toBe("cats site:arxiv.org -site:spam.example"); + }); + + it("normalizes the direction a split provider filters natively", async () => { + // `excludeDomainFilter` is decided separately from `domainFilter`, so the + // list left for the adapter to send must be reduced whichever side it is on. + const seen = vi.fn(); + WebSearchProviderRegistry.register( + fake("split", { domainFilter: "native", excludeDomainFilter: "query-operator" }, seen) + ); + await new WebSearchTask().run({ + query: "cats", + provider: "split", + includeDomains: ["https://arxiv.org/"], + excludeDomains: ["https://spam.example/"], + }); + const request = seen.mock.calls[0][0] as WebSearchRequest; + expect(request.includeDomains).toEqual(["arxiv.org"]); + expect(request.query).toBe("cats -site:spam.example"); + }); + it("clamps maxResults to the provider cap instead of refusing", async () => { const seen = vi.fn(); WebSearchProviderRegistry.register(fake("brave", { maxResultsCap: 5 }, seen)); diff --git a/packages/web-search/src/common.ts b/packages/web-search/src/common.ts index 206593ee0..e1ca0058e 100644 --- a/packages/web-search/src/common.ts +++ b/packages/web-search/src/common.ts @@ -15,6 +15,7 @@ import { WebSearchProviderRegistry } from "./WebSearchProviderRegistry"; import { WebSearchTask } from "./WebSearchTask"; export * from "./capabilityCheck"; +export * from "./domainInput"; export * from "./IWebSearchProvider"; export * from "./limitResults"; export * from "./providers/BraveWebSearchProvider"; diff --git a/packages/web-search/src/domainInput.ts b/packages/web-search/src/domainInput.ts index 6b9a0c628..f666a4a05 100644 --- a/packages/web-search/src/domainInput.ts +++ b/packages/web-search/src/domainInput.ts @@ -31,6 +31,21 @@ export function normalizeDomain(domain: string): string { return value; } +/** + * {@link normalizeDomain} over a whole list, preserving an absent one. + * + * Applied to every route rather than only the `site:` translation: the vendor + * APIs behind a native domain filter document bare hosts, so an entry the + * operator route reduces and the native route forwards raw is the same request + * filtered two different ways — and a vendor that skips an entry it cannot parse + * runs the search unrestricted and reports it as honored. + */ +export function normalizeDomains( + domains: readonly string[] | undefined +): readonly string[] | undefined { + return domains?.map(normalizeDomain); +} + /** * Whether a domain survives normalization as something a filter can express. *