diff --git a/README.md b/README.md index 0254ebf..d26e790 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # Guion Web Guion Web is a Node.js web research toolkit. It provides Exa or Brave search, -Context7 library documentation lookup, Sourcegraph public code search, and two -page-fetch backends through a CLI, stdio MCP server, Pi extension, and DeepSeek +Context7 library documentation lookup, Sourcegraph public code search, page-link +discovery, and two page-fetch backends through a CLI, stdio MCP server, Pi extension, and DeepSeek Harness (DSH) integration: direct HTML-to-Markdown extraction and explicit `agent-browser` rendering for client-rendered pages on supported hosts. @@ -44,6 +44,7 @@ document on stdout, which is useful for automation. web search --provider exa -- "Node AbortSignal" web fetch https://example.com/article --tree web fetch https://example.com/article --section introduction +web links https://example.com/article --limit 50 web docs resolve react web docs fetch /facebook/react --topic hooks --tokens 2000 web sgraph --count 10 -- "repo:^github\\.com/nodejs/node$ AbortSignal" @@ -51,7 +52,8 @@ web sgraph --count 10 -- "repo:^github\\.com/nodejs/node$ AbortSignal" Use `--` before a search or Sourcegraph query that begins with a hyphen. `fetch` supports `--full`, `--tree`, and `--section`; long extracted documents default to -a heading tree so a later request can retrieve a stable section ID. +a heading tree so a later request can retrieve a stable section ID. `links` lists +up to 100 unique HTTP(S) anchors from the original page DOM. ## MCP @@ -63,10 +65,10 @@ web mcp web mcp --provider brave ``` -The server exposes five read-only tools: `search`, `fetch`, `docs_resolve`, +The server exposes six read-only tools: `search`, `fetch`, `links`, `docs_resolve`, `docs_fetch`, and `source_search`. Its stdout is reserved for MCP protocol messages; diagnostics go to stderr. For a client-rendered page, explicitly call -`fetch` with `render: "agent-browser"` and an integer `waitMs`; this optional +`fetch` or `links` with `render: "agent-browser"` and an integer `waitMs`; this optional retry requires a host-installed executable and never happens automatically. ## Pi @@ -77,11 +79,13 @@ Install the independently bundled Pi extension: pi install npm:@guionai/pi-web ``` -It registers `web_search`, `web_fetch`, `web_docs`, and `web_source_search` and calls +It registers `web_search`, `web_fetch`, `web_links`, `web_docs`, and `web_source_search` and calls the bundled core in-process. Pi and TypeBox are peer dependencies supplied by the host; no CLI executable or MCP configuration is required. `web_fetch` uses direct fetch by default and can explicitly use `render: "agent-browser"` with an integer `waitMs` when its host provides that optional executable. +`web_links` uses the same explicit rendering contract and lists HTTP(S) anchors +from the original page DOM. ## DSH @@ -93,11 +97,13 @@ dsh plugin --profile web add @guionai/dsh-web The included profile patch routes stock PTC web search through the selected Exa or Brave provider. Its settings UI stores provider selection and manages -namespaced write-only credentials. Fetch, documentation, and Sourcegraph tools +namespaced write-only credentials. Fetch, link discovery, documentation, and Sourcegraph tools also run in-process. The host DSH packages and React are peers supplied by DSH. `web_fetch` uses direct fetch by default and can explicitly use `render: "agent-browser"` with an integer `waitMs` on a host that supplies the optional executable. +`web_links` uses the same explicit rendering contract and lists HTTP(S) anchors +from the original page DOM. ## Page-fetch backends @@ -114,11 +120,16 @@ web fetch https://example.com/app --render=agent-browser --wait=2000 web fetch https://example.com/app --render=agent-browser --wait=10000 ``` +`web links` uses the same direct or explicit browser-rendered source, but parses +the original DOM rather than Defuddle output so navigation and other links outside +the readable article remain discoverable. It returns only HTTP(S) `a[href]` +destinations, deduplicated and capped at 100 by default. + `--wait` is mandatory with `--render=agent-browser`, including `--wait=0`, and accepts only an integer from 0 through 30,000 milliseconds. Direct `fetch` -requests must not provide `--wait`. The same `render: "agent-browser"` and required -`waitMs` fields are available on the MCP `fetch`, Pi `web_fetch`, and DSH -`web_fetch` tools. A direct-fetch failure may return the structured +or `links` requests must not provide `--wait`. The same `render: "agent-browser"` and required +`waitMs` fields are available on the MCP `fetch`/`links`, Pi `web_fetch`/`web_links`, and DSH +`web_fetch`/`web_links` tools. A direct-fetch failure may return the structured `javascript_rendering_may_be_required` hint with the 2,000 ms suggestion; the agent decides whether to retry with a longer wait or abandon the page. diff --git a/packages/dsh-web/README.md b/packages/dsh-web/README.md index d064aac..f2d5cbb 100644 --- a/packages/dsh-web/README.md +++ b/packages/dsh-web/README.md @@ -18,7 +18,7 @@ settings expose only configured/source/writable metadata. The published package is a dual host/browser bundle. Its host and client artifacts, profile patch, and exact DSH `0.1.0-rc.8` peer contract are included -in the npm package. Search, direct page fetch, optional agent-browser rendering, +in the npm package. Search, direct page fetch, page-link discovery, optional agent-browser rendering, Context7 documentation, and Sourcegraph all run in-process through the bundled Guion Web core. `web_fetch` has two page-fetch backends: direct fetch (the default) and explicit `render: "agent-browser"` with required `waitMs` (an @@ -32,6 +32,11 @@ be directly runnable from `PATH` without a shell. The renderer is supported on macOS and Linux, is not an npm dependency, and never reuses persistent browser state or credentials. +`web_links` lists up to 100 unique HTTP(S) anchors from the original page DOM, +so it includes navigation and other links that readable-content extraction drops. +It uses the same direct default and explicit `render: "agent-browser"` / required +`waitMs` contract as `web_fetch`. + Rendered requests are bounded and constrained to the requested hostname, `*.` (the target and its subdomains), and this fixed common CDN list: `cdn.jsdelivr.net`, `unpkg.com`, `cdnjs.cloudflare.com`, diff --git a/packages/dsh-web/src/tools.ts b/packages/dsh-web/src/tools.ts index 937ca67..8f33638 100644 --- a/packages/dsh-web/src/tools.ts +++ b/packages/dsh-web/src/tools.ts @@ -1,13 +1,17 @@ import { createWebOperations, + DEFAULT_LINK_LIMIT, normalizeDocsToolInput, formatSize, + MAX_LINK_LIMIT, truncateHead, type Context7Credentials, type DocsFetchResult, type DocsResolveResult, type DocsToolInput, type FetchResult, + type LinksInput, + type LinksResult, type SGraphResult, type WebOperations, } from "@guionai/web-core"; @@ -62,6 +66,29 @@ const fetchParameters = { }, } as const; +const linksParameters = { + url: { + type: "string", + required: true, + description: "HTTP or HTTPS URL to inspect", + }, + limit: { + type: "integer", + default: DEFAULT_LINK_LIMIT, + description: `Maximum links to return (1-${MAX_LINK_LIMIT})`, + }, + render: { + type: "string", + enum: ["fetch", "agent-browser"], + description: "Page-fetch backend; defaults to direct fetch", + }, + waitMs: { + type: "integer", + description: + "Required post-load wait for agent-browser rendering (0-30000)", + }, +} as const; + const docsParameters = { action: { type: "string", @@ -131,6 +158,38 @@ const fetchOutput = { ], }; +const linksOutput = { + schema: { + type: "object", + additionalProperties: false, + properties: { + url: { type: "string", required: true }, + links: { + type: "array", + required: true, + items: { + type: "object", + additionalProperties: false, + properties: { + text: { type: "string", required: true }, + url: { type: "string", required: true }, + }, + }, + }, + truncated: { type: "boolean", required: true }, + }, + } as const, + render: (_args: unknown, value: LinksResult) => [ + { + type: "text" as const, + text: boundedToolText( + formatLinks(value), + "Use web_fetch to read a selected destination.", + ), + }, + ], +}; + const docsOutput = { schema: { oneOf: [ @@ -234,6 +293,49 @@ function requireString(input: unknown, field: string): string { return input[field]; } +function normalizeLinks(input: unknown): LinksInput { + if (!isRecord(input)) throw new Error("web_links input must be an object"); + rejectUnknownFields(input, Object.keys(linksParameters), "web_links"); + const url = requireString(input, "url"); + const limit = input.limit; + if ( + limit !== undefined && + (typeof limit !== "number" || + !Number.isInteger(limit) || + limit < 1 || + limit > MAX_LINK_LIMIT) + ) + throw new Error( + `limit must be an integer from 1 through ${MAX_LINK_LIMIT}`, + ); + + const render = input.render; + const waitMs = input.waitMs; + if (render !== undefined && render !== "fetch" && render !== "agent-browser") + throw new Error('render must be "fetch" or "agent-browser"'); + if (render !== "agent-browser") { + if (waitMs !== undefined) + throw new Error("waitMs is only valid with render agent-browser"); + } else { + if (waitMs === undefined) + throw new Error("waitMs is required with render agent-browser"); + if ( + typeof waitMs !== "number" || + !Number.isInteger(waitMs) || + waitMs < 0 || + waitMs > 30_000 + ) + throw new Error("waitMs must be an integer from 0 through 30000"); + } + + return { + url, + ...(limit !== undefined ? { limit } : {}), + ...(render !== undefined ? { render } : {}), + ...(waitMs !== undefined ? { waitMs } : {}), + }; +} + function normalizeDocs(input: unknown): DocsToolInput { if (!isRecord(input)) throw new Error("web_docs input must be an object"); return normalizeDocsToolInput(input); @@ -270,6 +372,15 @@ function formatDocsResolve(result: DocsResolveResult): string { return `Found ${result.libraries.length} libraries:\n${result.libraries.map((library) => `- ${library.id}: ${library.title}`).join("\n")}`; } +function formatLinks(result: LinksResult): string { + if (result.links.length === 0) return "No HTTP(S) links found."; + const lines = result.links.map( + (link, index) => + `${index + 1}. ${link.text || "(no text)"}\n URL: ${link.url}`, + ); + return `Found ${result.links.length} link${result.links.length === 1 ? "" : "s"}${result.truncated ? " (truncated)" : ""}:\n\n${lines.join("\n\n")}`; +} + function webFetchTool( dependencies: WebToolDependencies, operations: WebOperations, @@ -301,6 +412,25 @@ function webFetchTool( ); } +function webLinksTool( + dependencies: WebToolDependencies, + operations: WebOperations, +): ToolDefinition { + return strictDefinition( + defineTool({ + name: "web_links", + description: + "List HTTP(S) links from a page. Use direct fetch for static pages, or explicit agent-browser rendering with waitMs for client-rendered pages.", + parameters: linksParameters, + output: linksOutput, + isConcurrencySafe: () => true, + async execute(args, exec) { + return operations.links(normalizeLinks(args), exec.signal); + }, + }), + ); +} + function webDocsTool( dependencies: WebToolDependencies, operations: WebOperations, @@ -369,10 +499,11 @@ function webSgraphTool( export function createWebToolDefinitions( dependencies: WebToolDependencies, -): readonly [ToolDefinition, ToolDefinition, ToolDefinition] { +): readonly [ToolDefinition, ToolDefinition, ToolDefinition, ToolDefinition] { const operations = dependencies.operations ?? createWebOperations(); return [ webFetchTool(dependencies, operations), + webLinksTool(dependencies, operations), webDocsTool(dependencies, operations), webSgraphTool(dependencies, operations), ]; diff --git a/packages/dsh-web/test/artifact.test.ts b/packages/dsh-web/test/artifact.test.ts index 7e65223..0c7518f 100644 --- a/packages/dsh-web/test/artifact.test.ts +++ b/packages/dsh-web/test/artifact.test.ts @@ -92,7 +92,7 @@ if (command === "open" && args.some((value) => value.includes("/blocked"))) { process.exit(1); } if (command === "eval") - console.log(JSON.stringify({ success: true, data: { result: "

Packed DSH rendered fixture.

" } })); + console.log(JSON.stringify({ success: true, data: { result: JSON.stringify({ html: "

Packed DSH rendered fixture.

", url: "https://93.184.216.34/rendered" }) } })); else console.log(JSON.stringify({ success: true, data: {} })); `, ); @@ -195,6 +195,11 @@ describe("DSH rc.8 packed package contract", () => { "

Packed DSH browserless fixture.

", { headers: { "content-type": "text/html" } }, ); + if (String(url) === "https://93.184.216.34/links") + return new Response( + '', + { headers: { "content-type": "text/html" } }, + ); expect(String(url)).toBe("https://api.exa.ai/search"); expect( (init.headers as Headers).get?.("x-api-key") ?? @@ -236,6 +241,11 @@ describe("DSH rc.8 packed package contract", () => { ); if (!fetchTool) throw new Error("packed DSH artifact did not register web_fetch"); + const linksTool = tools.find( + (definition) => definition.name === "web_links", + ); + if (!linksTool) + throw new Error("packed DSH artifact did not register web_links"); process.env.PATH = `${browser.bin}:${originalPath ?? ""}`; const direct = await fetchTool.execute( { url: "https://93.184.216.34/direct", full: true }, @@ -243,6 +253,16 @@ describe("DSH rc.8 packed package contract", () => { ); expect(direct.content).toBe("Packed DSH browserless fixture.\n"); expect(() => readFileSync(browser.log, "utf8")).toThrow(); + const links = await linksTool.execute( + { url: "https://93.184.216.34/links" }, + { signal: new AbortController().signal }, + ); + expect(links.links).toEqual([ + { + text: "Packed link", + url: "https://93.184.216.34/destination", + }, + ]); const rendered = await fetchTool.execute( { url: "https://93.184.216.34/rendered", diff --git a/packages/dsh-web/test/tools.test.ts b/packages/dsh-web/test/tools.test.ts index ff92f27..3f9efaa 100644 --- a/packages/dsh-web/test/tools.test.ts +++ b/packages/dsh-web/test/tools.test.ts @@ -32,7 +32,7 @@ function dependencies( } describe("DSH direct web tools", () => { - it("registers direct fetch, docs, and Sourcegraph tools with current schemas and concurrent execution", () => { + it("registers fetch, links, docs, and Sourcegraph tools with current schemas and concurrent execution", () => { const definitions = createWebToolDefinitions(dependencies()); const registered: ToolDefinition[] = []; registerWebTools( @@ -45,22 +45,29 @@ describe("DSH direct web tools", () => { ); expect(definitions.map((definition) => definition.name)).toEqual([ "web_fetch", + "web_links", "web_docs", "web_source_search", ]); expect(registered.map((definition) => definition.name)).toEqual([ "web_fetch", + "web_links", "web_docs", "web_source_search", ]); expect([ definitions[0]!.isConcurrencySafe?.({ url: "https://example.test" }), definitions[1]!.isConcurrencySafe?.({ + url: "https://example.test", + render: "agent-browser", + waitMs: 0, + }), + definitions[2]!.isConcurrencySafe?.({ action: "resolve", query: "react", }), - definitions[2]!.isConcurrencySafe?.({ query: "repo:guionai" }), - ]).toEqual([true, true, true]); + definitions[3]!.isConcurrencySafe?.({ query: "repo:guionai" }), + ]).toEqual([true, true, true, true]); expect((definitions[0]!.parameters as any).additionalProperties).toBe( false, ); @@ -71,11 +78,18 @@ describe("DSH direct web tools", () => { expect((definitions[0]!.parameters as any).properties.waitMs.type).toBe( "integer", ); - expect((definitions[1]!.parameters as any).properties.action.enum).toEqual([ + expect((definitions[1]!.parameters as any).properties.limit.default).toBe( + 100, + ); + expect((definitions[1]!.parameters as any).properties.render.enum).toEqual([ + "fetch", + "agent-browser", + ]); + expect((definitions[2]!.parameters as any).properties.action.enum).toEqual([ "resolve", "fetch", ]); - expect(Object.keys((definitions[2]!.parameters as any).properties)).toEqual( + expect(Object.keys((definitions[3]!.parameters as any).properties)).toEqual( ["query", "count", "context", "timeout"], ); }); @@ -83,13 +97,21 @@ describe("DSH direct web tools", () => { it("calls the bundled operations once with current direct inputs and caller cancellation", async () => { const calls: unknown[] = []; const controller = new AbortController(); - const [fetch, docs, sgraph] = createWebToolDefinitions( + const [fetch, links, docs, sgraph] = createWebToolDefinitions( dependencies({ operations: operations({ fetch: async (input, abortSignal) => { calls.push({ kind: "fetch", input, abortSignal }); return { url: input.url, mode: "section", content: "selected" }; }, + links: async (input, abortSignal) => { + calls.push({ kind: "links", input, abortSignal }); + return { + url: input.url, + links: [{ text: "destination", url: "https://example.test/to" }], + truncated: false, + }; + }, docsResolve: async (input) => { calls.push({ kind: "resolve", input }); return { query: input.query, libraries: [] }; @@ -122,6 +144,20 @@ describe("DSH direct web tools", () => { controller.signal, ), ).resolves.toMatchObject({ mode: "section" }); + await expect( + call( + links!, + { + url: "https://example.test", + limit: 25, + render: "agent-browser", + waitMs: 2000, + }, + controller.signal, + ), + ).resolves.toMatchObject({ + links: [{ url: "https://example.test/to" }], + }); await expect( call(docs!, { action: "resolve", query: "react" }, controller.signal), ).resolves.toMatchObject({ query: "react" }); @@ -153,6 +189,16 @@ describe("DSH direct web tools", () => { }, abortSignal: controller.signal, }, + { + kind: "links", + input: { + url: "https://example.test", + limit: 25, + render: "agent-browser", + waitMs: 2000, + }, + abortSignal: controller.signal, + }, { kind: "resolve", input: { query: "react", credentials: {}, signal: controller.signal }, @@ -307,7 +353,7 @@ describe("DSH direct web tools", () => { }, }), }), - )[1]!; + )[2]!; await expect( call(docs, { action: "fetch", library_id: "/react" }), ).rejects.toThrow("web docs fetch failed"); @@ -318,13 +364,37 @@ describe("DSH direct web tools", () => { it("rejects undeclared and cross-action fields before invoking operations", async () => { const fetch = vi.fn(); + const links = vi.fn(); const docsResolve = vi.fn(); - const [fetchTool, docsTool, sgraphTool] = createWebToolDefinitions( - dependencies({ operations: operations({ fetch, docsResolve }) }), - ); + const [fetchTool, linksTool, docsTool, sgraphTool] = + createWebToolDefinitions( + dependencies({ operations: operations({ fetch, links, docsResolve }) }), + ); await expect( call(fetchTool!, { url: "https://example.test", extra: true }), ).rejects.toThrow(/does not accept field extra/); + await expect( + call(linksTool!, { url: "https://example.test", extra: true }), + ).rejects.toThrow(/does not accept field extra/); + await expect( + call(linksTool!, { url: "https://example.test", limit: 0 }), + ).rejects.toThrow("limit must be an integer from 1 through 100"); + await expect( + call(linksTool!, { url: "https://example.test", limit: 101 }), + ).rejects.toThrow("limit must be an integer from 1 through 100"); + await expect( + call(linksTool!, { + url: "https://example.test", + render: "agent-browser", + }), + ).rejects.toThrow("waitMs is required"); + await expect( + call(linksTool!, { + url: "https://example.test", + render: "agent-browser", + waitMs: -1, + }), + ).rejects.toThrow("waitMs must be an integer from 0 through 30000"); await expect( call(docsTool!, { action: "resolve", query: "x", library_id: "/wrong" }), ).rejects.toThrow(/does not accept library_id/); @@ -332,6 +402,7 @@ describe("DSH direct web tools", () => { call(sgraphTool!, { query: "x", extra: true }), ).rejects.toThrow(/does not accept field extra/); expect(fetch).not.toHaveBeenCalled(); + expect(links).not.toHaveBeenCalled(); expect(docsResolve).not.toHaveBeenCalled(); }); }); diff --git a/packages/pi-web/src/tool.ts b/packages/pi-web/src/tool.ts index 85838f5..f7a43e1 100644 --- a/packages/pi-web/src/tool.ts +++ b/packages/pi-web/src/tool.ts @@ -2,8 +2,11 @@ import { StringEnum } from "@earendil-works/pi-ai"; import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; import { createWebOperations, + DEFAULT_LINK_LIMIT, + MAX_LINK_LIMIT, normalizeDocsToolInput, type DocsToolInput, + type LinksResult, type WebCredentials, type WebOperations, type SearchResponse, @@ -81,6 +84,46 @@ export const webFetchSchema = Type.Union([ ), ]); +const linksProperties = { + url: Type.String({ description: "HTTP or HTTPS URL to inspect" }), + limit: Type.Optional( + Type.Integer({ + description: `Maximum links to return (1-${MAX_LINK_LIMIT})`, + minimum: 1, + maximum: MAX_LINK_LIMIT, + default: DEFAULT_LINK_LIMIT, + }), + ), +}; + +export const webLinksSchema = Type.Union([ + Type.Object( + { + ...linksProperties, + render: Type.Optional( + StringEnum(["fetch"] as const, { + description: "Use direct HTTP fetching (the default)", + }), + ), + }, + { additionalProperties: false }, + ), + Type.Object( + { + ...linksProperties, + render: StringEnum(["agent-browser"] as const, { + description: "Render the page through the host-installed agent-browser", + }), + waitMs: Type.Integer({ + description: "Additional post-load wait in milliseconds", + minimum: 0, + maximum: 30_000, + }), + }, + { additionalProperties: false }, + ), +]); + export const webDocsSchema = Type.Object( { action: StringEnum(["resolve", "fetch"] as const, { @@ -133,6 +176,7 @@ export const webSgraphSchema = Type.Object( export type WebSearchInput = Static; export type WebFetchInput = Static; +export type WebLinksInput = Static; export type WebDocsInput = Static; export type WebSgraphInput = Static; @@ -154,6 +198,11 @@ const FETCH_PROMPT_GUIDELINES = [ 'For a client-rendered or SPA page, or after javascript_rendering_may_be_required, retry explicitly with render: "agent-browser" and waitMs: 2000 only when the host has agent-browser installed. Increase waitMs explicitly or abandon an incomplete page; there is no automatic fallback.', "Never send waitMs with direct fetch. agent-browser is a host capability, not a package dependency.", ]; +const LINKS_PROMPT_GUIDELINES = [ + "Use web_links to discover HTTP(S) destinations from a page, including navigation and links outside the readable article body.", + 'Use direct fetch by default. For a client-rendered or SPA page, explicitly use render: "agent-browser" with waitMs; there is no automatic fallback.', + "Never send waitMs with direct fetch. agent-browser is a host capability, not a package dependency.", +]; const DOCS_PROMPT_GUIDELINES = [ "Use web_docs with action resolve, then action fetch, to read library documentation instead of fetching documentation sites page by page.", "For web_docs action fetch, provide the library_id returned by action resolve; use topic or tokens to narrow the result.", @@ -242,6 +291,46 @@ function normalizeFetch(input: unknown): WebFetchInput { return navigation; } +function normalizeLinks(input: unknown): WebLinksInput { + if (!isRecord(input)) throw new Error("web_links input must be an object"); + const url = requireString(input, "url"); + const render = input.render; + const waitMs = input.waitMs; + const limit = input.limit; + if (render !== undefined && render !== "fetch" && render !== "agent-browser") + throw new Error('render must be "fetch" or "agent-browser"'); + if (render !== "agent-browser") { + if (waitMs !== undefined) + throw new Error("waitMs is only valid with render agent-browser"); + } else { + if (waitMs === undefined) + throw new Error("waitMs is required with render agent-browser"); + if ( + typeof waitMs !== "number" || + !Number.isInteger(waitMs) || + waitMs < 0 || + waitMs > 30_000 + ) + throw new Error("waitMs must be an integer from 0 through 30000"); + } + if ( + limit !== undefined && + (typeof limit !== "number" || + !Number.isInteger(limit) || + limit < 1 || + limit > MAX_LINK_LIMIT) + ) + throw new Error( + `limit must be an integer from 1 through ${MAX_LINK_LIMIT}`, + ); + const typed = input as unknown as WebLinksInput; + const result = { url, limit: typed.limit }; + if (render === "agent-browser") + return { ...result, render, waitMs: waitMs as number }; + if (render === "fetch") return { ...result, render }; + return result; +} + function mergeSearchResults(responses: SearchResponse[]): SearchResponse { const results: SearchResponse["results"] = []; for (let resultIndex = 0; ; resultIndex += 1) { @@ -338,6 +427,25 @@ export function webFetchTool(dependencies: WebToolDependencies = {}) { }); } +export function webLinksTool(dependencies: WebToolDependencies = {}) { + const operations = dependencies.operations ?? createWebOperations(); + return makeTool({ + name: "web_links", + label: "Web links", + description: + "List HTTP(S) links from a web page, with direct fetch or explicit agent-browser rendering for client-rendered pages.", + promptSnippet: "List links from a web page with web_links", + promptGuidelines: LINKS_PROMPT_GUIDELINES, + parameters: webLinksSchema, + execute: async (params, signal) => { + const data = await operations.links(normalizeLinks(params), signal); + return modelTextResult(data, formatLinks(data), { + hint: "Use web_fetch to read a selected destination.", + }); + }, + }); +} + export function webDocsTool(dependencies: WebToolDependencies = {}) { const operations = dependencies.operations ?? createWebOperations(); const credentials = dependencies.credentials ?? environmentCredentials; @@ -435,6 +543,15 @@ function errorMessage(error: unknown): string { return error instanceof Error ? error.message : "web search failed"; } +function formatLinks(result: LinksResult): string { + if (result.links.length === 0) return "No HTTP(S) links found."; + const lines = result.links.map( + (link, index) => + `${index + 1}. ${link.text || "(no text)"}\n URL: ${link.url}`, + ); + return `Found ${result.links.length} link${result.links.length === 1 ? "" : "s"}${result.truncated ? " (truncated)" : ""}:\n\n${lines.join("\n\n")}`; +} + export function registerWebTools( pi: Pick, dependencies: WebToolDependencies = {}, @@ -445,6 +562,7 @@ export function registerWebTools( }; pi.registerTool(webSearchTool(shared)); pi.registerTool(webFetchTool(shared)); + pi.registerTool(webLinksTool(shared)); pi.registerTool(webDocsTool(shared)); pi.registerTool(webSgraphTool(shared)); } diff --git a/packages/pi-web/test/extension.test.ts b/packages/pi-web/test/extension.test.ts index 7666066..1182a51 100644 --- a/packages/pi-web/test/extension.test.ts +++ b/packages/pi-web/test/extension.test.ts @@ -10,6 +10,8 @@ import { webDocsTool, webFetchSchema, webFetchTool, + webLinksSchema, + webLinksTool, webSearchSchema, webSearchTool, webSgraphSchema, @@ -23,6 +25,7 @@ function operations(overrides: Partial): WebOperations { return { search: unused, fetch: unused, + links: unused, docsResolve: unused, docsFetch: unused, sgraphSearch: unused, @@ -173,6 +176,53 @@ describe("pi-web extension", () => { }), ).rejects.toThrow("waitMs is only valid"); expect(fetch).toHaveBeenCalledTimes(1); + + expect(Value.Check(webLinksSchema, { url: "https://fixture.test" })).toBe( + true, + ); + expect( + Value.Check(webLinksSchema, { + url: "https://fixture.test", + render: "agent-browser", + waitMs: 0, + limit: 25, + }), + ).toBe(true); + expect( + Value.Check(webLinksSchema, { + url: "https://fixture.test", + render: "agent-browser", + }), + ).toBe(false); + expect( + Value.Check(webLinksSchema, { + url: "https://fixture.test", + limit: 101, + }), + ).toBe(false); + + const links = vi.fn(async (input: { url: string }) => ({ + url: input.url, + links: [{ text: "Rendered link", url: "https://fixture.test/link" }], + truncated: false, + })); + const linksTool = webLinksTool({ operations: operations({ links }) }); + const linksResult = await call(linksTool, { + url: "https://fixture.test", + limit: 25, + render: "agent-browser", + waitMs: 1250, + }); + expect(links).toHaveBeenCalledWith( + { + url: "https://fixture.test", + limit: 25, + render: "agent-browser", + waitMs: 1250, + }, + undefined, + ); + expect(linksResult.content[0]?.text).toContain("Rendered link"); }); it("preserves structured fetch failures for Pi callers", async () => { @@ -329,6 +379,14 @@ describe("pi-web extension", () => { }), { url: "https://fixture.test" }, ], + [ + webLinksTool({ + operations: operations({ + links: (_input, signal) => abortable(signal), + }), + }), + { url: "https://fixture.test" }, + ], [ webDocsTool({ operations: operations({ diff --git a/packages/pi-web/test/packed-smoke.mjs b/packages/pi-web/test/packed-smoke.mjs index 4ed55b6..9aba947 100644 --- a/packages/pi-web/test/packed-smoke.mjs +++ b/packages/pi-web/test/packed-smoke.mjs @@ -110,9 +110,9 @@ try { extension.default({ registerTool: (tool) => registered.push(tool) }); if ( registered.map((tool) => tool.name).join(",") !== - "web_search,web_fetch,web_docs,web_source_search" + "web_search,web_fetch,web_links,web_docs,web_source_search" ) { - throw new Error("packed extension did not register exactly four web tools"); + throw new Error("packed extension did not register exactly five web tools"); } const fakeBin = join(root, "fake-bin"); @@ -129,7 +129,7 @@ if (command === "open" && args.some((value) => value.includes("/blocked"))) { console.log(JSON.stringify({ success: false, error: { message: "domain not allowed", hostname: "missing.cdn.test" } })); process.exit(1); } else if (command === "eval") { - console.log(JSON.stringify({ success: true, data: { result: "

Rendered fixture

JavaScript output from fake agent-browser.

" } })); + console.log(JSON.stringify({ success: true, data: { result: JSON.stringify({ html: "

Rendered fixture

JavaScript output from fake agent-browser.

", url: "https://93.184.216.34/rendered" }) } })); } else { console.log(JSON.stringify({ success: true, data: {} })); } @@ -154,6 +154,11 @@ if (command === "open" && args.some((value) => value.includes("/blocked"))) { "

Direct fixture

Browserless output.

", { headers: { "Content-Type": "text/html" } }, ); + if (target === "https://93.184.216.34/links") + return new Response( + '', + { headers: { "Content-Type": "text/html" } }, + ); if (target !== "https://api.exa.ai/search") throw new Error(`unexpected fixture URL ${target}`); if ( @@ -199,6 +204,15 @@ if (command === "open" && args.some((value) => value.includes("/blocked"))) { throw error; } + const linksTool = registered.find((tool) => tool.name === "web_links"); + if (!linksTool) + throw new Error("packed extension did not register web_links"); + const links = await linksTool.execute("test", { + url: "https://93.184.216.34/links", + }); + if (links.details.links?.[0]?.url !== "https://93.184.216.34/destination") + throw new Error("packed extension did not list browserless links"); + const rendered = await fetchTool.execute("test", { url: "https://93.184.216.34/rendered", render: "agent-browser", diff --git a/packages/web-core/src/fetch.ts b/packages/web-core/src/fetch.ts index dd45006..b76c88f 100644 --- a/packages/web-core/src/fetch.ts +++ b/packages/web-core/src/fetch.ts @@ -30,6 +30,8 @@ const MAX_BROWSER_STDOUT_BYTES = 10 * 1024 * 1024; const MAX_BROWSER_STDERR_BYTES = 64 * 1024; const MAX_BINARY_SCAN_BYTES = 8192; const WEB_FETCH_AGENT = "guionai-web/1.0"; +export const DEFAULT_LINK_LIMIT = 100; +export const MAX_LINK_LIMIT = 100; export const RENDER_REPORT_URL = "https://github.com/guionai/web/issues/new"; export const RENDER_CDN_ALLOWLIST = [ "cdn.jsdelivr.net", @@ -57,6 +59,24 @@ export type FetchResult = { content: string; }; +export type LinksInput = { + url: string; + limit?: number; + render?: "fetch" | "agent-browser"; + waitMs?: number; +}; + +export type PageLink = { + text: string; + url: string; +}; + +export type LinksResult = { + url: string; + links: PageLink[]; + truncated: boolean; +}; + export type FetchErrorDetails = { retryableWithRender?: boolean; suggestedArguments?: { render: "agent-browser"; waitMs: 2000 }; @@ -122,7 +142,26 @@ export async function fetchWebPage( return { url, mode: rendered.mode, content: rendered.content }; } -function validateRenderInput(input: FetchInput): "fetch" | "agent-browser" { +/** Lists HTTP(S) links from the original or browser-rendered page DOM. */ +export async function fetchWebLinks( + input: LinksInput, + callerSignal?: AbortSignal, + options?: FetchOptions, +): Promise { + const url = validateURL(input.url); + const render = validateRenderInput(input); + const limit = validateLinkLimit(input.limit); + const source = + render === "agent-browser" + ? await renderPageHTML(url, input.waitMs!, callerSignal, options) + : await fetchPageHTML(url, callerSignal, options); + throwIfAborted(callerSignal); + return listPageLinks(source.html, source.url, url, limit); +} + +function validateRenderInput( + input: Pick, +): "fetch" | "agent-browser" { const render = input.render ?? "fetch"; if (render !== "fetch" && render !== "agent-browser") throw new Error('render must be "fetch" or "agent-browser"'); @@ -142,6 +181,15 @@ function validateRenderInput(input: FetchInput): "fetch" | "agent-browser" { return render; } +function validateLinkLimit(limit: number | undefined): number { + if (limit === undefined) return DEFAULT_LINK_LIMIT; + if (!Number.isInteger(limit) || limit < 1 || limit > MAX_LINK_LIMIT) + throw new Error( + `limit must be an integer from 1 through ${MAX_LINK_LIMIT}`, + ); + return limit; +} + async function fetchCached( url: string, callerSignal: AbortSignal | undefined, @@ -169,6 +217,57 @@ async function fetchLocal( callerSignal?: AbortSignal, options?: FetchOptions, ): Promise { + try { + const response = await downloadPage(url, callerSignal, options); + if (response.contentType !== "" && response.contentType !== "text/html") + return truncateContent(new TextDecoder().decode(response.body)); + return extractHTML( + new TextDecoder().decode(response.body), + url, + callerSignal, + true, + ); + } catch (error) { + if ( + isOperationAborted(error) || + isRequestTimeout(error) || + error instanceof FetchCapabilityError || + (error instanceof Error && + (error.message.startsWith("binary content at ") || + error.message.startsWith(`fetch ${url}:`))) + ) + throw error; + throw new Error(`fetch ${url}: ${errorMessage(error)}`); + } +} + +type DownloadedPage = { + body: Uint8Array; + contentType: string; + url: string; +}; + +type PageHTML = { + html: string; + url: string; +}; + +async function fetchPageHTML( + url: string, + callerSignal: AbortSignal | undefined, + options: FetchOptions | undefined, +): Promise { + const response = await downloadPage(url, callerSignal, options); + if (response.contentType !== "" && response.contentType !== "text/html") + throw new Error(`links ${url}: page is not HTML`); + return { html: new TextDecoder().decode(response.body), url: response.url }; +} + +async function downloadPage( + url: string, + callerSignal: AbortSignal | undefined, + options: FetchOptions | undefined, +): Promise { throwIfAborted(callerSignal); const fetcher = options?.fetch ?? globalThis.fetch; try { @@ -200,16 +299,7 @@ async function fetchLocal( response.headers.get("content-type") ?? "", ); } - - if (contentType !== "" && contentType !== "text/html") { - return truncateContent(new TextDecoder().decode(body)); - } - return extractHTML( - new TextDecoder().decode(body), - url, - callerSignal, - true, - ); + return { body, contentType, url: response.url || url }; }, ); } catch (error) { @@ -228,12 +318,11 @@ async function fetchLocal( } } -async function renderPage( +async function validateRenderTarget( url: string, - waitMs: number, callerSignal: AbortSignal | undefined, options: FetchOptions | undefined, -): Promise { +): Promise { const target = new URL(url); if (target.username || target.password) throw new Error( @@ -241,7 +330,39 @@ async function renderPage( ); await validatePublicTarget(target, callerSignal, options); throwIfAborted(callerSignal); + return target; +} +async function renderPage( + url: string, + waitMs: number, + callerSignal: AbortSignal | undefined, + options: FetchOptions | undefined, +): Promise { + const target = await validateRenderTarget(url, callerSignal, options); + try { + const page = await renderPageHTML( + url, + waitMs, + callerSignal, + options, + target, + ); + return await extractHTML(page.html, url, callerSignal, false); + } catch (error) { + throw rendererFailure(error); + } +} + +async function renderPageHTML( + url: string, + waitMs: number, + callerSignal: AbortSignal | undefined, + options: FetchOptions | undefined, + target?: URL, +): Promise { + const renderTarget = + target ?? (await validateRenderTarget(url, callerSignal, options)); const workDirectory = await mkdtemp(join(tmpdir(), "guionai-web-render-")); const configPath = join(workDirectory, "agent-browser.json"); const session = randomUUID(); @@ -253,7 +374,7 @@ async function renderPage( "--config", configPath, "--allowed-domains", - renderAllowlist(target.hostname).join(","), + renderAllowlist(renderTarget.hostname).join(","), "--idle-timeout", RENDER_IDLE_TIMEOUT, ]; @@ -272,14 +393,17 @@ async function renderPage( assertCommandSuccess(opened.stdout); await waitForRender(waitMs, callerSignal); const capture = await runAgentBrowser( - [...commonArgs, "eval", "document.documentElement.outerHTML"], + [ + ...commonArgs, + "eval", + "JSON.stringify({html: document.documentElement.outerHTML, url: document.location.href})", + ], environment, workDirectory, callerSignal, RENDER_CAPTURE_TIMEOUT_MS, ); - const html = parseCapturedHTML(capture.stdout); - return await extractHTML(html, url, callerSignal, false); + return parseCapturedPage(capture.stdout, url); } catch (error) { throw rendererFailure(error); } finally { @@ -293,6 +417,68 @@ async function renderPage( } } +function listPageLinks( + html: string, + sourceURL: string, + resultURL: string, + limit: number, +): LinksResult { + const { document } = parseHTML(html); + const baseURL = resolveDocumentBaseURL(document, sourceURL); + const links: PageLink[] = []; + const seen = new Set(); + let truncated = false; + + for (const anchor of document.querySelectorAll("a[href]")) { + const url = resolvePageLink(anchor.getAttribute("href"), baseURL); + if (!url || seen.has(url)) continue; + seen.add(url); + if (links.length >= limit) { + truncated = true; + continue; + } + links.push({ text: pageLinkText(anchor), url }); + } + return { url: resultURL, links, truncated }; +} + +function resolveDocumentBaseURL(document: Document, sourceURL: string): string { + const href = document.querySelector("base[href]")?.getAttribute("href"); + if (!href) return sourceURL; + try { + return new URL(href, sourceURL).href; + } catch { + return sourceURL; + } +} + +function resolvePageLink( + href: string | null, + baseURL: string, +): string | undefined { + if (!href || href.trim() === "") return undefined; + try { + const url = new URL(href, baseURL); + return url.protocol === "http:" || url.protocol === "https:" + ? url.href + : undefined; + } catch { + return undefined; + } +} + +function pageLinkText(anchor: Element): string { + const text = (anchor.textContent ?? "").replace(/\s+/g, " ").trim(); + if (text) return text; + return ( + anchor.getAttribute("aria-label") ?? + anchor.getAttribute("title") ?? + "" + ) + .replace(/\s+/g, " ") + .trim(); +} + async function cleanupRenderer( commandAttempted: boolean, commonArgs: string[], @@ -445,9 +631,11 @@ function rendererEnvironment( workDirectory: string, configPath: string, ): NodeJS.ProcessEnv { + // Omit HOME rather than replacing it: agent-browser falls back to the host + // account to locate its installed Chrome runtime. A unique session without a + // profile or restore input still gives each render a fresh browser state. const environment: NodeJS.ProcessEnv = { PATH: process.env.PATH, - HOME: workDirectory, TMPDIR: workDirectory, TMP: workDirectory, TEMP: workDirectory, @@ -591,7 +779,7 @@ function runAgentBrowser( }); } -function parseCapturedHTML(stdout: string): string { +function parseCapturedPage(stdout: string, fallbackURL: string): PageHTML { const record = parseSuccessEnvelope(stdout); const data = record.data; if ( @@ -603,7 +791,30 @@ function parseCapturedHTML(stdout: string): string { ) { throw new FetchCapabilityError("render_capture_failed"); } - return (data as Record).result as string; + try { + const page = JSON.parse( + (data as Record).result as string, + ) as unknown; + if (!page || typeof page !== "object" || Array.isArray(page)) + throw new Error("invalid rendered page"); + const { html, url } = page as Record; + if (typeof html !== "string" || typeof url !== "string") + throw new Error("invalid rendered page"); + return { html, url: renderedPageURL(url, fallbackURL) }; + } catch { + throw new FetchCapabilityError("render_capture_failed"); + } +} + +function renderedPageURL(url: string, fallbackURL: string): string { + try { + const value = new URL(url); + return value.protocol === "http:" || value.protocol === "https:" + ? value.href + : fallbackURL; + } catch { + return fallbackURL; + } } function assertCommandSuccess(stdout: string): void { diff --git a/packages/web-core/src/index.ts b/packages/web-core/src/index.ts index dde8766..c57b16a 100644 --- a/packages/web-core/src/index.ts +++ b/packages/web-core/src/index.ts @@ -7,7 +7,14 @@ import { type DocsResolveInput, type DocsResolveResult, } from "./docs.js"; -import { fetchWebPage, type FetchInput, type FetchResult } from "./fetch.js"; +import { + fetchWebLinks, + fetchWebPage, + type FetchInput, + type FetchResult, + type LinksInput, + type LinksResult, +} from "./fetch.js"; import { sgraphSearch, type SGraphInput, type SGraphResult } from "./sgraph.js"; import { boundedRequest, @@ -19,14 +26,20 @@ import { export { fetchWebPage, + fetchWebLinks, + DEFAULT_LINK_LIMIT, FetchCapabilityError, RENDER_CDN_ALLOWLIST, RENDER_REPORT_URL, + MAX_LINK_LIMIT, type FetchCache, type FetchErrorDetails, type FetchInput, type FetchOptions, type FetchResult, + type LinksInput, + type LinksResult, + type PageLink, } from "./fetch.js"; export { renderMarkdown, @@ -101,6 +114,7 @@ export type SearchInput = { export type WebOperations = { search(input: SearchInput): Promise; fetch(input: FetchInput, signal?: AbortSignal): Promise; + links(input: LinksInput, signal?: AbortSignal): Promise; docsResolve(input: DocsResolveInput): Promise; docsFetch(input: DocsFetchInput): Promise; sgraphSearch(input: SGraphInput): Promise; @@ -111,6 +125,7 @@ export function createWebOperations(): WebOperations { return { search, fetch: fetchWebPage, + links: fetchWebLinks, docsResolve, docsFetch, sgraphSearch, diff --git a/packages/web-core/test/fetch.test.ts b/packages/web-core/test/fetch.test.ts index 23ad4df..bbdaec5 100644 --- a/packages/web-core/test/fetch.test.ts +++ b/packages/web-core/test/fetch.test.ts @@ -19,7 +19,7 @@ import { join } from "node:path"; import { describe, expect, it, vi } from "vitest"; -import { fetchWebPage } from "../src/index.js"; +import { fetchWebLinks, fetchWebPage } from "../src/index.js"; type FetchPage = typeof fetchWebPage; @@ -153,6 +153,47 @@ describe.sequential("browserless fetch migrated from Organon", () => { } }); + it("lists raw page links without Defuddle and resolves relative destinations", async () => { + const { server, url } = await startServer((_req, res) => { + res.setHeader("Content-Type", "text/html; charset=utf-8"); + res.end(` + + + + + + + + `); + }); + try { + await expect( + fetchWebLinks({ url: `${url}/page`, limit: 4 }), + ).resolves.toEqual({ + url: `${url}/page`, + links: [ + { text: "Navigation", url: `${url}/navigation` }, + { text: "Guide", url: `${url}/docs/guide` }, + { text: "Section", url: `${url}/docs/#section` }, + { text: "Outside", url: "https://outside.test/path" }, + ], + truncated: true, + }); + await expect( + fetchWebLinks({ url: `${url}/page`, limit: 0 }), + ).rejects.toThrow("limit must be an integer from 1 through 100"); + } finally { + await close(server); + } + }); + it("rejects empty HTML extraction, known binary media, and NUL-bearing bodies", async () => { const { server, url } = await startServer((req, res) => { if (req.url === "/empty") { @@ -361,7 +402,7 @@ describe.sequential("browserless fetch migrated from Organon", () => { ), ).toBe(true); expect(open.cwd).not.toBe(process.cwd()); - expect(open.home).toContain("guionai-web-render-"); + expect(open.home).toBeUndefined(); expect(open.config).toBe("{}\n"); expect(open.profile).toBeUndefined(); @@ -381,6 +422,70 @@ describe.sequential("browserless fetch migrated from Organon", () => { }); }); + it("lists links from an explicit agent-browser rendering", async () => { + await withFakeAgentBrowser( + async (logPath) => { + await expect( + fetchWebLinks( + { + url: "https://render.test/page", + render: "agent-browser", + waitMs: 0, + }, + undefined, + { resolveHost: async () => ["93.184.216.34"] }, + ), + ).resolves.toEqual({ + url: "https://render.test/page", + links: [ + { + text: "Rendered destination", + url: "https://render.test/destination", + }, + ], + truncated: false, + }); + expect( + readFakeLog(logPath).map((entry) => command(entry.args)), + ).toEqual(["open", "eval", "close"]); + }, + { + html: 'Rendered destination', + }, + ); + }); + + it("resolves rendered links from the page URL after navigation", async () => { + await withFakeAgentBrowser( + async () => { + await expect( + fetchWebLinks( + { + url: "https://render.test/start", + render: "agent-browser", + waitMs: 0, + }, + undefined, + { resolveHost: async () => ["93.184.216.34"] }, + ), + ).resolves.toEqual({ + url: "https://render.test/start", + links: [ + { + text: "Next", + url: "https://render.test/docs/next", + }, + ], + truncated: false, + }); + }, + { + html: 'Next', + url: "https://render.test/docs/index.html", + }, + ); + }); + it("cancels the post-load wait and still closes the isolated session", async () => { await withFakeAgentBrowser(async (logPath) => { const controller = new AbortController(); @@ -655,6 +760,7 @@ describe.sequential("browserless fetch migrated from Organon", () => { async function withFakeAgentBrowser( fn: (logPath: string) => Promise, + options: { html?: string; url?: string } = {}, ): Promise { const directory = mkdtempSync( join(tmpdir(), "guionai-web-fake-agent-browser-"), @@ -662,6 +768,12 @@ async function withFakeAgentBrowser( const executable = join(directory, "agent-browser"); const logPath = join(directory, "commands.jsonl"); const previousPath = process.env.PATH; + const renderedPage = JSON.stringify({ + html: + options.html ?? + "

SPA_MARKER_RENDERED

", + url: options.url ?? "https://render.test/page", + }); writeFileSync( executable, `#!/usr/bin/env node @@ -684,7 +796,7 @@ if (command === "open" && args.includes("https://blocked.test/page")) { process.exit(1); } if (command === "eval") { - console.log(JSON.stringify({ success: true, data: { result: "

SPA_MARKER_RENDERED

" } })); + console.log(JSON.stringify({ success: true, data: { result: ${JSON.stringify(renderedPage)} } })); } else { console.log(JSON.stringify({ success: true, data: {} })); } diff --git a/packages/web/src/mcp.ts b/packages/web/src/mcp.ts index ce4e637..aed1e13 100644 --- a/packages/web/src/mcp.ts +++ b/packages/web/src/mcp.ts @@ -8,7 +8,9 @@ import { serveStdio } from "@modelcontextprotocol/server/stdio"; import { Command } from "commander"; import { + DEFAULT_LINK_LIMIT, FetchCapabilityError, + MAX_LINK_LIMIT, RENDER_REPORT_URL, type FetchErrorDetails, type WebCredentials, @@ -27,6 +29,12 @@ type FetchToolInput = { render?: "fetch" | "agent-browser"; waitMs?: number; }; +type LinksToolInput = { + url: string; + limit?: number; + render?: "fetch" | "agent-browser"; + waitMs?: number; +}; type DocsResolveToolInput = { query: string }; type DocsFetchToolInput = { library_id: string; @@ -97,6 +105,44 @@ const fetchInputSchema = schema({ }, ], }); +const linksInputSchema = schema({ + type: "object", + additionalProperties: false, + properties: { + url: { type: "string", description: "HTTP or HTTPS URL to inspect" }, + limit: { + type: "integer", + minimum: 1, + maximum: MAX_LINK_LIMIT, + default: DEFAULT_LINK_LIMIT, + description: "maximum links to return", + }, + render: { + type: "string", + enum: ["fetch", "agent-browser"], + default: "fetch", + description: "optional page-fetch backend; direct fetch is the default", + }, + waitMs: { + type: "integer", + minimum: 0, + maximum: 30_000, + description: + "required post-load wait for agent-browser rendering (0-30000)", + }, + }, + required: ["url"], + oneOf: [ + { + properties: { render: { enum: ["fetch"] } }, + not: { required: ["waitMs"] }, + }, + { + properties: { render: { const: "agent-browser" } }, + required: ["render", "waitMs"], + }, + ], +}); const docsResolveInputSchema = schema({ type: "object", properties: { @@ -172,6 +218,25 @@ const fetchOutputSchema = schema({ }, required: ["url", "mode", "content"], }); +const linksOutputSchema = schema({ + type: "object", + properties: { + url: { type: "string" }, + links: { + type: "array", + items: { + type: "object", + properties: { + text: { type: "string" }, + url: { type: "string" }, + }, + required: ["text", "url"], + }, + }, + truncated: { type: "boolean" }, + }, + required: ["url", "links", "truncated"], +}); const docsResolveOutputSchema = schema({ type: "object", properties: { @@ -215,7 +280,7 @@ const sgraphOutputSchema = schema({ required: ["content"], }); -/** Creates the five-tool MCP server used by the stdio command and adapter tests. */ +/** Creates the six-tool MCP server used by the stdio command and adapter tests. */ export function createMcpServer(dependencies: McpDependencies): McpServer { const server = new McpServer({ name: "guionai-web", version: "0.1.0" }); @@ -240,6 +305,30 @@ export function createMcpServer(dependencies: McpDependencies): McpServer { ), ); + server.registerTool( + "links", + toolConfig( + "List page links", + "List HTTP or HTTPS links from a static page or explicit agent-browser rendering.", + linksInputSchema, + linksOutputSchema, + ), + async ({ url, limit, render, waitMs }, context) => + runTool( + () => + dependencies.operations.links( + { + url, + ...(limit !== undefined ? { limit } : {}), + ...(render !== undefined ? { render } : {}), + ...(waitMs !== undefined ? { waitMs } : {}), + }, + context.mcpReq.signal, + ), + dependencies.credentials(), + ), + ); + server.registerTool( "fetch", toolConfig( diff --git a/packages/web/src/program.ts b/packages/web/src/program.ts index 3812365..653ae0a 100644 --- a/packages/web/src/program.ts +++ b/packages/web/src/program.ts @@ -5,6 +5,7 @@ import { createMcpCommand } from "./mcp.js"; import { formatSearchResults, type DocsLibrary, + type LinksResult, type WebCredentials, type WebOperations, } from "@guionai/web-core"; @@ -24,6 +25,7 @@ export function createProgram(dependencies: ProgramDependencies): Command { .showHelpAfterError(false) .addCommand(createSearchCommand(dependencies)) .addCommand(createFetchCommand(dependencies)) + .addCommand(createLinksCommand(dependencies)) .addCommand(createDocsCommand(dependencies)) .addCommand(createSGraphCommand(dependencies)) .addCommand(createMcpCommand(dependencies)); @@ -246,3 +248,55 @@ function createFetchCommand(dependencies: ProgramDependencies): Command { }, ); } + +function createLinksCommand(dependencies: ProgramDependencies): Command { + const writeOut = + dependencies.writeOut ?? ((text: string) => process.stdout.write(text)); + return new Command("links") + .description("List HTTP(S) links from a web page") + .argument("", "HTTP or HTTPS URL") + .option("--limit ", "Maximum links to return (1-100)", Number) + .option( + "--render ", + "Rendering backend: fetch (default) or agent-browser", + ) + .option( + "--wait ", + "Required post-load wait for --render agent-browser (0-30000)", + Number, + ) + .option("--json", "Output the structured result as JSON") + .action( + async ( + url: string, + options: { + limit?: number; + render?: "fetch" | "agent-browser"; + wait?: number; + json?: boolean; + }, + ) => { + const result = await dependencies.operations.links({ + url, + limit: options.limit, + ...(options.render !== undefined ? { render: options.render } : {}), + ...(options.wait !== undefined ? { waitMs: options.wait } : {}), + }); + if (options.json) { + writeOut(JSON.stringify(result) + "\n"); + return; + } + writeOut(formatLinks(result)); + }, + ); +} + +function formatLinks(result: LinksResult): string { + if (result.links.length === 0) return "No HTTP(S) links found.\n"; + let output = `Found ${result.links.length} link${result.links.length === 1 ? "" : "s"}${result.truncated ? " (truncated)" : ""}:\n\n`; + for (const [index, link] of result.links.entries()) { + output += `${index + 1}. ${link.text || "(no text)"}\n`; + output += ` URL: ${link.url}\n\n`; + } + return output; +} diff --git a/packages/web/test/mcp.test.ts b/packages/web/test/mcp.test.ts index 68ef7b2..f0022fa 100644 --- a/packages/web/test/mcp.test.ts +++ b/packages/web/test/mcp.test.ts @@ -31,6 +31,11 @@ function webService(): WebOperations { mode: "tree" as const, content: "# Page", })), + links: vi.fn(async (input) => ({ + url: input.url, + links: [{ text: "MCP link", url: "https://example.test/link" }], + truncated: false, + })), docsResolve: vi.fn(async () => ({ query: "effect", libraries: [ @@ -204,7 +209,7 @@ describe("web stdio MCP adapter", () => { } }); - it("lists exactly five typed read-only, idempotent, open-world tools", async () => { + it("lists exactly six typed read-only, idempotent, open-world tools", async () => { const { client } = await connect(); const { tools } = await client.listTools(); @@ -212,6 +217,7 @@ describe("web stdio MCP adapter", () => { "docs_fetch", "docs_resolve", "fetch", + "links", "search", "source_search", ]); @@ -231,6 +237,10 @@ describe("web stdio MCP adapter", () => { string, unknown >; + const linksProperties = byName.links!.inputSchema.properties! as Record< + string, + unknown + >; const docsFetchProperties = byName.docs_fetch!.inputSchema .properties! as Record; const sgraphProperties = byName.source_search!.inputSchema @@ -246,6 +256,15 @@ describe("web stdio MCP adapter", () => { maximum: 30000, }); expect(fetchProperties.timeout).toBeUndefined(); + expect(linksProperties.limit).toMatchObject({ + default: 100, + minimum: 1, + maximum: 100, + }); + expect(linksProperties.render).toMatchObject({ + enum: ["fetch", "agent-browser"], + default: "fetch", + }); expect(docsFetchProperties.tokens).toMatchObject({ default: 0 }); expect(sgraphProperties).toMatchObject({ count: { default: 10 }, @@ -270,6 +289,15 @@ describe("web stdio MCP adapter", () => { waitMs: 125, }, }); + const links = await client.callTool({ + name: "links", + arguments: { + url: "https://example.test/page", + limit: 25, + render: "agent-browser", + waitMs: 125, + }, + }); const resolve = await client.callTool({ name: "docs_resolve", arguments: { query: "effect" }, @@ -289,6 +317,9 @@ describe("web stdio MCP adapter", () => { expect(search.structuredContent).toMatchObject({ provider: "Brave" }); expect(fetch.structuredContent).toMatchObject({ mode: "tree" }); + expect(links.structuredContent).toMatchObject({ + links: [{ url: "https://example.test/link" }], + }); expect(resolve.structuredContent).toMatchObject({ query: "effect" }); expect(docs.structuredContent).toMatchObject({ content: "Effect docs" }); expect(sgraph.structuredContent).toMatchObject({ @@ -316,6 +347,15 @@ describe("web stdio MCP adapter", () => { }, expect.any(AbortSignal), ); + expect(operations.links).toHaveBeenCalledWith( + { + url: "https://example.test/page", + limit: 25, + render: "agent-browser", + waitMs: 125, + }, + expect.any(AbortSignal), + ); expect(operations.docsFetch).toHaveBeenCalledWith( expect.objectContaining({ library_id: "effect-ts/effect", diff --git a/packages/web/test/packed-smoke.mjs b/packages/web/test/packed-smoke.mjs index ae3fdcb..9fd18f6 100644 --- a/packages/web/test/packed-smoke.mjs +++ b/packages/web/test/packed-smoke.mjs @@ -47,14 +47,16 @@ async function pnpm(args, cwd) { } const server = createServer((request, response) => { - if (request.url !== "/page") { + if (request.url !== "/page" && request.url !== "/links") { response.statusCode = 404; response.end(); return; } response.setHeader("content-type", "text/html; charset=utf-8"); response.end( - "

Packed fetch fixture.

", + request.url === "/links" + ? '' + : "

Packed fetch fixture.

", ); }); @@ -128,7 +130,7 @@ if (command === "open" && args.some((value) => value.includes("/blocked"))) { process.exit(1); } if (command === "eval") - console.log(JSON.stringify({ success: true, data: { result: "

Packed rendered fixture.

" } })); + console.log(JSON.stringify({ success: true, data: { result: JSON.stringify({ html: "

Packed rendered fixture.

", url: "https://93.184.216.34/rendered" }) } })); else console.log(JSON.stringify({ success: true, data: {} })); `, ); @@ -162,6 +164,7 @@ else console.log(JSON.stringify({ success: true, data: {} })); "docs_fetch", "docs_resolve", "fetch", + "links", "search", "source_search", ]) @@ -181,6 +184,18 @@ else console.log(JSON.stringify({ success: true, data: {} })); throw new Error("packed MCP stdio could not fetch the local fixture"); } + const mcpLinks = await client.callTool({ + name: "links", + arguments: { url: `http://127.0.0.1:${port}/links` }, + }); + if ( + mcpLinks.isError || + mcpLinks.structuredContent?.links?.[0]?.url !== + `http://127.0.0.1:${port}/destination` + ) { + throw new Error("packed MCP stdio could not list local links"); + } + const result = await execFileAsync( binary, ["fetch", `http://127.0.0.1:${port}/page`, "--full", "--json"], @@ -193,6 +208,18 @@ else console.log(JSON.stringify({ success: true, data: {} })); ) { throw new Error("installed web CLI could not fetch the local fixture"); } + + const linkResult = await execFileAsync( + binary, + ["links", `http://127.0.0.1:${port}/links`, "--json"], + { cwd: root }, + ); + if ( + JSON.parse(linkResult.stdout).links?.[0]?.url !== + `http://127.0.0.1:${port}/destination` + ) { + throw new Error("installed web CLI could not list local links"); + } } finally { await client.close(); } diff --git a/packages/web/test/program.test.ts b/packages/web/test/program.test.ts index 555f7b5..22e0efb 100644 --- a/packages/web/test/program.test.ts +++ b/packages/web/test/program.test.ts @@ -24,6 +24,11 @@ function setup() { mode: "full" as const, content: "# Fixture page\n", })), + links: vi.fn(async (input: { url: string }) => ({ + url: input.url, + links: [{ text: "Fixture link", url: "https://example.test/link" }], + truncated: false, + })), docsResolve: vi.fn(), docsFetch: vi.fn(), sgraphSearch: vi.fn(async () => ({ @@ -243,6 +248,40 @@ describe("web search Commander adapter", () => { expect(output().stdout.endsWith("\n")).toBe(true); }); + it("forwards link discovery options and supports concise and JSON output", async () => { + const { program, operations, output } = setup(); + await program.parseAsync( + [ + "links", + "https://example.test/page", + "--limit", + "25", + "--render=agent-browser", + "--wait=0", + ], + { from: "user" }, + ); + + expect(operations.links).toHaveBeenCalledWith({ + url: "https://example.test/page", + limit: 25, + render: "agent-browser", + waitMs: 0, + }); + expect(output()).toEqual({ + stdout: + "Found 1 link:\n\n1. Fixture link\n URL: https://example.test/link\n\n", + stderr: "", + }); + + await program.parseAsync(["links", "https://example.test/page", "--json"], { + from: "user", + }); + expect(JSON.parse(output().stdout.split("\n").at(-2)!)).toEqual( + await operations.links.mock.results[1]!.value, + ); + }); + it("formats the browser retry guidance without partial stdout", async () => { const operations = { search: vi.fn(), @@ -252,6 +291,7 @@ describe("web search Commander adapter", () => { suggestedArguments: { render: "agent-browser", waitMs: 2000 }, }); }), + links: vi.fn(), docsResolve: vi.fn(), docsFetch: vi.fn(), sgraphSearch: vi.fn(), @@ -287,6 +327,7 @@ describe("web search Commander adapter", () => { ); }), fetch: vi.fn(), + links: vi.fn(), docsResolve: vi.fn(), docsFetch: vi.fn(), sgraphSearch: vi.fn(),