diff --git a/README.md b/README.md index b4a377c..80fac49 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,10 @@ # Guion Web Guion Web is a Node.js web research toolkit. It provides Exa or Brave search, -browserless HTML-to-Markdown extraction, Context7 library documentation lookup, -and Sourcegraph public code search through a CLI, stdio MCP server, Pi extension, -and DeepSeek Harness (DSH) integration. +Context7 library documentation lookup, Sourcegraph public code search, and two +page-fetch backends through a CLI, stdio MCP server, Pi extension, and DeepSeek +Harness (DSH) integration: direct HTML-to-Markdown extraction and explicit +`agent-browser` rendering for client-rendered pages on supported hosts. ## Install and configure @@ -64,7 +65,9 @@ web mcp --provider brave The server exposes five read-only tools: `search`, `fetch`, `docs_resolve`, `docs_fetch`, and `sgraph_search`. Its stdout is reserved for MCP protocol -messages; diagnostics go to stderr. +messages; diagnostics go to stderr. For a client-rendered page, explicitly call +`fetch` with `render: "agent-browser"` and an integer `waitMs`; this optional +retry requires a host-installed executable and never happens automatically. ## Pi @@ -76,7 +79,9 @@ pi install npm:@guionai/pi-web It registers `web_search`, `web_fetch`, `web_docs`, and `web_sgraph` and calls the bundled core in-process. Pi and TypeBox are peer dependencies supplied by -the host; no CLI executable or MCP configuration is required. +the host; no CLI executable or MCP configuration is required. `web_fetch` uses +direct fetch by default and can explicitly use `render: "agent-browser"` with +an integer `waitMs` when its host provides that optional executable. ## DSH @@ -90,13 +95,18 @@ The included profile patch routes stock PTC web search through the selected Exa or Brave provider. Its settings UI stores provider selection and manages namespaced write-only credentials. Fetch, documentation, and Sourcegraph tools also run in-process. The host DSH packages and React are peers supplied by DSH. +`web_fetch` uses direct fetch by default and can explicitly use +`render: "agent-browser"` with an integer `waitMs` on a host that supplies the +optional executable. -## Optional JavaScript rendering +## Page-fetch backends -`web fetch` is browserless by default. It uses Node `fetch`, `linkedom`, and -Defuddle, never launches a subprocess for an ordinary fetch, and does not -execute page JavaScript. If the response is an application shell, retry -explicitly; there is no automatic browser fallback: +`web fetch` has two backends. `fetch` (the default) uses Node `fetch`, +`linkedom`, and Defuddle for direct HTML-to-Markdown extraction from static, +SSR, and pre-rendered pages. `agent-browser` renders client-side pages through +a separately installed host executable. Direct fetch is used by default; choose +agent-browser explicitly when needed. The implementation never falls back +automatically: ```bash web fetch https://example.com/app --render=agent-browser --wait=2000 @@ -105,18 +115,27 @@ web fetch https://example.com/app --render=agent-browser --wait=10000 ``` `--wait` is mandatory with `--render=agent-browser`, including `--wait=0`, and -accepts only an integer from 0 through 30,000 milliseconds. Browserless fetches -must not provide `--wait`. The same `render: "agent-browser"` and required +accepts only an integer from 0 through 30,000 milliseconds. Direct `fetch` +requests must not provide `--wait`. The same `render: "agent-browser"` and required `waitMs` fields are available on the MCP `fetch`, Pi `web_fetch`, and DSH -`web_fetch` tools. A browserless failure may return the structured +`web_fetch` tools. A direct-fetch failure may return the structured `javascript_rendering_may_be_required` hint with the 2,000 ms suggestion; the agent decides whether to retry with a longer wait or abandon the page. -Rendering is an optional host capability. The host must already have a -compatible `agent-browser` executable directly runnable from `PATH`; this -project does not install it, download Chromium, or reuse browser credentials. -The renderer is supported on macOS and Linux hosts. The three npm packages -remain browserless and installable when `agent-browser` is absent. +Rendering is an optional host capability. If you choose to use it, install +[agent-browser](https://github.com/vercel-labs/agent-browser) separately on the +host: + +```bash +npm install --global agent-browser +agent-browser install +``` + +`agent-browser install` manages its own browser runtime; Guion packages never +run it, bundle it, or reuse browser credentials. A compatible executable must be +directly runnable from `PATH` without a shell. The renderer is supported on +macOS and Linux hosts. Direct fetch remains available, and the three npm +packages remain installable when `agent-browser` is absent. A rendered session is fresh and non-persistent. Before launch, the target must be an HTTP(S) public hostname or address. The browser allowlist then contains diff --git a/packages/dsh-web/README.md b/packages/dsh-web/README.md index 60482fa..d064aac 100644 --- a/packages/dsh-web/README.md +++ b/packages/dsh-web/README.md @@ -18,13 +18,19 @@ settings expose only configured/source/writable metadata. The published package is a dual host/browser bundle. Its host and client artifacts, profile patch, and exact DSH `0.1.0-rc.8` peer contract are included -in the npm package. Search, browserless page fetch, Context7 documentation, -and Sourcegraph all run in-process through the bundled Guion Web core. -`web_fetch` remains browserless by default; an agent may explicitly request -`render: "agent-browser"` with a required integer `waitMs` from 0 through -30,000 to render a client-side page through a host-installed `agent-browser` -executable. The optional renderer is supported on macOS and Linux, is not an -npm dependency, and never reuses persistent browser state or credentials. +in the npm package. Search, direct page fetch, optional agent-browser rendering, +Context7 documentation, and Sourcegraph all run in-process through the bundled +Guion Web core. `web_fetch` has two page-fetch backends: direct fetch (the +default) and explicit `render: "agent-browser"` with required `waitMs` (an +integer from 0 through 30,000) for client-rendered pages through a host-installed +`agent-browser` +executable. To enable that optional capability, install +[agent-browser](https://github.com/vercel-labs/agent-browser) separately with +`npm install --global agent-browser` followed by `agent-browser install`. Its +browser runtime is managed outside this package; the compatible executable must +be directly runnable from `PATH` without a shell. The renderer is supported on +macOS and Linux, is not an npm dependency, and never reuses persistent browser +state or credentials. Rendered requests are bounded and constrained to the requested hostname, `*.` (the target and its subdomains), and this fixed common diff --git a/packages/dsh-web/package.json b/packages/dsh-web/package.json index 5ec082c..ab6d47f 100644 --- a/packages/dsh-web/package.json +++ b/packages/dsh-web/package.json @@ -1,7 +1,7 @@ { "name": "@guionai/dsh-web", "version": "0.1.0", - "description": "Guion Web provider and settings for DeepSeek Harness rc.8.", + "description": "DeepSeek Harness rc.8 provider for web research with direct and optional agent-browser page-fetch backends.", "homepage": "https://github.com/guionai/web#readme", "bugs": { "url": "https://github.com/guionai/web/issues" diff --git a/packages/dsh-web/src/tools.ts b/packages/dsh-web/src/tools.ts index 1144e5a..564a920 100644 --- a/packages/dsh-web/src/tools.ts +++ b/packages/dsh-web/src/tools.ts @@ -53,7 +53,7 @@ const fetchParameters = { render: { type: "string", enum: ["fetch", "agent-browser"], - description: "Rendering backend; defaults to browserless fetch", + description: "Page-fetch backend; defaults to direct fetch", }, waitMs: { type: "integer", @@ -278,7 +278,7 @@ function webFetchTool( defineTool({ name: "web_fetch", description: - "Fetch and read an HTTP or HTTPS web page as Markdown with heading-tree and section navigation.", + "Use direct fetch for static, SSR, and pre-rendered pages. For client-rendered or SPA pages, set render: agent-browser with required waitMs on a host that has agent-browser installed; there is no automatic fallback.", parameters: fetchParameters, output: fetchOutput, isConcurrencySafe: () => true, diff --git a/packages/pi-web/package.json b/packages/pi-web/package.json index a57bf7d..c920776 100644 --- a/packages/pi-web/package.json +++ b/packages/pi-web/package.json @@ -1,7 +1,7 @@ { "name": "@guionai/pi-web", "version": "0.1.0", - "description": "Pi extension for in-process web search, page fetch, documentation, and Sourcegraph code search.", + "description": "Pi extension for web research with direct and optional agent-browser page-fetch backends.", "homepage": "https://github.com/guionai/web#readme", "bugs": { "url": "https://github.com/guionai/web/issues" diff --git a/packages/pi-web/src/tool.ts b/packages/pi-web/src/tool.ts index 800df52..9f4d73d 100644 --- a/packages/pi-web/src/tool.ts +++ b/packages/pi-web/src/tool.ts @@ -59,7 +59,7 @@ export const webFetchSchema = Type.Union([ ...fetchNavigationProperties, render: Type.Optional( StringEnum(["fetch"] as const, { - description: "Use browserless HTTP fetching (the default)", + description: "Use direct HTTP fetching (the default)", }), ), }, @@ -150,7 +150,9 @@ const SEARCH_PROMPT_GUIDELINES = [ ]; const FETCH_PROMPT_GUIDELINES = [ "Use web_fetch to read a web page; large pages are truncated with a continuation notice, so follow up with tree or section_id to navigate.", - 'Browserless fetching is the default. If the result says JavaScript rendering may be required, retry explicitly with render: "agent-browser" and waitMs from 0 through 30000.', + 'web_fetch has two backends: omit render or use render: "fetch" for direct HTML-to-Markdown (the default for static, SSR, and pre-rendered pages).', + 'For a client-rendered or SPA page, or after javascript_rendering_may_be_required, retry explicitly with render: "agent-browser" and waitMs: 2000 only when the host has agent-browser installed. Increase waitMs explicitly or abandon an incomplete page; there is no automatic fallback.', + "Never send waitMs with direct fetch. agent-browser is a host capability, not a package dependency.", ]; const DOCS_PROMPT_GUIDELINES = [ "Use web_docs with action resolve, then action fetch, to read library documentation instead of fetching documentation sites page by page.", @@ -323,7 +325,7 @@ export function webFetchTool(dependencies: WebToolDependencies = {}) { name: "web_fetch", label: "Web fetch", description: - "Fetch and read an HTTP or HTTPS web page as Markdown, with heading-tree navigation. Browserless fetching is the default; explicitly choose render agent-browser with waitMs 0 through 30000 for JavaScript pages. Text output is limited to 2,000 lines or 50KB; truncated output is saved to a temporary file.", + "Fetch and read an HTTP or HTTPS web page as Markdown, with direct fetch or explicit agent-browser rendering for client-rendered pages. Rendered fetch requires waitMs 0 through 30000. Text output is limited to 2,000 lines or 50KB; truncated output is saved to a temporary file.", promptSnippet: "Fetch a web page with web_fetch", promptGuidelines: FETCH_PROMPT_GUIDELINES, parameters: webFetchSchema, diff --git a/packages/web/package.json b/packages/web/package.json index 1ebed2e..b59b596 100644 --- a/packages/web/package.json +++ b/packages/web/package.json @@ -1,7 +1,7 @@ { "name": "@guionai/web", "version": "0.1.0", - "description": "Browserless web research CLI and stdio MCP server.", + "description": "Web research CLI and stdio MCP server with direct and optional agent-browser page-fetch backends.", "homepage": "https://github.com/guionai/web#readme", "bugs": { "url": "https://github.com/guionai/web/issues" diff --git a/packages/web/src/mcp.ts b/packages/web/src/mcp.ts index 5fef3da..4c7b634 100644 --- a/packages/web/src/mcp.ts +++ b/packages/web/src/mcp.ts @@ -75,7 +75,7 @@ const fetchInputSchema = schema({ type: "string", enum: ["fetch", "agent-browser"], default: "fetch", - description: "optional renderer; browserless fetch is the default", + description: "optional page-fetch backend; direct fetch is the default", }, waitMs: { type: "integer", @@ -244,7 +244,7 @@ export function createMcpServer(dependencies: McpDependencies): McpServer { "fetch", toolConfig( "Fetch a web page", - "Fetch a web page and return rendered Markdown content.", + "Use direct fetch (omit render or set render: fetch) for static, SSR, and pre-rendered pages. For client-rendered or SPA pages, set render: agent-browser with required waitMs on a host that has agent-browser installed; there is no automatic fallback.", fetchInputSchema, fetchOutputSchema, ),