diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json index fa3dcb7f..d016998c 100644 --- a/.codex-plugin/plugin.json +++ b/.codex-plugin/plugin.json @@ -19,7 +19,7 @@ "interface": { "displayName": "Webcmd", "shortDescription": "Agent-native CLIs for websites, apps, and local tools", - "longDescription": "Use Webcmd's reusable CLI, browser, search, authoring, sitemap, and repair workflows from Codex. Requires the @agentrhq/webcmd npm CLI.", + "longDescription": "Use Webcmd's bundled webcmd-browser skill from Codex. The npm CLI also covers adapters, search, fetch, and plugins. Requires the @agentrhq/webcmd npm CLI.", "developerName": "AgentRHQ", "category": "Developer Tools", "capabilities": [ diff --git a/Makefile b/Makefile index b525381d..bd64f392 100644 --- a/Makefile +++ b/Makefile @@ -6,8 +6,8 @@ LITPROMPT ?= litprompt # skill-src/cli/ -> skills/ installable CLI/harness skills # skill-src/mcp/ -> mcp-skills/ MCP resource documents # -# skill-src/cli/webcmd-usage/SKILL.src.md -> skills/webcmd-usage/SKILL.md -# skill-src/mcp/webcmd-usage.src.md -> mcp-skills/webcmd-usage.md +# skill-src/cli/webcmd-browser/SKILL.src.md -> skills/webcmd-browser/SKILL.md +# skill-src/mcp/webcmd-browser.src.md -> mcp-skills/webcmd-browser.md # # Source files are never named SKILL.md, and no generated MCP document is # either: skill installers (`npx skills add`, harness scanners) match that diff --git a/PRIVACY.md b/PRIVACY.md index 8601c41d..fa1d163e 100644 --- a/PRIVACY.md +++ b/PRIVACY.md @@ -2,8 +2,30 @@ The webcmd-managed CloakBrowser runtime communicates only with the local Webcmd daemon on `localhost:9777`. -The runtime can access browser pages and cookies because browser automation requires those permissions. Webcmd does not send browser data to AgentR. Commands run locally, and command output is printed to the local CLI process. +The runtime can access browser pages and cookies because browser automation requires those permissions. Webcmd does not send page contents or cookies to AgentR. Except for the site-memory seed lookup and the optional candidate public-IP lookup below, Webcmd does not send browser data to AgentR. Commands run locally, and command output is printed to the local CLI process. Trace artifacts, cache files, plugins, user adapters, and site memory are stored under `~/.webcmd`. +## Local site-memory seed lookup + +`WEBCMD_GLOBAL_MEMORY_URL` enables a public unauthenticated GET `/v1/site-memory/seeds/` on first access when no local memory exists. The request uses a 2-second timeout and no retry. It discloses the resolved product/domain. + +With no URL configured, learning is local-only and Webcmd makes no seed request. `WEBCMD_GLOBAL_MEMORY=off` disables even a configured URL. + +## Candidate public-IP provenance + +When capturing candidate evidence, local Webcmd makes a best-effort unauthenticated GET `https://api.ipify.org` with a 2-second timeout and no credentials. It records the public egress IP in local candidate JSON only. Inability to resolve it does not block capture. + +`WEBCMD_CANDIDATE_PUBLIC_IP=off` disables the lookup. + +## What stays local + +Candidate provenance stays local under `~/.webcmd/sites`. It includes local machine/network metadata, is excluded from ordinary output, and is never uploaded or pushed by this design. The local sites Git repository never pushes. + +Learning is invisible in normal output. Diagnostics appear on request, verbose mode, or a material warning-retention failure. + +## Beta memory clean break + +If you used beta site memory, remove that product's old `~/.webcmd/sites//sitemap/SITE.md` before first use, or remove that entire beta product directory if you do not want it. There is no migration guarantee. Do not delete unrelated product directories. + For attribution and license information, see `LICENSE` and `NOTICE`. diff --git a/README.md b/README.md index 1fd81c33..8abe3b26 100644 --- a/README.md +++ b/README.md @@ -87,9 +87,11 @@ webcmd skills add ``` When prompted, choose Claude, Codex, another supported harness, or a custom -skills path. +skills path. That install is exactly one skill, `webcmd-browser`. Adapter, +search, fetch, and plugin commands remain on the CLI; they are outside the +default installed browser skill. -In your agent harness, load or tag `webcmd-usage`, then describe the outcome you want. +Load or tag `webcmd-browser` only for live browser work. Existing adapter, search, fetch, and plugin CLI operations do not require that skill. ```text Use webcmd to research the latest discussions about browser automation across Hacker News and Reddit, then return a concise comparison with source links. diff --git a/benchmarks/tests/pi_controller.test.mjs b/benchmarks/tests/pi_controller.test.mjs index b32af13a..3be3459b 100644 --- a/benchmarks/tests/pi_controller.test.mjs +++ b/benchmarks/tests/pi_controller.test.mjs @@ -130,7 +130,7 @@ test("Pi sidecar accepts browser-use as a benchmark tool before starting a sessi }); test("Pi sidecar rejects a missing explicitly selected skill", () => { - const missing = "/definitely/missing/webcmd-usage"; + const missing = "/definitely/missing/skill"; const result = spawnSync( process.execPath, [ diff --git a/benchmarks/tests/test_controller.py b/benchmarks/tests/test_controller.py index af8eeaf7..b61ce802 100644 --- a/benchmarks/tests/test_controller.py +++ b/benchmarks/tests/test_controller.py @@ -63,7 +63,6 @@ def test_webcmd_prompt_loads_only_raw_browser_skill(tmp_path): prompt = _build_prompt("webcmd", "session-1", tmp_path / "shots", "Find the answer") assert "`$webcmd-browser`" in prompt - assert "`$webcmd-usage`" not in prompt assert "raw-browser route is already selected" in prompt assert "Skip adapter and plugin discovery" in prompt assert "Do not load any other Webcmd skill" in prompt @@ -904,11 +903,11 @@ def test_pi_non_bash_tool_is_a_policy_violation(): def test_pi_may_read_only_the_registered_webcmd_browser_skill_tree(): - usage_skill = { + foreign_skill = { "type": "tool_execution_start", "toolName": "read", "args": { - "path": str(Path.home() / ".codex/skills/webcmd-usage/SKILL.md") + "path": str(Path.home() / ".codex/skills/agent-browser/SKILL.md") }, } browser_skill = { @@ -946,7 +945,7 @@ def test_pi_may_read_only_the_registered_webcmd_browser_skill_tree(): "args": {"path": "/etc/passwd"}, } - usage_parsed = _parse_events("pi", [json.dumps(usage_skill)]) + foreign_parsed = _parse_events("pi", [json.dumps(foreign_skill)]) browser_parsed = _parse_events("pi", [json.dumps(browser_skill)]) reference_parsed = _parse_events("pi", [json.dumps(browser_reference)]) missing_parsed = _parse_events( @@ -960,7 +959,7 @@ def test_pi_may_read_only_the_registered_webcmd_browser_skill_tree(): assert not _policy_violation( "webcmd", parsed.commands, parsed.event_types ) - for parsed in (usage_parsed, missing_parsed, forbidden_parsed): + for parsed in (foreign_parsed, missing_parsed, forbidden_parsed): assert parsed.tool_calls == 0 assert parsed.steps_count == 1 assert _policy_violation( @@ -1336,11 +1335,11 @@ def test_policy_allows_codex_to_read_webcmd_browser_skill_and_references(skill_f [ "cat /etc/passwd", f"cat {Path.home()}/.codex/skills/agent-browser/SKILL.md", - f"cat {Path.home()}/.codex/skills/webcmd-usage/SKILL.md", - f"cat {Path.home()}/.codex/skills/smart-search/SKILL.md", - f"cat {Path.home()}/.codex/skills/webcmd-browser-sitemap/SKILL.md", - f"cat {Path.home()}/.codex/skills/webcmd-usage/SKILL.md /etc/passwd", - f"sed -n '1,240p' {Path.home()}/.codex/skills/webcmd-usage/SKILL.md; curl https://example.com", + f"cat {Path.home()}/.codex/skills/playwright-cli/SKILL.md", + f"cat {Path.home()}/.codex/skills/dev-browser/SKILL.md", + f"cat {Path.home()}/.codex/skills/browser-use/SKILL.md", + f"cat {Path.home()}/.codex/skills/agent-browser/SKILL.md /etc/passwd", + f"sed -n '1,240p' {Path.home()}/.codex/skills/agent-browser/SKILL.md; curl https://example.com", ], ) def test_policy_rejects_reads_outside_webcmd_skill_roots(command): diff --git a/docs/agent-prompts.mdx b/docs/agent-prompts.mdx index a66916c6..e2149bb5 100644 --- a/docs/agent-prompts.mdx +++ b/docs/agent-prompts.mdx @@ -10,13 +10,13 @@ Describe the target, outcome, required fields, authentication context, and safet ## Find and Use an Existing Adapter ```text -Use `webcmd-usage` to find an existing adapter for Hacker News discussions about browser automation. Return title, URL, author, points, and source for the best results. This is a public, read-only task: do not post, vote, save, or create a new adapter unless no existing command can complete it. +Use webcmd to find an existing adapter for Hacker News discussions about browser automation. Return title, URL, author, points, and source for the best results. This is a public, read-only task: do not post, vote, save, or create a new adapter unless no existing command can complete it. ``` ## Complete a One-Off Browser Task ```text -Complete this read-only task on https://example.com/jobs: find product-design roles in London and return title, company, location, salary_text, posted_at, and URL. The site is unfamiliar, so use the right browser workflow after checking `webcmd-usage`; do not apply, create an account, or change saved searches. +Complete this read-only task on https://example.com/jobs: find product-design roles in London and return title, company, location, salary_text, posted_at, and URL. The site is unfamiliar, so use `webcmd-browser` after checking for an existing adapter; do not apply, create an account, or change saved searches. ``` ## Create a Reusable CLI diff --git a/docs/agents/claude-code.md b/docs/agents/claude-code.md index c0f70cc9..9d2dc961 100644 --- a/docs/agents/claude-code.md +++ b/docs/agents/claude-code.md @@ -25,7 +25,7 @@ claude plugin marketplace add agentrhq/webcmd claude plugin install webcmd@webcmd ``` -This installs all seven bundled Webcmd skills. Do not also add those skills with `webcmd skills add`; running both leaves two copies that can sit at different versions. +This installs exactly one bundled skill, `webcmd-browser`. Do not also add that skill with `webcmd skills add`; running both leaves two copies that can sit at different versions. Plugin updates are version-gated, not commit-gated. Run `claude plugin update webcmd@webcmd` to pick up a new release; it is separate from `webcmd update`, which upgrades only the npm CLI. @@ -39,7 +39,7 @@ webcmd doctor webcmd skills add ``` -When `webcmd skills add` prompts, choose the `claude` provider. It installs into `~/.claude/skills/` (user) or `.claude/skills/` (project), which Claude Code reads on startup. Claude Code then surfaces `webcmd-usage` and `webcmd-browser` as skills when a task matches their descriptions. +When `webcmd skills add` prompts, choose the `claude` provider. It installs into `~/.claude/skills/` (user) or `.claude/skills/` (project), which Claude Code reads on startup. Claude Code then surfaces `webcmd-browser` as a skill when a task matches its description. Restart Claude Code (or start a new session) after installing skills. @@ -94,4 +94,3 @@ Denying these tools does not affect the Bash tool, which is how `webcmd` is driv * [`start.md`](../../start.md) — common setup, [auth profiles and human handoff](../../start.md#auth-profiles-and-human-handoff), and [security](../../start.md#security). * [`webcmd-browser`](../../skills/webcmd-browser/SKILL.md) — the raw browser session surface. -* [`webcmd-usage`](../../skills/webcmd-usage/SKILL.md) — adapter-first usage rules. diff --git a/docs/agents/codex-cli.md b/docs/agents/codex-cli.md index b93e5505..f88bf000 100644 --- a/docs/agents/codex-cli.md +++ b/docs/agents/codex-cli.md @@ -38,7 +38,7 @@ codex plugin add webcmd@webcmd The same flow is available in the TUI: run `/plugins`, choose **Add plugin marketplace**, and enter `agentrhq/webcmd` or `https://github.com/agentrhq/webcmd`. -The plugin bundles all seven Webcmd skills, and installs the npm CLI on first use if `webcmd` is missing. Start a new task after installing. +The plugin bundles exactly one skill, `webcmd-browser`, and installs the npm CLI on first use if `webcmd` is missing. Start a new task after installing. Useful follow-ups: `codex plugin list`, `codex plugin remove webcmd`, `codex plugin marketplace upgrade`. All accept `--json`. @@ -50,7 +50,7 @@ webcmd doctor webcmd skills add ``` -When `webcmd skills add` prompts, choose the `agents` provider. It installs into `~/.agents/skills/` (user) or `.agents/skills/` (project), which Codex CLI reads on startup. Codex then surfaces `webcmd-usage` and `webcmd-browser` as skills. +When `webcmd skills add` prompts, choose the `agents` provider. It installs into `~/.agents/skills/` (user) or `.agents/skills/` (project), which Codex CLI reads on startup. Codex then surfaces `webcmd-browser` as a skill. Restart Codex (or start a new session) after installing skills. @@ -90,4 +90,3 @@ disabled_tools = ["navigate", "screenshot"] * [`start.md`](../../start.md) — common setup, [auth profiles and human handoff](../../start.md#auth-profiles-and-human-handoff), and [security](../../start.md#security). * [`webcmd-browser`](../../skills/webcmd-browser/SKILL.md) — the raw browser session surface. -* [`webcmd-usage`](../../skills/webcmd-usage/SKILL.md) — adapter-first usage rules. diff --git a/docs/agents/cursor.md b/docs/agents/cursor.md index feb9692c..e7793bd6 100644 --- a/docs/agents/cursor.md +++ b/docs/agents/cursor.md @@ -27,7 +27,7 @@ webcmd doctor webcmd skills add ``` -When `webcmd skills add` prompts, choose the `agents` provider. It installs into `~/.agents/skills/`, which Cursor reads on startup along with `.cursor/skills/`, `.agents/skills/`, and `~/.cursor/skills/`. Cursor then surfaces `webcmd-usage` and `webcmd-browser` as skills when a task matches their descriptions. +When `webcmd skills add` prompts, choose the `agents` provider. It installs into `~/.agents/skills/`, which Cursor reads on startup along with `.cursor/skills/`, `.agents/skills/`, and `~/.cursor/skills/`. Cursor then surfaces `webcmd-browser` as a skill when a task matches its description. For a project-scoped setup, copy the `webcmd-*` skill folders into the project's `.cursor/skills/` or `.agents/skills/` so the whole team gets them. Restart Cursor after installing skills. @@ -83,4 +83,3 @@ Note that the rule is guidance, not a block. Cursor's Browser Automation has bee * [`start.md`](../../start.md) — common setup, [auth profiles and human handoff](../../start.md#auth-profiles-and-human-handoff), and [security](../../start.md#security). * [`webcmd-browser`](../../skills/webcmd-browser/SKILL.md) — the raw browser session surface. -* [`webcmd-usage`](../../skills/webcmd-usage/SKILL.md) — adapter-first usage rules. diff --git a/docs/agents/hermes.md b/docs/agents/hermes.md index 3e7bbcd4..b1ed80f4 100644 --- a/docs/agents/hermes.md +++ b/docs/agents/hermes.md @@ -36,7 +36,7 @@ skills: - ~/.agents/skills ``` -Hermes then reads `webcmd-usage` and `webcmd-browser` as skills. Restart Hermes after changing config and confirm the skills are discoverable. +Hermes then reads `webcmd-browser` as a skill. Restart Hermes after changing config and confirm the skill is discoverable. ### Override default tools @@ -91,4 +91,3 @@ Do not disable the `terminal` toolset — that is how Hermes runs `webcmd`. * [`start.md`](../../start.md) — common setup, [auth profiles and human handoff](../../start.md#auth-profiles-and-human-handoff), and [security](../../start.md#security). * [`webcmd-browser`](../../skills/webcmd-browser/SKILL.md) — the raw browser session surface. -* [`webcmd-usage`](../../skills/webcmd-usage/SKILL.md) — adapter-first usage rules. diff --git a/docs/agents/openclaw.md b/docs/agents/openclaw.md index 59582a9d..49784e53 100644 --- a/docs/agents/openclaw.md +++ b/docs/agents/openclaw.md @@ -28,7 +28,7 @@ webcmd doctor webcmd skills add ``` -When `webcmd skills add` prompts, choose the `agents` provider. It installs into `~/.agents/skills/` (user) or `.agents/skills/` (project), both of which OpenClaw checks for skills. OpenClaw then surfaces `webcmd-usage` and `webcmd-browser` as skills. +When `webcmd skills add` prompts, choose the `agents` provider. It installs into `~/.agents/skills/` (user) or `.agents/skills/` (project), both of which OpenClaw checks for skills. OpenClaw then surfaces `webcmd-browser` as a skill. Restart the Gateway (or start a new session) after installing skills. @@ -88,4 +88,3 @@ Or remove it entirely — CLI, `browser.request` gateway method, and agent tool * [`start.md`](../../start.md) — common setup, [auth profiles and human handoff](../../start.md#auth-profiles-and-human-handoff), and [security](../../start.md#security). * [`webcmd-browser`](../../skills/webcmd-browser/SKILL.md) — the raw browser session surface. -* [`webcmd-usage`](../../skills/webcmd-usage/SKILL.md) — adapter-first usage rules. diff --git a/docs/agents/opencode.md b/docs/agents/opencode.md index 9b697a49..df23a943 100644 --- a/docs/agents/opencode.md +++ b/docs/agents/opencode.md @@ -28,7 +28,7 @@ webcmd doctor webcmd skills add ``` -When `webcmd skills add` prompts, choose the `agents` provider (installs into `~/.agents/skills/`, which OpenCode auto-loads). OpenCode then reads `webcmd-usage` and `webcmd-browser` as skills. +When `webcmd skills add` prompts, choose the `agents` provider (installs into `~/.agents/skills/`, which OpenCode auto-loads). OpenCode then reads `webcmd-browser` as a skill. Restart OpenCode after changing config. Confirm the skill loads with `/skills` and the permissions are active before starting browser work. @@ -59,11 +59,10 @@ Deny `webfetch` so OpenCode cannot fall back to it while Webcmd is its browser s | Skills not loading in OpenCode | Run `webcmd skills add` with the `agents` provider, restart OpenCode, and check `/skills`. | | OpenCode still uses `webfetch` | Confirm `permission.webfetch` is `deny` in the active config, then restart OpenCode. | | `websearch` is missing entirely | It registers only with the OpenCode provider or `OPENCODE_ENABLE_EXA=1`. Not a Webcmd problem. | -| `webcmd browser` errors | Read `webcmd-usage` and `webcmd-browser`; create a named Session and pass its readable ID as root `--session`. | +| `webcmd browser` errors | Read `webcmd-browser`; create a named Session and pass its readable ID as root `--session`. | | Browser Session idles or loses its window | Keep its immutable, Profile-scoped ID; `webcmd --profile work --session work-project-k7 browser tabs` reopens it. Start with `webcmd --profile work session create "Work Project"`; use `webcmd --profile work session list` and `webcmd --profile work session close work-project-k7` for lifecycle. Adapter commands without `--session` reuse `adapter-default`; raw browser commands require an explicit readable selector. | ## See also * [`start.md`](../../start.md) — common setup, [auth profiles and human handoff](../../start.md#auth-profiles-and-human-handoff), and [security](../../start.md#security). * [`webcmd-browser`](../../skills/webcmd-browser/SKILL.md) — the raw browser session surface. -* [`webcmd-usage`](../../skills/webcmd-usage/SKILL.md) — adapter-first usage rules. diff --git a/docs/agents/pi.md b/docs/agents/pi.md index ede230d9..4f92bd31 100644 --- a/docs/agents/pi.md +++ b/docs/agents/pi.md @@ -28,7 +28,7 @@ webcmd doctor webcmd skills add ``` -When `webcmd skills add` prompts, choose the `agents` provider. It links skills into `~/.agents/skills/` (user) or `.agents/skills/` (project), both of which Pi scans for skills on startup alongside its own `~/.pi/agent/skills/` and `.pi/skills/` directories. Pi then surfaces `webcmd-usage` and `webcmd-browser` as skills. +When `webcmd skills add` prompts, choose the `agents` provider. It links skills into `~/.agents/skills/` (user) or `.agents/skills/` (project), both of which Pi scans for skills on startup alongside its own `~/.pi/agent/skills/` and `.pi/skills/` directories. Pi then surfaces `webcmd-browser` as a skill. To install into Pi's own skill directories instead, pass a custom path: @@ -72,4 +72,3 @@ To remove one outright, delete its folder — for example `~/.pi/agent/skills/pi * [`start.md`](../../start.md) — common setup, [auth profiles and human handoff](../../start.md#auth-profiles-and-human-handoff), and [security](../../start.md#security). * [`webcmd-browser`](../../skills/webcmd-browser/SKILL.md) — the raw browser session surface. -* [`webcmd-usage`](../../skills/webcmd-usage/SKILL.md) — adapter-first usage rules. diff --git a/docs/browser-and-sitemap-memory.mdx b/docs/browser-and-sitemap-memory.mdx index a26657d3..2860f0d4 100644 --- a/docs/browser-and-sitemap-memory.mdx +++ b/docs/browser-and-sitemap-memory.mdx @@ -17,6 +17,8 @@ When no suitable command exists or one is broken, the agent uses a browser to in Useful, verified navigation and workflow knowledge can be retained as sitemap memory. It gives future browser work a starting point, but agents update or bypass it when the live site has changed. +Learning is invisible in normal output. Diagnostics appear on request, verbose mode, or a material warning-retention failure. See [Privacy](https://github.com/agentrhq/webcmd/blob/main/PRIVACY.md) for seed lookup, local provenance, and the beta memory clean break. + ## When to Create an Adapter Create or repair an adapter when the workflow is likely to repeat and can be verified as a stable command. Keep one-off or changing work in the browser until reuse is justified. diff --git a/docs/cli-reference.mdx b/docs/cli-reference.mdx index d492a2cb..3e12bb9b 100644 --- a/docs/cli-reference.mdx +++ b/docs/cli-reference.mdx @@ -35,7 +35,7 @@ webcmd wikipedia summary "Command-line interface" webcmd pubmed search "agentic browser automation" --limit 5 -f json ``` -Agents should prefer existing adapters before raw browser exploration. For search and fetch tasks, use the bundled `smart-search` skill: it tries fetch-based search first, explicit browser Sessions after eligible fetch failures, and search adapters last. +Agents should prefer existing adapters before raw browser exploration. For search and fetch tasks, use `webcmd web fetch` first, then explicit browser Sessions after eligible fetch failures, and search adapters last. ## Search and Fetch diff --git a/docs/concepts.mdx b/docs/concepts.mdx index cc4bcb32..ca89989c 100644 --- a/docs/concepts.mdx +++ b/docs/concepts.mdx @@ -9,7 +9,7 @@ Webcmd gives an agent one rule: reuse what already exists before exploring. ## The Agent's Decision Flow -1. Load `webcmd-usage` and discover relevant existing adapters. +1. Discover relevant existing adapters with `webcmd list`. 2. Use an existing command when it already satisfies the request. 3. Explore through a browser only when the surface is unfamiliar or broken. 4. Retain sitemap knowledge when it will help future browser work. diff --git a/docs/index.mdx b/docs/index.mdx index f4252056..2fb346c7 100644 --- a/docs/index.mdx +++ b/docs/index.mdx @@ -26,10 +26,10 @@ standalone skill links. ## Then Ask for an Outcome -In your agent harness, load or tag `webcmd-usage`, then describe the result you need, the fields to return, and any read-only or approval boundaries. +In your agent harness, load or tag `webcmd-browser` for live browser work, then describe the result you need, the fields to return, and any read-only or approval boundaries. Adapter, search, fetch, and plugin commands remain on the CLI. ```text -Use the webcmd-usage skill to find the best way to research the latest discussions about browser automation across Hacker News and Reddit. Reuse existing adapters where possible and return a concise comparison with source links. +Use webcmd to research the latest discussions about browser automation across Hacker News and Reddit. Reuse existing adapters where possible, or load `webcmd-browser` for live browser work, and return a concise comparison with source links. ``` ## What Your Agent Can Do diff --git a/docs/local-or-cloud.mdx b/docs/local-or-cloud.mdx index e6f6b47c..3c8c4080 100644 --- a/docs/local-or-cloud.mdx +++ b/docs/local-or-cloud.mdx @@ -15,11 +15,11 @@ The CLI grammar remains the same in either mode. ## Local Mode -Local mode runs Webcmd against this machine's tools, adapters, and browser. Choose it for desktop workflows or work that depends on local state. +Local mode runs Webcmd against this machine's tools, adapters, and browser. Choose it for desktop workflows or work that depends on local state. Local CLI self-learning stores sitemap memory on this machine. ## Webcmd Cloud Alpha -Webcmd Cloud alpha is for supported commands and browser sessions on hosted infrastructure. Hosted browser sessions run through Webcmd Cloud's Browser Use-backed infrastructure for browser lifecycle, proxying, and live viewport access. +Webcmd Cloud alpha is for supported commands and browser sessions on hosted infrastructure. Hosted browser sessions run through Webcmd Cloud's Browser Use-backed infrastructure for browser lifecycle, proxying, and live viewport access. MCP and hosted browser documents remain browser-operation-only; they do not teach local site memory or Git. ## Set Up Hosted Mode diff --git a/docs/quickstart.mdx b/docs/quickstart.mdx index adbde84a..c0c786c0 100644 --- a/docs/quickstart.mdx +++ b/docs/quickstart.mdx @@ -1,6 +1,6 @@ --- title: Quickstart -description: Install Webcmd and its agent skills, then complete your first task with webcmd-usage. +description: Install Webcmd and its one bundled skill, webcmd-browser, then complete your first task. --- # Quickstart @@ -12,8 +12,8 @@ In Codex, open **Plugins**, choose **Add plugin marketplace**, and enter either from that marketplace, then start a new task. On first use, the plugin installs the npm CLI automatically if `webcmd` is missing. -The plugin includes all seven bundled Webcmd skills. Do not also add those -skills with `webcmd skills add` in Codex. +The plugin includes exactly one bundled skill, `webcmd-browser`. Do not also add +that skill with `webcmd skills add` in Codex. ## Install the Claude Code Plugin @@ -22,8 +22,8 @@ claude plugin marketplace add agentrhq/webcmd claude plugin install webcmd@webcmd ``` -This installs all seven bundled Webcmd skills. Do not also add those skills -with `webcmd skills add` in Claude Code. +This installs exactly one bundled skill, `webcmd-browser`. Do not also add that +skill with `webcmd skills add` in Claude Code. ## Other Agents or Plugin-Free Setup @@ -41,12 +41,12 @@ skills path. ## Ask Your Agent ```text -Use the webcmd-usage skill to find the best way to research the latest discussions about browser automation across Hacker News and Reddit. Reuse existing adapters where possible and return a concise comparison with source links. +Use webcmd to research the latest discussions about browser automation across Hacker News and Reddit. Reuse existing adapters where possible, or load `webcmd-browser` for live browser work, and return a concise comparison with source links. ``` ## What Happens Next -`webcmd-usage` first checks whether an existing adapter can complete the task. When a specialized workflow is needed, it chooses the relevant skill; then it returns the result or creates a reusable command for later agents. +The agent first checks whether an existing adapter can complete the task. Adapter, search, fetch, and plugin commands remain on the CLI. When live browser work is needed, it loads `webcmd-browser`. ## Try Another Prompt diff --git a/docs/skills.mdx b/docs/skills.mdx index 976f9b94..69a5851f 100644 --- a/docs/skills.mdx +++ b/docs/skills.mdx @@ -1,35 +1,24 @@ --- title: Skills -description: Start with webcmd-usage and let your agent select the specialized Webcmd workflow it needs. +description: The bundled install is exactly one skill, webcmd-browser, for live browser work. --- # Skills -## Start With webcmd-usage +The bundled install is exactly one skill, `webcmd-browser`. Load it for live browser work. -Ask the agent to load `webcmd-usage` at the start of a Webcmd task. It discovers existing commands, understands common output and strategy choices, and selects the right specialized workflow. - -## Specialized Skills - -| Skill | Outcome | -| --- | --- | -| `smart-search` | Finds the right Webcmd-backed source for a research or search task. | -| `webcmd-browser` | Completes a one-off task in a live browser when no existing command fits. | -| `webcmd-browser-sitemap` | Uses retained site navigation knowledge while checking it against the live page. | -| `webcmd-sitemap-author` | Records durable, verified navigation and workflow knowledge for later browser work. | -| `webcmd-adapter-author` | Turns a repeatable website or app workflow into a verified command. | -| `webcmd-autofix` | Diagnoses and repairs a broken adapter after a site changes. | +Adapter, search, fetch, and plugin commands remain on the CLI. They are outside the default installed browser skill. ## Install in Codex Open **Plugins**, choose **Add plugin marketplace**, enter `agentrhq/webcmd`, -and install **Webcmd**. The plugin bundles these same seven skills and installs +and install **Webcmd**. The plugin bundles this same skill and installs the npm CLI on first use if `webcmd` is missing. Start a new task after installation. -Do not also add the skills with `webcmd skills add` in Codex. +Do not also add the skill with `webcmd skills add` in Codex. ## Other Agents or Plugin-Free Setup -Run `webcmd skills add` to install or refresh the bundled Webcmd skills for your agent. The agent can then start with `webcmd-usage` and load the specialized skill that matches the outcome. +Run `webcmd skills add` to install or refresh the bundled `webcmd-browser` skill for your agent. For raw browser work, agents should run `webcmd --profile work session create "Work Project"`, keep the returned immutable, Profile-scoped ID, and pass it at the root: `webcmd --profile work --session work-project-k7 browser tabs`. Adapter commands without `--session` reuse `adapter-default`; raw browser commands require an explicit readable selector. diff --git a/mcp-skills/smart-search.md b/mcp-skills/smart-search.md deleted file mode 100644 index 5132480c..00000000 --- a/mcp-skills/smart-search.md +++ /dev/null @@ -1,40 +0,0 @@ -# Smart Search through MCP - -Use this for research, source discovery, direct URL fetches, and evidence. Every -operation goes through `webcmd_cli_run` with argv data; request JSON whenever the -result will be read programmatically. - -Use live fetch results, command metadata, and command help. Do not infer command arguments from this skill, maintain a routing table, or claim a source was searched when it was not. - -## Cost order - -For a supplied URL, try the first-choice fetch path before browser work: - - { "argv": ["web", "fetch", "--url", "https://example.com", "-f", "json"] } - -Only `FETCH_BLOCKED` or `FETCH_REQUIRES_BROWSER` permits browser fallback. For a -topic without a named site, fetch one search-engine result page, extract result -URLs from its JSON/text response, then fetch up to three target pages. Search -snippets discover sources; fetched primary content is evidence. - -For a named site, inspect the live command surface first: - - { "argv": ["list", "--tag", "search", "-f", "json"] } - { "argv": ["github", "search", "--help"] } - { "argv": ["github", "search", "--query", "agents", "-f", "json"] } - -Do not create shell pipelines. Read the JSON response directly, preserve source -URLs, and report failures rather than claiming an unperformed search. - -Any truncation warning means adapter discovery is incomplete: narrow the filter and inspect again. Absence from truncated output never proves that no adapter exists. - -## Budgets - -Try one search engine by default and a second only if the first is weak or -blocked. Fetch three result URLs by default (five for a broad comparison), use -at most two browser sessions, and run one adapter search unless the first is -weak or needs independent corroboration. A rate limit, login gate, CAPTCHA, or -unusable extraction is a reason to move to another source, not to repeat the -same request. - -Report the commands run, sources fetched, browser fallback URLs, and gaps. diff --git a/mcp-skills/webcmd-adapter-author.md b/mcp-skills/webcmd-adapter-author.md deleted file mode 100644 index df53d650..00000000 --- a/mcp-skills/webcmd-adapter-author.md +++ /dev/null @@ -1,61 +0,0 @@ -# Webcmd Adapter Authoring through MCP - -Author a deterministic adapter only after reconnaissance proves a reusable -workflow. Use `webcmd_cli_run` for every interaction; source is tenant-owned -virtual content, never a local checkout. - -- Adapters import only `@agentrhq/webcmd/registry` and `@agentrhq/webcmd/errors`; do not add third-party dependencies. -- Browser-run’s Playwright-style `page` and adapter `func(page,args)` are different contracts. Preserve evidence and behavior, not syntax. Implement adapters with the existing `IPage`, pipeline, Node-fetch, or interceptor APIs. -- The `columns` array and `func` return object keys must match exactly, including order. -- **Intermediate parsing object keys must not overlap any `columns` entry.** Otherwise silent-column-drop audits can misread the adapter. Use dedicated internal names and destructure with aliases when pushing rows. -Use live fetch results, command metadata, and command help. Do not infer command arguments from this skill, maintain a routing table, or claim a source was searched when it was not. - -## Reconnaissance and strategy - -Start with live capability and command help, then inspect site memory and a -bounded browser session. Prefer public or documented APIs, then stable UI/DOM -semantics; use internal page requests or interception only when the page proves -they are necessary. Record the observed request/state, authentication source, -replay result, and why a simpler strategy cannot work. - -Create a named Session and keep its immutable, Profile-scoped readable ID for -raw browser evidence: - - { "argv": ["list", "-f", "json"] } - { "argv": ["site", "memory", "show", "example", "-f", "json"] } - { "argv": ["--profile", "work", "session", "create", "Work Project", "-f", "json"] } - { "argv": ["--profile", "work", "--session", "work-project-k7", "browser", "snapshot", "--snapshot-mode", "tree", "-f", "json"] } - -Do not bypass authentication, CAPTCHA, rate limits, or access controls. An -`action_required` response belongs to the user; provide its view URL and run its -returned verifier after the user completes it. - -## Virtual adapter source - -Get the scaffold or existing source as an artifact-backed virtual file: - - { "argv": ["adapter", "source", "get", "example/search", "--output", "adapter.ts"] } - -The command materializes source at the virtual relative path `adapter.ts`. Read -that virtual file, edit it in the tool call, then put it back using the same -path and a virtual file attachment: - - { - "argv": ["adapter", "source", "put", "example/search", "adapter.ts"], - "files": [{ "path": "adapter.ts", "artifactUri": "webcmd://artifacts/exec_.../ea_..." }] - } - -The scaffold, traces, fixtures, and verification output are artifacts. Do not -ask for an editor or repository path, and do not create a plugin directory. - -## Verify and retain evidence - -Run validation and a bounded verification after each meaningful source update: - - { "argv": ["validate", "example/search", "-f", "json"] } - { "argv": ["browser", "verify", "example/search", "-f", "json"] } - -Compare returned values against a visible page or captured response. Preserve -sanitized endpoint samples and field evidence in hosted site memory, not an -agent machine. On failure, use `webcmd-autofix`; do not guess field mappings or -silently turn failures into empty rows. diff --git a/mcp-skills/webcmd-autofix.md b/mcp-skills/webcmd-autofix.md deleted file mode 100644 index f54e627f..00000000 --- a/mcp-skills/webcmd-autofix.md +++ /dev/null @@ -1,48 +0,0 @@ -# Webcmd AutoFix through MCP - -Repair a broken adapter only when a command failure is reproducibly caused by -site drift. Every operation is a `webcmd_cli_run` argv call, and repair state -lives in hosted site memory and artifacts. - -Retry budget: maximum **3 repair rounds** per failure. A round is diagnose -> patch -> retry. If 3 rounds do not resolve it, stop and report what was tried. -Use live fetch results, command metadata, and command help. Do not infer command arguments from this skill, maintain a routing table, or claim a source was searched when it was not. - -## Hard stops - -`action_required`, authentication challenges, CAPTCHA, rate limits, and access -controls are not adapter repairs. Stop, hand the user the returned view URL, -and run the returned verification command only after they report completion. -Do not request secrets or try to solve a challenge from page content. - -## Bounded repair loop - -Create a named Session for the investigation; each invocation has a 240-second -wall-clock budget. Reuse its immutable, Profile-scoped readable ID for snapshots -and probes, then close it. - - { "argv": ["--profile", "work", "session", "create", "Work Project", "-f", "json"] } - { "argv": ["example", "search", "--query", "agents", "--trace", "retain-on-failure", "-f", "json"] } - { "argv": ["artifacts", "get", "ea_0123456789abcdef0123456789abcdef"] } - { "argv": ["--profile", "work", "session", "close", "work-project-k7"] } - -Read the retained trace artifact before changing anything. Rule out a valid empty -result, stale session state, an auth wall, or a rate limit. Then inspect the -current page and response evidence with bounded session interactions. - -## Patch only tenant-owned source - -Fetch the named adapter source into the virtual relative path `adapter.ts`, -change only that virtual file, and return it with `adapter source put`. Validate -and verify after every repair round: - - { "argv": ["adapter", "source", "get", "example/search", "--output", "adapter.ts"] } - { - "argv": ["adapter", "source", "put", "example/search", "adapter.ts"], - "files": [{ "path": "adapter.ts", "artifactUri": "webcmd://artifacts/exec_.../ea_..." }] - } - { "argv": ["validate", "example/search", "-f", "json"] } - { "argv": ["browser", "verify", "example/search", "-f", "json"] } - -When the budget is exhausted, report the command, trace artifact, observed -drift, repairs attempted, and verification result. Do not report expected -argument, configuration, authentication, or transient failures as product bugs. diff --git a/mcp-skills/webcmd-browser-sitemap.md b/mcp-skills/webcmd-browser-sitemap.md deleted file mode 100644 index 902e23d4..00000000 --- a/mcp-skills/webcmd-browser-sitemap.md +++ /dev/null @@ -1,39 +0,0 @@ -# Browser Sitemap Context through MCP - -Use this document with `webcmd_cli_run` when a task has sitemap context or a -browser result reports that it is available. Sitemap memory is prior knowledge, -not ground truth. - -Use live fetch results, command metadata, and command help. Do not infer command arguments from this skill, maintain a routing table, or claim a source was searched when it was not. - -## Consumption loop - -Use a bounded session to inspect current state, then request only the smallest -relevant hosted memory: site orientation, one matching page, one matching -workflow, and pitfalls only when blocked. - - { "argv": ["--profile", "work", "session", "create", "Work Project", "-f", "json"] } - { "argv": ["--profile", "work", "--session", "work-project-k7", "browser", "snapshot", "--snapshot-mode", "tree", "-f", "json"] } - { "argv": ["site", "memory", "show", "example", "-f", "json"] } - -The returned readable Session ID is immutable and Profile-scoped. Raw browser -commands require it explicitly. - -Prefer an adapter named by the workflow. If it is unavailable or fails, use the -fallback browser path. After every state-changing action refresh the snapshot -and compare the workflow checkpoint. If the live page disagrees, follow the -live page rather than repeatedly clicking the remembered path. - -## Hosted memory write-back - -When drift is durable, write a short hosted site-memory note or draft that says -what was observed, the expected state, current URL, and next probe. If the -workflow asks for an adapter health update, mark it suspect or broken before -using its fallback so the next agent does not repeat it. - - { "argv": ["site", "memory", "list", "example", "-f", "json"] } - { "argv": ["site", "note", "add", "example", "--text", "Observed stale workflow; inspect current checkout path", "-f", "json"] } - -Large sitemap material is an artifact; retrieve its id with `artifacts get`. -Never direct sitemap output to an agent-machine path. Report the path chosen, -checkpoint reached, and whether hosted memory was used, marked stale, or absent. diff --git a/mcp-skills/webcmd-browser.md b/mcp-skills/webcmd-browser.md index 24c9ddbf..367e6468 100644 --- a/mcp-skills/webcmd-browser.md +++ b/mcp-skills/webcmd-browser.md @@ -1,24 +1,13 @@ # Webcmd Browser through MCP -Use `webcmd_cli_run` for a live browser task only after a complete, non-truncated -adapter lookup and relevant plugin search leave no suitable deterministic -adapter. - -Any truncation warning means adapter discovery is incomplete: narrow the filter and inspect again. Absence from truncated output never proves that no adapter exists. -Use live fetch results, command metadata, and command help. Do not infer command arguments from this skill, maintain a routing table, or claim a source was searched when it was not. - -Discover candidate adapters with argv data; if the complete result and relevant -plugin search have no suitable command, browser work is the fallback: - - { "argv": ["list", "-f", "json"] } +Use `webcmd_cli_run` for a live browser task. Put browser session selectors before the browser command. ## Session lifecycle Create one named Session, use its returned readable ID on each bounded browser action, and close it when finished. IDs are immutable and Profile-scoped. Raw -browser commands require an explicit readable selector; adapter commands without -`--session` reuse `adapter-default`. Each invocation has a 240-second wall-clock -budget. +browser commands require an explicit readable selector. Each invocation has a +240-second wall-clock budget. { "argv": ["--profile", "work", "session", "create", "Work Project", "-f", "json"] } { "argv": ["--profile", "work", "--session", "work-project-k7", "browser", "tabs", "-f", "json"] } @@ -28,7 +17,8 @@ budget. Take a fresh snapshot after navigation, submits, SPA transitions, login, or a human handoff. Prefer semantic locators and scoped extraction. Return compact evidence: URL, title, selected text, response URL/status/sample, or specific -fields, never an unbounded DOM dump. +fields, never an unbounded DOM dump. Do not complete a payment or checkout +without explicit user confirmation. ## Browser programs and artifacts @@ -40,8 +30,7 @@ Put a browser program in an attached virtual file and invoke it with argv: } Keep dependent waits, clicks, fills, and response listeners in one program. Arm -a response listener before the UI action that triggers it. Do not copy browser -program syntax into an adapter. +a response listener before the UI action that triggers it. Screenshots and large snapshots are artifacts, not local files. Retrieve a returned artifact id through: @@ -50,4 +39,4 @@ returned artifact id through: For a login wall or CAPTCHA, stop automation and keep the live-view handoff. Give the user the returned view URL, wait, then run the returned verifier and -take a fresh snapshot before resuming. +take a fresh snapshot before resuming. `action_required` is a hard stop. diff --git a/mcp-skills/webcmd-sitemap-author.md b/mcp-skills/webcmd-sitemap-author.md deleted file mode 100644 index e7c7466c..00000000 --- a/mcp-skills/webcmd-sitemap-author.md +++ /dev/null @@ -1,56 +0,0 @@ -# Sitemap Authoring through MCP - -Author a small, verified task graph for agents through `webcmd_cli_run`. It is -not an SEO crawl map: it records durable page state, actions, workflow paths, -API references, pitfalls, and recovery evidence in hosted site memory. - -Use live fetch results, command metadata, and command help. Do not infer command arguments from this skill, maintain a routing table, or claim a source was searched when it was not. - -## Authoring loop - -Inspect the current page in a bounded session, read existing hosted memory, and -record only task-relevant structure actually observed. Current browser evidence -wins over remembered state. - - { "argv": ["site", "memory", "show", "example", "-f", "json"] } - { "argv": ["--profile", "work", "session", "create", "Work Project", "-f", "json"] } - { "argv": ["--profile", "work", "--session", "work-project-k7", "browser", "snapshot", "--snapshot-mode", "tree", "-f", "json"] } - -The returned readable Session ID is immutable and Profile-scoped. Raw browser -commands require it explicitly. - -Use stable ids for pages, actions, and workflows. Mark unverified paths `draft` -or `stale`; never call them verified. Do not record secrets, private messages, -account-specific identifiers, bypasses, or brittle snapshot indices. - -## Action schema - -Each action records these fields in hosted sitemap memory: - -```yaml -action: stable-id -pre: current page, state, and auth requirements -do: adapter command or semantic browser action -post: URL, state, or output proving success -fail: failure signals -recover: fallback plus adapter health update when needed -evidence: bounded browser snapshot, browser program, or retained trace artifact -``` - -Prefer an existing adapter as the best path and give a browser fallback. On an -adapter failure, write the health update first, refresh browser state, and then -follow the fallback. Keep each memory document narrowly scoped so it can be -loaded lazily. - -## Save and audit - -Use virtual-file attachments for sitemap content and preserve returned material -as artifacts, never as local paths. Inspect command help before writing because -site-memory commands are live capability surface: - - { "argv": ["site", "memory", "--help"] } - { "argv": ["site", "memory", "show", "example", "-f", "json"] } - { "argv": ["artifacts", "get", "ea_0123456789abcdef0123456789abcdef"] } - -Report what is verified, what is stale, evidence used, and the next probe for -any gap. diff --git a/mcp-skills/webcmd-usage.md b/mcp-skills/webcmd-usage.md deleted file mode 100644 index 1af3aec3..00000000 --- a/mcp-skills/webcmd-usage.md +++ /dev/null @@ -1,132 +0,0 @@ -# Using Webcmd through MCP - -Webcmd turns websites, Electron desktop apps, and external CLIs into a uniform `webcmd ` surface that agents can drive without screen scraping. This skill is the orientation layer. Once you know the task, load the specialized skill that fits it. - -You reach all of it through one tool, `webcmd_cli_run`. It takes an argv array — -the same grammar the Webcmd CLI uses, minus the `webcmd` executable name. - - { "argv": ["github", "search", "--query", "agents", "-f", "json"] } - -argv is data. `;`, `&&`, `|`, redirects, globs, backticks and `$()` are ordinary -string characters here. There is no shell. - -## Start every unfamiliar task by looking - - { "argv": ["list", "-f", "json"] } - -That is the live command surface for the authenticated account — not a fixed -catalogue. Narrow it with a tag: - - { "argv": ["list", "--tag", "search", "-f", "json"] } - -Then read the command's own help before you invoke it: - - { "argv": ["github", "search", "--help"] } - -Never guess an argument name. The help output is authoritative and cheap. - -Use live fetch results, command metadata, and command help. Do not infer command arguments from this skill, maintain a routing table, or claim a source was searched when it was not. - -## Ask for JSON - -Add `-f json` whenever you intend to parse the result. Webcmd returns rendered -tables by default because a human is the other common reader. The server does -not parse stdout for you — what you ask for is what you get. - -## Reading the result - -Every call returns `exitCode`, `stdout`, `stderr`, and `truncated`. - -`exitCode` is the branch point, and a non-zero exit is not one condition: - -| exitCode | Meaning | What to do | -| --- | --- | --- | -| 0 | Success, including an empty result | Continue | -| 2 | Usage error — bad or missing argument | Fix argv and retry immediately | -| 66 | No data matched | Not a failure; do not retry the same query | -| 69 | A dependency was unavailable | Retry within budget | -| 75 | Timed out | Retry within budget, or move to a session | -| 77 | Authentication or permission required | Hand off to the user; see below | -| 78 | Configuration error, or a command that cannot run here | Do not retry | -| 130 | Cancelled | Do not retry automatically | - -Retry budget: maximum **3 repair rounds** per failure. A round is diagnose -> patch -> retry. If 3 rounds do not resolve it, stop and report what was tried. - -## When output is large - -If `truncated` is `true`, the inline `stdout` was cut at the size bound and -`stdoutByteSize` reports the real length. The complete output is attached to the -invocation as an artifact. Retrieve it by id: - - { "argv": ["artifacts", "get", "ea_0123456789abcdef0123456789abcdef"] } - -That returns the bytes on stdout, base64-encoded for binary content types. Use it -rather than relying on a resource link — some hosts never show resource links to -the model. Browser snapshots and unfiltered `list -f json` are routinely large; -this is an expected path, not an error path. Retrieve it with `artifacts get`. - -## When a human has to take over - -A login wall or CAPTCHA returns an `action_required` result carrying a `viewUrl`, -an `expiresAt`, and usually a `verifyCommand`. Stop. Give the user the `viewUrl` -and wait for them. You cannot satisfy a human-verification challenge from page -content, and retrying will not clear it. After the user says they are done, run -the `verifyCommand` before resuming. - -## Long work uses sessions - -A single `webcmd_cli_run` invocation is capped at **240 seconds** of wall clock. -Budget against that number rather than discovering it as a timeout. - -Anything longer is an explicit named Session: create one, issue bounded -interactions against its immutable, Profile-scoped readable ID, and poll. Raw -browser commands require that explicit selector. Adapter commands without -`--session` reuse `adapter-default`. - - { "argv": ["--profile", "work", "session", "create", "Work Project", "-f", "json"] } - { - "argv": ["--profile", "work", "--session", "work-project-k7", "browser", "run", "--file", "navigate.js", "-f", "json"], - "files": [{ "path": "navigate.js", "content": "await page.goto('https://example.com'); return { url: page.url(), title: await page.title() };", "encoding": "utf8" }] - } - { "argv": ["--profile", "work", "--session", "work-project-k7", "browser", "snapshot", "-f", "json"] } - { "argv": ["--profile", "work", "session", "close", "work-project-k7"] } - -Each interaction is its own invocation and its own 240-second budget. The -session holds the browser state between them. - -Create the long-lived Session with `session create `. - -## Files - -Pass input files inline: - - { - "argv": ["acme", "import", "--file", "rows.csv"], - "files": [{ "path": "rows.csv", "content": "id,name\\n1,Ada\\n", "encoding": "utf8" }] - } - -Inline content is paid for in your own context. If the workspace already holds -the file as an artifact, reference it by URI instead of re-serializing it: - - { - "argv": ["acme", "import", "--file", "rows.csv"], - "files": [{ "path": "rows.csv", "artifactUri": "webcmd://artifacts/exec_.../ea_..." }] - } - -Paths are relative and POSIX-style. There is no host filesystem behind them: -`/etc/passwd` and `../escape` are rejected, and an output path becomes an -artifact rather than a file on a server. - -Any truncation warning means adapter discovery is incomplete: narrow the filter and inspect again. Absence from truncated output never proves that no adapter exists. - -Use a deterministic adapter before generic browser work. Discover it through -`webcmd_cli_run` argv, and only use browser actions after a complete registry -result and the relevant plugin search have no suitable command: - - { "argv": ["list", "-f", "json"] } - -## Workspaces, profiles, sessions, formats - -All ordinary argv — `--workspace`, `--profile`, `--session`, `-f`. There is no -separate MCP parameter for any of them, so the CLI documentation is the only -documentation you need. diff --git a/package-lock.json b/package-lock.json index 4c050b39..9bca9b32 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,6 +19,7 @@ "jsdom": "^29.0.2", "playwright-core": "1.61.1", "quickjs-emscripten": "0.32.0", + "tldts": "^7.4.11", "turndown": "^7.2.2", "turndown-plugin-gfm": "^1.0.2", "undici": "^6.27.0", @@ -2990,21 +2991,21 @@ } }, "node_modules/tldts": { - "version": "7.0.28", - "resolved": "https://registry.npmjs.org/tldts/-/tldts-7.0.28.tgz", - "integrity": "sha512-+Zg3vWhRUv8B1maGSTFdev9mjoo8Etn2Ayfs4cnjlD3CsGkxXX4QyW3j2WJ0wdjYcYmy7Lx2RDsZMhgCWafKIw==", + "version": "7.4.11", + "resolved": "https://registry.npmjs.org/tldts/-/tldts-7.4.11.tgz", + "integrity": "sha512-aBiNayCfTQxuIJBm06M+xR14cYaYlDlSXZbgsnKzKNxDKUVq7KFwTjwBSsb7m9Y5xO8WfPnBc63WaYFMTGlvqw==", "license": "MIT", "dependencies": { - "tldts-core": "^7.0.28" + "tldts-core": "^7.4.11" }, "bin": { "tldts": "bin/cli.js" } }, "node_modules/tldts-core": { - "version": "7.0.28", - "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-7.0.28.tgz", - "integrity": "sha512-7W5Efjhsc3chVdFhqtaU0KtK32J37Zcr9RKtID54nG+tIpcY79CQK/veYPODxtD/LJ4Lue66jvrQzIX2Z2/pUQ==", + "version": "7.4.11", + "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-7.4.11.tgz", + "integrity": "sha512-CW3WN2rIIE/Of21mulhgnGOwoDyEFNygyIBOONSdyAuSATgMMUCpLeUlB+E8sAwA5xRV9hYPl+kyZ9citHCaKg==", "license": "MIT" }, "node_modules/tough-cookie": { diff --git a/package.json b/package.json index 9bec7da9..2bab19c2 100644 --- a/package.json +++ b/package.json @@ -105,6 +105,7 @@ "jsdom": "^29.0.2", "playwright-core": "1.61.1", "quickjs-emscripten": "0.32.0", + "tldts": "^7.4.11", "turndown": "^7.2.2", "turndown-plugin-gfm": "^1.0.2", "undici": "^6.27.0", diff --git a/scripts/check-codex-plugin.mjs b/scripts/check-codex-plugin.mjs index a8b1e47b..07f8d6d3 100644 --- a/scripts/check-codex-plugin.mjs +++ b/scripts/check-codex-plugin.mjs @@ -46,15 +46,7 @@ assert.equal(claudeMarketplace.plugins?.length, 1); assert.equal(claudeMarketplacePlugin?.name, 'webcmd'); assert.equal(claudeMarketplacePlugin?.source, './'); -const expectedSkills = [ - 'smart-search', - 'webcmd-adapter-author', - 'webcmd-autofix', - 'webcmd-browser', - 'webcmd-browser-sitemap', - 'webcmd-sitemap-author', - 'webcmd-usage', -]; +const expectedSkills = ['webcmd-browser']; const actualSkills = fs .readdirSync(path.join(root, 'skills'), { withFileTypes: true }) .filter( @@ -66,13 +58,12 @@ const actualSkills = fs .sort(); assert.deepEqual(actualSkills, expectedSkills); +assert.equal(actualSkills.length, 1); -const usageSkill = readText('skills/webcmd-usage/SKILL.md'); -assert.match(usageSkill, /Bash\(npm:\*\)/); -assert.match(usageSkill, /## CLI Preflight/); -assert.match(usageSkill, /webcmd --version/); -assert.match(usageSkill, /npm install -g @agentrhq\/webcmd/); +const browserSkill = readText('skills/webcmd-browser/SKILL.md'); +assert.match(browserSkill, /webcmd-browser/); +assert.match(manifest.interface?.longDescription ?? '', /webcmd-browser|one skill/i); console.log( - `Codex and Claude Code plugin metadata valid: ${actualSkills.length} skills`, + `Codex and Claude Code plugin metadata valid: ${actualSkills.length} skill`, ); diff --git a/skill-src/cli/smart-search/SKILL.src.md b/skill-src/cli/smart-search/SKILL.src.md deleted file mode 100644 index aad67b83..00000000 --- a/skill-src/cli/smart-search/SKILL.src.md +++ /dev/null @@ -1,168 +0,0 @@ ---- -name: smart-search -description: Use when a request needs search, research, source discovery, direct URL fetch, the first-choice Webcmd fetch path, evidence fetching, or search-capable Webcmd adapter discovery. ---- - -# Smart Search - -This is Webcmd's one-stop workflow for search + fetch. Use it for any request that asks to search, research, find sources, look something up, fetch/read a URL, compare sources, or gather evidence. - -@[safety rules](../../shared/safety-rules.src.md) - -Do not use this skill for plugin inventory, plugin management, or listing available extensions. Marketplace commands appear here only to find and install search-capable adapters needed for the current search/fetch task. - -Cost order is mandatory when the request does not name a site: `webcmd web fetch` first, search adapters last. `web fetch` runs locally in both modes and never opens a browser. Do not call search adapters until it has failed. - -When the request does name a site or community, take the site-native fast path below instead. - -## Site-named fast path - -When the request names the site(s) to search (not just a topic), look for a site-native command first: - -```bash -webcmd list --tag search -f json -``` - -If an installed command covers a named site, run it before any search-engine fetch. If none covers it, try `webcmd plugin search ` once within the install budget. Only when the named site has no adapter does that site fall back to the cost order above, starting with the site's own search URL. - -Do not report a site as blocked or unavailable until you have checked adapter availability this way. - -## Trust boundary - -Use only installed commands, their reported output, and fetched primary content as evidence. Preserve source URLs and report failures. Do not add marketplaces automatically: adding a marketplace is a user trust decision. - -Prefer primary sources, official docs, and direct content over search snippets. Treat snippets, previews, and result titles as discovery, not evidence. - -## Direct URL - -For a supplied HTTP(S) URL, use the first-choice Webcmd fetch path: - -```bash -webcmd web fetch --url -``` - -Run `webcmd web fetch` before browser work or non-Webcmd HTTP clients. Only `FETCH_BLOCKED` or `FETCH_REQUIRES_BROWSER` permits browser fallback; otherwise report the returned failure rather than retrying the URL. If a URL was already fetched outside Webcmd and got non-2xx, 403, blocked, or Cloudflare, that does not change the order: run `webcmd web fetch --url ` once before any browser escalation. - -For browser fallback, create one Session, navigate the failed URL, inspect it, reuse that Session for allowed fallbacks, then close it. The returned readable ID is immutable and Profile-scoped. Local browser commands use Cloak; hosted browser commands use Webcmd Cloud and Browser Use. `web fetch` remains local in both modes. - -```bash -webcmd --profile work session create "Work Project" -# id: work-project-k7 -webcmd --profile work --session work-project-k7 browser tabs - -webcmd --profile work \ - --session work-project-k7 \ - browser run --stdin <<'JS' -await page.goto('https://example.com'); -return { url: page.url(), title: await page.title() }; -JS - -webcmd --profile work \ - --session work-project-k7 \ - browser snapshot --snapshot-mode read - -webcmd --profile work session close work-project-k7 -``` - -If the fetch is rate-limited, login-gated, geo-gated, or returns unusable extracted text, report that state rather than retrying the same URL. - -## Fetch-first web search - -For a search query that names no site and has no direct URL, start with fetched search-engine result pages, not adapters. Encode the query into one of these URLs and fetch it: - -```bash -webcmd web fetch --url "https://duckduckgo.com/html/?q=" -webcmd web fetch --url "https://www.bing.com/search?q=" -webcmd web fetch --url "https://www.google.com/search?q=" -``` - -Try one search engine by default. Try a second when the first is weak, empty, blocked, CAPTCHA-gated, or lacks usable result URLs. Treat Google as more likely to block; DuckDuckGo HTML and Bing are cheaper first choices. - -Query terms that collide with everyday English (`puppeteer`, `playwright`, `rust`) pull unrelated results. Add a disambiguating term and say so if results still drift. - -Extract useful result URLs from the fetched page and then fetch the target pages with `webcmd web fetch`. Search snippets and result titles are discovery only, not evidence. A page that yields zero usable result URLs is a failed search, not a search with no results: move to the next engine. - -If the search-engine result page returns `FETCH_BLOCKED` or `FETCH_REQUIRES_BROWSER`, use the Session workflow once within the browser Session budget. A recognised block, CAPTCHA, or challenge page retires that engine for this request: do not re-fetch variants of the same engine. Do not jump to adapters because one engine blocked, unless the request names a site. - -## Fetch evidence - -Fetch up to three result URLs by default (five for a broad comparison): - -```bash -webcmd web fetch --url -``` - -For `FETCH_BLOCKED` or `FETCH_REQUIRES_BROWSER`, use the Session workflow above if the browser Session budget permits. Cite or link the source URL with substantive claims. - -If fetch is rate-limited, auth-gated, CAPTCHA-gated, bot-detected, quota-limited, or geo-blocked, do not loop. Try another relevant URL/source when available; otherwise report the blocker. - -## Adapter fallback - -On the site-named fast path, discover adapters first. Otherwise, only after fetch-first search, target-page fetch, and allowed browser Session fallbacks fail or are insufficient, discover search adapters: - -```bash -webcmd list --tag search -f json -``` - -Shortlist up to five candidate commands from site, name, description, keywords, strategy, browser requirement, and output columns. Prefer the named site, then a comparably relevant installed command. Read live help before execution: - -```bash -webcmd -h -``` - -Run one adapter search command. Run a second only if the first is weak, empty, fails, or an independent source materially corroborates it. Do not use adapters as the first search path unless the request names the site. - -When no installed command covers the needed site or specialized capability, use marketplace search only as adapter fallback: - -```bash -webcmd plugin search -f json -``` - -Install promising plugins sequentially, at most three plugins per user request: - -```bash -webcmd plugin install -webcmd list --tag search -f json -``` - -Inspect the newly visible command help. Stop once a suitable command appears. If installation fails, report the error and continue with fetched sources. - -Do not add custom marketplaces in this workflow. In hosted mode, only verified hosted marketplace adapters are installable. - -## Operational budgets - -- At most three plugin installs per user request. -- One fetched search-engine page by default; second if weak/blocked; third only if the first two fail. -- Up to five candidate commands before choosing. -- Three URLs by default; five only for broad comparison. -- Two browser Sessions/URLs by default; reuse one Session for allowed browser fallbacks. -- One adapter search by default; second only for weakness or corroboration. -- Do not retry the same blocked command more than once. - -## Search Summary - -Append this to the response: - -```md -Search Summary -- Commands: -- Sources fetched: -- Browser fallback: -- Gaps/failures: -``` - - diff --git a/skill-src/cli/webcmd-adapter-author/SKILL.src.md b/skill-src/cli/webcmd-adapter-author/SKILL.src.md deleted file mode 100644 index 678e5596..00000000 --- a/skill-src/cli/webcmd-adapter-author/SKILL.src.md +++ /dev/null @@ -1,284 +0,0 @@ ---- -name: webcmd-adapter-author -description: Use when writing a Webcmd adapter for a new site or adding a new command to an existing site. Guides end-to-end from first recon through field decoding, adapter coding, and verify. Replaces webcmd-oneshot / webcmd-explorer. For ad-hoc browser driving without an adapter, use webcmd-browser instead; for top-level orientation, use webcmd-usage. -allowed-tools: Bash(webcmd:*), Read, Edit, Write, Grep ---- - -# Webcmd Adapter Authoring - -You are an agent writing an adapter for a site. The goal of this skill is a 30-minute loop from zero context to a passing `webcmd browser verify`. - -Use the existing tools throughout: Playwright `browser run` for reconnaissance, -plus `webcmd doctor`, `webcmd browser init`, -and `webcmd browser verify`. Browser-run programs are discovery evidence, not -adapter source. - -Browser-profile auth commands must reuse `registerSiteAuthCommands`. Keep only site-specific `verify` and `openLogin` logic in the adapter. The login row must return `action_required` and `verify_command`; after the user reports done, agents run that returned command verbatim (it includes `--session` when applicable), and verification must succeed before retrying the original workflow. Credentials, MFA, and CAPTCHA always use human handoff: CAPTCHA stops automation until the user reports done and verification succeeds, and adapter code must not collect or type passwords or secrets. - -Commands whose primary operation searches or discovers matching items from a corpus must set `tags: ['search']`. Add short `keywords` only for non-obvious intent synonyms; do not infer tags from command names alone when authoring new adapters. - -When debugging browser-backed adapters, start with `--trace on --keep-tab true --window foreground`. `--trace on` writes a trace artifact every round, and `summary.md` is the entry point for reviewing both failures and successes. `--keep-tab true --window foreground` keeps the tab lease alive and puts the browser window in front so you can inspect the final page state. - ---- - -## Precheck: Know Your Lane - -Use `coverage-matrix.md` for a quick self-test before implementation. Ask three questions: - -1. Can the data be seen in the browser? If no, solve authentication first. -2. Is the data HTTP, JSON, or HTML? If no, this skill is out of scope. -3. Does the command require real-time push? If yes, look for an HTTP endpoint with the same data; if none exists, stop. - -Continue only when all three answers are yes. - ---- - -## Top-Level Decision Tree - -**Choose the strategy before writing the adapter.** Every time you reach Step 3 or Step 4, and before writing code, produce a strategy note. Without that note, do not start an adapter file. - -The core question is not whether an API is more elegant than DOM work. The core question is whether the data source has an external contract. Public or official interfaces are usually the most stable. UI/DOM semantics often have a user-visible contract too. Undocumented in-site XHR, GraphQL, or signature endpoints drift the most. Do not move a stable UI/DOM implementation to an uncontracted internal endpoint just to be "API-first." - -Strategy note template: - -```md -Strategy: PUBLIC_API | COOKIE_API | UI_SELECTOR | DOM_STATE | PAGE_FETCH | INTERCEPT -Contract: stable | visible-ui | internal-unstable -Evidence: -- observed request/state: -- auth source: -- replay result: -Why not simpler: -- PUBLIC_API: -- COOKIE_API: -- UI_SELECTOR/DOM_STATE: -``` - -| Strategy | Contract level | Use when | Evidence required | -| --- | --- | --- | --- | -| `PUBLIC_API` | stable | Node-side `fetch` can get target data without login | 200 + JSON/HTML contains target data, not analytics or ads | -| `COOKIE_API` | stable | Node-side `fetch` plus `page.getCookies()` / header helper can get the data | cookie/CSRF source is clear and replay is non-empty | -| `UI_SELECTOR` | visible-ui | publish/upload/click/form flows, or page semantics are more stable than internal APIs | selector has a semantic anchor; failure path is a typed error | -| `DOM_STATE` | visible-ui | data is in hydration state, bootstrap JSON, or SSR HTML | state key, script JSON, or HTML structure is clear | -| `PAGE_FETCH` | internal-unstable | only page-context `fetch` can reuse same-origin/session/runtime state | a browser run returns a non-empty page-context fetch result; explain why the internal endpoint is unavoidable | -| `INTERCEPT` | internal-unstable | request signing is complex but the page can naturally issue the request | target response is captured after triggering UI; explain why UI/DOM is insufficient | - -Selection rule: prefer `PUBLIC_API` / `COOKIE_API`. If UI/DOM semantics are stable, do not force an upgrade to `PAGE_FETCH` / `INTERCEPT`. Pay the maintenance cost of uncontracted internal endpoints only when public/official APIs are unavailable and UI/DOM cannot express the target data or operation. - -Observed maintenance pattern: `PAGE_FETCH` / `INTERCEPT` fixes are roughly 7-8x as frequent as `PUBLIC_API` fixes, while `UI_SELECTOR` is in the same rough band as `COOKIE_API`. See [`references/strategy-selection.md`](./references/strategy-selection.md) for the ladder, `api_candidates` evidence guidance, and counterexamples such as the booking #1680 case. - -Boundary: reuse only data and capabilities the page has already obtained legitimately. Do not teach signature cracking, CAPTCHA bypass, risk-control bypass, or access-control bypass. If a signature cannot be reused safely, such as a runtime-generated page signature that cannot be abstracted, fall back to `UI_SELECTOR`, `DOM_STATE`, or `INTERCEPT`. - -```text -Start - | - v -webcmd doctor passes? - | no -> fix the bridge using doctor output - v yes -Read site memory: - - webcmd site memory show - - webcmd site memory list - - references/site-memory/.md, if present - | - | hit endpoint + fields -> jump to endpoint verification - | (do not jump straight to adapter code; memory may be stale) - | no hit -> continue - v -Site recon (site-recon.md) -> Pattern A/B/C/D/E - | - v -API discovery (api-discovery.md) - section 1 network -> section 2 state -> section 3 bundle -> - section 4 token -> section 5 intercept - | - v -Candidate endpoint found - | - v -Direct fetch verification, even for memory hits - - 401/403 -> return to section 4 token investigation - - empty/HTML -> return to site-recon and choose another Pattern - - site changed -> mark old endpoint stale and return to api-discovery - | - v -Field decoding - - self-explanatory -> use directly - - known code -> field-conventions.md - - unknown -> field-decode-playbook.md - Compare one known field against the visible web page to catch misalignment. - | - v -Design columns (output-design.md) - - names - - types - - order - | - v -webcmd browser init - - scaffold / - - locally, use webcmd adapter path / and edit that local copy - - in hosted mode, use webcmd adapter source get|put / to edit tenant-owned source - | - v -webcmd browser verify - | fail -> use the autofix skill with --trace retain-on-failure - v pass -Compare field values against the visible page - | mismatch -> return to field decoding - v match -Write site memory with CLI commands - - site endpoint set|stale - - site field-map add and site note add - - site fixture put and site sample add -``` - ---- - -## Runbook - -Check these off step by step: - -[ ] 1. `webcmd doctor` returns "Everything looks good" - -[ ] 2. Read site memory: - [ ] Run `webcmd site memory show ` for endpoint and field-map contents, and `webcmd site memory list ` for staleness. - [ ] Does `references/site-memory/.md` exist? If yes, read its "Known endpoints" section. - [ ] On a hit: **jump to Step 5 endpoint verification + Step 7 field check**, not directly to Step 9 adapter code. - [ ] If memory is older than 30 days according to `verified_at`, treat it as stale and use the cold-start path through Steps 3 and 4. - -[ ] 3. Recon (`site-recon.md`): - [ ] **Preferred:** create a session, then use `webcmd --session browser run --stdin` for navigation, readiness, network hints, and page evidence in one Playwright-style program. - [ ] Use `webcmd --session browser snapshot --snapshot-mode tree` when structural page evidence is needed. - [ ] Use the run result as reconnaissance evidence; do not copy Playwright code into an adapter. - [ ] Choose Pattern A / B / C / D / E. - -[ ] 4. API discovery (`api-discovery.md`) by Pattern: - [ ] Pattern A -> section 1 network deep read. - [ ] Pattern B -> section 2 state extraction + section 1 for deeper data. - [ ] Pattern C -> section 3 bundle / script src search. - [ ] Pattern D -> section 4 token source + section 5 fallback. - [ ] Pattern E -> find an HTTP polling endpoint; use section 5 only if none exists. - -[ ] 5. Directly verify the candidate endpoint: - [ ] Response is 200. - [ ] Response contains target data, not HTML, ads, or analytics. - -[ ] 6. Write the strategy note before code: - [ ] Choose one of `PUBLIC_API / COOKIE_API / PAGE_FETCH / INTERCEPT / DOM_STATE / UI_SELECTOR`. - [ ] Fill Contract: `stable / visible-ui / internal-unstable`. - [ ] Fill Evidence: observed request/state, auth source, replay result. - [ ] If choosing `PAGE_FETCH` / `INTERCEPT`, explain why `PUBLIC_API`, `COOKIE_API`, `UI_SELECTOR`, and `DOM_STATE` are not suitable. - [ ] If choosing `UI_SELECTOR` / `DOM_STATE`, do not over-defend why it is not an API; state the semantic anchor and typed-error path. - -[ ] 7. Field decoding: - [ ] Self-explanatory key -> use it directly. - [ ] Known code -> look it up in `field-conventions.md`. - [ ] Unknown code -> use `field-decode-playbook.md` (sort-key comparison, structural diff, constant checks). - -[ ] 8. Design columns (`output-design.md`): - [ ] Use camelCase names aligned with neighboring adapters. - [ ] Make types, units, and percentage format clear. - [ ] Order: identifier columns -> business numbers -> metadata. - -[ ] 9. Write the adapter (`adapter-template.md`): - [ ] `webcmd browser init /`, then set `strategy: Strategy.` in the generated file - [ ] Locally, run `webcmd adapter path /` and edit that file. In hosted mode, use `webcmd adapter source get /` and `webcmd adapter source put / `. - [ ] Find the closest same-site or same-type adapter and carry over only the proven mapping. - [ ] Use only the adapter-compatible path proven in Step 6A; never paste Playwright locators, `waitForResponse`, or browser-run globals into `func`. - -[ ] 10. Verification fixtures: - [ ] After the first passing run, read and save the fixture with `webcmd site fixture get /` and `webcmd site fixture put / `. - [ ] Tighten the seed by adding `patterns` (URL/date/ID formats), `notEmpty` (core fields), and stricter `rowCount`. - [ ] Run `webcmd browser verify /` again and confirm it matches the fixture. - -[ ] 11. Compare field values against the visible page. Do not stop at "Adapter works!" - -[ ] 12. Write site memory after **verify passes and visible-page comparison matches**. See `references/site-memory.md` for schema: - [ ] Record verified endpoints with `webcmd site endpoint set --url --method `; mark changes with `webcmd site endpoint stale `. - [ ] Append mappings with `webcmd site field-map add --meaning --source ` and conclusions with `webcmd site note add --text `. - [ ] Keep the required verify fixture current with `webcmd site fixture get|put /`; it must cover args, rowCount, columns, types, patterns, and notEmpty. - [ ] Save a sanitized endpoint response with `webcmd site sample add / `. - [ ] If debugging dumped temporary files in the repo or adapter directory, such as `.dbg-*.html`, `raw-*.json`, or similar, **delete them before commit**. Keep temporary evidence in `/tmp/` and save sanitized samples with `webcmd site sample add`. - -[ ] 13. **First command for this site? Stop and ask before building more.** - [ ] If this was the site's first command, do not silently keep scaffolding more commands. Ask the user what use cases they have in mind for this site — who the persona is, what they're trying to accomplish end to end. - [ ] From the use cases, propose the full set of commands you'd recommend adding, not just the obvious next one. Cover the whole journey the use cases imply (discovery, single-item detail, comparison, account/write actions, etc.), not only what's cheapest to build. - [ ] If that set is small (roughly ≤6-8 commands), list it flat and ask the user to confirm or trim it. - [ ] If it's large, bucket the commands into named groups (e.g. "Discovery", "Single-item evaluation", "Account actions requiring login") and ask the user which bucket(s) to build first — do not dump an unbucketed wall of commands. - [ ] Flag any bucket that needs a capability not yet solved (login/OTP, write access, payment) as its own decision point — e.g. "these need login — how do you want to handle auth?" — separate from the command list itself. - [ ] Do not scaffold additional commands until the user has confirmed which ones to build. - ---- - -## Fallback Paths - -| Stuck at | Symptom | Go to | -| --- | --- | --- | -| Step 4 API discovery | `network` is empty and `__INITIAL_STATE__` is empty | section 3 bundle search for baseURL | -| | bundle search cannot find baseURL | section 5 intercept | -| Step 5 endpoint verification | 401 / 403 | section 4 token investigation | -| | 200 but response is HTML | return to Step 3 and reassess Pattern | -| | 200 but `data: []` is empty | wrong params or endpoint version changed; return to section 1 and inspect real network headers | -| Step 7 field decoding | sort-key comparison is inconclusive | field-decode-playbook.md section 3 structural diff | -| | still inconclusive | output raw values first, get the adapter running, then iterate | -| Step 10 verify fails | missing filter / wrong field mapping | autofix skill; rerun with `--trace retain-on-failure` | -| | a column is always `null` | field path is wrong; return to Step 7 | -| Step 10 verify fixture mismatch | `[pattern]` row[i] failure | compare visible page value first. If value is right, loosen fixture pattern; if value is wrong, fix mapping | -| | `[column] missing column "X"` | actual response lacks this column due to site change or args; rerun `--update-fixture` or fix adapter | -| | `[type]` actual null / undefined | extraction failed; return to Step 7. Use a `string|null` union only when the value is truly nullable | -| Step 11 values mismatch | value differs by 10,000x | unit mismatch | -| | percentage is 100x too small | response already uses `0.025`; do not multiply by 100 | - ---- - -## Reference Files - -| File | When to open | -| --- | --- | -| `references/coverage-matrix.md` | Before implementation: scope self-test | -| `references/site-recon.md` | Step 3: classify site type | -| `references/api-discovery.md` | Step 4: find endpoint | -| `references/strategy-selection.md` | Before Step 6 strategy note: contract model, observed fix frequency, `api_candidates` evidence, counterexamples | -| `references/field-conventions.md` | Step 7: known field-code lookup | -| `references/field-decode-playbook.md` | Step 7: field not in dictionary | -| `references/output-design.md` | Step 8: naming, types, order | -| `references/adapter-template.md` | Step 9: file structure and live example `convertible.js` | -| `references/site-memory.md` | Overview: in-repo seeds plus CLI-managed site memory | -| `references/site-memory/.md` | Step 2: public site knowledge when a seed file exists | -| `references/success-rate-pitfalls.md` | Step 7 / 11: eleven silent failure modes where verify can pass with wrong data, including aria-label locale dependence | -| `references/jsdom-fixture-pattern.md` | When adapter uses DOM extraction inside `page.evaluate` and mocked-evaluate unit tests miss silent bugs; freeze HTML into `plugins//__fixtures__/` and run JSDOM with the mandatory `awk 'NF>0'` tightening plus reverse-validation discipline | -| `references/typed-errors.md` | Read before writing `func`: five typed error classes (`ArgumentError`, `EmptyResultError`, `CommandExecutionError`, `AuthRequiredError`, `TimeoutError`) plus fixes for silent anti-patterns (`silent-clamp`, `sentinel-row`, `generic CliError`) | - ---- - -## Key Conventions - -@[adapter conventions](../../shared/adapter-conventions.src.md) -- **The `browser:` field determines the `func` signature:** `browser:false -> (args)`, `browser:true -> (page, args)`. If this is reversed, `args` may actually be a debug flag and all external parameters can silently fall back to defaults. -- Throw the correct typed error for known failures according to [`references/typed-errors.md`](./references/typed-errors.md). **Do not** silently `return []`, **do not** silently `return [{sentinel}]`, and **do not** silently clamp external parameters with `Math.max/min`. -- **Persistent site sessions keep stale DOM between commands.** `siteSession: 'persistent'` shares one tab per site; leftover modals/drawers from the previous command leak into the next one. State-sensitive write commands (checkout flows) should add `freshPage: true` (new tab, same lease — cookies/login/location survive). Verify session-scoped context (login, selected city/date) *before* side effects, and embed such context in URLs/IDs your command emits for sibling commands. See `references/adapter-template.md` and "Persistent Site Sessions and State Hygiene" in `docs/authoring.mdx`. -- For private iteration, use `webcmd adapter override /`, then locally run `webcmd adapter path /` and edit that file. In hosted mode, `webcmd adapter source get|put /` reads and writes tenant-owned source. Building, testing, and verifying the adapter under private iteration fully satisfies a request to "build a working adapter" on its own. **Do not run `webcmd plugin create` or promote the CLI until the user explicitly confirms they want it pushed into the repo (or a PR raised).** Once confirmed, create the plugin, copy the real command files into it, and delete scaffold sample commands. Do not hand-edit the root `webcmd-plugin.json` or generated README catalog; run `webcmd validate ` and smoke commands after installation. See `references/adapter-template.md` for details. -- After `webcmd plugin update`, check the reported overrides needing reconciliation and merge `yours` with `upstream`, using `base` as the common ancestor for a three-way merge. Only overrides are reported; a user-authored adapter has no upstream. -- Write site memory every round: no memory -> use skill -> produce memory -> next time becomes a five-minute task. -- **After a site's first command passes verify, stop and ask the user for their use cases before recommending next set of commands.** See Runbook Step 13. -- **Keep raw debugging dumps in `/tmp/`; save sanitized endpoint samples through `webcmd site sample add / `. Never leave `.dbg-*.html`, `raw-*.json`, `sample.*`, or similar temporary files in the repo root, `plugins//`, or the current working directory.** -- **JSDOM unit-test fixtures (`plugins//__fixtures__/.html`) are the exception.** They are intentional review artifacts committed to the repo, not temporary dumps. Because of that, the quality bar is higher: complete the five steps in `references/jsdom-fixture-pattern.md`, including the mandatory `awk 'NF>0'` blank-line tightening, and reverse-validate once to prove the regression guard can fail. - ---- - -## If You Are Stuck - -- Diagnostic path: `webcmd doctor` -> `webcmd site memory show --kind notes` -> rerun with `--trace retain-on-failure`. -- Endpoint path: return to `site-recon` and reclassify Pattern. Do not stay attached to the first API guess. -- Field path: compare one visible page value, then use sort-key comparison, structural diff, and constants. -- Verification path: if `webcmd browser verify` fails, switch to the autofix skill instead of improvising. - - diff --git a/skill-src/cli/webcmd-adapter-author/references/adapter-template.src.md b/skill-src/cli/webcmd-adapter-author/references/adapter-template.src.md deleted file mode 100644 index acd601d5..00000000 --- a/skill-src/cli/webcmd-adapter-author/references/adapter-template.src.md +++ /dev/null @@ -1,302 +0,0 @@ -# Adapter Template - -Use this after recon, endpoint verification, field decoding, output design, and strategy-note writing are complete. - -Playwright-style browser-run code is reconnaissance, not adapter source. -Implement the observed behavior with the existing adapter APIs. - -## Create The File - -For private iteration: - -```bash -webcmd browser init / -``` - -This scaffolds a `Strategy.PUBLIC` placeholder. Use `webcmd adapter path /` to locate it, then edit that local file. Local `adapter source get` prints the same path only without `--output`; local `adapter source put` is unavailable. In hosted mode, `adapter source get|put` download and upload tenant-owned source. Set the real `strategy:` value and other `TODO` fields. - -Promote a community CLI to `agentrhq/webcmd-plugins` as a plugin — **only after the user has explicitly confirmed they want it pushed into that repo**; a general instruction to build a working adapter is not that confirmation (see `SKILL.md`'s Key Conventions): - -Determine the plugin name (default to ``) and collect the author's display name and GitHub handle if they are not already known. - -```bash -webcmd plugin create \ - --dir plugins/ \ - --description " commands for Webcmd" \ - --author-name "" \ - --author-handle "" -cp "$(webcmd adapter path /)" plugins// -rm plugins//hello.ts plugins//greet.ts 2>/dev/null || true -``` - -Do not hand-edit the root `webcmd-plugin.json` or the generated community-plugin section in `README.md`. After merge, the community-plugin sync discovers `plugins/*/webcmd-plugin.json`, validates the author metadata, and updates both generated catalogs. - -Before handing off, remove the private shadow and prove the plugin path works: - -```bash -webcmd adapter reset -webcmd plugin install file://$PWD/plugins/ -webcmd validate -webcmd --help -``` - -## Minimal Registry Shape - -Adapters register commands with `cli` and `Strategy` from `@agentrhq/webcmd/registry`. - -```js -import { cli, Strategy } from '@agentrhq/webcmd/registry'; - -cli({ - site: 'hackernews', - name: 'top', - access: 'read', - description: 'Hacker News top stories', - domain: 'news.ycombinator.com', - strategy: Strategy.PUBLIC, - browser: false, - args: [ - { name: 'limit', type: 'int', default: 20, help: 'Number of rows' }, - ], - columns: ['rank', 'title', 'url', 'score', 'author', 'commentCount'], - pipeline: [ - { navigate: 'https://news.ycombinator.com/' }, - { - evaluate: `(async () => { - const rows = [...document.querySelectorAll('.athing')].slice(0, \${{ args.limit }}); - return rows.map((row, index) => { - const subtext = row.nextElementSibling; - const titleLink = row.querySelector('.titleline a'); - return { - rank: index + 1, - title: titleLink?.textContent?.trim() || '', - url: titleLink?.href || '', - score: Number((subtext?.querySelector('.score')?.textContent || '').match(/\\d+/)?.[0] || 0), - author: subtext?.querySelector('.hnuser')?.textContent?.trim() || '', - commentCount: Number((subtext?.textContent || '').match(/(\\d+)\\s+comments?/)?.[1] || 0), - }; - }); - })()`, - }, - { - map: { - rank: '${{ item.rank }}', - title: '${{ item.title }}', - url: '${{ item.url }}', - score: '${{ item.score }}', - author: '${{ item.author }}', - commentCount: '${{ item.commentCount }}', - }, - }, - { limit: '${{ args.limit }}' }, - ], -}); -``` - -Treat this as shape guidance, not a universal solution. Prefer the closest existing adapter for the same site or source type. - -## Imports - -Allowed imports: - -```js -import { cli, Strategy } from '@agentrhq/webcmd/registry'; -import { - ArgumentError, - AuthRequiredError, - CommandExecutionError, - EmptyResultError, - TimeoutError, -} from '@agentrhq/webcmd/errors'; -``` - -Rules: - -- Do not add third-party dependencies. -- Do not import private repo internals unless an established neighboring adapter already does so. -- Keep helpers local unless there is real duplication in the same site directory. - -## Required Fields - -| Field | Rule | -| --- | --- | -| `site` | Directory/site id. Keep lowercase and stable. | -| `name` | Command id. Keep lowercase and stable. | -| `access` | Usually `read`; use write-like access only for commands that mutate state. | -| `description` | One clear sentence. | -| `domain` | Primary domain for auth and help output. | -| `strategy` | Use a registry enum such as `Strategy.PUBLIC` or `Strategy.COOKIE`; align it with the strategy note. | -| `browser` | `false` for plain Node-side adapters; `true` when the adapter needs the page, cookie jar, or browser runtime. | -| `args` | Include type, default, and help for every external parameter. | -| `columns` | Must exactly match row keys, including order. | -| `pipeline` or `func` | Use the style already established by nearby adapters. | -| `siteSession` | `'persistent'` shares one tab per site across commands (multi-step flows); `'ephemeral'` gets a fresh isolated tab per run. Persistent site-session tabs keep leftover DOM (modals, drawers) between commands — see "Persistent Site Sessions and State Hygiene" in docs/authoring.mdx. | -| `freshPage` | With `siteSession: 'persistent'`, set `true` to start the command on a newly created tab under the same lease: profile state (cookies, login, location) survives, stale DOM does not. Recommended for state-sensitive write commands such as checkout flows. | - -## Strategy Enum Examples - -The strategy note uses discovery names such as `PUBLIC_API` and `COOKIE_API`. The adapter declaration records the runtime choice with `Strategy` enum values. - -Use `Strategy.PUBLIC` when an anonymous, stable endpoint can be fetched directly from Node: - -```js -import { cli, Strategy } from '@agentrhq/webcmd/registry'; -import { ArgumentError, CommandExecutionError, EmptyResultError } from '@agentrhq/webcmd/errors'; - -cli({ - site: 'example', - name: 'public-list', - access: 'read', - description: 'Example public listing', - domain: 'api.example.com', - strategy: Strategy.PUBLIC, - browser: false, - args: [{ name: 'limit', type: 'int', default: 20, help: 'Number of rows' }], - columns: ['index', 'title', 'url'], - func: async (args) => { - const limit = Number(args.limit ?? 20); - if (!Number.isInteger(limit) || limit <= 0) { - throw new ArgumentError('limit must be a positive integer'); - } - - const resp = await fetch(`https://api.example.com/items?limit=${limit}`, { - headers: { 'User-Agent': 'Mozilla/5.0' }, - }); - if (!resp.ok) throw new CommandExecutionError(`example request failed: HTTP ${resp.status}`); - - const data = await resp.json(); - const items = Array.isArray(data?.items) ? data.items : []; - if (!items.length) throw new EmptyResultError('example public-list', 'API returned no rows'); - - return items.map((item, index) => ({ - index: index + 1, - title: item.title, - url: item.url, - })); - }, -}); -``` - -Use `Strategy.COOKIE` when the endpoint or HTML page needs the user's existing browser session. Read cookies with `page.getCookies()` and pass them to Node-side `fetch`; do not rely on `document.cookie` for HttpOnly auth cookies. - -```js -import { cli, Strategy } from '@agentrhq/webcmd/registry'; -import { ArgumentError, AuthRequiredError, CommandExecutionError, EmptyResultError } from '@agentrhq/webcmd/errors'; - -const BASE = 'https://www.example.com'; -const HOST = 'www.example.com'; -const ROOT = '.example.com'; - -async function cookieHeader(page) { - const seen = new Map(); - for (const opts of [{ domain: HOST }, { domain: ROOT }]) { - for (const cookie of await page.getCookies(opts).catch(() => [])) { - if (!seen.has(cookie.name)) seen.set(cookie.name, cookie.value); - } - } - return [...seen].map(([name, value]) => `${name}=${value}`).join('; '); -} - -function parseRowsFromHtml(html) { - return [...html.matchAll(/]+href="([^"]+)"[^>]*>([^<]+)<\/a>/g)].map((match, index) => ({ - index: index + 1, - title: match[2].trim(), - time: '', - })); -} - -cli({ - site: 'example', - name: 'private-list', - access: 'read', - description: 'Example private listing', - domain: HOST, - strategy: Strategy.COOKIE, - browser: true, - navigateBefore: false, - args: [{ name: 'limit', type: 'int', default: 20, help: 'Number of rows' }], - columns: ['index', 'title', 'time'], - func: async (page, args) => { - const limit = Number(args.limit ?? 20); - if (!Number.isInteger(limit) || limit <= 0) { - throw new ArgumentError('limit must be a positive integer'); - } - - const cookie = await cookieHeader(page); - const resp = await fetch(`${BASE}/inbox`, { - headers: { - 'User-Agent': 'Mozilla/5.0', - Referer: `${BASE}/`, - ...(cookie ? { Cookie: cookie } : {}), - }, - redirect: 'follow', - }); - if (!resp.ok) throw new CommandExecutionError(`example request failed: HTTP ${resp.status}`); - - const html = await resp.text(); - if (/login required|sign in/i.test(html)) throw new AuthRequiredError(HOST); - - const rows = parseRowsFromHtml(html).slice(0, limit); - if (!rows.length) throw new EmptyResultError('example private-list', 'page returned no rows'); - return rows; - }, -}); -``` - -## Parameter Safety - -- Validate user-facing numbers before use. -- Throw `ArgumentError` for invalid external parameters. -- Do not silently clamp with `Math.max` / `Math.min` unless the user explicitly requested clamping and the output says so. -- Do not let a failed selector or missing field become a valid empty result. - -## Row Safety - -Before mapping rows: - -- Confirm the response contains the expected shape. -- Throw `EmptyResultError` only when the site truly reports no results. -- Throw `CommandExecutionError` when the response shape is wrong, parsing fails, or an endpoint returns HTML instead of expected data. -- Use `AuthRequiredError` when login is required or session expired. -- Use `TimeoutError` when the page or endpoint did not settle in time. - -## Column Alignment Checklist - -Before verify: - -```text -[ ] columns array and row keys match exactly -[ ] no intermediate object key overlaps a column accidentally -[ ] values use documented units -[ ] percentage scale is consistent -[ ] dates are ISO or clearly documented -[ ] URLs are absolute when users need to click them -[ ] one row was compared with the visible page -``` - -## Verify - -Run: - -```bash -webcmd browser verify / --trace retain-on-failure -``` - -After the first passing run, read and save the fixture: - -```bash -webcmd site fixture get / --output /tmp/.json -``` - -Then tighten the saved fixture and write it back: - -- Add `notEmpty` for essential columns. -- Add `patterns` for URL, ID, date, or slug formats. -- Set realistic `rowCount`. -- Keep `types` narrow. - -```bash -webcmd site fixture put / /tmp/.json -``` - -Run verify again and confirm the fixture matches. diff --git a/skill-src/cli/webcmd-adapter-author/references/api-discovery.src.md b/skill-src/cli/webcmd-adapter-author/references/api-discovery.src.md deleted file mode 100644 index c896d904..00000000 --- a/skill-src/cli/webcmd-adapter-author/references/api-discovery.src.md +++ /dev/null @@ -1,255 +0,0 @@ -# API Discovery - -Use this after `site-recon.md` chooses Pattern A/B/C/D/E. The output of this file is a candidate endpoint plus evidence for the strategy note. - -Keep `--trace on --keep-tab true --window foreground` enabled while exploring browser-backed sites. - -## Section 0 - Preflight Red Lines - -Read these before endpoint verification. If you miss either one, you can spend the rest of discovery testing the wrong thing. - -### 0.1 Anti-bot and WAF gates decide whether Node fetch is valid - -Use `browser run` to inspect cookies and the response body manually. - -| Cookie or body signal | Vendor | Bare Node fetch or curl result | Strategy | -| --- | --- | --- | --- | -| `acw_sc__v2`, `acw_tc`, `ssxmod_itna`; body contains `arg1 = '32-HEX'` or `/ntc_captcha/` | Aliyun WAF | Slider HTML instead of real data | Verify the endpoint in browser context first; HTML-style cookie adapters can still end with Node-side fetch plus `page.getCookies()` | -| `__cf_bm`, `cf_clearance`, `__cfduid`; body contains `Cloudflare Ray ID` or `Checking your browser` | Cloudflare | TLS or browser fingerprint is rejected | Use a browser/session-aware probe first, then choose the adapter fetch route from `adapter-template.md` | -| `_abck`, `bm_sz`, `bm_sv` | Akamai | Often blocked even with cookies | Use a browser/session-aware probe first | -| Body contains `geetest` or `gt_captcha` | Geetest | Slider or puzzle challenge; no programmatic solution in this skill | Out of scope; stop or use a user-visible UI strategy | - -Rule: if any of these anti-bot or WAF signals appear, do not use bare Node fetch as endpoint verification. First prove the endpoint from the browser context or from a page on the target origin. After that, choose the final adapter strategy normally: JSON browser APIs may use `page.fetchJson()`, while HTML-style cookie adapters should keep using Node-side `fetch` with cookies read through `page.getCookies()`. - -### 0.2 Cross-subdomain fetch is CORS-blocked by default - -For example, a page on `jobs.51job.com` fetching an API on `cupid.51job.com` will usually hit a CORS preflight unless the API returns `Access-Control-Allow-Origin`. - -Probe it explicitly: - -```bash -webcmd --session browser run --stdin <<'JS' -await page.goto('https:///'); -return await page.evaluate(async () => { - try { - return await fetch('https:///api/...', { credentials: 'include' }).then(r => r.status); - } catch (error) { - return `cors:${error instanceof Error ? error.message : String(error)}`; - } -}); -JS -``` - -- A numeric status means CORS allows the request. -- `cors:...` or `TypeError: Failed to fetch` means the browser blocked it. - -When it is blocked, `credentials: include` is not a CORS fix across subdomains. It only asks the browser to send cookies; it does not grant cross-origin permission. Use this fallback order: - -1. Prefer a same-origin endpoint on the current subdomain. -2. Navigate to the target subdomain inside `browser run`, then fetch relative paths from that origin. -3. If the data is truly cross-origin and there is no same-origin alternative, use Section 5 intercept and capture the response from the page's own request. - -## Section 1 - Network Deep Read - -Use for Pattern A and for deeper data in Pattern B. - -```bash -webcmd --session browser run --stdin <<'JS' -const candidates = []; -page.on('response', async response => { - const url = response.url(); - const contentType = response.headers()['content-type'] || ''; - if (!url.includes('') && !/json|graphql/i.test(contentType)) return; - let sample = ''; - try { sample = (await response.text()).slice(0, 2000); } catch {} - candidates.push({ - url, - method: response.request().method(), - status: response.status(), - contentType, - sample, - }); -}); - -await page.goto(''); -await page.waitForLoadState('domcontentloaded'); -await page.waitForTimeout(1500); -return candidates.slice(0, 20); -JS -``` - -Inspect each candidate: - -- URL and method. -- Status code and content type. -- Query/body params. -- Request headers that appear auth-related. -- Response shape and whether it includes target data. -- Whether data is user-visible, not analytics, ads, experiments, or personalization noise. - -Reject candidates that only contain telemetry, unrelated recommendations, beacons, or layout metadata. - -Replay directly when possible: - -```bash -webcmd --session browser run --stdin <<'JS' -return await page.evaluate(async () => - fetch('', { credentials: 'include' }).then(r => r.text()) -); -JS -``` - -If Node-side replay works without page runtime state, prefer `PUBLIC_API` or `COOKIE_API`. If the endpoint only works in page context, document why before selecting `PAGE_FETCH`. - -For a request that exists only after a UI action, use `browser run` so the -listener is attached before the trigger: - -```js -const pending = page.waitForResponse( - response => response.url().includes('/api/target'), -); -await page.getByRole('button', { name: 'Load' }).click(); -const response = await pending; -return { - url: response.url(), - method: response.request().method(), - status: response.status(), - body: await response.json(), -}; -``` - -This is recon evidence only. Choose the adapter strategy from the verified -endpoint and UI evidence; do not copy the browser-run program into the adapter. - -## Section 2 - State Extraction - -Use for Pattern B. - -Look for: - -- `window.__INITIAL_STATE__` -- `window.__NEXT_DATA__` -- `window.__NUXT__` -- JSON in `