From 9d9c18f665fed9bf0a117423d4212372dfd5249b Mon Sep 17 00:00:00 2001 From: James Sesler Date: Sun, 30 Aug 2026 15:29:15 -0400 Subject: [PATCH 1/4] fix: make the Broad-Side reading guide reachable and document the feature MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Broad-Side shipped across seven PRs with its user-facing paper trail lagging the code. Three gaps, one of them a defect: - .codecarto/broadside/SKILL.md was written but unreachable. codecarto_skill resolves .codecarto/skills//SKILL.md, so {name: "broadside"} returned "Unknown skill" and codecarto_list_skills never mentioned it. Both surfaces now serve it under the name `broadside` via the new readBroadsideSkill, exempt from the post-pipeline completion gate — a scout run is read before the pipeline and during it, not after — and readable on a repository with scout state and no workspace (the packaged copy answers when there is no workspace copy). list_skills reports it apart from the post-pipeline set, and the unknown-skill error names the exemption. - The packaged agent skill never mentioned Broad-Side, so an agent with the tool had no doctrine for it. New references/broadside.md covers when to scout, the cost guardrails, how to read a run, and the leads-never-evidence rule; it is served as the `broadside` topic of codecarto_guide and linked from the skill's drive loop and reference list. - config.yaml lagged the tool: incremental, retry_truncated, include_synthesis, include_triage and wait_seconds were call-parameters only, so a repo could not fix its own scouting policy. All five are now config keys with repo defaults. An explicit tool parameter still wins, and a malformed value falls back to the shipped default rather than failing a run mid-flight. Also: a README section, a MANUAL step, and MCP-quickstart coverage in place of a single table row; broadside/ in the README repo-structure block and core/broadside.ts in CLAUDE.md's core-modules list; and npm run smoke:broadside for the previously orphaned opt-in live smoke script. Tests: config defaults and their malformed-value fallbacks, a check that the documented config keys and the parsed config keys are the same set, skill reachability on both the MCP and Pi surfaces, and the guide reference's content rules. 413 pass. Co-Authored-By: Claude Opus 5 --- .codecarto/broadside/SKILL.md | 11 ++ .codecarto/broadside/config.yaml | 34 +++++- CHANGELOG.md | 3 + CLAUDE.md | 4 +- MANUAL.md | 18 +++ README.md | 24 ++++ ROADMAP.md | 9 +- agent-skill/codecartographer/SKILL.md | 4 +- .../codecartographer/references/broadside.md | 105 ++++++++++++++++++ core/broadside.ts | 58 +++++++++- docs/mcp-quickstart.md | 11 ++ extensions/codecarto/index.ts | 31 +++++- mcp-server/server.ts | 72 +++++++++--- package.json | 1 + tests/broadside.test.mjs | 84 ++++++++++++++ tests/guide.test.mjs | 13 +++ tests/mcp-server.test.mjs | 34 ++++++ tests/pi-extension-activation.test.mjs | 20 +++- 18 files changed, 510 insertions(+), 26 deletions(-) create mode 100644 agent-skill/codecartographer/references/broadside.md diff --git a/.codecarto/broadside/SKILL.md b/.codecarto/broadside/SKILL.md index e42d896..11ad38c 100644 --- a/.codecarto/broadside/SKILL.md +++ b/.codecarto/broadside/SKILL.md @@ -81,6 +81,17 @@ Collect runs two cross-lens post-passes by default: **synthesis** (the executive report) and **triage** (the prioritized work order). Pass `include_synthesis: false` or `include_triage: false` on collect to skip one. +Every run knob — `incremental`, `retry_truncated`, `include_synthesis`, +`include_triage`, `wait_seconds` — also has a repository default under the same +name in this directory's `config.yaml`, alongside `model`, `api_key`, +`default_lenses`, `max_cost`, and the `pricing` overrides. An explicit +parameter on the call always wins over the file. + +This file is also served directly: `codecarto_skill {cwd, name: "broadside"}` +returns it. Unlike the post-pipeline skills under `.codecarto/skills/`, it is +not gated on a completed pipeline — a scout run is meant to be read before the +pipeline starts and while it runs. + It works on any git repository — no initialized workspace required — and needs an OpenRouter API key via the `api_key` parameter, the `OPENROUTER_API_KEY` environment variable, or `api_key` in this directory's `config.yaml`. diff --git a/.codecarto/broadside/config.yaml b/.codecarto/broadside/config.yaml index 93604e0..391af45 100644 --- a/.codecarto/broadside/config.yaml +++ b/.codecarto/broadside/config.yaml @@ -51,4 +51,36 @@ # # pricing: # input_per_m: 0.1875 -# output_per_m: 0.9375 \ No newline at end of file +# output_per_m: 0.9375 +# --------------------------------------------------------------------------- +# Run defaults. Each key below mirrors a codecarto_broadside parameter of the +# same name and sets this repository's default for it; an explicit parameter on +# the call always wins. Set them here when a repo's scouting policy is stable, +# so it does not have to be restated on every submit and collect. + +# Scan only the modules whose files changed since the previous run's git HEAD. +# Falls back to a full scan on a dirty tree or when no prior run exists. +# +# incremental: false + +# Re-submit lens results that came back truncated at the output token limit, +# once, with a doubled output cap. Truncation is always reported either way. +# +# retry_truncated: true + +# Run the cross-lens synthesis pass (synthesis.md, the executive report) once +# every lens batch completes. +# +# include_synthesis: true + +# Run the triage pass (triage.md, the P0-P3 work order) once every lens batch +# completes. +# +# include_triage: true + +# Default poll budget in seconds. 0 returns as soon as the batches are +# submitted or the recorded state is read; a positive value polls that long +# before returning with whatever is done. Batch jobs routinely take tens of +# minutes, so a submit-then-collect-later rhythm is normal. +# +# wait_seconds: 0 diff --git a/CHANGELOG.md b/CHANGELOG.md index ba8b2c8..36c316d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ All notable changes to this project are documented here. The format is based on ### Added +- **Broad-Side: the reading guide is reachable, and the docs say the feature exists.** Broad-Side shipped across seven PRs with its user-facing paper trail lagging behind the code. `.codecarto/broadside/SKILL.md` was written but unreachable: `codecarto_skill` resolves `.codecarto/skills//SKILL.md`, so `{name: "broadside"}` returned "Unknown skill" and `codecarto_list_skills` never mentioned it. Both surfaces now serve it under the name `broadside`, exempt from the post-pipeline completion gate — a scout run is read *before* the pipeline and during it — and readable on a repository that has scout state and no workspace at all (the packaged copy answers when the workspace has none). `codecarto_list_skills` lists it apart from the post-pipeline set, and an unknown-skill error names the exemption. New `references/broadside.md` in the packaged agent skill teaches when to scout, the cost guardrails, and the leads-never-evidence rule, served as the `broadside` topic of `codecarto_guide`; the skill overview, README, MANUAL, and the MCP quickstart now cover the feature instead of leaving a single table row as its only mention. +- **Broad-Side: every run knob has a repository default** (`.codecarto/broadside/config.yaml`). `incremental`, `retry_truncated`, `include_synthesis`, `include_triage`, and `wait_seconds` were call-parameters only, so a repo could not fix its own scouting policy without restating it on every submit and collect. All five are now config keys documented alongside `model`, `api_key`, `default_lenses`, `max_cost`, and the `pricing` overrides; an explicit tool parameter always wins, and a malformed value falls back to the shipped default rather than failing a run. A test asserts the documented key set and the parsed key set are the same, so a key can no longer be documented into existence without being read. +- **`npm run smoke:broadside`** wires the existing opt-in live Broad-Side smoke script to a script name. It still skips cleanly without `OPENROUTER_API_KEY` and still spends real money when it runs. - **Broad-Side: incremental re-scouting** (#142). Submit records the git HEAD (and dirty flag) of each run, and `incremental: true` diffs against the previous run's HEAD to scan only the modules whose files changed — unchanged modules are skipped, so recurring scouting costs O(delta) instead of O(repo). Falls back to a full scan on a dirty tree, a non-git tree, or when no prior run exists. Repo-info lenses (architecture) always run. - **Broad-Side: zero-config slicing** (#140). The defect, conventions, and porting lenses now slice by `auto` instead of always per-directory: a repo whose matching files fit within the lens's char cap collapses to a single whole-repo slice (one request instead of one per module), while a repo too large for one slice still splits by top-level directory. Small repos stop paying for per-module request overhead; large repos keep full coverage. The decision is deterministic from file sizes and recorded implicitly in the slice layout. - **Broad-Side: truncated slices re-submit automatically** (#133). Submit now persists every request body to `.codecarto/broadside//requests.json`, and collect re-submits each truncated result once with a doubled output cap (bounded by the model's completion ceiling) — recovering coverage lost to a `max_tokens` cutoff instead of leaving the module silently unscouted. Recovered slices rewrite their JSON/markdown, clear their truncation flag, and report in the collect summary (`↻ N recovered`); anything still truncated after the retry stays flagged. Opt out with `retry_truncated: false` on collect. diff --git a/CLAUDE.md b/CLAUDE.md index 6911a6a..ace3bad 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -9,6 +9,7 @@ npm ci # install (lockfile-strict) npm run build # tsc compile to dist/ (required before publish; CI runs this) npm test # node --experimental-strip-types --test tests/*.test.mjs npm run smoke # end-to-end MCP smoke test against the published npm package +npm run smoke:broadside # opt-in live Broad-Side run (spends real money; needs OPENROUTER_API_KEY + a target repo) # Run a single test file: node --experimental-strip-types --disable-warning=ExperimentalWarning --test tests/pipeline-invariants.test.mjs @@ -25,7 +26,7 @@ CodeCartographer is one framework shipped through **three delivery surfaces** th 1. **`.codecarto/`** — the drop-in template (Markdown + YAML, no executable code). This is both the source of truth committed in this repo *and* the directory that gets copied into a user's target repo on `codecarto-init`. `core/workspace.ts` exposes it as `packagedWorkspaceDir`, resolved at runtime by walking up from `core/` to find the nearest `package.json`. 2. **`extensions/codecarto/`** — Pi extension. Registers `/codecarto-*` slash commands, runs phases as isolated `AgentSession` sub-agents (`auto-runner.ts`), renders the live widget (`agent-widget.ts`), and writes the HTML dashboard (`dashboard-writer.ts`). The `tool_call` hook in `index.ts` blocks `bash` outright and confines `edit`/`write` to `.codecarto/` plus the configured library path. -3. **`mcp-server/`** — MCP server exposing ten workflow and library operations as JSON-RPC tools over stdio. Never spawns sub-agents; returns prompt text for the host to dispatch. +3. **`mcp-server/`** — MCP server exposing the workflow, library, and Broad-Side operations as JSON-RPC tools over stdio. Never spawns sub-agents; returns prompt text for the host to dispatch. Both wrappers import everything they share from `core/index.ts` (barrel re-export). **If you add a primitive used by both surfaces, it goes in `core/` and must be re-exported through `index.ts`.** Wrapper-specific logic (UI, sub-agent lifecycle, MCP plumbing) stays in the wrapper. @@ -50,6 +51,7 @@ When adding a feature, the question to ask is "does this work on all three surfa - `core/orchestrator-config.ts` — loads `.codecarto/workflow/config.yaml` (the `orchestrator.llm_steer_next_phase` flag lives here). - `core/library.ts` — versioned library discovery, publication, reads, listing, reindexing, and optional git commits shared by Pi and MCP. - `core/synthesis.ts` — vision/library/proposal preflight and exact confirmed-version resolution for the four-phase synthesis workflow. +- `core/broadside.ts` — Broad-Side batch reconnaissance: lens registry and prompts, repo slicing, OpenRouter Batch API submit/poll/collect, model catalog and cost pre-flight, synthesis and triage post-passes, and `.codecarto/broadside/` state. Executable-surface only (MCP today); the template carries just the reading guide. ### Pipeline shape diff --git a/MANUAL.md b/MANUAL.md index 9eaa140..a882efb 100644 --- a/MANUAL.md +++ b/MANUAL.md @@ -53,6 +53,24 @@ your-repo/ The LLM reads your source code directly from the repository root. No symlinking or copying required. +## Optional: Scout the Repository First + +On a repository too large to skim, a **Broad-Side** batch reconnaissance run is worth firing before you choose a pipeline. It sends six analysis lenses at the code as cheap asynchronous batch jobs and produces an executive report plus a P0-P3 work order, which turns the pipeline choice below into an informed one instead of a guess. + +Broad-Side needs the MCP server and an OpenRouter API key: + +``` +codecarto_broadside {cwd: "/path/to/repo", action: "submit"} +codecarto_broadside {cwd: "/path/to/repo", action: "collect"} +``` + +Results land in `.codecarto/broadside//`; read `synthesis.md` and `triage.md` first. Submit prices the run before it fires and refuses anything over `max_cost`, so there is no silent spend. + +**These findings are leads, not evidence.** Each lens is one shot with no cross-file traversal and no runtime verification. Every finding is a `file:line` pointer for the real analysis to confirm — never cite a Broad-Side report as a source in a phase artifact. See [README.md](README.md#broad-side-batch-reconnaissance) and `.codecarto/broadside/SKILL.md`. + +Skip this entirely on a repository you can read directly; a sweep that costs more than the reading it saves is waste. + + ## Step 2: Choose a Pipeline The default is `workflow/pipeline-full-with-deep-audit.yaml` — the full 7-phase pipeline with split defect scan. If that's what you want, skip to Step 3. diff --git a/README.md b/README.md index 02ce3c3..6736bae 100644 --- a/README.md +++ b/README.md @@ -363,6 +363,29 @@ Each workflow tool accepts an absolute `cwd` for the target repository. `codecar --- +## Broad-Side (batch reconnaissance) + +Broad-Side is the cheap sweep you run *before* the expensive interactive run. It fires six analysis lenses — architecture, API surface, security, mechanical defect scan, convention extraction, porting — at a repository as single-turn prompts over the [OpenRouter Batch API](https://openrouter.ai/docs), then cross-references them into one executive report (`synthesis.md`) and a prioritized P0–P3 work order (`triage.md`). + +**Broad-Side findings are unverified scouting leads, not evidence.** Each lens is one shot: no cross-file traversal, no runtime verification, no builds, no tests. Every finding is a `file:line` pointer that the interactive pipeline — or you — must confirm before it is a fact. That division of labor is the point: a sub-dollar unattended sweep that tells the expensive run where to look. Nothing downstream may cite a Broad-Side report as a source. + +It runs on any git repository — no initialized workspace required — and needs an OpenRouter API key (`api_key` parameter, `OPENROUTER_API_KEY` environment variable, or `api_key` in `.codecarto/broadside/config.yaml`). + +``` +codecarto_broadside {cwd, action: "models"} # compare batch models and pricing +codecarto_broadside {cwd, action: "submit", lenses: [...]} # fire the batches, priced first +codecarto_broadside {cwd, action: "status"} # what is in flight +codecarto_broadside {cwd, action: "collect"} # poll, save, synthesize, triage +``` + +Submit and collect are separate because batch jobs routinely take tens of minutes; collect is resumable and picks up whatever is still in flight. Submit prices the run from the collected file sizes against the model's live per-token pricing (cached 24h) and refuses when the estimate exceeds `max_cost` unless `force: true` is passed — a pre-flight estimate, not a runtime stop. Actual spend lands in each run's `run-meta.json`. + +Repository defaults live in `.codecarto/broadside/config.yaml` (`model`, `api_key`, `default_lenses`, `max_cost`, `pricing` overrides, `incremental`, `retry_truncated`, `include_synthesis`, `include_triage`, `wait_seconds`); an explicit tool parameter always wins. `codecarto_skill {cwd, name: "broadside"}` returns the reading guide for a completed run, and unlike post-pipeline skills it is not gated on a finished pipeline. + +Broad-Side is an executable-surface feature and today ships on the **MCP server only** — the Pi command ([#138](https://github.com/HuginnIndustries/CodeCartographer/issues/138)) and the `broadside-scout` pipeline phase ([#139](https://github.com/HuginnIndustries/CodeCartographer/issues/139)) are on the roadmap. See [ROADMAP.md](ROADMAP.md) for what has shipped and what is next. + +--- + ## Compatible environments | Environment | Recommended surface | @@ -507,6 +530,7 @@ If you're testing a new model, start with `pipeline-architecture-only.yaml` on a scratch/ # Disposable notes plus checkpoints and structured phase handoffs. templates/ # Output structure templates. workflow/ # Pipeline definitions, status, validation, config. + broadside/ # Broad-Side batch reconnaissance: config, state, run results. closeouts/ # Per-session closeout files. THREAD_LOG.md # Cross-session summary log. dashboard.html # Generated; gitignored. diff --git a/ROADMAP.md b/ROADMAP.md index bb6f85b..5e2a988 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -27,7 +27,14 @@ file only moves when a tier completes. and clamps lens `max_tokens` to the provider's completion ceiling. - Triage post-pass on collect: findings scored by impact × difficulty into a P0–P3 work order with effort estimates, saved as triage.json/md. -- Tests: 35 unit tests (fake-fetcher based), opt-in live smoke script. +- Reading guide reachable as `codecarto_skill {name: "broadside"}` on both + executable surfaces, exempt from the post-pipeline completion gate; agent + doctrine as the `broadside` topic of `codecarto_guide`; README / MANUAL / + MCP quickstart coverage. +- Repository defaults in `config.yaml` for every per-call run knob + (`incremental`, `retry_truncated`, `include_synthesis`, `include_triage`, + `wait_seconds`); an explicit parameter always wins. +- Tests: fake-fetcher unit suite, opt-in live smoke script (`npm run smoke:broadside`). ## Tier 1 — make Broad-Side better at what it does diff --git a/agent-skill/codecartographer/SKILL.md b/agent-skill/codecartographer/SKILL.md index 566d270..9303cba 100644 --- a/agent-skill/codecartographer/SKILL.md +++ b/agent-skill/codecartographer/SKILL.md @@ -38,6 +38,7 @@ codecarto_status → codecarto_init (first time only) status reports "complete" ``` +0. **`codecarto_broadside`** — optional, and only worth it on a repository too large to read directly. A cheap batch reconnaissance sweep that produces unverified leads telling the pipeline where to look, before you commit to a pipeline choice. Costs real money and is priced before it fires — `references/broadside.md`. 1. **`codecarto_status`** — always start here. It reports the active pipeline, progress, next action, and any scaffold-staleness warning. Every tool takes an absolute `cwd` pointing at the target repository. 2. **`codecarto_init`** — only when no `.codecarto/` exists. Choose the pipeline deliberately (see `references/pipeline-selection.md`). Never pass `force: true` without the user's explicit approval; it moves an existing workspace, findings and all, to a backup directory. 3. **`codecarto_next`** — returns the prompt for the next eligible phase. It returns *text*; it does not execute anything. Use `codecarto_phase` only to force a specific phase out of order, and only when the user asked for that. @@ -46,7 +47,7 @@ codecarto_status → codecarto_init (first time only) 6. **`codecarto_validate`** — parses the validation block you appended to the primary output. Returns `PASS`, `PASS WITH GAPS`, `FAIL`, or `MISSING`. 7. **`codecarto_complete`** — marks the phase done, applies your handoff to canonical state, and writes the closeout and `THREAD_LOG.md` entry. It refuses anything worse than `PASS WITH GAPS`. -When `codecarto_status` reports all phases complete, post-pipeline skills become available via `codecarto_list_skills` and `codecarto_skill`. +When `codecarto_status` reports all phases complete, post-pipeline skills become available via `codecarto_list_skills` and `codecarto_skill`. One name that tool answers to is exempt from that gate: `codecarto_skill {name: "broadside"}` returns the reading guide for a batch reconnaissance run, which is meant to be read *before* the pipeline and during it. ## The handoff contract @@ -133,6 +134,7 @@ If the goal is to rebuild or refactor rather than to understand, two phases carr Running the pipeline: +- `references/broadside.md` — batch reconnaissance: when to scout, cost guardrails, and why its findings are leads rather than evidence - `references/orchestration.md` — the orchestrator's duties, inline vs delegated execution, and the session-by-session fallback's real costs - `references/pipeline-selection.md` — choosing a variant, and switching without losing work - `references/executors.md` — the executor contract, adapters, and model selection diff --git a/agent-skill/codecartographer/references/broadside.md b/agent-skill/codecartographer/references/broadside.md new file mode 100644 index 0000000..7671b62 --- /dev/null +++ b/agent-skill/codecartographer/references/broadside.md @@ -0,0 +1,105 @@ +# Broad-Side: scouting a repository before you spend on it + +Broad-Side is CodeCartographer's batch reconnaissance pass. It fires six +analysis lenses — architecture, API surface, security, mechanical defect scan, +convention extraction, and porting — at a repository as single-turn prompts +over the OpenRouter Batch API, then cross-references them into one executive +report and a prioritized work order. + +It is not a phase, and it does not replace one. It is the cheap sweep that +tells the expensive interactive run where to look. + +## Leads, never evidence + +Every Broad-Side finding is an **unverified scouting signal**. Each lens is one +shot: no cross-file traversal, no runtime verification, no builds, no tests, no +follow-up questions. The batch model is chosen for price, not strength. + +This is the rule the whole feature rests on: a finding is a `file:line` lead +that a phase — or you — must confirm against the source before it is a fact. +Cite the source you confirmed it from, never the Broad-Side report. A "high" +you can neither confirm nor dismiss becomes an open question in your handoff, +not a finding in your report. + +## When to fire it + +- **Before `codecarto_init`,** on a repository nobody on the team knows. The + synthesis report is a map of where the risk sits, which makes the pipeline + choice an informed one instead of a guess. +- **Before an expensive phase,** when the repository is large enough that the + architecture or defect phases would otherwise read blind. +- **Not at all,** when the repository is small enough to read directly. A sweep + that costs more than the reading it saves is waste. + +It works on any git repository — no workspace required — and `codecarto_init` +tolerates a `.codecarto/` that holds nothing but scout state. + +## Driving it + +``` +codecarto_broadside {cwd, action: "models"} # compare batch models first +codecarto_broadside {cwd, action: "submit", lenses: [...]} # fire the batches +codecarto_broadside {cwd, action: "status"} # what is in flight +codecarto_broadside {cwd, action: "collect"} # poll, save, synthesize, triage +``` + +Submit and collect are separate on purpose: batch jobs routinely take tens of +minutes, and nothing is lost by returning between them. Pass `wait_seconds` to +poll inline when you would rather block. Collect is resumable — call it again +and it picks up the batches the recorded state still lists as in flight. + +Requires an OpenRouter API key via the `api_key` parameter, the +`OPENROUTER_API_KEY` environment variable, or `api_key` in +`.codecarto/broadside/config.yaml`. + +## Cost is a first-class parameter + +Submit prices the run before it fires: it estimates from the collected file +sizes against the model's live per-token pricing (cached 24h) and refuses when +the estimate exceeds `max_cost`, printing the per-lens breakdown. `force: true` +overrides. This is a pre-flight estimate, not a runtime stop — actual spend +lands in the run's `run-meta.json`. + +**Never pass `force: true` on the user's behalf without telling them what the +estimate was.** The guardrail exists because the expensive end of the batch +model catalog runs past $80 per million output tokens. + +Two more economies worth knowing: + +- `incremental: true` diffs against the previous run's git HEAD and scans only + the modules whose files changed, falling back to a full scan on a dirty tree. +- Every knob above has a repository default in `.codecarto/broadside/config.yaml` + (`model`, `default_lenses`, `max_cost`, `incremental`, `retry_truncated`, + `include_synthesis`, `include_triage`, `wait_seconds`). An explicit parameter + on the call always wins. + +## Reading a run + +Results land in `.codecarto/broadside//`. Read them in this order: + +1. `synthesis.md` — executive summary, severity counts, top cross-lens + findings, per-module risk. +2. `triage.md` — the same findings scored by impact × difficulty into a P0–P3 + work order with effort estimates. A starting point for re-verification, not + a commitment. +3. The per-lens `*.json` / `*.md` behind whatever matters to the phase you are + about to run: `architecture-*` seeds the architecture phase, `api-*` the + contracts and protocols phases, `security-*` and `defect-*` the defect + scans, `conventions-*` the convention candidates, `porting-*` the porting + phase. +4. `run-meta.json` — scope: which lenses ran, at what cost, with what coverage + caps. + +`codecarto_skill {cwd, name: "broadside"}` returns the full reading guide. +Unlike post-pipeline skills, that one is not gated on a completed pipeline, +because a scout run is meant to be read before the pipeline and during it. + +## Coverage is spoken, not implied + +- A lens output whose JSON does not parse is saved verbatim and marked + `truncated`. The collect summary counts it, `run-meta.json` records it, and + the synthesis prompt is told that module is unrepresented — not clean. By + default a truncated slice is resubmitted once with a doubled output cap. +- A skipped lens, a capped slice, and unscanned scope are all reported. Scope + outside the sweep is **unscouted, not clean**, and saying otherwise in a phase + report is the one way Broad-Side can actively mislead a run. diff --git a/core/broadside.ts b/core/broadside.ts index b4c4f3d..88256d0 100644 --- a/core/broadside.ts +++ b/core/broadside.ts @@ -27,8 +27,10 @@ // (openrouter:web_search etc.), this invariant becomes load-bearing and the // resubmit path must gate on whether any tool executed. // -// Deliberately not in .codecarto/ template prose: Broad-Side requires runtime -// code, so it lives on the executable surfaces (MCP today, Pi on the roadmap). +// Broad-Side requires runtime code, so the feature itself lives on the +// executable surfaces (MCP today, Pi on the roadmap). What the template does +// carry is the reading guide for its output — `.codecarto/broadside/SKILL.md`, +// served by codecarto_skill under the name `broadside` (see readBroadsideSkill). import { mkdir, readFile, readdir, stat, writeFile } from "node:fs/promises"; import { execFile } from "node:child_process"; @@ -36,6 +38,7 @@ import { promisify } from "node:util"; import { dirname, join } from "node:path"; import { pathExists, sleep } from "./utils.ts"; import { loadYamlFile } from "./yaml.ts"; +import { packagedWorkspaceDir } from "./workspace.ts"; const execFileAsync = promisify(execFile); @@ -44,6 +47,8 @@ const execFileAsync = promisify(execFile); export const BROADSIDE_MODEL = "google/gemini-3.7-flash:batch"; export const BROADSIDE_BATCH_URL = "https://openrouter.ai/api/beta/batches"; export const BROADSIDE_DIR = "broadside"; // relative to .codecarto/ +/** Name Broad-Side answers to on the skill surfaces. Not a post-pipeline skill — see readBroadsideSkill. */ +export const BROADSIDE_SKILL_NAME = "broadside"; export const BROADSIDE_STATE_FILE = "state.json"; export const BROADSIDE_CONFIG_FILE = "config.yaml"; export const BROADSIDE_STATE_SCHEMA_VERSION = 1; @@ -224,6 +229,18 @@ export type BroadsideConfig = { maxCost: number; /** Manual pricing overrides (USD per million). Live lookup is preferred. */ pricing: { inputPerM: number; outputPerM: number } | null; + /** + * Repo defaults for the per-call run knobs. Each mirrors a tool parameter + * of the same name; an explicit parameter always wins. They live here so a + * repository can fix its own scouting policy once instead of restating it + * on every submit and collect. + */ + incremental: boolean; + retryTruncated: boolean; + includeSynthesis: boolean; + includeTriage: boolean; + /** Default poll budget in seconds; 0 means "return immediately". */ + waitSeconds: number; }; export type BroadsideSubmitResult = { @@ -1441,6 +1458,33 @@ export function broadsideDirFor(cwd: string): string { return join(cwd, ".codecarto", BROADSIDE_DIR); } +/** + * Read the Broad-Side reading guide. + * + * It is deliberately not a post-pipeline skill under `.codecarto/skills/`: a + * scout run is read *before* or *during* the interactive pipeline, and the + * post-pipeline machinery gates on a completed run and wraps its prompt in + * post-pipeline framing that would be false here. It is also readable on a + * repository that has scout state and no workspace at all, which is why this + * falls back to the packaged copy. + * + * @param cwd - Absolute path to the target repository. + * @returns the skill text and the path it came from. + * @throws when neither the workspace copy nor the packaged copy exists. + */ +export async function readBroadsideSkill(cwd: string): Promise<{ path: string; content: string }> { + const candidates = [ + join(broadsideDirFor(cwd), "SKILL.md"), + join(packagedWorkspaceDir, BROADSIDE_DIR, "SKILL.md"), + ]; + for (const path of candidates) { + if (await pathExists(path)) return { path, content: await readFile(path, "utf8") }; + } + throw new Error( + `Broad-Side skill not found at ${candidates.join(" or ")}. Reinstall codecartographer-pi.`, + ); +} + export function defaultBroadsideState(): BroadsideStateFile { return { schema_version: BROADSIDE_STATE_SCHEMA_VERSION, runs: [] }; } @@ -1478,6 +1522,11 @@ export async function loadBroadsideConfig(broadsideDir: string): Promise; const inputOverride = typeof rawPricing.input_per_m === "number" ? rawPricing.input_per_m : undefined; const outputOverride = typeof rawPricing.output_per_m === "number" ? rawPricing.output_per_m : undefined; + // A malformed value falls back to the shipped default rather than failing + // the run: config.yaml is hand-edited, and a typo in a poll budget must not + // cost a user their batches. + const flag = (key: string, fallback: boolean): boolean => + typeof raw[key] === "boolean" ? (raw[key] as boolean) : fallback; return { model: typeof raw.model === "string" && raw.model.trim() ? raw.model.trim() : BROADSIDE_MODEL, apiKey: typeof raw.api_key === "string" ? raw.api_key.trim() : "", @@ -1487,6 +1536,11 @@ export async function loadBroadsideConfig(broadsideDir: string): Promise 0 ? raw.wait_seconds : 0, }; } diff --git a/docs/mcp-quickstart.md b/docs/mcp-quickstart.md index 1fa7bdd..1ef0b02 100644 --- a/docs/mcp-quickstart.md +++ b/docs/mcp-quickstart.md @@ -153,6 +153,17 @@ Pass `pipeline: ""` to `codecarto_init` to choose. See the [pipeline va Every finding is tagged: `observed fact`, `strong inference`, `portability hazard`, or `open question`. +## Optional: scout first with Broad-Side + +On a repository too large to skim, `codecarto_broadside` fires six analysis lenses at it as cheap asynchronous batch jobs over the OpenRouter Batch API and produces an executive report plus a prioritized work order — a map of where the expensive interactive run should spend its attention. + +``` +codecarto_broadside {cwd: "/abs/path/to/repo", action: "submit"} +codecarto_broadside {cwd: "/abs/path/to/repo", action: "collect"} +``` + +It needs an OpenRouter API key (`api_key` parameter, `OPENROUTER_API_KEY`, or `.codecarto/broadside/config.yaml`) and works on any git repository, with or without a workspace. Submit prices the run first and refuses anything over `max_cost`. Its findings are **unverified leads, not evidence** — see the [Broad-Side section](../README.md#broad-side-batch-reconnaissance) in the README. + ## No agent? Use the drop-in template If your tool doesn't speak MCP, copy the template directly: diff --git a/extensions/codecarto/index.ts b/extensions/codecarto/index.ts index 1188516..6ec09a9 100644 --- a/extensions/codecarto/index.ts +++ b/extensions/codecarto/index.ts @@ -29,6 +29,7 @@ import { getWorkspaceState, isWithinPath, isWithinPathResolved, + BROADSIDE_SKILL_NAME, listSkillNames, loadCodecartoConfig, loadUsage, @@ -37,6 +38,7 @@ import { packagedWorkspaceDir, pathExists, PACKAGE_VERSION, + readBroadsideSkill, PhasePreflightError, type PhasePreflightResult, PIPELINE_ALIASES, @@ -680,6 +682,30 @@ export default function codeCartographerExtension(pi: ExtensionAPI) { return; } + // Broad-Side is a reading guide for batch reconnaissance output, not a + // post-pipeline skill: it is read before or during the pipeline and works + // on a repository with scout state and no workspace. Same exemption the + // MCP surface makes in handleSkill. + if (skillName === BROADSIDE_SKILL_NAME) { + const skill = await readBroadsideSkill(ctx.cwd).catch(() => null); + if (!skill) { + ctx.ui.notify("Broad-Side reading guide not found. Reinstall codecartographer-pi.", "error"); + return; + } + const message = [ + "Read the Broad-Side reading guide below and apply it to the batch reconnaissance results in .codecarto/broadside/.", + "", + skill.content, + ].join("\n"); + if (ctx.isIdle()) { + pi.sendUserMessage(message); + } else { + pi.sendUserMessage(message, { deliverAs: "followUp" }); + } + ctx.ui.notify("Queued the Broad-Side reading guide", "info"); + return; + } + const state = await ensureWorkspaceState(ctx); if (!state) return; @@ -696,7 +722,10 @@ export default function codeCartographerExtension(pi: ExtensionAPI) { if (!(await pathExists(skillFile))) { const available = await listSkillNames(state.workspaceDir); const hint = available.length > 0 ? ` (available: ${available.join(", ")})` : " (no skills installed)"; - ctx.ui.notify(`Unknown skill: ${skillName}${hint}`, "error"); + ctx.ui.notify( + `Unknown skill: ${skillName}${hint}. The Broad-Side reading guide is served as \`${BROADSIDE_SKILL_NAME}\` and is not pipeline-gated.`, + "error", + ); return; } diff --git a/mcp-server/server.ts b/mcp-server/server.ts index 1e557e5..3935570 100644 --- a/mcp-server/server.ts +++ b/mcp-server/server.ts @@ -29,6 +29,7 @@ import { BROADSIDE_DIR, broadsideDirFor, BROADSIDE_LENS_IDS, + BROADSIDE_SKILL_NAME, type BroadsideLensId, canonicalPath, collectResultText, @@ -78,6 +79,7 @@ import { refreshScaffold, resolvePhase, resolvePipelineChoice, + readBroadsideSkill, runBroadsideCollect, runBroadsideStatus, runBroadsideSubmit, @@ -434,6 +436,17 @@ export async function handleSkill(args: { cwd: string; name: string }) { throw new McpError(ErrorCode.InvalidParams, "name is required"); } const cwd = await validateCwd(args.cwd); + + // Broad-Side is a reading guide for batch reconnaissance output, not a + // post-pipeline skill: it is useful before the pipeline starts and on a + // repository with no workspace at all, so it is served ahead of both gates. + if (args.name.trim() === BROADSIDE_SKILL_NAME) { + const skill = await readBroadsideSkill(cwd).catch((error) => { + throw new McpError(ErrorCode.InvalidRequest, error instanceof Error ? error.message : String(error)); + }); + return textResult(skill.content, { skill: BROADSIDE_SKILL_NAME, path: skill.path, postPipeline: false }); + } + const state = await requireWorkspace(cwd); const nextPhase = getNextEligiblePhase(state); if (nextPhase) { @@ -446,7 +459,10 @@ export async function handleSkill(args: { cwd: string; name: string }) { if (!(await pathExists(skillFile))) { const available = await listSkillNames(state.workspaceDir); const hint = available.length > 0 ? ` Available: ${available.join(", ")}.` : " No skills installed."; - throw new McpError(ErrorCode.InvalidParams, `Unknown skill: ${args.name}.${hint}`); + throw new McpError( + ErrorCode.InvalidParams, + `Unknown skill: ${args.name}.${hint} The Broad-Side reading guide is served as \`${BROADSIDE_SKILL_NAME}\` and is not pipeline-gated.`, + ); } const prompt = await buildSkillPrompt(state, args.name); return textResult(prompt, { skill: args.name }); @@ -925,7 +941,17 @@ export async function handleListSkills(args: { cwd: string }) { const lines = skills.length > 0 ? [`Available skills (${skills.length}):`, ...skills.map((s) => ` - ${s}`)] : ["No skills installed."]; - return textResult(lines.join("\n"), { skills }); + + // Broad-Side is listed apart from the post-pipeline set because it answers + // to codecarto_skill without the completion gate. + const broadsideAvailable = await readBroadsideSkill(cwd).then(() => true, () => false); + if (broadsideAvailable) { + lines.push( + "", + `Also served by codecarto_skill (not pipeline-gated): ${BROADSIDE_SKILL_NAME} — how to read a Broad-Side batch reconnaissance run.`, + ); + } + return textResult(lines.join("\n"), { skills, broadside: broadsideAvailable }); } export async function handleRefreshScaffold(args: { cwd: string }) { @@ -1022,7 +1048,16 @@ export async function handleBroadside(args: { } const apiKey = resolveBroadsideApiKey(args.api_key, config); - const waitMs = typeof args.wait_seconds === "number" && args.wait_seconds > 0 ? args.wait_seconds * 1000 : undefined; + // Every run knob resolves the same way: explicit parameter, else the repo's + // config.yaml default, else the shipped default baked into loadBroadsideConfig. + const waitSeconds = typeof args.wait_seconds === "number" && args.wait_seconds > 0 + ? args.wait_seconds + : config.waitSeconds; + const waitMs = waitSeconds > 0 ? waitSeconds * 1000 : undefined; + const includeSynthesis = args.include_synthesis ?? config.includeSynthesis; + const includeTriage = args.include_triage ?? config.includeTriage; + const retryTruncated = args.retry_truncated ?? config.retryTruncated; + const incremental = args.incremental ?? config.incremental; if (action === "models") { const { entries, benchmarks } = await listBatchModels(broadsideDirFor(cwd), config, apiKey, { @@ -1056,7 +1091,7 @@ export async function handleBroadside(args: { model: config.model, maxCost, force: args.force === true, - incremental: args.incremental === true, + incremental, }).catch((error) => { throw new McpError(ErrorCode.InvalidRequest, error instanceof Error ? error.message : String(error)); }); @@ -1066,9 +1101,9 @@ export async function handleBroadside(args: { lines.push("", "Waiting for batches to complete..."); const collect = await runBroadsideCollect(cwd, apiKey, { waitMs, - includeSynthesis: args.include_synthesis !== false, - includeTriage: args.include_triage !== false, - retryTruncated: args.retry_truncated !== false, + includeSynthesis, + includeTriage, + retryTruncated, onStatus: (lensId, status, counts) => lines.push(` ${lensId}: ${status} (${counts.completed ?? 0}/${counts.total ?? "?"})`), }); @@ -1087,9 +1122,9 @@ export async function handleBroadside(args: { // action === "collect" const collect = await runBroadsideCollect(cwd, apiKey, { waitMs, - includeSynthesis: args.include_synthesis !== false, - includeTriage: args.include_triage !== false, - retryTruncated: args.retry_truncated !== false, + includeSynthesis, + includeTriage, + retryTruncated, }).catch((error) => { throw new McpError(ErrorCode.InvalidRequest, error instanceof Error ? error.message : String(error)); }); @@ -1208,12 +1243,12 @@ const TOOLS = [ { name: "codecarto_skill", description: - "Return the prompt text for a post-pipeline skill (only callable after all phases are complete). Use codecarto_status to confirm completion first.", + "Return the prompt text for a post-pipeline skill (only callable after all phases are complete). Use codecarto_status to confirm completion first. One name is exempt from the completion gate: \"broadside\" returns the reading guide for a Broad-Side batch reconnaissance run, which is meant to be read before or during the pipeline and works without a workspace.", inputSchema: { type: "object", properties: { cwd: { type: "string", description: "Absolute path to the target repository." }, - name: { type: "string", description: "Skill name (a directory under .codecarto/skills/)." }, + name: { type: "string", description: "Skill name (a directory under .codecarto/skills/), or \"broadside\" for the Broad-Side reading guide." }, }, required: ["cwd", "name"], }, @@ -1371,7 +1406,8 @@ const TOOLS = [ }, { name: "codecarto_list_skills", - description: "List available post-pipeline skills installed in the workspace.", + description: + "List available post-pipeline skills installed in the workspace, plus the Broad-Side reading guide when it is present (that one is not pipeline-gated).", inputSchema: { type: "object", properties: { cwd: { type: "string", description: "Absolute path to the target repository." } }, @@ -1425,21 +1461,21 @@ const TOOLS = [ }, wait_seconds: { type: "number", - description: "For submit: after submitting, poll up to this many seconds before returning. For collect: poll up to this many seconds before returning with partial state.", + description: "For submit: after submitting, poll up to this many seconds before returning. For collect: poll up to this many seconds before returning with partial state. Falls back to wait_seconds in .codecarto/broadside/config.yaml.", }, include_synthesis: { type: "boolean", - description: "Run the cross-lens synthesis pass once all lens batches complete (default true).", + description: "Run the cross-lens synthesis pass once all lens batches complete. Falls back to include_synthesis in .codecarto/broadside/config.yaml (default true).", }, include_triage: { type: "boolean", description: - "Run the triage pass once all lens batches complete: turns the findings into a prioritized work order (impact × difficulty, P0-P3, effort estimates). Default true.", + "Run the triage pass once all lens batches complete: turns the findings into a prioritized work order (impact × difficulty, P0-P3, effort estimates). Falls back to include_triage in .codecarto/broadside/config.yaml (default true).", }, retry_truncated: { type: "boolean", description: - "Re-submit lens results that came back truncated at the output token limit, once, with a doubled output cap. Default true.", + "Re-submit lens results that came back truncated at the output token limit, once, with a doubled output cap. Falls back to retry_truncated in .codecarto/broadside/config.yaml (default true).", }, max_cost: { type: "number", @@ -1453,7 +1489,7 @@ const TOOLS = [ incremental: { type: "boolean", description: - "Diff against the previous run's git HEAD and scan only the modules whose files changed (falls back to a full scan on a dirty tree or when no prior run exists). Default false.", + "Diff against the previous run's git HEAD and scan only the modules whose files changed (falls back to a full scan on a dirty tree or when no prior run exists). Falls back to incremental in .codecarto/broadside/config.yaml (default false).", }, include_benchmarks: { type: "boolean", diff --git a/package.json b/package.json index 2cf794f..7f13d85 100644 --- a/package.json +++ b/package.json @@ -49,6 +49,7 @@ "prepublishOnly": "npm run build", "test": "node --experimental-strip-types --disable-warning=ExperimentalWarning --test tests/*.test.mjs", "smoke": "node scripts/smoke-mcp.mjs", + "smoke:broadside": "node scripts/smoke-broadside.mjs", "demo:synthesis": "npm run build && node scripts/create-synthesis-demo.mjs" }, "dependencies": { diff --git a/tests/broadside.test.mjs b/tests/broadside.test.mjs index 67f51cc..34dfbdc 100644 --- a/tests/broadside.test.mjs +++ b/tests/broadside.test.mjs @@ -311,6 +311,90 @@ test("config falls back to defaults and honors overrides", async () => { } }); +test("config carries repo defaults for every per-call run knob", async () => { + const dir = await mkdtemp(join(tmpdir(), "broadside-run-defaults-")); + try { + const defaults = await loadBroadsideConfig(dir); + assert.equal(defaults.incremental, false); + assert.equal(defaults.retryTruncated, true, "truncation repair is on unless a repo turns it off"); + assert.equal(defaults.includeSynthesis, true); + assert.equal(defaults.includeTriage, true); + assert.equal(defaults.waitSeconds, 0, "0 means submit and collect later"); + + await writeFile( + join(dir, "config.yaml"), + [ + "incremental: true", + "retry_truncated: false", + "include_synthesis: false", + "include_triage: false", + "wait_seconds: 600", + "", + ].join("\n"), + ); + const set = await loadBroadsideConfig(dir); + assert.equal(set.incremental, true); + assert.equal(set.retryTruncated, false); + assert.equal(set.includeSynthesis, false); + assert.equal(set.includeTriage, false); + assert.equal(set.waitSeconds, 600); + + // config.yaml is hand-edited: a typo must not cost a user their batches. + await writeFile(join(dir, "config.yaml"), "incremental: yes-please\nwait_seconds: soon\n"); + const malformed = await loadBroadsideConfig(dir); + assert.equal(malformed.incremental, false, "a non-boolean flag falls back to the shipped default"); + assert.equal(malformed.waitSeconds, 0, "a non-numeric poll budget falls back to the shipped default"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("every documented config key is one loadBroadsideConfig actually reads", async () => { + // The commented-out keys in the shipped config.yaml are the only + // documentation a user gets. A key documented but not parsed reads as a + // working setting that silently does nothing. + const shipped = await readFile(join(REPO_ROOT, ".codecarto", "broadside", "config.yaml"), "utf8"); + const documented = [...shipped.matchAll(/^#\s{0,3}([a-z_]+):/gm)].map((match) => match[1]); + const parsed = new Set([ + "model", + "api_key", + "default_lenses", + "max_cost", + "pricing", + "input_per_m", + "output_per_m", + "incremental", + "retry_truncated", + "include_synthesis", + "include_triage", + "wait_seconds", + ]); + const undocumented = [...parsed].filter((key) => !documented.includes(key)); + assert.deepEqual(undocumented, [], `config.yaml does not document: ${undocumented.join(", ")}`); + for (const key of documented) { + assert.ok(parsed.has(key), `config.yaml documents ${key}, which loadBroadsideConfig does not read`); + } +}); + +test("the Broad-Side reading guide is readable from a repo with no workspace", async () => { + const dir = await mkdtemp(join(tmpdir(), "broadside-skill-")); + try { + // No .codecarto/ at all: the packaged copy answers. + const packaged = await core.readBroadsideSkill(dir); + assert.match(packaged.content, /# Broad-Side/); + assert.ok(packaged.path.includes(join("broadside", "SKILL.md"))); + + // A workspace copy wins, so a user's edits to their own scaffold are served. + await mkdir(join(dir, ".codecarto", "broadside"), { recursive: true }); + await writeFile(join(dir, ".codecarto", "broadside", "SKILL.md"), "# Local guide\n"); + const local = await core.readBroadsideSkill(dir); + assert.equal(local.content, "# Local guide\n"); + assert.equal(local.path, join(dir, ".codecarto", "broadside", "SKILL.md")); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + // ---------- pricing resolution & expense limits ---------- function modelsCatalog(body) { diff --git a/tests/guide.test.mjs b/tests/guide.test.mjs index 8a4480c..6cc84c9 100644 --- a/tests/guide.test.mjs +++ b/tests/guide.test.mjs @@ -90,6 +90,19 @@ test("the rewrite references carry the disposition vocabulary the porting phase assert.match(kernel, /fake/i, "must describe the fake-driven acceptance harness"); }); +test("the Broad-Side reference states the rule the feature rests on", async () => { + // Broad-Side output is a cheap model's one-shot guesses. The single way it + // can actively harm a run is an agent citing a lead as a finding, so the + // reference that teaches the tool must say so outright. + const topics = await core.listGuideTopics(); + assert.ok(topics.includes("broadside"), "broadside must be a guide topic"); + const broadside = (await core.readGuide("broadside")).content; + assert.match(broadside, /unverified/i, "must mark findings unverified"); + assert.match(broadside, /never (be )?evidence|not evidence|leads, never evidence/i); + assert.match(broadside, /max_cost/, "must name the cost guardrail"); + assert.match(broadside, /codecarto_broadside/, "must name the tool an agent calls"); +}); + test("SKILL.md carries frontmatter a skill installer can read", async () => { const raw = await readFile(join(SKILL_DIR, "SKILL.md"), "utf8"); assert.match(raw, /^---\r?\n/, "SKILL.md must open with frontmatter"); diff --git a/tests/mcp-server.test.mjs b/tests/mcp-server.test.mjs index b0e3096..0e65554 100644 --- a/tests/mcp-server.test.mjs +++ b/tests/mcp-server.test.mjs @@ -23,6 +23,7 @@ const { handleValidate, handleComplete, handleSkill, + handleListSkills, } = await import(pathToFileURL(`${REPO_ROOT}/mcp-server/server.ts`).href); const { buildPhasePrompt, getNextEligiblePhase, getWorkspaceState } = await import(pathToFileURL(`${REPO_ROOT}/core/index.ts`).href); const { McpError, ErrorCode } = await import("@modelcontextprotocol/sdk/types.js"); @@ -104,6 +105,39 @@ test("handleSkill refuses while pipeline is incomplete", async () => { ); }); +test("handleSkill serves the Broad-Side reading guide without the completion gate", async () => { + // Scout leads are read before and during the pipeline, so the one skill + // name that is not a post-pipeline skill must not be gated on completion. + const result = await handleSkill({ cwd: WORKSPACE, name: "broadside" }); + assert.match(result.content[0].text, /# Broad-Side/); + assert.match(result.content[0].text, /unverified scouting signals/); + assert.equal(result.structuredContent.skill, "broadside"); + assert.equal(result.structuredContent.postPipeline, false); +}); + +test("handleListSkills advertises Broad-Side apart from the post-pipeline set", async () => { + const result = await handleListSkills({ cwd: WORKSPACE }); + assert.equal(result.structuredContent.broadside, true); + assert.match(result.content[0].text, /not pipeline-gated.*broadside/s); + assert.ok( + !result.structuredContent.skills.includes("broadside"), + "broadside is not a post-pipeline skill and must not be listed as one", + ); +}); + +test("an unknown skill name points at the Broad-Side exemption", async () => { + await assert.rejects( + handleSkill({ cwd: WORKSPACE, name: "no-such-skill" }), + (error) => { + assert.ok(error instanceof McpError, "expected McpError"); + // The completion gate fires first for a workspace mid-pipeline; either + // message is acceptable, but the name must never dead-end silently. + assert.match(error.message, /pipeline is not complete|Unknown skill/); + return true; + }, + ); +}); + test("handleSkill reports unknown skill names with available list", async () => { // Hard-stub by emptying status.yaml's phase_order — easier path: just expect // the "pipeline not complete" guard to fire first if no phases are done. diff --git a/tests/pi-extension-activation.test.mjs b/tests/pi-extension-activation.test.mjs index 7de7ef6..5056b36 100644 --- a/tests/pi-extension-activation.test.mjs +++ b/tests/pi-extension-activation.test.mjs @@ -22,7 +22,10 @@ function createHarness(cwd) { pi.sessionName = name; }, sendMessage: () => {}, - sendUserMessage: () => {}, + sentUserMessages: [], + sendUserMessage: (message) => { + pi.sentUserMessages.push(message); + }, activeTools: undefined, sessionName: undefined, }; @@ -105,6 +108,21 @@ test("/codecarto-init activates the CodeCartographer UI and read-only tool polic }); }); +test("/codecarto-skill broadside serves the reading guide without a workspace", async () => { + // Parity with the MCP surface: Broad-Side is not a post-pipeline skill, so + // neither the completion gate nor the workspace requirement applies to it. + await withTempRepo(async (cwd) => { + const { commands, pi, ctx, ui } = createHarness(cwd); + + await commands.get("codecarto-skill").handler("broadside", ctx); + + assert.equal(pi.sentUserMessages.length, 1, "the guide should be queued as one message"); + assert.match(pi.sentUserMessages[0], /# Broad-Side/); + assert.match(pi.sentUserMessages[0], /unverified scouting signals/); + assert.equal(ui.notifications.at(-1).level, "info"); + }); +}); + test("/codecarto-open activates an existing workspace without resetting durable state", async () => { await withTempRepo(async (cwd) => { await cp(join(REPO_ROOT, ".codecarto"), join(cwd, ".codecarto"), { recursive: true }); From a18d39445ce54b1cc6f84809df09f0c03db40f33 Mon Sep 17 00:00:00 2001 From: James Sesler Date: Sun, 30 Aug 2026 15:36:08 -0400 Subject: [PATCH 2/4] feat: /codecarto-broadside brings batch reconnaissance to the Pi surface (#138) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Broad-Side shipped MCP-first by agreement, leaving the recommended user surface without it. The Pi command exposes the same four actions with tab-completion for actions, lens names, and flags, and a live per-lens progress widget while batches poll. Two divergences from MCP, both deliberate: - The spend decision is interactive. MCP cannot ask a human, so it refuses a run over max_cost until the caller passes force. Pi shows the per-lens breakdown, the rates, the limit and whether the run exceeds it, then asks — and an approval IS the force flag. runBroadsideSubmit grows an optional confirm hook that fires after slicing and before any state write or submission; declining throws BroadsideCancelledError and nothing is submitted. Surfaces with no human omit the hook and keep the old path. - The command takes no API key argument. A key typed into a slash command lands in the session transcript, so it is OPENROUTER_API_KEY or config.yaml only, and the error message says why. Like the MCP tool it runs on a repository with no workspace; there the result renders into its own widget rather than the phase widget, which has no state to draw. Argument parsing lives in broadside-flags.ts following the parseNextFlags idiom: never throws, collects unknown tokens, and refuses flags that mean nothing for the chosen action rather than ignoring them — silently dropping --incremental on a collect would read as "collected incrementally". Tests: the confirm hook's decline-and-approve paths at the core level, the flag grammar including a check that every completion token the command offers is one the parser accepts, and the command end-to-end through a fake Pi harness with stubbed fetch. Two new invariants: the extension must register a command for every framework operation (broadside included), and the README table must name every registered command. 430 pass. Co-Authored-By: Claude Opus 5 --- .codecarto/broadside/SKILL.md | 11 +- CHANGELOG.md | 1 + CLAUDE.md | 2 +- MANUAL.md | 11 +- README.md | 9 +- ROADMAP.md | 5 +- .../codecartographer/references/broadside.md | 7 +- core/broadside.ts | 68 ++++- extensions/codecarto/broadside-flags.ts | 121 +++++++++ extensions/codecarto/index.ts | 237 ++++++++++++++++++ tests/broadside-flags.test.mjs | 100 ++++++++ tests/broadside.test.mjs | 60 +++++ tests/default-pipeline.test.mjs | 13 +- tests/pi-broadside.test.mjs | 187 ++++++++++++++ 14 files changed, 819 insertions(+), 13 deletions(-) create mode 100644 extensions/codecarto/broadside-flags.ts create mode 100644 tests/broadside-flags.test.mjs create mode 100644 tests/pi-broadside.test.mjs diff --git a/.codecarto/broadside/SKILL.md b/.codecarto/broadside/SKILL.md index 11ad38c..3322138 100644 --- a/.codecarto/broadside/SKILL.md +++ b/.codecarto/broadside/SKILL.md @@ -59,7 +59,16 @@ not replace any phase; it tells phases where to look. ## Running Broad-Side -Broad-Side is an executable-surface feature (MCP today): +Broad-Side is an executable-surface feature. On the Pi extension: + +``` +/codecarto-broadside submit [lenses…] # prices the run, asks, then fires +/codecarto-broadside collect # poll, save, synthesize +/codecarto-broadside status # show recorded runs +/codecarto-broadside models # compare batch models +``` + +On the MCP server: ``` codecarto_broadside {cwd, action: "submit", lenses: [...]} # fire the batches diff --git a/CHANGELOG.md b/CHANGELOG.md index 36c316d..66b0b72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to this project are documented here. The format is based on ### Added +- **Broad-Side on the Pi extension** (#138). New `/codecarto-broadside [submit|collect|status|models] [lenses…]` with tab-completion for actions, lens names, and flags (`--incremental`, `--max-cost=N`, `--wait=SECONDS`, `--no-synthesis`, `--no-triage`, `--no-retry-truncated`, `--benchmarks`), and a live per-lens progress widget while batches poll. Two deliberate divergences from MCP: the spend decision is interactive — Pi shows the per-lens breakdown, the rates, the limit, and whether the run exceeds it, and an approval *is* the force flag — where MCP has to refuse and wait for `force: true`; and the command takes no API key argument, because a key typed into a slash command lands in the session transcript (`OPENROUTER_API_KEY` or `config.yaml` only). Like the MCP tool, it runs on a repository with no CodeCartographer workspace, and on such a repository the result renders into its own widget rather than the phase widget. `runBroadsideSubmit` grows an optional `confirm` hook that receives the pre-flight estimate after slicing and before any state write or submission; declining throws `BroadsideCancelledError` and nothing is submitted. Surfaces without a human keep the refuse-unless-force path unchanged. - **Broad-Side: the reading guide is reachable, and the docs say the feature exists.** Broad-Side shipped across seven PRs with its user-facing paper trail lagging behind the code. `.codecarto/broadside/SKILL.md` was written but unreachable: `codecarto_skill` resolves `.codecarto/skills//SKILL.md`, so `{name: "broadside"}` returned "Unknown skill" and `codecarto_list_skills` never mentioned it. Both surfaces now serve it under the name `broadside`, exempt from the post-pipeline completion gate — a scout run is read *before* the pipeline and during it — and readable on a repository that has scout state and no workspace at all (the packaged copy answers when the workspace has none). `codecarto_list_skills` lists it apart from the post-pipeline set, and an unknown-skill error names the exemption. New `references/broadside.md` in the packaged agent skill teaches when to scout, the cost guardrails, and the leads-never-evidence rule, served as the `broadside` topic of `codecarto_guide`; the skill overview, README, MANUAL, and the MCP quickstart now cover the feature instead of leaving a single table row as its only mention. - **Broad-Side: every run knob has a repository default** (`.codecarto/broadside/config.yaml`). `incremental`, `retry_truncated`, `include_synthesis`, `include_triage`, and `wait_seconds` were call-parameters only, so a repo could not fix its own scouting policy without restating it on every submit and collect. All five are now config keys documented alongside `model`, `api_key`, `default_lenses`, `max_cost`, and the `pricing` overrides; an explicit tool parameter always wins, and a malformed value falls back to the shipped default rather than failing a run. A test asserts the documented key set and the parsed key set are the same, so a key can no longer be documented into existence without being read. - **`npm run smoke:broadside`** wires the existing opt-in live Broad-Side smoke script to a script name. It still skips cleanly without `OPENROUTER_API_KEY` and still spends real money when it runs. diff --git a/CLAUDE.md b/CLAUDE.md index ace3bad..942d8b7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -51,7 +51,7 @@ When adding a feature, the question to ask is "does this work on all three surfa - `core/orchestrator-config.ts` — loads `.codecarto/workflow/config.yaml` (the `orchestrator.llm_steer_next_phase` flag lives here). - `core/library.ts` — versioned library discovery, publication, reads, listing, reindexing, and optional git commits shared by Pi and MCP. - `core/synthesis.ts` — vision/library/proposal preflight and exact confirmed-version resolution for the four-phase synthesis workflow. -- `core/broadside.ts` — Broad-Side batch reconnaissance: lens registry and prompts, repo slicing, OpenRouter Batch API submit/poll/collect, model catalog and cost pre-flight, synthesis and triage post-passes, and `.codecarto/broadside/` state. Executable-surface only (MCP today); the template carries just the reading guide. +- `core/broadside.ts` — Broad-Side batch reconnaissance: lens registry and prompts, repo slicing, OpenRouter Batch API submit/poll/collect, model catalog and cost pre-flight, synthesis and triage post-passes, and `.codecarto/broadside/` state. Executable surfaces only (Pi via `/codecarto-broadside`, MCP via `codecarto_broadside`); the template carries just the reading guide. ### Pipeline shape diff --git a/MANUAL.md b/MANUAL.md index a882efb..e1e7ff9 100644 --- a/MANUAL.md +++ b/MANUAL.md @@ -57,14 +57,19 @@ The LLM reads your source code directly from the repository root. No symlinking On a repository too large to skim, a **Broad-Side** batch reconnaissance run is worth firing before you choose a pipeline. It sends six analysis lenses at the code as cheap asynchronous batch jobs and produces an executive report plus a P0-P3 work order, which turns the pipeline choice below into an informed one instead of a guess. -Broad-Side needs the MCP server and an OpenRouter API key: +Broad-Side needs Pi or the MCP server, plus an OpenRouter API key: ``` -codecarto_broadside {cwd: "/path/to/repo", action: "submit"} +/codecarto-broadside submit # Pi: prices the run and asks before spending +/codecarto-broadside collect +``` + +``` +codecarto_broadside {cwd: "/path/to/repo", action: "submit"} # MCP codecarto_broadside {cwd: "/path/to/repo", action: "collect"} ``` -Results land in `.codecarto/broadside//`; read `synthesis.md` and `triage.md` first. Submit prices the run before it fires and refuses anything over `max_cost`, so there is no silent spend. +Results land in `.codecarto/broadside//`; read `synthesis.md` and `triage.md` first. Submit prices the run before it fires: Pi shows the breakdown and asks, MCP refuses anything over `max_cost` until you pass `force`. Either way there is no silent spend. **These findings are leads, not evidence.** Each lens is one shot with no cross-file traversal and no runtime verification. Every finding is a `file:line` pointer for the real analysis to confirm — never cite a Broad-Side report as a source in a phase artifact. See [README.md](README.md#broad-side-batch-reconnaissance) and `.codecarto/broadside/SKILL.md`. diff --git a/README.md b/README.md index 6736bae..7b17ae3 100644 --- a/README.md +++ b/README.md @@ -318,7 +318,8 @@ Beyond the slash commands, the Pi extension layers on: | `/codecarto-phase ` | Force a specific phase, even out of pipeline order | | `/codecarto-validate [phase]` | Validate a phase output against completion criteria | | `/codecarto-complete [phase]` | Validate and atomically apply the phase handoff, canonical status, closeout, and log entry | -| `/codecarto-skill ` | Run a post-pipeline skill once all phases are complete | +| `/codecarto-skill ` | Run a post-pipeline skill once all phases are complete (or `broadside` any time, for the scout reading guide) | +| `/codecarto-broadside [action] [lenses…]` | Batch reconnaissance (Broad-Side). Actions: `submit`, `collect`, `status`, `models`. Prices the run and asks before spending; works with or without a workspace | | `/codecarto-publish` | Publish the reimplementation spec to the configured library after reviewing an explicit confirmation preview | | `/codecarto-library-init [--namespace ]` | Create a library directory with marker and write the config — fixes the first-publish dead end | | `/codecarto-config` | Show the effective merged configuration (global + workspace) and library marker status | @@ -357,7 +358,7 @@ Implements MCP spec revision [`2025-11-25`](https://modelcontextprotocol.io/spec | `codecarto_publish` | MCP-only library publish | | `codecarto_library_list` | MCP-only library listing | | `codecarto_library_reindex` | MCP-only library reindex | -| `codecarto_broadside` | MCP-only batch reconnaissance (Broad-Side) | +| `codecarto_broadside` | `/codecarto-broadside` | Each workflow tool accepts an absolute `cwd` for the target repository. `codecarto_init` requires `force: true` to overwrite an existing `.codecarto/` (instead of Pi's interactive confirmation). The library tools accept an explicit absolute `library_path` or resolve `library.path` from `.codecarto/workflow/config.yaml` / `~/.codecarto/config.yaml`. The library schema is experimental and may break before v2. @@ -382,7 +383,9 @@ Submit and collect are separate because batch jobs routinely take tens of minute Repository defaults live in `.codecarto/broadside/config.yaml` (`model`, `api_key`, `default_lenses`, `max_cost`, `pricing` overrides, `incremental`, `retry_truncated`, `include_synthesis`, `include_triage`, `wait_seconds`); an explicit tool parameter always wins. `codecarto_skill {cwd, name: "broadside"}` returns the reading guide for a completed run, and unlike post-pipeline skills it is not gated on a finished pipeline. -Broad-Side is an executable-surface feature and today ships on the **MCP server only** — the Pi command ([#138](https://github.com/HuginnIndustries/CodeCartographer/issues/138)) and the `broadside-scout` pipeline phase ([#139](https://github.com/HuginnIndustries/CodeCartographer/issues/139)) are on the roadmap. See [ROADMAP.md](ROADMAP.md) for what has shipped and what is next. +On the Pi extension the same run is `/codecarto-broadside [submit|collect|status|models] [lenses…]`, with tab-completion for actions and lens names and live per-lens progress while batches poll. The two surfaces differ in one deliberate place: MCP cannot ask a human, so it refuses a run over `max_cost` until you pass `force`; Pi shows the per-lens breakdown and asks, and your approval *is* the force flag. Neither surface takes an API key as a command argument — a key typed into a slash command lands in the session transcript. + +Broad-Side needs runtime code, so it is an executable-surface feature: Pi and MCP have it, the pure drop-in template does not (it carries only the reading guide). The `broadside-scout` pipeline phase ([#139](https://github.com/HuginnIndustries/CodeCartographer/issues/139)) is still on the roadmap — nothing in a pipeline YAML consumes scout output yet; you read it and decide. See [ROADMAP.md](ROADMAP.md). --- diff --git a/ROADMAP.md b/ROADMAP.md index 5e2a988..4f876bf 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -27,6 +27,9 @@ file only moves when a tier completes. and clamps lens `max_tokens` to the provider's completion ceiling. - Triage post-pass on collect: findings scored by impact × difficulty into a P0–P3 work order with effort estimates, saved as triage.json/md. +- Pi surface: `/codecarto-broadside [submit|collect|status|models] [lenses…]` + with a spend confirmation (`confirm` hook on `runBroadsideSubmit`), so Pi asks + where MCP must refuse. - Reading guide reachable as `codecarto_skill {name: "broadside"}` on both executable surfaces, exempt from the post-pipeline completion gate; agent doctrine as the `broadside` topic of `codecarto_guide`; README / MANUAL / @@ -49,7 +52,7 @@ file only moves when a tier completes. | Item | Issue | Notes | |---|---|---| -| **Pi extension** — `/codecarto-broadside` command with lens picker and live progress | [#138](https://github.com/HuginnIndustries/CodeCartographer/issues/138) | Agreed order: MCP first (shipped), Pi second | +| **Pi extension** — `/codecarto-broadside` command with lens picker and live progress | [#138](https://github.com/HuginnIndustries/CodeCartographer/issues/138) | **Shipped**: four actions with tab-completed lens picker, live per-lens progress widget, and an interactive spend confirmation in place of MCP's refuse-unless-`force` | | **Pipeline phase** — `broadside-scout` phase feeding later phases via `required_reads` | [#139](https://github.com/HuginnIndustries/CodeCartographer/issues/139) | SKILL.md contract stays: leads, never evidence | | **Zero-config executive** — meta-pass picks lenses and slicing resolution from repo shape | [#140](https://github.com/HuginnIndustries/CodeCartographer/issues/140) | **Shipped**: `auto` slicing collapses small repos to one slice, directory-splits large ones | diff --git a/agent-skill/codecartographer/references/broadside.md b/agent-skill/codecartographer/references/broadside.md index 7671b62..105d427 100644 --- a/agent-skill/codecartographer/references/broadside.md +++ b/agent-skill/codecartographer/references/broadside.md @@ -43,6 +43,8 @@ codecarto_broadside {cwd, action: "status"} # what is in flig codecarto_broadside {cwd, action: "collect"} # poll, save, synthesize, triage ``` +(The Pi extension exposes the same four actions as `/codecarto-broadside [lenses…]`.) + Submit and collect are separate on purpose: batch jobs routinely take tens of minutes, and nothing is lost by returning between them. Pass `wait_seconds` to poll inline when you would rather block. Collect is resumable — call it again @@ -62,7 +64,10 @@ lands in the run's `run-meta.json`. **Never pass `force: true` on the user's behalf without telling them what the estimate was.** The guardrail exists because the expensive end of the batch -model catalog runs past $80 per million output tokens. +model catalog runs past $80 per million output tokens. On MCP the refusal is +the only protection there is — the server cannot ask, which is exactly why +`force` must be the user's decision rather than your retry. (Pi has a human to +ask, so it shows the breakdown and prompts instead of refusing.) Two more economies worth knowing: diff --git a/core/broadside.ts b/core/broadside.ts index 88256d0..7197361 100644 --- a/core/broadside.ts +++ b/core/broadside.ts @@ -28,7 +28,7 @@ // resubmit path must gate on whether any tool executed. // // Broad-Side requires runtime code, so the feature itself lives on the -// executable surfaces (MCP today, Pi on the roadmap). What the template does +// executable surfaces (Pi and MCP), not the pure template. What the template does // carry is the reading guide for its output — `.codecarto/broadside/SKILL.md`, // served by codecarto_skill under the name `broadside` (see readBroadsideSkill). @@ -243,6 +243,37 @@ export type BroadsideConfig = { waitSeconds: number; }; +/** + * The pre-flight facts a caller needs to decide whether a run is worth its + * price: what each lens would cost, at what rates, against which limit. Handed + * to {@link BroadsideSubmitOptions.confirm} before anything is submitted. + */ +export type BroadsideEstimate = { + model: string; + pricing: ModelPricing; + lenses: Array<{ lensId: BroadsideLensId; name: string; slices: number; maxTokens: number; cost: number }>; + totalCost: number; + inputTokens: number; + outputTokens: number; + /** Run expense limit in USD; 0 means no limit. */ + maxCost: number; + /** True when totalCost is over a non-zero maxCost. */ + exceedsLimit: boolean; + /** Set when incremental scouting found a baseline to diff against. */ + baseHead: string | null; + sourceDirty: boolean; + /** The provider's completion ceiling, when the catalog advertises one. */ + outputCap?: number; +}; + +/** Thrown when a confirm hook declines a run. Nothing was submitted. */ +export class BroadsideCancelledError extends Error { + constructor(message = "Broad-Side submission cancelled. Nothing was submitted.") { + super(message); + this.name = "BroadsideCancelledError"; + } +} + export type BroadsideSubmitResult = { runId: string; outputDir: string; @@ -1897,6 +1928,16 @@ export async function runBroadsideSubmit( force?: boolean; /** Diff against the previous run's HEAD and scan only changed modules (#142). */ incremental?: boolean; + /** + * Called with the pre-flight estimate after slicing and before any state + * write or submission. Returning false throws {@link BroadsideCancelledError} + * and nothing is submitted; returning true proceeds even past maxCost, + * because an interactive approval of a priced run *is* the force flag. + * + * A surface that cannot ask a human (MCP) omits this and keeps the + * refuse-unless-force behavior. + */ + confirm?: (estimate: BroadsideEstimate) => boolean | Promise; } = {}, ): Promise { const info = await collectRepoInfo(cwd); @@ -1975,7 +2016,30 @@ export async function runBroadsideSubmit( perLensEstimate.push({ lens, cost: estimate.cost, maxTokens }); } - if (limit > 0 && !opts.force && estimatedTotalCost > limit) { + const exceedsLimit = limit > 0 && estimatedTotalCost > limit; + + if (opts.confirm) { + const approved = await opts.confirm({ + model, + pricing, + lenses: perLensEstimate.map(({ lens, cost, maxTokens }) => ({ + lensId: lens.id, + name: lens.name, + slices: (slicesByLens.get(lens.id) ?? []).length, + maxTokens, + cost, + })), + totalCost: estimatedTotalCost, + inputTokens: estimatedInputTokens, + outputTokens: estimatedOutputTokens, + maxCost: limit, + exceedsLimit, + baseHead, + sourceDirty, + ...(outputCap !== undefined && { outputCap }), + }); + if (!approved) throw new BroadsideCancelledError(); + } else if (exceedsLimit && !opts.force) { const breakdown = perLensEstimate .map(({ lens, cost }) => ` ${lens.name}: ~$${cost.toFixed(4)}`) .join("\n"); diff --git a/extensions/codecarto/broadside-flags.ts b/extensions/codecarto/broadside-flags.ts new file mode 100644 index 0000000..f433c02 --- /dev/null +++ b/extensions/codecarto/broadside-flags.ts @@ -0,0 +1,121 @@ +// Argument parser for /codecarto-broadside. The grammar is one optional +// action followed by lens names and flags, in any order: +// +// /codecarto-broadside → submit, default lenses +// /codecarto-broadside submit architecture security → submit, two lenses +// /codecarto-broadside collect --wait=900 +// /codecarto-broadside status +// /codecarto-broadside models --benchmarks +// +// Flags mirror the codecarto_broadside tool parameters, with the negative +// forms spelled out because a slash command has no place to pass `false`: +// --incremental --no-synthesis +// --max-cost=N --no-triage +// --wait=SECONDS --no-retry-truncated +// --benchmarks (models only) +// +// The parser never throws. index.ts decides how to surface unknown tokens and +// invalid combinations, matching parseNextFlags. + +import { BROADSIDE_LENS_IDS, type BroadsideLensId } from "../../core/index.ts"; + +export type BroadsideAction = "submit" | "collect" | "status" | "models"; + +export interface BroadsideFlags { + action: BroadsideAction; + /** Empty means "the repository's default lens set". */ + lenses: BroadsideLensId[]; + incremental: boolean; + includeSynthesis?: boolean; + includeTriage?: boolean; + retryTruncated?: boolean; + /** Undefined means "use the repository's config default". */ + maxCost?: number; + waitSeconds?: number; + benchmarks: boolean; + unknown: string[]; + /** Set on an invalid combination. The caller surfaces it as an error. */ + error?: string; +} + +const ACTIONS = new Set(["submit", "collect", "status", "models"]); + +/** Every token the completer offers, in the order it offers them. */ +export const KNOWN_BROADSIDE_TOKENS = [ + "submit", + "collect", + "status", + "models", + ...BROADSIDE_LENS_IDS, + "--incremental", + "--max-cost=", + "--wait=", + "--no-synthesis", + "--no-triage", + "--no-retry-truncated", + "--benchmarks", +] as const; + +// A numeric flag with a missing or unparseable value is an error, not a +// silent fallback to the config default: "--max-cost=" almost certainly means +// the user meant to cap the spend and mistyped it. +function parseNumeric(token: string, name: string, result: BroadsideFlags): number | undefined { + const raw = token.slice(name.length + 1); + const value = Number(raw); + if (!raw || !Number.isFinite(value) || value < 0) { + result.error = `${name} needs a non-negative number (got "${raw}").`; + return undefined; + } + return value; +} + +export function parseBroadsideFlags(args: string): BroadsideFlags { + const tokens = args.trim().split(/\s+/).filter((token) => token.length > 0); + const result: BroadsideFlags = { + action: "submit", + lenses: [], + incremental: false, + benchmarks: false, + unknown: [], + }; + + let actionSeen = false; + for (const token of tokens) { + if (!actionSeen && ACTIONS.has(token as BroadsideAction)) { + result.action = token as BroadsideAction; + actionSeen = true; + continue; + } + if (BROADSIDE_LENS_IDS.includes(token as BroadsideLensId)) { + // A lens named twice is one lens, not two batches of it. + if (!result.lenses.includes(token as BroadsideLensId)) result.lenses.push(token as BroadsideLensId); + continue; + } + if (token === "--incremental") { result.incremental = true; continue; } + if (token === "--no-synthesis") { result.includeSynthesis = false; continue; } + if (token === "--no-triage") { result.includeTriage = false; continue; } + if (token === "--no-retry-truncated") { result.retryTruncated = false; continue; } + if (token === "--benchmarks") { result.benchmarks = true; continue; } + if (token.startsWith("--max-cost=")) { result.maxCost = parseNumeric(token, "--max-cost", result); continue; } + if (token.startsWith("--wait=")) { result.waitSeconds = parseNumeric(token, "--wait", result); continue; } + result.unknown.push(token); + } + + // Flags that only mean something for one action are refused rather than + // ignored: silently dropping --incremental on a collect would read as + // "collected incrementally", which is not a thing. + if (result.lenses.length > 0 && result.action !== "submit") { + result.error ??= `Lens names are only meaningful for submit (got action "${result.action}").`; + } + if (result.incremental && result.action !== "submit") { + result.error ??= `--incremental is only meaningful for submit (got action "${result.action}").`; + } + if (result.benchmarks && result.action !== "models") { + result.error ??= `--benchmarks is only meaningful for models (got action "${result.action}").`; + } + if (result.action === "status" && result.waitSeconds !== undefined) { + result.error ??= "--wait is only meaningful for submit and collect; status reads recorded state."; + } + + return result; +} diff --git a/extensions/codecarto/index.ts b/extensions/codecarto/index.ts index 6ec09a9..312313a 100644 --- a/extensions/codecarto/index.ts +++ b/extensions/codecarto/index.ts @@ -8,6 +8,7 @@ import { disposeAgentsWidget } from "./agent-widget.ts"; import { parseDashboardFlags } from "./dashboard-flags.ts"; import { narrateDashboard } from "./dashboard-narrator.ts"; import { writeDashboard } from "./dashboard-writer.ts"; +import { parseBroadsideFlags, KNOWN_BROADSIDE_TOKENS } from "./broadside-flags.ts"; import { parseNextFlags } from "./next-flags.ts"; import { phaseCompactionExtension } from "./phase-compaction.ts"; @@ -29,8 +30,22 @@ import { getWorkspaceState, isWithinPath, isWithinPathResolved, + BROADSIDE_LENS_IDS, BROADSIDE_SKILL_NAME, + BroadsideCancelledError, + type BroadsideEstimate, + broadsideDirFor, + collectResultText, + estimateSubmitText, + getLens, + listBatchModels, listSkillNames, + loadBroadsideConfig, + modelsText, + runBroadsideCollect, + runBroadsideStatus, + runBroadsideSubmit, + statusText, loadCodecartoConfig, loadUsage, loadYamlFile, @@ -59,6 +74,9 @@ import { initLibrary } from "../../core/library.ts"; import { resolveUserConfigPath, USER_CONFIG_DIR } from "../../core/orchestrator-config.ts"; const STATUS_WIDGET_ID = "codecarto-widget"; +// Broad-Side gets its own widget id: a scout run is legal on a repository with +// no workspace, where the phase widget has nothing to render. +const BROADSIDE_WIDGET_ID = "codecarto-broadside"; const STATUS_LINE_ID = "codecarto-status"; const SAFE_TOOL_NAMES = ["read", "grep", "find", "ls", "edit", "write"]; @@ -145,6 +163,46 @@ function setUiState(ctx: ExtensionContext | ExtensionCommandContext, state: Work ctx.ui.setWidget(STATUS_WIDGET_ID, buildStatusLines(state, extraLines)); } +/** + * Resolve the OpenRouter key for a Broad-Side run. Deliberately no slash-command + * parameter: a key typed as a command argument lands in the session transcript. + */ +function resolveBroadsideKey(configuredKey: string): string | null { + const fromEnv = process.env.OPENROUTER_API_KEY?.trim(); + if (fromEnv) return fromEnv; + return configuredKey.trim() || null; +} + +/** The spend decision, rendered for a human about to approve it. */ +function describeBroadsideEstimate(estimate: BroadsideEstimate): string { + const lines = [ + `Model: ${estimate.model} (pricing: ${estimate.pricing.source})`, + `Rates: $${estimate.pricing.inputPerM.toFixed(4)}/M in · $${estimate.pricing.outputPerM.toFixed(4)}/M out`, + "", + "Per lens:", + ...estimate.lenses.map( + ({ name, slices, cost }) => ` ${name}: ${slices} slice${slices === 1 ? "" : "s"} — ~$${cost.toFixed(4)}`, + ), + "", + `Estimated total: ~$${estimate.totalCost.toFixed(4)} ` + + `(~${Math.round(estimate.inputTokens / 1000)}k in, ~${Math.round(estimate.outputTokens / 1000)}k out)`, + ]; + if (estimate.maxCost > 0) { + lines.push( + estimate.exceedsLimit + ? `This EXCEEDS the configured max_cost of $${estimate.maxCost.toFixed(2)}. Approving here overrides it for this run.` + : `Within the configured max_cost of $${estimate.maxCost.toFixed(2)}.`, + ); + } + if (estimate.baseHead) { + lines.push(`Incremental: only modules changed since ${estimate.baseHead.slice(0, 8)} are included.`); + } else if (estimate.sourceDirty) { + lines.push("Incremental was requested but the tree is dirty — this is a full scan."); + } + lines.push("", "The estimate is a pre-flight prediction from file sizes; OpenRouter bills actual usage."); + return lines.join("\n"); +} + export default function codeCartographerExtension(pi: ExtensionAPI) { phaseCompactionExtension(pi); let lastFeedbackLines: string[] = []; @@ -742,6 +800,185 @@ export default function codeCartographerExtension(pi: ExtensionAPI) { }, }); + pi.registerCommand("codecarto-broadside", { + description: "Batch reconnaissance (Broad-Side): /codecarto-broadside [submit|collect|status|models] [lenses…] [flags]", + getArgumentCompletions: (prefix) => { + const items = KNOWN_BROADSIDE_TOKENS + .filter((value) => value.startsWith(prefix)) + .map((value) => ({ value, label: value })); + return items.length > 0 ? items : null; + }, + handler: async (args, ctx) => { + const flags = parseBroadsideFlags(args); + if (flags.unknown.length > 0) { + ctx.ui.notify( + `Unknown /codecarto-broadside argument: ${flags.unknown.join(" ")}. ` + + `Actions: submit, collect, status, models. Lenses: ${BROADSIDE_LENS_IDS.join(", ")}.`, + "error", + ); + return; + } + if (flags.error) { + ctx.ui.notify(flags.error, "error"); + return; + } + + // Broad-Side runs on any git repository, with or without a workspace — + // so this command never goes through ensureWorkspaceState. + const broadsideDir = broadsideDirFor(ctx.cwd); + const config = await loadBroadsideConfig(broadsideDir); + + const finish = (lines: string[], notice: string, level: "info" | "warning" = "info"): void => { + lastFeedbackLines = lines; + if (codecartoModeActive) { + // A workspace session already has a widget; fold the result into it. + if (ctx.hasUI) ctx.ui.setWidget(BROADSIDE_WIDGET_ID, undefined); + void refreshWorkspaceUi(ctx, lines); + } else if (ctx.hasUI) { + // Scout-only repository: the Broad-Side widget is the only place + // the result can live, so it holds it instead of being cleared. + ctx.ui.setWidget(BROADSIDE_WIDGET_ID, ["Broad-Side", ...lines]); + } + ctx.ui.notify(notice, level); + }; + + if (flags.action === "status") { + const { state } = await runBroadsideStatus(ctx.cwd); + const runs = state.runs.length; + finish( + statusText(state).split("\n"), + runs > 0 ? `Broad-Side: ${runs} recorded run${runs === 1 ? "" : "s"}` : "Broad-Side: no runs recorded yet", + ); + return; + } + + const apiKey = resolveBroadsideKey(config.apiKey); + if (!apiKey) { + ctx.ui.notify( + "No OpenRouter API key. Set OPENROUTER_API_KEY in the environment, or add api_key to " + + ".codecarto/broadside/config.yaml. (A slash command takes no key: it would land in the transcript.)", + "error", + ); + return; + } + + if (flags.action === "models") { + ctx.ui.notify("Fetching the OpenRouter batch-model catalog…", "info"); + try { + const { entries, benchmarks } = await listBatchModels(broadsideDir, config, apiKey, { + includeBenchmarks: flags.benchmarks, + }); + finish( + modelsText(entries, { benchmarks, defaultModel: config.model }).split("\n"), + `Broad-Side: ${entries.length} batch model${entries.length === 1 ? "" : "s"} listed`, + ); + } catch (error) { + ctx.ui.notify(`Model catalog lookup failed: ${error instanceof Error ? error.message : String(error)}`, "error"); + } + return; + } + + // Live per-lens progress. Poll callbacks fire often, so they render + // into a widget rather than a notification stream. + const progress = new Map(); + const renderProgress = (heading: string): void => { + if (!ctx.hasUI) return; + ctx.ui.setWidget(BROADSIDE_WIDGET_ID, [ + "Broad-Side", + heading, + ...[...progress.entries()].map(([lensId, line]) => ` ${lensId}: ${line}`), + ]); + }; + const onStatus = (lensId: string, status: string, counts: Record): void => { + progress.set(lensId, `${status} (${counts.completed ?? 0}/${counts.total ?? "?"})`); + renderProgress("Polling batches…"); + }; + + const waitSeconds = flags.waitSeconds ?? config.waitSeconds; + const waitMs = waitSeconds > 0 ? waitSeconds * 1000 : undefined; + const includeSynthesis = flags.includeSynthesis ?? config.includeSynthesis; + const includeTriage = flags.includeTriage ?? config.includeTriage; + const retryTruncated = flags.retryTruncated ?? config.retryTruncated; + + if (flags.action === "submit") { + const lenses = flags.lenses.length > 0 ? flags.lenses : config.defaultLenses; + renderProgress("Slicing the repository and pricing the run…"); + let submit; + try { + submit = await runBroadsideSubmit(ctx.cwd, apiKey, { + lenses, + model: config.model, + maxCost: flags.maxCost ?? config.maxCost, + incremental: flags.incremental || config.incremental, + // Pi can ask, so it asks instead of refusing over max_cost the + // way MCP has to. An approval here IS the force flag. + confirm: (estimate) => + ctx.ui.confirm( + `Broad-Side will spend about $${estimate.totalCost.toFixed(4)}`, + describeBroadsideEstimate(estimate), + ), + }); + } catch (error) { + if (ctx.hasUI) ctx.ui.setWidget(BROADSIDE_WIDGET_ID, undefined); + if (error instanceof BroadsideCancelledError) { + ctx.ui.notify("Broad-Side cancelled. Nothing was submitted.", "info"); + return; + } + ctx.ui.notify(`Broad-Side submit failed: ${error instanceof Error ? error.message : String(error)}`, "error"); + return; + } + + const lines = estimateSubmitText(submit, lenses.map(getLens)).split("\n"); + if (!waitMs) { + lines.push("", "Batches are in flight. Run /codecarto-broadside collect when they finish."); + finish(lines, `Broad-Side submitted: run ${submit.runId} (~$${submit.estimatedTotalCost.toFixed(4)})`); + return; + } + + ctx.ui.notify(`Broad-Side submitted run ${submit.runId}; polling for up to ${waitSeconds}s…`, "info"); + try { + const collect = await runBroadsideCollect(ctx.cwd, apiKey, { + waitMs, + includeSynthesis, + includeTriage, + retryTruncated, + onStatus, + }); + finish([...lines, "", ...collectResultText(collect).split("\n")], `Broad-Side ${collect.status}: run ${collect.runId}`); + } catch (error) { + if (ctx.hasUI) ctx.ui.setWidget(BROADSIDE_WIDGET_ID, undefined); + // The batches are submitted and paid for either way — say so, so + // nobody re-submits a run that is already in flight. + ctx.ui.notify( + `Broad-Side submitted run ${submit.runId}, but collect failed: ` + + `${error instanceof Error ? error.message : String(error)}. Retry with /codecarto-broadside collect.`, + "error", + ); + } + return; + } + + // action === "collect" + renderProgress("Polling batches…"); + try { + const collect = await runBroadsideCollect(ctx.cwd, apiKey, { + waitMs, + includeSynthesis, + includeTriage, + retryTruncated, + onStatus, + }); + const lines = collectResultText(collect).split("\n"); + const done = collect.status === "completed"; + if (!done) lines.push("", "Still in flight. Run /codecarto-broadside collect again to resume."); + finish(lines, `Broad-Side ${collect.status}: ${collect.resultCount} result${collect.resultCount === 1 ? "" : "s"} saved`, done ? "info" : "warning"); + } catch (error) { + if (ctx.hasUI) ctx.ui.setWidget(BROADSIDE_WIDGET_ID, undefined); + ctx.ui.notify(`Broad-Side collect failed: ${error instanceof Error ? error.message : String(error)}`, "error"); + } + }, + }); + pi.registerCommand("codecarto-publish", { description: "Publish the completed reimplementation spec to the configured CodeCartographer library", handler: async (_args, ctx) => { diff --git a/tests/broadside-flags.test.mjs b/tests/broadside-flags.test.mjs new file mode 100644 index 0000000..88c88ef --- /dev/null +++ b/tests/broadside-flags.test.mjs @@ -0,0 +1,100 @@ +// Tests for /codecarto-broadside argument parsing. The grammar mixes an +// action, bare lens names, and flags in any order, so the parse is where a +// mistyped command becomes either a clear error or a surprise batch of spend. + +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const { parseBroadsideFlags, KNOWN_BROADSIDE_TOKENS } = await import( + pathToFileURL(`${REPO_ROOT}/extensions/codecarto/broadside-flags.ts`).href +); + +test("no arguments means submit with the repository's default lenses", () => { + const r = parseBroadsideFlags(""); + assert.equal(r.action, "submit"); + assert.deepEqual(r.lenses, [], "empty means 'let config decide', not 'no lenses'"); + assert.equal(r.incremental, false); + assert.equal(r.maxCost, undefined); + assert.equal(r.waitSeconds, undefined); + assert.deepEqual(r.unknown, []); + assert.equal(r.error, undefined); +}); + +test("an action and lens names parse in either order", () => { + assert.deepEqual(parseBroadsideFlags("submit architecture security").lenses, ["architecture", "security"]); + const reversed = parseBroadsideFlags("architecture security submit"); + assert.equal(reversed.action, "submit", "a lens before the action still leaves submit the action"); + assert.deepEqual(reversed.lenses, ["architecture", "security"]); +}); + +test("a lens named twice is one lens", () => { + assert.deepEqual(parseBroadsideFlags("submit defect defect").lenses, ["defect"]); +}); + +test("only the first action token is the action; a second is unknown", () => { + const r = parseBroadsideFlags("collect status"); + assert.equal(r.action, "collect"); + assert.deepEqual(r.unknown, ["status"]); +}); + +test("negative flags carry false, and absence carries undefined", () => { + const bare = parseBroadsideFlags("collect"); + assert.equal(bare.includeSynthesis, undefined, "absence must defer to config, not force true"); + assert.equal(bare.includeTriage, undefined); + assert.equal(bare.retryTruncated, undefined); + + const off = parseBroadsideFlags("collect --no-synthesis --no-triage --no-retry-truncated"); + assert.equal(off.includeSynthesis, false); + assert.equal(off.includeTriage, false); + assert.equal(off.retryTruncated, false); +}); + +test("numeric flags parse their value", () => { + const r = parseBroadsideFlags("submit --max-cost=2.50 --wait=900"); + assert.equal(r.maxCost, 2.5); + assert.equal(r.waitSeconds, 900); +}); + +test("a malformed numeric flag is an error, never a silent config fallback", () => { + // "--max-cost=" almost certainly means the user meant to cap the spend. + for (const args of ["submit --max-cost=", "submit --max-cost=abc", "submit --max-cost=-1"]) { + const r = parseBroadsideFlags(args); + assert.match(r.error ?? "", /--max-cost needs a non-negative number/, `${args} must error`); + } + assert.match(parseBroadsideFlags("collect --wait=soon").error ?? "", /--wait needs a non-negative number/); +}); + +test("flags that mean nothing for the chosen action are refused, not ignored", () => { + assert.match(parseBroadsideFlags("collect --incremental").error ?? "", /--incremental is only meaningful for submit/); + assert.match(parseBroadsideFlags("collect architecture").error ?? "", /Lens names are only meaningful for submit/); + assert.match(parseBroadsideFlags("submit --benchmarks").error ?? "", /--benchmarks is only meaningful for models/); + assert.match(parseBroadsideFlags("status --wait=60").error ?? "", /--wait is only meaningful for submit and collect/); + assert.equal(parseBroadsideFlags("models --benchmarks").error, undefined); +}); + +test("unknown tokens are collected for the caller to surface", () => { + const r = parseBroadsideFlags("submit --bogus architecture nonsense"); + assert.deepEqual(r.unknown, ["--bogus", "nonsense"]); + assert.deepEqual(r.lenses, ["architecture"]); +}); + +test("extra whitespace produces no empty unknowns", () => { + assert.deepEqual(parseBroadsideFlags(" collect ").unknown, []); +}); + +test("every completion token the command offers is one the parser accepts", () => { + // A completer that suggests a token the parser rejects teaches the user a + // command that fails. + for (const token of KNOWN_BROADSIDE_TOKENS) { + // Value-taking flags are offered as a prefix ("--max-cost="); complete + // them with a value before parsing. + const arg = token.endsWith("=") ? `${token}1` : token; + const context = token === "--benchmarks" ? "models " : token === "--wait=" ? "collect " : ""; + const r = parseBroadsideFlags(`${context}${arg}`); + assert.deepEqual(r.unknown, [], `completion token ${token} parses as unknown`); + assert.equal(r.error, undefined, `completion token ${token} errors: ${r.error}`); + } +}); diff --git a/tests/broadside.test.mjs b/tests/broadside.test.mjs index 34dfbdc..531891a 100644 --- a/tests/broadside.test.mjs +++ b/tests/broadside.test.mjs @@ -515,6 +515,66 @@ test("submit refuses over-budget runs and creates no run entry; force bypasses", } }); +test("a confirm hook decides the run, and declining submits nothing", async () => { + const dir = await mkdtemp(join(tmpdir(), "broadside-confirm-")); + try { + await writeFile(join(dir, "go.mod"), "module x\n"); + await mkdir(join(dir, "big"), { recursive: true }); + for (let i = 0; i < 40; i++) { + await writeFile(join(dir, "big", `file${i}.go`), "package big\n" + `// ${"y".repeat(2000)}\n`); + } + let posted = 0; + const fetcher = async (url, init) => { + if (init.method === "POST") { + posted += 1; + return fakeResponse(202, { id: "batch-ok", status: "validating" }); + } + return fakeResponse(200, { id: "x", status: "in_progress" }); + }; + const stateDir = join(dir, ".codecarto", "broadside"); + + // Declining is not an error condition to paper over: nothing is spent, + // nothing is recorded, and the caller can tell it apart from a failure. + const seen = []; + await assert.rejects( + () => runBroadsideSubmit(dir, "sk-fake", { + lenses: ["defect"], + fetcher, + confirm: (estimate) => { seen.push(estimate); return false; }, + }), + (error) => { + assert.equal(error.name, "BroadsideCancelledError"); + assert.match(error.message, /Nothing was submitted/); + return true; + }, + ); + assert.equal(posted, 0, "a declined run must not submit a batch"); + assert.equal((await loadBroadsideState(stateDir)).runs.length, 0, "a declined run must not be recorded"); + + assert.equal(seen.length, 1, "the hook is called once, before submission"); + assert.equal(seen[0].lenses.length, 1); + assert.equal(seen[0].lenses[0].lensId, "defect"); + assert.ok(seen[0].totalCost > 0, "the estimate must carry a price"); + assert.ok(seen[0].lenses[0].slices > 0, "the estimate must say how much work was sliced"); + + // An interactive approval is the force flag: it carries the run past a + // max_cost the non-interactive path would refuse. + const approved = []; + const result = await runBroadsideSubmit(dir, "sk-fake", { + lenses: ["defect"], + fetcher, + maxCost: 0.0001, + confirm: (estimate) => { approved.push(estimate); return true; }, + }); + assert.equal(approved[0].exceedsLimit, true, "the hook must be told it is over budget"); + assert.equal(approved[0].maxCost, 0.0001); + assert.equal(result.batches.defect.status, "validating"); + assert.equal((await loadBroadsideState(stateDir)).runs.length, 1); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + test("submit passes the configured model into batch payloads", async () => { const dir = await mkdtemp(join(tmpdir(), "broadside-model-")); try { diff --git a/tests/default-pipeline.test.mjs b/tests/default-pipeline.test.mjs index 7cbd246..5d63fb2 100644 --- a/tests/default-pipeline.test.mjs +++ b/tests/default-pipeline.test.mjs @@ -61,7 +61,7 @@ test("every PIPELINE_ALIASES target resolves to a real file", async () => { test("Pi extension registers a command for every CodeCartographer operation", async () => { // The set of operations the framework exposes; both wrappers must surface them. - const expected = ["init", "open", "vision", "switch-pipeline", "status", "next", "phase", "validate", "complete", "skill", "publish", "library-init", "config", "usage", "dashboard"]; + const expected = ["init", "open", "vision", "switch-pipeline", "status", "next", "phase", "validate", "complete", "skill", "publish", "library-init", "config", "usage", "dashboard", "broadside"]; const indexSrc = await readFile(join(REPO_ROOT, "extensions", "codecarto", "index.ts"), "utf8"); const missing = expected.filter((op) => !indexSrc.includes(`pi.registerCommand("codecarto-${op}"`)); assert.deepEqual( @@ -70,3 +70,14 @@ test("Pi extension registers a command for every CodeCartographer operation", as `Pi extension is missing command registration for: ${missing.join(", ")}`, ); }); + +test("the README slash-command table names every command the extension registers", async () => { + // Pi is the recommended surface, so its command table is the first place a + // user looks. A command that ships unlisted there effectively did not ship. + const indexSrc = await readFile(join(REPO_ROOT, "extensions", "codecarto", "index.ts"), "utf8"); + const registered = [...indexSrc.matchAll(/pi\.registerCommand\("(codecarto-[\w-]+)"/g)].map((m) => m[1]); + assert.ok(registered.length > 0, "expected the extension to register commands"); + const readme = await readFile(join(REPO_ROOT, "README.md"), "utf8"); + const undocumented = registered.filter((name) => !readme.includes(`/${name}`)); + assert.deepEqual(undocumented, [], `README does not document: ${undocumented.join(", ")}`); +}); diff --git a/tests/pi-broadside.test.mjs b/tests/pi-broadside.test.mjs new file mode 100644 index 0000000..c5bf96c --- /dev/null +++ b/tests/pi-broadside.test.mjs @@ -0,0 +1,187 @@ +// The Pi surface for Broad-Side (/codecarto-broadside). Drives the registered +// command through a fake Pi harness with a stubbed global fetch — no network, +// no spend. What matters here is the surface's two divergences from MCP: it +// asks a human about the money instead of refusing over max_cost, and it runs +// on a repository with no CodeCartographer workspace. + +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const { default: codeCartographerExtension } = await import( + pathToFileURL(`${REPO_ROOT}/extensions/codecarto/index.ts`).href +); + +function createHarness(cwd, { confirm = async () => true } = {}) { + const commands = new Map(); + const pi = { + on: () => {}, + registerCommand: (name, command) => commands.set(name, command), + setActiveTools: () => {}, + setSessionName: () => {}, + sendMessage: () => {}, + sendUserMessage: () => {}, + }; + const ui = { + widgets: [], + notifications: [], + confirmations: [], + theme: { fg: (_name, text) => text }, + setStatus: () => {}, + setWidget: (id, value) => ui.widgets.push({ id, value }), + notify: (message, level) => ui.notifications.push({ message, level }), + confirm: async (title, body) => { + ui.confirmations.push({ title, body }); + return confirm(title, body); + }, + }; + const ctx = { cwd, hasUI: true, ui, signal: new AbortController().signal, isIdle: () => true, reload: async () => {} }; + codeCartographerExtension(pi); + return { commands, ctx, ui }; +} + +function fakeResponse(status, body) { + return new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json" } }); +} + +/** A repository big enough that the defect lens produces a priced slice. */ +async function scoutableRepo() { + const cwd = await mkdtemp(join(tmpdir(), "cc-pi-broadside-")); + await writeFile(join(cwd, "go.mod"), "module x\n"); + await mkdir(join(cwd, "big"), { recursive: true }); + for (let i = 0; i < 20; i++) { + await writeFile(join(cwd, "big", `file${i}.go`), `package big\n// ${"y".repeat(2000)}\n`); + } + return cwd; +} + +async function withStubbedFetch(handler, fn) { + const original = globalThis.fetch; + const posts = []; + globalThis.fetch = async (url, init = {}) => { + if (init.method === "POST") posts.push({ url: String(url), body: JSON.parse(init.body ?? "{}") }); + return handler(String(url), init) ?? fakeResponse(200, { id: "x", status: "in_progress" }); + }; + try { + return await fn(posts); + } finally { + globalThis.fetch = original; + } +} + +function lastNotification(ui) { + return ui.notifications.at(-1); +} + +test("status works with no workspace, no API key, and no runs recorded", async () => { + const cwd = await mkdtemp(join(tmpdir(), "cc-pi-broadside-empty-")); + const key = process.env.OPENROUTER_API_KEY; + delete process.env.OPENROUTER_API_KEY; + try { + const { commands, ctx, ui } = createHarness(cwd); + await commands.get("codecarto-broadside").handler("status", ctx); + + assert.match(lastNotification(ui).message, /no runs recorded/i); + assert.equal(lastNotification(ui).level, "info", "an empty scout history is not an error"); + // With no workspace there is no phase widget, so the result lives in the + // Broad-Side widget rather than vanishing into a one-line notification. + assert.equal(ui.widgets.at(-1).id, "codecarto-broadside"); + assert.equal(ui.widgets.at(-1).value[0], "Broad-Side"); + } finally { + if (key !== undefined) process.env.OPENROUTER_API_KEY = key; + await rm(cwd, { recursive: true, force: true }); + } +}); + +test("a missing API key names both ways to supply one and never takes it as an argument", async () => { + const cwd = await mkdtemp(join(tmpdir(), "cc-pi-broadside-nokey-")); + const key = process.env.OPENROUTER_API_KEY; + delete process.env.OPENROUTER_API_KEY; + try { + const { commands, ctx, ui } = createHarness(cwd); + await commands.get("codecarto-broadside").handler("submit architecture", ctx); + + const notice = lastNotification(ui); + assert.equal(notice.level, "error"); + assert.match(notice.message, /OPENROUTER_API_KEY/); + assert.match(notice.message, /config\.yaml/); + assert.match(notice.message, /transcript/, "must say why the command takes no key argument"); + assert.equal(ui.confirmations.length, 0, "no spend prompt without a key"); + } finally { + if (key !== undefined) process.env.OPENROUTER_API_KEY = key; + await rm(cwd, { recursive: true, force: true }); + } +}); + +test("submit prices the run, asks before spending, and submits nothing when declined", async () => { + const cwd = await scoutableRepo(); + const key = process.env.OPENROUTER_API_KEY; + process.env.OPENROUTER_API_KEY = "sk-fake"; + try { + await withStubbedFetch(() => fakeResponse(202, { id: "batch-1", status: "validating" }), async (posts) => { + const { commands, ctx, ui } = createHarness(cwd, { confirm: async () => false }); + await commands.get("codecarto-broadside").handler("submit defect", ctx); + + assert.equal(ui.confirmations.length, 1, "the user must be asked before any spend"); + assert.match(ui.confirmations[0].title, /will spend about \$\d/); + assert.match(ui.confirmations[0].body, /Per lens:/); + assert.match(ui.confirmations[0].body, /pre-flight prediction/, "must not present the estimate as the bill"); + assert.equal(posts.length, 0, "declining must submit no batch"); + assert.equal(lastNotification(ui).level, "info", "a cancel is not an error"); + assert.match(lastNotification(ui).message, /cancelled/i); + }); + } finally { + if (key === undefined) delete process.env.OPENROUTER_API_KEY; + else process.env.OPENROUTER_API_KEY = key; + await rm(cwd, { recursive: true, force: true }); + } +}); + +test("an approved submit fires the batch and reports the run id", async () => { + const cwd = await scoutableRepo(); + const key = process.env.OPENROUTER_API_KEY; + process.env.OPENROUTER_API_KEY = "sk-fake"; + try { + await withStubbedFetch(() => fakeResponse(202, { id: "batch-1", status: "validating" }), async (posts) => { + const { commands, ctx, ui } = createHarness(cwd, { confirm: async () => true }); + await commands.get("codecarto-broadside").handler("submit defect", ctx); + + assert.equal(posts.length, 1, "exactly one lens batch should be submitted"); + assert.match(lastNotification(ui).message, /Broad-Side submitted: run /); + const widget = ui.widgets.at(-1).value.join("\n"); + assert.match(widget, /collect/, "the result must say how to finish the run"); + }); + } finally { + if (key === undefined) delete process.env.OPENROUTER_API_KEY; + else process.env.OPENROUTER_API_KEY = key; + await rm(cwd, { recursive: true, force: true }); + } +}); + +test("a bad argument is refused before anything is priced or spent", async () => { + const cwd = await scoutableRepo(); + const key = process.env.OPENROUTER_API_KEY; + process.env.OPENROUTER_API_KEY = "sk-fake"; + try { + await withStubbedFetch(() => fakeResponse(202, { id: "b", status: "validating" }), async (posts) => { + const { commands, ctx, ui } = createHarness(cwd); + await commands.get("codecarto-broadside").handler("submit --max-cost=", ctx); + assert.equal(lastNotification(ui).level, "error"); + assert.match(lastNotification(ui).message, /--max-cost needs a non-negative number/); + + await commands.get("codecarto-broadside").handler("sumbit", ctx); + assert.match(lastNotification(ui).message, /Unknown \/codecarto-broadside argument: sumbit/); + + assert.equal(ui.confirmations.length, 0); + assert.equal(posts.length, 0); + }); + } finally { + if (key === undefined) delete process.env.OPENROUTER_API_KEY; + else process.env.OPENROUTER_API_KEY = key; + await rm(cwd, { recursive: true, force: true }); + } +}); From d3da9862eee023aae2bb0cad18ceb66b4d1dcd5b Mon Sep 17 00:00:00 2001 From: James Sesler Date: Sun, 30 Aug 2026 15:41:57 -0400 Subject: [PATCH 3/4] feat: the scout-first pipeline routes Broad-Side leads into phases (#139) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Broad-Side produced reports nothing in a pipeline pointed at. The scout-first variant is full-with-deep-audit with a broadside-scout phase in front. Two constraints shaped the design, and both are worth stating: - A required_read must be an upstream phase's primary_output, and run directories are timestamped and gitignored, so no pipeline YAML can ever name broadside//synthesis.md. The scout phase therefore distills: it reads whatever a prior run wrote and produces one stable, reviewable artifact, findings/broadside-scout/scout-brief.md. That indirection is also where the judgment lives — deciding which leads are worth six phases' attention is not a file copy. - status.yaml phases must match phase_order exactly, so a phase cannot be optional. Scouting therefore ships as its own variant rather than as a toggle on the default pipeline, which would otherwise put a paid API scan in front of every run. Architecture, defect-scan-mechanical, contracts, protocols, defect-scan-semantic and porting read the brief, and each carries a completion criterion requiring every lead routed to it to be confirmed against the source, dismissed with a reason, or carried forward — never reported as a finding on the brief's authority. That criterion is where "leads, never evidence" stops being a docstring and becomes a gate. reimplementation-spec deliberately does not read the brief; the porting bundle stays its compression boundary. The phase never submits a batch and never spends. With no run on disk it writes an explicitly empty brief with coverage disposition NONE and the pipeline proceeds exactly as full-with-deep-audit would — which matters because /codecarto-next --auto runs it unattended. Tests pin who reads the brief, who must account for it, that the spec phase does not, that the skill never tells an executor to fire a run, and that the variant matches the deep-audit pipeline it wraps field by field apart from the scout additions. 441 pass. Co-Authored-By: Claude Opus 5 --- .codecarto/.gitignore | 1 + .codecarto/GUIDE.md | 2 + .codecarto/README.md | 3 + .codecarto/findings/broadside-scout/README.md | 20 ++ .codecarto/findings/broadside-scout/SKILL.md | 101 +++++++ .codecarto/templates/broadside-scout-brief.md | 97 +++++++ .codecarto/workflow/pipeline-scout-first.yaml | 271 ++++++++++++++++++ CHANGELOG.md | 1 + CLAUDE.md | 2 +- MANUAL.md | 5 + README.md | 8 +- ROADMAP.md | 6 +- .../references/pipeline-selection.md | 14 + core/pipeline.ts | 1 + tests/broadside-scout-pipeline.test.mjs | 114 ++++++++ 15 files changed, 643 insertions(+), 3 deletions(-) create mode 100644 .codecarto/findings/broadside-scout/README.md create mode 100644 .codecarto/findings/broadside-scout/SKILL.md create mode 100644 .codecarto/templates/broadside-scout-brief.md create mode 100644 .codecarto/workflow/pipeline-scout-first.yaml create mode 100644 tests/broadside-scout-pipeline.test.mjs diff --git a/.codecarto/.gitignore b/.codecarto/.gitignore index 39f6ab6..619f0b3 100644 --- a/.codecarto/.gitignore +++ b/.codecarto/.gitignore @@ -5,6 +5,7 @@ findings/contracts/behavioral-contracts.md findings/protocols/protocols-and-state.md findings/porting/reverse-engineering-bundle.md findings/reimplementation-spec/reimplementation-spec.md +findings/broadside-scout/scout-brief.md # Secondary / optional outputs findings/public-surfaces/public-surfaces.md diff --git a/.codecarto/GUIDE.md b/.codecarto/GUIDE.md index 9194c78..9e21ca3 100644 --- a/.codecarto/GUIDE.md +++ b/.codecarto/GUIDE.md @@ -99,6 +99,7 @@ Seven pipeline variants are available. Check the `pipeline` field in `workflow/s | Variant | File | Phases | When to use | |---|---|---|---| | Full with deep audit (default) | `workflow/pipeline-full-with-deep-audit.yaml` | architecture → defect-scan-mechanical → contracts → protocols → defect-scan-semantic → porting → reimplementation-spec | Complete analysis with defect scan split into an early mechanical pass and a deep semantic pass; reimplementation designs around defects with full context | +| Scout first | `workflow/pipeline-scout-first.yaml` | broadside-scout → architecture → defect-scan-mechanical → contracts → protocols → defect-scan-semantic → porting → reimplementation-spec | The deep-audit run behind a Broad-Side routing brief: the scout phase distills an existing batch reconnaissance run into leads addressed to later phases, each of which must confirm, dismiss, or carry them forward. Leads are never evidence | | Full with audit | `workflow/pipeline-full-with-audit.yaml` | architecture → defect-scan → contracts → protocols → porting → reimplementation-spec | Single early defect scan; cheaper than the deep variant when you do not need contracts/protocols-grounded defect findings | | Full | `workflow/pipeline.yaml` | architecture → contracts → protocols → porting → reimplementation-spec | Porting bundle without defect scan | | Defect scan | `workflow/pipeline-defect-scan.yaml` | architecture → defect-scan | Maintenance audit to surface latent problems | @@ -304,6 +305,7 @@ your-repo/ SKILL.md workflow/ pipeline-full-with-deep-audit.yaml # 7-phase pipeline with split defect scan (default). + pipeline-scout-first.yaml # 8-phase: deep audit behind a broadside-scout routing brief. pipeline-full-with-audit.yaml # 6-phase pipeline with single early defect scan. pipeline.yaml # 5-phase (no defect scan). pipeline-defect-scan.yaml # 2-phase (architecture + defect scan). diff --git a/.codecarto/README.md b/.codecarto/README.md index 8016a15..dca0945 100644 --- a/.codecarto/README.md +++ b/.codecarto/README.md @@ -31,6 +31,7 @@ The default is the 7-phase **full-with-deep-audit** pipeline, which splits the d ```yaml pipeline: workflow/pipeline-full-with-deep-audit.yaml # 7-phase with split defect scan (default; depth-first) +pipeline: workflow/pipeline-scout-first.yaml # 8-phase: the deep-audit run behind a broadside-scout brief (Pi/MCP only) pipeline: workflow/pipeline-full-with-audit.yaml # 6-phase with single early defect scan — adjust phases to use one defect-scan pipeline: workflow/pipeline.yaml # 5-phase without defect scan — remove defect-scan phases pipeline: workflow/pipeline-defect-scan.yaml # 2-phase defect audit — remove contracts through reimplementation-spec @@ -39,4 +40,6 @@ pipeline: workflow/pipeline-architecture-only.yaml # 1-phase quick overview pipeline: workflow/pipeline-synthesis.yaml # 4-phase forward synthesis — vision + confirmed library specs → project plan (Pi/MCP only) ``` +The scout-first pipeline is the deep-audit run with one phase in front of it: `broadside-scout` distills a completed Broad-Side batch reconnaissance run into `findings/broadside-scout/scout-brief.md`, and the six phases after it read that brief and must account for the leads routed to them. Firing the reconnaissance run itself needs Pi or MCP; the scout phase only reads what a run already wrote, so with no run on disk it produces an explicitly empty brief and the pipeline proceeds. + The synthesis pipeline is different from the analysis variants: it requires Pi or MCP, a configured non-empty CodeCartographer library, and a completed `inputs/vision.md`. It pauses after proposing candidate specs and will not merge or finalize until the user changes at least one proposal checkbox from `[ ]` to `[x]`. diff --git a/.codecarto/findings/broadside-scout/README.md b/.codecarto/findings/broadside-scout/README.md new file mode 100644 index 0000000..3fbeea6 --- /dev/null +++ b/.codecarto/findings/broadside-scout/README.md @@ -0,0 +1,20 @@ +# Broad-Side Scout + +Distills a completed Broad-Side batch reconnaissance run into a routing brief. +Runs first, before architecture, in the `pipeline-scout-first` workflow. + +**Primary output:** `scout-brief.md` + +**Depends on:** nothing in the pipeline. It reads what a prior +`/codecarto-broadside` (Pi) or `codecarto_broadside` (MCP) run wrote under +`broadside//`. It never submits a batch and never spends; with no run on +disk it produces an explicitly empty brief and the pipeline proceeds. + +**Consumed by:** architecture, defect-scan-mechanical, contracts, protocols, +defect-scan-semantic, and porting, each of which must account for the leads +routed to it at validation. `reimplementation-spec` deliberately does not read +it — the porting bundle is that phase's compression boundary. + +Everything in the brief is an unverified scouting lead. No phase may cite it, +or any file under `broadside/`, as a source. See `SKILL.md`, and +`broadside/SKILL.md` for how to read the underlying run. diff --git a/.codecarto/findings/broadside-scout/SKILL.md b/.codecarto/findings/broadside-scout/SKILL.md new file mode 100644 index 0000000..9cd9263 --- /dev/null +++ b/.codecarto/findings/broadside-scout/SKILL.md @@ -0,0 +1,101 @@ +--- +name: broadside-scout +description: Distill a completed Broad-Side batch reconnaissance run into a routing brief the later phases read. Runs first, before architecture, in the scout-first pipeline. Produces leads with a target phase for each — never findings, never evidence. +--- + +# Broad-Side Scout + +This phase turns a Broad-Side batch reconnaissance run into a **routing brief**: +a short document that tells each later phase where to spend its attention +first. It runs before architecture, and everything downstream reads it. + +The source code to analyze is in the parent directory (`../` relative to +`.codecarto/`). + +## This phase spends no money + +Broad-Side itself is fired by `/codecarto-broadside submit` (Pi) or +`codecarto_broadside` (MCP), and it is priced and confirmed there. This phase +only reads what those already wrote under `broadside//`. It never submits +a batch, and it must never instruct anyone to. + +If no run exists, that is a legitimate outcome — see "When there is no run." + +## What you are reading, and what it is worth + +Broad-Side findings are **unverified scouting signals** produced by a cheap +batch model in a single shot: no cross-file traversal, no runtime +verification, no builds, no tests, no follow-up questions. + +The entire value of this phase is routing attention. The entire risk is that a +lead gets copied forward as a fact. So the brief you write is a list of +*places to look*, each addressed to a phase, and every entry carries the +source pointer that phase must confirm for itself. + +Nothing you write here is evidence. No later phase may cite this brief, or any +file under `broadside/`, as a source for a finding. A later phase cites the +code it confirmed. + +## Reading the run + +1. Find the most recent run directory under `broadside/`. If several exist, + use the newest and say which one you used. +2. `broadside//synthesis.md` — the executive summary, severity counts, + top cross-lens findings, per-module risk. Start here. +3. `broadside//triage.md` — the same findings scored by impact × + difficulty into a P0–P3 order with effort estimates. +4. `broadside//run-meta.json` — which lenses ran, at what cost, with what + coverage caps. This is where you learn what was *not* scanned. +5. The per-lens files only when a lead matters enough to need its detail. + +## Routing + +Each lead goes to exactly one phase. Use the lens it came from as the default +routing, and override when the content says otherwise: + +| Lens | Default target phase | +|---|---| +| architecture | `architecture` | +| api | `contracts`, or `protocols` for wire formats | +| security | `defect-scan-semantic` | +| defect | `defect-scan-mechanical` | +| porting | `porting` | +| conventions | none — these are candidates for the orchestrator's `CONVENTIONS.md`, not a phase | + +A lead you cannot route to a phase in the active pipeline is not a lead for +this run. Drop it and say you dropped it. + +## Cutting the list down + +A brief that forwards everything routes nothing. Keep the leads that would +change where a phase starts looking, and drop the rest. Two filters: + +- **Would this phase find it anyway in its first pass?** If yes, it is not + worth a lead — the phase's own rubric already covers it. +- **Is it specific enough to check?** A lead without a file or a module is not + actionable. Note the theme in coverage notes instead of forwarding noise. + +Prefer 3–8 leads per target phase. If a lens produced far more than that, say +so in the coverage notes and forward the strongest. + +## Coverage is spoken, not implied + +`run-meta.json` records truncated slices, skipped lenses, and coverage caps. +Everything outside the sweep is **unscouted, not clean**, and the brief must +say which parts of the repository were never looked at. A later phase that +reads "no leads for module X" must be able to tell "the scout found nothing +there" from "the scout never looked." + +## When there is no run + +If `broadside/` holds no completed run, do not submit one and do not stall the +pipeline. Write the brief with an empty lead table, state plainly under +Coverage and limits that no run exists and therefore no module was scouted +(coverage disposition `NONE`), and validate the coverage criteria against that. Every later phase then proceeds on its own +rubric, exactly as it would in a pipeline without this phase. + +## Output + +Write the brief to the primary output using +`templates/broadside-scout-brief.md`. Keep it short: it is read at the top of +six later phases, and every line costs each of them context. diff --git a/.codecarto/templates/broadside-scout-brief.md b/.codecarto/templates/broadside-scout-brief.md new file mode 100644 index 0000000..94a784e --- /dev/null +++ b/.codecarto/templates/broadside-scout-brief.md @@ -0,0 +1,97 @@ +# Broad-Side Scout Brief — [project_name] + + + +## Scout Context + +- **Run:** `broadside/[run-id]/` (or: no completed run — see Scout Coverage) +- **Model:** [batch model id] +- **Lenses that ran:** [list] +- **Recorded cost:** [from run-meta.json] +- **Pipeline:** [pipeline variant name] +- **Date:** [date] + +> These are unverified scouting leads. Each one is a place to look, not a +> fact. The receiving phase confirms it against the source and cites the +> source — never this brief. + +--- + +## Leads by Phase + + + +| # | Target phase | Lead | Source pointer | Lens | Scout confidence | +|---|--------------|------|----------------|------|------------------| +| 1 | | | | | | + +--- + +## Convention Candidates + + + +| # | Candidate convention | Where the scout saw it | +|---|----------------------|------------------------| +| 1 | | | + +--- + +## Leads Dropped + + + +| # | Lead | Why dropped | +|---|------|-------------| +| 1 | | | + +--- + +## Coverage and limits + + + +- Inspected scope: [modules scanned, or "whole repository in one slice"] +- Skipped scope: [modules the lens globs, slicing cap, or incremental diff excluded; lenses skipped, with reason] +- Evidence basis: batch-model scouting signals only — no source inspection, no tests, no runtime verification +- Known blind spots: [truncated slices and the modules they covered; everything under Skipped scope is unscouted, not clean] +- Coverage disposition: COMPLETE | PARTIAL | NONE (no completed Broad-Side run) + +## Validation + + + +| # | Criterion | Result | Evidence | +|---|-----------|--------|----------| +| 1 | Every forwarded lead names a target phase in this pipeline and a source pointer the target phase can start from. | PASS / PARTIAL / FAIL | | +| 2 | Every lead is marked as an unverified scouting signal; none is stated as a fact or cited as evidence. | PASS / PARTIAL / FAIL | | +| 3 | Leads dropped rather than forwarded are recorded with a reason. | PASS / PARTIAL / FAIL | | +| 4 | Convention candidates are routed to the orchestrator's CONVENTIONS.md review, not to a phase. | PASS / PARTIAL / FAIL | | +| 5 | When no completed Broad-Side run exists, the brief says so explicitly and forwards no leads. | PASS / PARTIAL / FAIL | | +| 6 | Coverage and limits name inspected scope, skipped scope, evidence basis, and blind spots. | PASS / PARTIAL / FAIL | | + +**Validated by:** [session identifier or date] +**Overall:** PASS / PASS WITH GAPS / FAIL diff --git a/.codecarto/workflow/pipeline-scout-first.yaml b/.codecarto/workflow/pipeline-scout-first.yaml new file mode 100644 index 0000000..aeb967e --- /dev/null +++ b/.codecarto/workflow/pipeline-scout-first.yaml @@ -0,0 +1,271 @@ +workflow_name: codebase-reverse-engineering-scout-first +workflow_version: 1 +workflow_goal: The full deep-audit reverse-engineering run, preceded by a Broad-Side scout phase that distills an existing batch reconnaissance run into a routing brief every later phase reads. Identical to pipeline-full-with-deep-audit.yaml apart from that first phase and the brief each phase must account for. +source_location: ../ +validation_protocol: workflow/VALIDATE.md +phase_order: + - broadside-scout + - architecture + - defect-scan-mechanical + - contracts + - protocols + - defect-scan-semantic + - porting + - reimplementation-spec +phases: + - id: broadside-scout + purpose: Distill a completed Broad-Side batch reconnaissance run into a short routing brief that tells each later phase where to look first. Reads only what a prior Broad-Side run already wrote; it never submits a batch and never spends. + skill_path: findings/broadside-scout/SKILL.md + output_template: templates/broadside-scout-brief.md + depends_on: [] + primary_output: findings/broadside-scout/scout-brief.md + secondary_outputs: [] + required_reads: + - GUIDE.md + - workflow/status.yaml + completion_criteria: + - Every forwarded lead names a target phase in this pipeline and a source pointer the target phase can start from. + - Every lead is marked as an unverified scouting signal; none is stated as a fact or cited as evidence. + - Coverage and limits name inspected scope, skipped scope, evidence basis, and blind spots, so unscouted scope is distinguishable from clean scope. + - Leads dropped rather than forwarded are recorded with a reason. + - Convention candidates are routed to the orchestrator's CONVENTIONS.md review, not to a phase. + - When no completed Broad-Side run exists, the brief says so explicitly, forwards no leads, and does not block the pipeline. + handoff_requirements: + - Run validation per workflow/VALIDATE.md. Append validation block to primary output. + - Write the phase handoff to scratch/handoffs/.yaml with owner notes, open questions, and carry-forward routings; completion applies it to workflow/status.yaml. + - Provide closeout_summary and optional closeout_content in the handoff; completion writes the closeout and THREAD_LOG.md entry. + - id: architecture + purpose: Build the layer map, dependency direction, public surfaces, and high-level system structure. + skill_path: findings/architecture/SKILL.md + output_template: templates/architecture-map.md + depends_on: + - broadside-scout + primary_output: findings/architecture/architecture-map.md + secondary_outputs: + - path: findings/public-surfaces/public-surfaces.md + mode: append + - path: findings/runtime-lifecycle/runtime-lifecycle.md + mode: append + - path: findings/state-and-storage/state-and-storage.md + mode: append + - path: findings/build-and-deploy/build-and-deploy.md + mode: append + - path: findings/config-model/config-model.md + mode: append + required_reads: + - GUIDE.md + - workflow/status.yaml + - findings/broadside-scout/scout-brief.md + completion_criteria: + - Every Broad-Side lead the scout brief routed to this phase is confirmed against the source, dismissed with a reason, or carried forward; none is reported as a finding on the brief's authority alone. + - The system intent is documented. + - The layer map and dependency direction are documented. + - Public surfaces are identified. + - Runtime lifecycle, concurrency model, and porting priorities are summarized. + - Findings are marked with evidence levels. + - Coverage and limits name inspected scope, skipped scope, evidence basis, and blind spots. + handoff_requirements: + - Run validation per workflow/VALIDATE.md. Append validation block to primary output. + - Write the phase handoff to scratch/handoffs/.yaml with owner notes, open questions, and carry-forward routings; completion applies it to workflow/status.yaml. + - Provide closeout_summary and optional closeout_content in the handoff; completion writes the closeout and THREAD_LOG.md entry. + - id: defect-scan-mechanical + purpose: Early, context-light defect pass covering logic, error handling, and configuration hazards. Runs before contracts so mechanical bugs are surfaced for the contracts and porting phases to reference. + skill_path: findings/defect-scan-mechanical/SKILL.md + output_template: templates/mechanical-defects.md + depends_on: + - broadside-scout + - architecture + primary_output: findings/defect-scan-mechanical/mechanical-defects.md + secondary_outputs: [] + required_reads: + - GUIDE.md + - workflow/status.yaml + - findings/broadside-scout/scout-brief.md + - findings/architecture/architecture-map.md + completion_criteria: + - Every Broad-Side lead the scout brief routed to this phase is confirmed against the source, dismissed with a reason, or carried forward; none is reported as a finding on the brief's authority alone. + - At least two of the three mechanical passes (1, 2, 6) produced findings or documented "no defects found." + - Each finding has location, severity, evidence level, and recommended action. + - Findings are organized by pass and sorted by severity. + - Summary tables are complete and counts match the detailed findings. + - Items spotted that are actually semantic in nature are routed onward via a carry_forward entry in the phase handoff targeting defect-scan-semantic. + - Findings are marked with evidence levels. + - Coverage and limits name inspected scope, skipped scope, evidence basis, and blind spots. + handoff_requirements: + - Run validation per workflow/VALIDATE.md. Append validation block to primary output. + - Write the phase handoff to scratch/handoffs/.yaml with owner notes, open questions, and carry-forward routings; completion applies it to workflow/status.yaml. + - Provide closeout_summary and optional closeout_content in the handoff; completion writes the closeout and THREAD_LOG.md entry. + - id: contracts + purpose: Recover user-visible behavior, defaults, side effects, error behavior, and black-box acceptance checks. + skill_path: findings/contracts/SKILL.md + output_template: templates/behavioral-contracts.md + depends_on: + - broadside-scout + - architecture + primary_output: findings/contracts/behavioral-contracts.md + secondary_outputs: + - path: findings/public-surfaces/public-surfaces.md + mode: append + - path: findings/runtime-lifecycle/runtime-lifecycle.md + mode: append + - path: findings/state-and-storage/state-and-storage.md + mode: append + - path: findings/config-model/config-model.md + mode: append + required_reads: + - GUIDE.md + - workflow/status.yaml + - findings/broadside-scout/scout-brief.md + - findings/architecture/architecture-map.md + - findings/defect-scan-mechanical/mechanical-defects.md + completion_criteria: + - Every Broad-Side lead the scout brief routed to this phase is confirmed against the source, dismissed with a reason, or carried forward; none is reported as a finding on the brief's authority alone. + - User-facing surfaces are split by surface type. + - Feature contracts record trigger, defaults, outputs, side effects, persisted state, error behavior, and recovery behavior. + - Security and authorization model is documented (if applicable). + - Contract ownership is mapped back to a layer or package. + - A black-box acceptance list is included. + - Findings are marked with evidence levels. + - Coverage and limits name inspected scope, skipped scope, evidence basis, and blind spots. + handoff_requirements: + - Run validation per workflow/VALIDATE.md. Append validation block to primary output. + - Write the phase handoff to scratch/handoffs/.yaml with owner notes, open questions, and carry-forward routings; completion applies it to workflow/status.yaml. + - Provide closeout_summary and optional closeout_content in the handoff; completion writes the closeout and THREAD_LOG.md entry. + - id: protocols + purpose: Capture event catalogs, state machines, persistence formats, and compatibility hazards. + skill_path: findings/protocols/SKILL.md + output_template: templates/protocols-and-state.md + depends_on: + - broadside-scout + - architecture + primary_output: findings/protocols/protocols-and-state.md + secondary_outputs: + - path: findings/public-surfaces/public-surfaces.md + mode: append + - path: findings/runtime-lifecycle/runtime-lifecycle.md + mode: append + - path: findings/state-and-storage/state-and-storage.md + mode: append + - path: findings/config-model/config-model.md + mode: append + required_reads: + - GUIDE.md + - workflow/status.yaml + - findings/broadside-scout/scout-brief.md + - findings/architecture/architecture-map.md + - findings/defect-scan-mechanical/mechanical-defects.md + completion_criteria: + - Every Broad-Side lead the scout brief routed to this phase is confirmed against the source, dismissed with a reason, or carried forward; none is reported as a finding on the brief's authority alone. + - An event catalog is documented. + - A state machine is documented. + - Persistent schema notes are documented. + - Compatibility hazards are documented. + - Findings are marked with evidence levels. + - Coverage and limits name inspected scope, skipped scope, evidence basis, and blind spots. + handoff_requirements: + - Run validation per workflow/VALIDATE.md. Append validation block to primary output. + - Write the phase handoff to scratch/handoffs/.yaml with owner notes, open questions, and carry-forward routings; completion applies it to workflow/status.yaml. + - Provide closeout_summary and optional closeout_content in the handoff; completion writes the closeout and THREAD_LOG.md entry. + - id: defect-scan-semantic + purpose: Deep, context-rich defect pass covering concurrency, security, and API contract violations. Runs after protocols so contracts/protocols context is available, and before porting so the porting phase can synthesize all defects with full understanding of the system. + skill_path: findings/defect-scan-semantic/SKILL.md + output_template: templates/semantic-defects.md + depends_on: + - broadside-scout + - architecture + - contracts + - protocols + - defect-scan-mechanical + primary_output: findings/defect-scan-semantic/semantic-defects.md + secondary_outputs: [] + required_reads: + - GUIDE.md + - workflow/status.yaml + - findings/broadside-scout/scout-brief.md + - findings/architecture/architecture-map.md + - findings/contracts/behavioral-contracts.md + - findings/protocols/protocols-and-state.md + - findings/defect-scan-mechanical/mechanical-defects.md + completion_criteria: + - Every Broad-Side lead the scout brief routed to this phase is confirmed against the source, dismissed with a reason, or carried forward; none is reported as a finding on the brief's authority alone. + - All three semantic passes (3, 4, 5) produced findings or documented "no defects found." + - Each finding has location, severity, evidence level, and recommended action. + - Pass 5 findings cite the contract or protocol reference they violate. + - Findings are organized by pass and sorted by severity; summary tables match the detailed findings. + - Any carry_forward entries that targeted defect-scan-semantic have been resolved or explicitly re-routed. + - Findings are marked with evidence levels. + - Coverage and limits name inspected scope, skipped scope, evidence basis, and blind spots. + handoff_requirements: + - Run validation per workflow/VALIDATE.md. Append validation block to primary output. + - Write the phase handoff to scratch/handoffs/.yaml with owner notes, open questions, and carry-forward routings; completion applies it to workflow/status.yaml. + - Provide closeout_summary and optional closeout_content in the handoff; completion writes the closeout and THREAD_LOG.md entry. + - id: porting + purpose: Synthesize architecture, contracts, protocols, and both defect passes into a reverse-engineering bundle oriented toward porting. Fill the Defect Synthesis section by consolidating mechanical-defects.md and semantic-defects.md into a single porting-oriented view. + skill_path: findings/porting/SKILL.md + output_template: templates/reverse-engineering-bundle.md + depends_on: + - broadside-scout + - architecture + - contracts + - protocols + - defect-scan-mechanical + - defect-scan-semantic + primary_output: findings/porting/reverse-engineering-bundle.md + secondary_outputs: + - path: findings/public-surfaces/public-surfaces.md + mode: append + - path: findings/runtime-lifecycle/runtime-lifecycle.md + mode: append + - path: findings/state-and-storage/state-and-storage.md + mode: append + - path: findings/build-and-deploy/build-and-deploy.md + mode: append + - path: findings/config-model/config-model.md + mode: append + required_reads: + - GUIDE.md + - workflow/status.yaml + - findings/broadside-scout/scout-brief.md + - findings/architecture/architecture-map.md + - findings/contracts/behavioral-contracts.md + - findings/protocols/protocols-and-state.md + - findings/defect-scan-mechanical/mechanical-defects.md + - findings/defect-scan-semantic/semantic-defects.md + completion_criteria: + - Every Broad-Side lead the scout brief routed to this phase is confirmed against the source, dismissed with a reason, or carried forward; none is reported as a finding on the brief's authority alone. + - The system summary, layer map, contract table, protocol notes, and porting findings are synthesized. + - Portability hazards and open questions are separated from facts. + - Feature importance is sorted for porting. + - Defect Synthesis consolidates mechanical-defects.md and semantic-defects.md with porting recommendations (fix before porting / port differently / leave behind). + - Findings are marked with evidence levels. + - Coverage and limits name inspected scope, skipped scope, evidence basis, and blind spots. + - The Source Index makes the bundle a self-contained compression boundary and identifies targeted deep-read triggers. + handoff_requirements: + - Run validation per workflow/VALIDATE.md. Append validation block to primary output. + - Write the phase handoff to scratch/handoffs/.yaml with owner notes, open questions, and carry-forward routings; completion applies it to workflow/status.yaml. + - Provide closeout_summary and optional closeout_content in the handoff; completion writes the closeout and THREAD_LOG.md entry. + - id: reimplementation-spec + purpose: Produce the final language-agnostic reimplementation plan and acceptance spec. The Strategic Alignment Hook in GUIDE.md decides between the default and opinionated template variants. + skill_path: findings/reimplementation-spec/SKILL.md + output_template: templates/reimplementation-spec.md + depends_on: + - porting + primary_output: findings/reimplementation-spec/reimplementation-spec.md + secondary_outputs: [] + required_reads: + - GUIDE.md + - workflow/status.yaml + - findings/porting/reverse-engineering-bundle.md + completion_criteria: + - Concept-level modules are defined. + - Required behaviors are stated. + - Protocol and persisted state expectations are stated. + - Acceptance scenarios and known unknowns are included. + - Defects identified in either scan are explicitly designed-around or noted as "left behind", with the choice cited. + - Findings are marked with evidence levels. + - Coverage and limits name inspected scope, skipped scope, evidence basis, and blind spots. + - Lower-level findings are deep-read only when the porting bundle identifies a gap, conflict, missing acceptance detail, or defect rationale. + handoff_requirements: + - Run validation per workflow/VALIDATE.md. Append validation block to primary output. + - Write the phase handoff to scratch/handoffs/.yaml with owner notes, open questions, and carry-forward routings; completion applies it to workflow/status.yaml. + - Provide closeout_summary and optional closeout_content in the handoff; completion writes the closeout and THREAD_LOG.md entry. diff --git a/CHANGELOG.md b/CHANGELOG.md index 66b0b72..f01477e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to this project are documented here. The format is based on ### Added +- **Broad-Side: the `scout-first` pipeline** (#139). New `workflow/pipeline-scout-first.yaml` (alias `scout-first`) is the deep-audit run with a `broadside-scout` phase in front. That phase distills a completed batch reconnaissance run into `findings/broadside-scout/scout-brief.md`: leads addressed to a specific later phase, each with the source pointer that phase starts from, plus the coverage accounting that distinguishes "the scout found nothing there" from "the scout never looked." Architecture, defect-scan-mechanical, contracts, protocols, defect-scan-semantic, and porting read the brief and carry a completion criterion requiring every lead routed to them to be confirmed against the source, dismissed with a reason, or carried forward — none may be reported as a finding on the brief's authority. `reimplementation-spec` deliberately does not read it; the porting bundle stays its compression boundary. The distillation step exists because run directories are timestamped and gitignored, so no pipeline YAML could name one as a `required_reads` path — and because deciding which leads are worth six phases' attention is judgment work, not a file copy. The scout phase never submits a batch and never spends: with no run on disk it writes an explicitly empty brief and the pipeline proceeds exactly as `full-with-deep-audit` would. A drift test pins the new variant to the deep-audit pipeline it wraps, so the two cannot diverge silently. - **Broad-Side on the Pi extension** (#138). New `/codecarto-broadside [submit|collect|status|models] [lenses…]` with tab-completion for actions, lens names, and flags (`--incremental`, `--max-cost=N`, `--wait=SECONDS`, `--no-synthesis`, `--no-triage`, `--no-retry-truncated`, `--benchmarks`), and a live per-lens progress widget while batches poll. Two deliberate divergences from MCP: the spend decision is interactive — Pi shows the per-lens breakdown, the rates, the limit, and whether the run exceeds it, and an approval *is* the force flag — where MCP has to refuse and wait for `force: true`; and the command takes no API key argument, because a key typed into a slash command lands in the session transcript (`OPENROUTER_API_KEY` or `config.yaml` only). Like the MCP tool, it runs on a repository with no CodeCartographer workspace, and on such a repository the result renders into its own widget rather than the phase widget. `runBroadsideSubmit` grows an optional `confirm` hook that receives the pre-flight estimate after slicing and before any state write or submission; declining throws `BroadsideCancelledError` and nothing is submitted. Surfaces without a human keep the refuse-unless-force path unchanged. - **Broad-Side: the reading guide is reachable, and the docs say the feature exists.** Broad-Side shipped across seven PRs with its user-facing paper trail lagging behind the code. `.codecarto/broadside/SKILL.md` was written but unreachable: `codecarto_skill` resolves `.codecarto/skills//SKILL.md`, so `{name: "broadside"}` returned "Unknown skill" and `codecarto_list_skills` never mentioned it. Both surfaces now serve it under the name `broadside`, exempt from the post-pipeline completion gate — a scout run is read *before* the pipeline and during it — and readable on a repository that has scout state and no workspace at all (the packaged copy answers when the workspace has none). `codecarto_list_skills` lists it apart from the post-pipeline set, and an unknown-skill error names the exemption. New `references/broadside.md` in the packaged agent skill teaches when to scout, the cost guardrails, and the leads-never-evidence rule, served as the `broadside` topic of `codecarto_guide`; the skill overview, README, MANUAL, and the MCP quickstart now cover the feature instead of leaving a single table row as its only mention. - **Broad-Side: every run knob has a repository default** (`.codecarto/broadside/config.yaml`). `incremental`, `retry_truncated`, `include_synthesis`, `include_triage`, and `wait_seconds` were call-parameters only, so a repo could not fix its own scouting policy without restating it on every submit and collect. All five are now config keys documented alongside `model`, `api_key`, `default_lenses`, `max_cost`, and the `pricing` overrides; an explicit tool parameter always wins, and a malformed value falls back to the shipped default rather than failing a run. A test asserts the documented key set and the parsed key set are the same, so a key can no longer be documented into existence without being read. diff --git a/CLAUDE.md b/CLAUDE.md index 942d8b7..1cc9961 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -55,7 +55,7 @@ When adding a feature, the question to ask is "does this work on all three surfa ### Pipeline shape -Pipelines are YAML DAGs in `.codecarto/workflow/pipeline*.yaml`. Each `PipelinePhase` declares `depends_on`, `primary_output`, `required_reads`, `completion_criteria`, etc. The active variant is the `pipeline:` field in `status.yaml`; six analysis variants plus the forward-synthesis pipeline ship today, and the default is `pipeline-full-with-deep-audit.yaml`. Phases form a DAG (contracts and protocols run in parallel after architecture), not a linear chain — `getNextEligiblePhase` walks `phase_order` and picks the first non-`complete` phase whose deps are all `complete`. +Pipelines are YAML DAGs in `.codecarto/workflow/pipeline*.yaml`. Each `PipelinePhase` declares `depends_on`, `primary_output`, `required_reads`, `completion_criteria`, etc. The active variant is the `pipeline:` field in `status.yaml`; seven analysis variants plus the forward-synthesis pipeline ship today, and the default is `pipeline-full-with-deep-audit.yaml`. Phases form a DAG (contracts and protocols run in parallel after architecture), not a linear chain — `getNextEligiblePhase` walks `phase_order` and picks the first non-`complete` phase whose deps are all `complete`. ### Invariant tests are the load-bearing guardrail diff --git a/MANUAL.md b/MANUAL.md index e1e7ff9..8aba291 100644 --- a/MANUAL.md +++ b/MANUAL.md @@ -73,6 +73,8 @@ Results land in `.codecarto/broadside//`; read `synthesis.md` and `triage.m **These findings are leads, not evidence.** Each lens is one shot with no cross-file traversal and no runtime verification. Every finding is a `file:line` pointer for the real analysis to confirm — never cite a Broad-Side report as a source in a phase artifact. See [README.md](README.md#broad-side-batch-reconnaissance) and `.codecarto/broadside/SKILL.md`. +To have the pipeline itself consume the results, choose the `scout-first` variant in Step 2. Its first phase distills the run into `findings/broadside-scout/scout-brief.md`, and the six phases after it must account for the leads routed to them — confirm against the source, dismiss with a reason, or carry forward. Without that variant the results are still yours to read; nothing in the pipeline points at them. + Skip this entirely on a repository you can read directly; a sweep that costs more than the reading it saves is waste. @@ -88,6 +90,9 @@ To use a different pipeline, you have two options: Here's how to decide: +**"I ran Broad-Side and want the phases to use it."** +Use `workflow/pipeline-scout-first.yaml` (8 phases). The deep-audit run with a `broadside-scout` phase in front that turns the reconnaissance run into a routing brief every later phase reads. With no run on disk the brief is empty and the pipeline behaves exactly like the deep-audit variant. + **"I want the full analysis with defect triage."** (default) Keep `workflow/pipeline-full-with-audit.yaml` (6 phases). Produces architecture, defect report, behavioral contracts, protocol notes, a porting synthesis, and a reimplementation spec. The defect findings feed into the porting phase so you can decide what to fix, port differently, or leave behind. diff --git a/README.md b/README.md index 7b17ae3..1e52f73 100644 --- a/README.md +++ b/README.md @@ -239,6 +239,7 @@ The default is a 7-phase run that splits the defect scan into a mechanical early | Variant | Phases | Use when | |---|---|---| | **Full with deep audit** (default) | 7 | Complete analysis with split defect scan; reimplementation grounded in contracts/protocols-aware defect findings | +| **Scout first** | 8 | The deep-audit run preceded by a `broadside-scout` phase that distills an existing [Broad-Side](#broad-side-batch-reconnaissance) run into a routing brief every later phase reads | | **Full with audit** | 6 | Single early defect scan; cheaper than the deep variant when defects are mostly mechanical | | **Full** | 5 | Porting or reimplementation without any defect scan | | **Defect scan** | 2 | Maintenance audit to surface latent problems | @@ -253,6 +254,7 @@ Switch the active pipeline with `/codecarto-switch-pipeline ` (Pi) or ` | Variant | Pipeline file | |---|---| | Full with deep audit (**default**) | `workflow/pipeline-full-with-deep-audit.yaml` | +| Scout first | `workflow/pipeline-scout-first.yaml` | | Full with audit | `workflow/pipeline-full-with-audit.yaml` | | Full | `workflow/pipeline.yaml` | | Defect scan | `workflow/pipeline-defect-scan.yaml` | @@ -385,7 +387,11 @@ Repository defaults live in `.codecarto/broadside/config.yaml` (`model`, `api_ke On the Pi extension the same run is `/codecarto-broadside [submit|collect|status|models] [lenses…]`, with tab-completion for actions and lens names and live per-lens progress while batches poll. The two surfaces differ in one deliberate place: MCP cannot ask a human, so it refuses a run over `max_cost` until you pass `force`; Pi shows the per-lens breakdown and asks, and your approval *is* the force flag. Neither surface takes an API key as a command argument — a key typed into a slash command lands in the session transcript. -Broad-Side needs runtime code, so it is an executable-surface feature: Pi and MCP have it, the pure drop-in template does not (it carries only the reading guide). The `broadside-scout` pipeline phase ([#139](https://github.com/HuginnIndustries/CodeCartographer/issues/139)) is still on the roadmap — nothing in a pipeline YAML consumes scout output yet; you read it and decide. See [ROADMAP.md](ROADMAP.md). +Broad-Side needs runtime code, so firing a run is an executable-surface feature: Pi and MCP have it, the pure drop-in template does not (it carries only the reading guide). + +**Feeding a run into the pipeline** is the `scout-first` variant. Its first phase, `broadside-scout`, distills a completed run into `findings/broadside-scout/scout-brief.md` — a short list of leads, each routed to a specific later phase with the source pointer that phase should start from. Six later phases read the brief, and each must account for the leads addressed to it at validation: confirmed against the source, dismissed with a reason, or carried forward. None may be reported as a finding on the brief's authority. The scout phase itself never submits a batch and never spends; with no run on disk it writes an explicitly empty brief and the pipeline proceeds unchanged. + +The indirection is deliberate. Run directories are timestamped and gitignored, so no pipeline YAML could name one as a `required_reads` path; the brief is the stable, reviewable artifact that a phase contract can point at — and distilling into it is where the "which of these is worth anyone's attention" judgment happens. --- diff --git a/ROADMAP.md b/ROADMAP.md index 4f876bf..431cd16 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -27,6 +27,10 @@ file only moves when a tier completes. and clamps lens `max_tokens` to the provider's completion ceiling. - Triage post-pass on collect: findings scored by impact × difficulty into a P0–P3 work order with effort estimates, saved as triage.json/md. +- `scout-first` pipeline variant: a `broadside-scout` phase distills a completed + run into `findings/broadside-scout/scout-brief.md`; architecture, both defect + scans, contracts, protocols, and porting read it and account for their leads at + validation. The phase never submits and never spends. - Pi surface: `/codecarto-broadside [submit|collect|status|models] [lenses…]` with a spend confirmation (`confirm` hook on `runBroadsideSubmit`), so Pi asks where MCP must refuse. @@ -53,7 +57,7 @@ file only moves when a tier completes. | Item | Issue | Notes | |---|---|---| | **Pi extension** — `/codecarto-broadside` command with lens picker and live progress | [#138](https://github.com/HuginnIndustries/CodeCartographer/issues/138) | **Shipped**: four actions with tab-completed lens picker, live per-lens progress widget, and an interactive spend confirmation in place of MCP's refuse-unless-`force` | -| **Pipeline phase** — `broadside-scout` phase feeding later phases via `required_reads` | [#139](https://github.com/HuginnIndustries/CodeCartographer/issues/139) | SKILL.md contract stays: leads, never evidence | +| **Pipeline phase** — `broadside-scout` phase feeding later phases via `required_reads` | [#139](https://github.com/HuginnIndustries/CodeCartographer/issues/139) | **Shipped**: `pipeline-scout-first.yaml`. The phase distills a run into a stable brief (run dirs are timestamped, so no YAML could name one), six phases read it, and each must confirm, dismiss, or carry forward every lead routed to it. Leads, never evidence — enforced at validation | | **Zero-config executive** — meta-pass picks lenses and slicing resolution from repo shape | [#140](https://github.com/HuginnIndustries/CodeCartographer/issues/140) | **Shipped**: `auto` slicing collapses small repos to one slice, directory-splits large ones | ## Tier 3 — cost and coverage economics diff --git a/agent-skill/codecartographer/references/pipeline-selection.md b/agent-skill/codecartographer/references/pipeline-selection.md index 8cd7f7e..a9da1d0 100644 --- a/agent-skill/codecartographer/references/pipeline-selection.md +++ b/agent-skill/codecartographer/references/pipeline-selection.md @@ -10,6 +10,7 @@ Pass an alias to `codecarto_init` as `pipeline`. The default is `full-with-deep- | `full` | architecture → contracts → protocols → porting → reimplementation-spec | porting or rewriting, no defect audit | | `full-with-audit` | adds a single defect-scan after architecture | porting, with defects surfaced once | | `full-with-deep-audit` *(default)* | splits the scan: mechanical after architecture, semantic after protocols | porting or rewriting where correctness matters | +| `scout-first` | `full-with-deep-audit` behind a `broadside-scout` brief | a repository large enough that a Broad-Side sweep already ran and should steer the phases | | `synthesis` | vision-capture → goal-synthesis-propose → spec-merge → goal-synthesis-finalize | forward synthesis of a *new* product, not reverse-engineering | ## Deep audit versus plain audit @@ -23,6 +24,19 @@ The mechanical pass routes anything it cannot settle locally to the semantic pas Choose `full-with-audit` when one combined pass is enough and you want fewer phases. Choose `full-with-deep-audit` when the output will drive a rewrite, since a semantic pass without protocols context will miss the findings that most change a port. +## Scout-first needs a scout run + +`scout-first` is `full-with-deep-audit` with one phase in front: `broadside-scout` +distills a completed Broad-Side batch reconnaissance run into a routing brief, +and the six phases after it read that brief and must account for the leads +addressed to them — confirmed, dismissed with a reason, or carried forward. +See `references/broadside.md` for the sweep itself. + +The scout phase never submits a batch and never spends; it reads only what a +prior run wrote. Choosing this variant without having run Broad-Side gets you an +explicitly empty brief and, from there on, exactly `full-with-deep-audit`. So +fire the sweep first, or choose the plain deep-audit variant. + ## Synthesis is a different workspace The `synthesis` pipeline plans a new product from a vision brief and a library of reusable specs. It does **not** treat the surrounding repository as source evidence. It has preflight gates: a completed `inputs/vision.md`, a valid non-empty library, and — for merge and finalization — at least one human-confirmed selection. `codecarto_vision` runs the guided interview that produces the brief. diff --git a/core/pipeline.ts b/core/pipeline.ts index 2ad3b1d..4974549 100644 --- a/core/pipeline.ts +++ b/core/pipeline.ts @@ -13,6 +13,7 @@ import { pathExists } from "./utils.ts"; export const PIPELINE_ALIASES: Record = { "full-with-audit": "workflow/pipeline-full-with-audit.yaml", "full-with-deep-audit": "workflow/pipeline-full-with-deep-audit.yaml", + "scout-first": "workflow/pipeline-scout-first.yaml", full: "workflow/pipeline.yaml", "defect-scan": "workflow/pipeline-defect-scan.yaml", lite: "workflow/pipeline-lite.yaml", diff --git a/tests/broadside-scout-pipeline.test.mjs b/tests/broadside-scout-pipeline.test.mjs new file mode 100644 index 0000000..4d4f3e1 --- /dev/null +++ b/tests/broadside-scout-pipeline.test.mjs @@ -0,0 +1,114 @@ +// The scout-first pipeline (#139). Its whole point is routing Broad-Side leads +// into phases without letting a cheap batch model's guesses become findings, +// so the contract worth pinning is: who reads the brief, who must account for +// it, and that the variant has not silently drifted from the deep-audit +// pipeline it wraps. + +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const CODECARTO = join(REPO_ROOT, ".codecarto"); +const { parseSimpleYaml } = await import(pathToFileURL(`${REPO_ROOT}/core/yaml.ts`).href); +const { PIPELINE_ALIASES } = await import(pathToFileURL(`${REPO_ROOT}/core/pipeline.ts`).href); + +const readPipeline = async (file) => parseSimpleYaml(await readFile(join(CODECARTO, "workflow", file), "utf8")); +const scoutFirst = await readPipeline("pipeline-scout-first.yaml"); +const deepAudit = await readPipeline("pipeline-full-with-deep-audit.yaml"); + +const BRIEF = "findings/broadside-scout/scout-brief.md"; +const RECEIVERS = ["architecture", "defect-scan-mechanical", "contracts", "protocols", "defect-scan-semantic", "porting"]; + +const phase = (pipeline, id) => pipeline.phases.find((p) => p.id === id); + +test("scout-first is reachable by alias and leads with the scout phase", () => { + assert.equal(PIPELINE_ALIASES["scout-first"], "workflow/pipeline-scout-first.yaml"); + assert.equal(scoutFirst.phase_order[0], "broadside-scout"); + assert.deepEqual(phase(scoutFirst, "broadside-scout").depends_on, [], "the scout phase gates on nothing"); + assert.equal(phase(scoutFirst, "broadside-scout").primary_output, BRIEF); +}); + +test("every phase that reads the brief must account for its leads at validation", () => { + for (const id of RECEIVERS) { + const p = phase(scoutFirst, id); + assert.ok(p, `${id} should exist in scout-first`); + assert.ok(p.required_reads.includes(BRIEF), `${id} must read the scout brief`); + assert.ok(p.depends_on.includes("broadside-scout"), `${id} must depend on the scout phase`); + // Reading the brief without a criterion that forces confirmation is how a + // batch model's guess becomes a cited finding. + const accounting = p.completion_criteria.filter((c) => /Broad-Side lead/i.test(c)); + assert.equal(accounting.length, 1, `${id} must carry exactly one lead-accounting criterion`); + assert.match(accounting[0], /confirmed against the source|dismissed|carried forward/i); + assert.match(accounting[0], /none is reported as a finding/i, `${id}'s criterion must forbid citing the brief`); + } +}); + +test("the reimplementation phase keeps the porting bundle as its only compression boundary", () => { + // The spec phase deliberately reads one upstream artifact. Feeding it raw + // scout leads would reopen exactly the boundary that phase exists to hold. + const spec = phase(scoutFirst, "reimplementation-spec"); + assert.ok(!spec.required_reads.includes(BRIEF), "reimplementation-spec must not read the scout brief"); + assert.ok(!spec.depends_on.includes("broadside-scout")); +}); + +test("scout-first differs from full-with-deep-audit only by the scout additions", () => { + // The variant was derived from the deep-audit pipeline. If the two drift, + // a user picking scout-first quietly gets a different analysis run. + assert.deepEqual( + scoutFirst.phase_order, + ["broadside-scout", ...deepAudit.phase_order], + "scout-first must be the deep-audit order with the scout phase in front", + ); + + for (const id of deepAudit.phase_order) { + const mine = phase(scoutFirst, id); + const theirs = phase(deepAudit, id); + assert.equal(mine.purpose, theirs.purpose, `${id}: purpose drifted`); + assert.equal(mine.skill_path, theirs.skill_path, `${id}: skill_path drifted`); + assert.equal(mine.output_template, theirs.output_template, `${id}: output_template drifted`); + assert.equal(mine.primary_output, theirs.primary_output, `${id}: primary_output drifted`); + assert.deepEqual(mine.secondary_outputs ?? [], theirs.secondary_outputs ?? [], `${id}: secondary_outputs drifted`); + assert.deepEqual(mine.handoff_requirements, theirs.handoff_requirements, `${id}: handoff_requirements drifted`); + + assert.deepEqual( + mine.depends_on.filter((dep) => dep !== "broadside-scout"), + theirs.depends_on, + `${id}: dependencies drifted beyond the scout edge`, + ); + assert.deepEqual( + mine.required_reads.filter((path) => path !== BRIEF), + theirs.required_reads, + `${id}: required_reads drifted beyond the scout brief`, + ); + assert.deepEqual( + mine.completion_criteria.filter((c) => !/Broad-Side lead/i.test(c)), + theirs.completion_criteria, + `${id}: completion_criteria drifted beyond the lead-accounting rule`, + ); + } +}); + +test("the scout phase reads a run rather than paying for one", async () => { + // A phase prompt that told an executor to submit batches would spend money + // inside an unattended /codecarto-next --auto run. + // Prose assertions run against a whitespace-normalized copy: the source is + // hard-wrapped, so a sentence can straddle a newline. + const skill = (await readFile(join(CODECARTO, "findings", "broadside-scout", "SKILL.md"), "utf8")) + .replace(/\s+/g, " "); + assert.match(skill, /never submits a batch/i, "the skill must state that it does not submit"); + assert.match(skill, /no completed run/i, "the skill must handle the no-run case"); + assert.match(skill, /do not submit one/i, "the no-run path must forbid firing a run to fill the gap"); + + const criteria = phase(scoutFirst, "broadside-scout").completion_criteria; + assert.ok( + criteria.some((c) => /no completed Broad-Side run exists/i.test(c) && /does not block/i.test(c)), + "a missing run must be a documented outcome, not a stall", + ); + assert.ok( + criteria.some((c) => /unverified scouting signal/i.test(c)), + "the brief's own criteria must carry the leads-never-evidence rule", + ); +}); From fd179e30010ffb2c89843a6d983f95fb793c082a Mon Sep 17 00:00:00 2001 From: James Sesler Date: Sun, 30 Aug 2026 15:46:37 -0400 Subject: [PATCH 4/4] feat: per-lens model overrides for Broad-Side (#141) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The multi-model item was half-built: the catalog, the models action, and the pricing/capability pre-flight all shipped, but a run still had exactly one model for all six lenses. That is the wrong granularity. A stronger model changes what the security and defect lenses find far more than it changes an architecture map, so the useful knob is per lens, not per run. lens_models in config.yaml routes individual lenses to their own batch model. Each distinct model is resolved and pre-flighted independently: priced from the live catalog, refused without structured-output support, and clamped to its own completion ceiling. The estimate carries a model and its rates per lens plus a mixedModels flag; the Pi confirmation names the overridden model on the affected rows, so a mixed-model run cannot be approved without seeing which lens costs what. Submission now fires from the priced rows rather than recomputing them — the request that goes out is the one the user approved. run-meta.json records lens_models, and collect's truncation retry re-submits on the lens's own model and ceiling rather than the run default, which would otherwise change the model mid-run. An override naming an unknown lens id is dropped, not carried: it can only be a typo, and a key that looks applied but is not is worse than one that never appears. What this deliberately does NOT do is change the default. The roadmap's remaining item was "a stronger default for semantic lenses," and that is a comparative-evaluation question on real repositories with real spend — picking one here would spend every user's money on our guess. The config says so and points at the models action. Per-model prompt tweaks stay open for the same reason: they need the same evidence. Tests: override parsing including the dropped-typo case, and an end-to-end run asserting the override reaches the estimate with its own rates, both levels of the batch payload, and the recorded per-lens entry, while the other lens stays on the default. The config-key drift test now separates top-level keys from nested examples instead of conflating them. 443 pass. Co-Authored-By: Claude Opus 5 --- .codecarto/broadside/SKILL.md | 9 + .codecarto/broadside/config.yaml | 18 ++ CHANGELOG.md | 1 + README.md | 2 +- ROADMAP.md | 5 +- .../codecartographer/references/broadside.md | 11 +- core/broadside.ts | 183 +++++++++++++----- extensions/codecarto/index.ts | 11 +- tests/broadside.test.mjs | 123 +++++++++++- 9 files changed, 298 insertions(+), 65 deletions(-) diff --git a/.codecarto/broadside/SKILL.md b/.codecarto/broadside/SKILL.md index 3322138..68f44e9 100644 --- a/.codecarto/broadside/SKILL.md +++ b/.codecarto/broadside/SKILL.md @@ -90,6 +90,15 @@ Collect runs two cross-lens post-passes by default: **synthesis** (the executive report) and **triage** (the prioritized work order). Pass `include_synthesis: false` or `include_triage: false` on collect to skip one. +Lenses do not all have to run on the same model. `lens_models` in `config.yaml` +routes individual lenses to their own batch model — the usual reason being that +a stronger model changes security and defect findings more than it changes an +architecture map. Each override is priced, capability-checked, and clamped like +the default, the submit estimate breaks cost out per lens, and `run-meta.json` +records which lens ran on what. No stronger default is shipped: which model is +worth the money depends on the repository and the budget, so compare with the +`models` action and decide. + Every run knob — `incremental`, `retry_truncated`, `include_synthesis`, `include_triage`, `wait_seconds` — also has a repository default under the same name in this directory's `config.yaml`, alongside `model`, `api_key`, diff --git a/.codecarto/broadside/config.yaml b/.codecarto/broadside/config.yaml index 391af45..8c37721 100644 --- a/.codecarto/broadside/config.yaml +++ b/.codecarto/broadside/config.yaml @@ -32,6 +32,24 @@ # - conventions # - porting +# Per-lens model overrides. A lens listed here runs on its own model; every +# other lens uses the `model` above. This is how you spend more where it pays: +# the cheap default is right for architecture and conventions, while security +# and defect findings are the ones a stronger model most changes. Each override +# is pre-flighted like the default — priced from the live catalog, refused +# without structured-output support, and clamped to its own completion ceiling +# — and the submit estimate breaks the cost out per lens so a mixed-model run +# cannot be approved without seeing which lens costs what. +# +# There is deliberately no stronger default shipped here: which model is worth +# the money for the semantic lenses depends on your repository and your budget, +# and picking one for you would spend your money on our guess. Compare +# candidates with the `models` action first. +# +# lens_models: +# security: anthropic/claude-opus-4.5:batch +# defect: anthropic/claude-opus-4.5:batch + # Approximate run expense limit in USD (0 = no limit). Before submitting, # Broad-Side estimates the run cost from the collected file sizes and the # model's per-token pricing — fetched live from OpenRouter's model catalog diff --git a/CHANGELOG.md b/CHANGELOG.md index f01477e..7452a88 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to this project are documented here. The format is based on ### Added +- **Broad-Side: per-lens model overrides** (#141). `lens_models` in `.codecarto/broadside/config.yaml` runs individual lenses on their own batch model, so a repository can spend more where it pays — a stronger model changes security and defect findings far more than it changes an architecture map — without raising the price of every lens. Each distinct model is resolved and pre-flighted independently: priced from the live catalog, refused without structured-output support, and clamped to its own completion ceiling. The submit estimate carries a model and its rates per lens (and a `mixedModels` flag), the Pi confirmation names the overridden model on each affected row so a mixed-model run cannot be approved without seeing which lens costs what, submission fires from the priced rows rather than recomputing, `run-meta.json` records `lens_models`, and collect's truncation retry re-submits on the lens's own model and ceiling instead of the run default. An override naming an unknown lens id is dropped rather than silently ignored. **No stronger default ships with this**: which model earns its price for the semantic lenses is a comparative-evaluation question on real repositories, and choosing one for every user would spend their money on our guess — compare with the `models` action and set `lens_models` yourself. - **Broad-Side: the `scout-first` pipeline** (#139). New `workflow/pipeline-scout-first.yaml` (alias `scout-first`) is the deep-audit run with a `broadside-scout` phase in front. That phase distills a completed batch reconnaissance run into `findings/broadside-scout/scout-brief.md`: leads addressed to a specific later phase, each with the source pointer that phase starts from, plus the coverage accounting that distinguishes "the scout found nothing there" from "the scout never looked." Architecture, defect-scan-mechanical, contracts, protocols, defect-scan-semantic, and porting read the brief and carry a completion criterion requiring every lead routed to them to be confirmed against the source, dismissed with a reason, or carried forward — none may be reported as a finding on the brief's authority. `reimplementation-spec` deliberately does not read it; the porting bundle stays its compression boundary. The distillation step exists because run directories are timestamped and gitignored, so no pipeline YAML could name one as a `required_reads` path — and because deciding which leads are worth six phases' attention is judgment work, not a file copy. The scout phase never submits a batch and never spends: with no run on disk it writes an explicitly empty brief and the pipeline proceeds exactly as `full-with-deep-audit` would. A drift test pins the new variant to the deep-audit pipeline it wraps, so the two cannot diverge silently. - **Broad-Side on the Pi extension** (#138). New `/codecarto-broadside [submit|collect|status|models] [lenses…]` with tab-completion for actions, lens names, and flags (`--incremental`, `--max-cost=N`, `--wait=SECONDS`, `--no-synthesis`, `--no-triage`, `--no-retry-truncated`, `--benchmarks`), and a live per-lens progress widget while batches poll. Two deliberate divergences from MCP: the spend decision is interactive — Pi shows the per-lens breakdown, the rates, the limit, and whether the run exceeds it, and an approval *is* the force flag — where MCP has to refuse and wait for `force: true`; and the command takes no API key argument, because a key typed into a slash command lands in the session transcript (`OPENROUTER_API_KEY` or `config.yaml` only). Like the MCP tool, it runs on a repository with no CodeCartographer workspace, and on such a repository the result renders into its own widget rather than the phase widget. `runBroadsideSubmit` grows an optional `confirm` hook that receives the pre-flight estimate after slicing and before any state write or submission; declining throws `BroadsideCancelledError` and nothing is submitted. Surfaces without a human keep the refuse-unless-force path unchanged. - **Broad-Side: the reading guide is reachable, and the docs say the feature exists.** Broad-Side shipped across seven PRs with its user-facing paper trail lagging behind the code. `.codecarto/broadside/SKILL.md` was written but unreachable: `codecarto_skill` resolves `.codecarto/skills//SKILL.md`, so `{name: "broadside"}` returned "Unknown skill" and `codecarto_list_skills` never mentioned it. Both surfaces now serve it under the name `broadside`, exempt from the post-pipeline completion gate — a scout run is read *before* the pipeline and during it — and readable on a repository that has scout state and no workspace at all (the packaged copy answers when the workspace has none). `codecarto_list_skills` lists it apart from the post-pipeline set, and an unknown-skill error names the exemption. New `references/broadside.md` in the packaged agent skill teaches when to scout, the cost guardrails, and the leads-never-evidence rule, served as the `broadside` topic of `codecarto_guide`; the skill overview, README, MANUAL, and the MCP quickstart now cover the feature instead of leaving a single table row as its only mention. diff --git a/README.md b/README.md index 1e52f73..837324e 100644 --- a/README.md +++ b/README.md @@ -383,7 +383,7 @@ codecarto_broadside {cwd, action: "collect"} # poll, save, synt Submit and collect are separate because batch jobs routinely take tens of minutes; collect is resumable and picks up whatever is still in flight. Submit prices the run from the collected file sizes against the model's live per-token pricing (cached 24h) and refuses when the estimate exceeds `max_cost` unless `force: true` is passed — a pre-flight estimate, not a runtime stop. Actual spend lands in each run's `run-meta.json`. -Repository defaults live in `.codecarto/broadside/config.yaml` (`model`, `api_key`, `default_lenses`, `max_cost`, `pricing` overrides, `incremental`, `retry_truncated`, `include_synthesis`, `include_triage`, `wait_seconds`); an explicit tool parameter always wins. `codecarto_skill {cwd, name: "broadside"}` returns the reading guide for a completed run, and unlike post-pipeline skills it is not gated on a finished pipeline. +Repository defaults live in `.codecarto/broadside/config.yaml` (`model`, `api_key`, `default_lenses`, `max_cost`, `pricing` overrides, `lens_models`, `incremental`, `retry_truncated`, `include_synthesis`, `include_triage`, `wait_seconds`); an explicit tool parameter always wins. `lens_models` routes individual lenses to their own batch model — a stronger model changes security and defect findings far more than it changes an architecture map — and each override is priced, capability-checked, and clamped exactly like the default, with the estimate broken out per lens so a mixed-model run cannot be approved without seeing which lens costs what. CodeCartographer ships no stronger default: which model earns its price depends on your repository and budget, so compare candidates with the `models` action and choose. `codecarto_skill {cwd, name: "broadside"}` returns the reading guide for a completed run, and unlike post-pipeline skills it is not gated on a finished pipeline. On the Pi extension the same run is `/codecarto-broadside [submit|collect|status|models] [lenses…]`, with tab-completion for actions and lens names and live per-lens progress while batches poll. The two surfaces differ in one deliberate place: MCP cannot ask a human, so it refuses a run over `max_cost` until you pass `force`; Pi shows the per-lens breakdown and asks, and your approval *is* the force flag. Neither surface takes an API key as a command argument — a key typed into a slash command lands in the session transcript. diff --git a/ROADMAP.md b/ROADMAP.md index 431cd16..ee8fd85 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -27,6 +27,9 @@ file only moves when a tier completes. and clamps lens `max_tokens` to the provider's completion ceiling. - Triage post-pass on collect: findings scored by impact × difficulty into a P0–P3 work order with effort estimates, saved as triage.json/md. +- Per-lens model overrides (`lens_models`): each lens can run on its own batch + model, pre-flighted and priced independently, recorded in `run-meta.json`, and + honored by the truncation retry. - `scout-first` pipeline variant: a `broadside-scout` phase distills a completed run into `findings/broadside-scout/scout-brief.md`; architecture, both defect scans, contracts, protocols, and porting read it and account for their leads at @@ -64,7 +67,7 @@ file only moves when a tier completes. | Item | Issue | Notes | |---|---|---| -| **Multi-model** — DeepSeek/Anthropic batch endpoints behind the lens registry | [#141](https://github.com/HuginnIndustries/CodeCartographer/issues/141) | Partially shipped: catalog lookup, `models` action, pricing + capability pre-flight. Remaining: per-model prompt tweaks and a stronger default for semantic lenses | +| **Multi-model** — DeepSeek/Anthropic batch endpoints behind the lens registry | [#141](https://github.com/HuginnIndustries/CodeCartographer/issues/141) | **Mechanism shipped**: catalog lookup, `models` action, pricing + capability pre-flight, and per-lens model overrides (`lens_models`) priced, clamped, and retried per lens. **Deliberately not shipped**: a stronger default for the semantic lenses. Which model earns its price is a comparative-evaluation question on real repositories, and picking one for every user spends their money on our guess. Per-model prompt tweaks stay open pending that same evidence | | **Incremental re-scouting** — diff against previous run's HEAD, rescan changed modules only | [#142](https://github.com/HuginnIndustries/CodeCartographer/issues/142) | **Shipped**: `incremental: true` diffs against the prior run's HEAD; dirty tree falls back to full scan | | **CodeCartoShow pipeline stage** — BATCH-SCOUT between SELECT and the interactive run | [CodeCartoShow#1](https://github.com/HuginnIndustries/CodeCartoShow/issues/1) | `scripts/batch-analyze.py` proved it; evidence rules apply unchanged | diff --git a/agent-skill/codecartographer/references/broadside.md b/agent-skill/codecartographer/references/broadside.md index 105d427..c2d9a21 100644 --- a/agent-skill/codecartographer/references/broadside.md +++ b/agent-skill/codecartographer/references/broadside.md @@ -74,9 +74,14 @@ Two more economies worth knowing: - `incremental: true` diffs against the previous run's git HEAD and scans only the modules whose files changed, falling back to a full scan on a dirty tree. - Every knob above has a repository default in `.codecarto/broadside/config.yaml` - (`model`, `default_lenses`, `max_cost`, `incremental`, `retry_truncated`, - `include_synthesis`, `include_triage`, `wait_seconds`). An explicit parameter - on the call always wins. + (`model`, `default_lenses`, `max_cost`, `lens_models`, `incremental`, + `retry_truncated`, `include_synthesis`, `include_triage`, `wait_seconds`). An + explicit parameter on the call always wins. +- `lens_models` runs individual lenses on their own model. Spending more on the + security and defect lenses while the cheap default carries architecture and + conventions is usually a better trade than raising the model for everything. + Overrides are priced and capability-checked individually, and the estimate + breaks cost out per lens. ## Reading a run diff --git a/core/broadside.ts b/core/broadside.ts index 7197361..fa40515 100644 --- a/core/broadside.ts +++ b/core/broadside.ts @@ -167,6 +167,10 @@ export type BroadsideBatchEntry = { cost?: number; resultCount?: number; error?: unknown; + /** Set when this lens used a model other than the run default. */ + model?: string; + /** The completion ceiling of this lens's model; bounds the truncation retry. */ + outputCap?: number; }; export type BroadsideSynthesisEntry = { @@ -229,6 +233,14 @@ export type BroadsideConfig = { maxCost: number; /** Manual pricing overrides (USD per million). Live lookup is preferred. */ pricing: { inputPerM: number; outputPerM: number } | null; + /** + * Per-lens model overrides. A lens absent here uses `model`. This is how a + * repository routes the semantic lenses (security, defect) to a stronger + * batch model while the cheap default carries the rest — the whole point of + * the cheap model is telling the expensive one where to look, and that + * trade-off is not the same for every lens. + */ + lensModels: Partial>; /** * Repo defaults for the per-call run knobs. Each mirrors a tool parameter * of the same name; an explicit parameter always wins. They live here so a @@ -251,7 +263,18 @@ export type BroadsideConfig = { export type BroadsideEstimate = { model: string; pricing: ModelPricing; - lenses: Array<{ lensId: BroadsideLensId; name: string; slices: number; maxTokens: number; cost: number }>; + lenses: Array<{ + lensId: BroadsideLensId; + name: string; + slices: number; + maxTokens: number; + cost: number; + /** The model this lens would use — `model` unless a per-lens override applies. */ + model: string; + pricing: ModelPricing; + }>; + /** True when at least one lens uses a model other than the run default. */ + mixedModels: boolean; totalCost: number; inputTokens: number; outputTokens: number; @@ -1558,6 +1581,15 @@ export async function loadBroadsideConfig(broadsideDir: string): Promise typeof raw[key] === "boolean" ? (raw[key] as boolean) : fallback; + // An override for an unknown lens id is dropped rather than carried: it can + // only be a typo, and a silently-ignored key that looks applied is worse + // than one that never appears. + const lensModels: Partial> = {}; + const rawLensModels = (raw.lens_models ?? {}) as Record; + for (const lensId of BROADSIDE_LENS_IDS) { + const value = rawLensModels[lensId]; + if (typeof value === "string" && value.trim()) lensModels[lensId] = value.trim(); + } return { model: typeof raw.model === "string" && raw.model.trim() ? raw.model.trim() : BROADSIDE_MODEL, apiKey: typeof raw.api_key === "string" ? raw.api_key.trim() : "", @@ -1567,6 +1599,7 @@ export async function loadBroadsideConfig(broadsideDir: string): Promise - ["structured_outputs", "json_schema", "response_format", "structuredoutputs"].includes(p.toLowerCase()), - ); - if (!supportsStructuredOutputs) { - throw new Error( - `Batch model "${model}" does not advertise structured-output support ` + - `(supported_parameters: ${entry.supportedParameters.join(", ") || "unknown"}), but every ` + - "Broad-Side lens requires json_schema response_format. Choose another batch model " + - "(codecarto_broadside action 'models') or pass a pricing override only if you know it works.", - ); + const modelForLens = (lensId: BroadsideLensId): string => config.lensModels[lensId] ?? model; + const resolved = new Map< + string, + { pricing: ModelPricing; outputCap?: number; entry: CatalogEntry; supportsStructuredOutputs: boolean } + >(); + for (const candidate of new Set([model, ...lensIds.map(modelForLens)])) { + const catalog = await resolveCatalogEntry(broadsideDir, config, candidate, apiKey, opts.fetcher); + const entry = catalog.entry!; + const supportsStructuredOutputs = + entry.supportedParameters.length === 0 || + entry.supportedParameters.some((p) => + ["structured_outputs", "json_schema", "response_format", "structuredoutputs"].includes(p.toLowerCase()), + ); + if (!supportsStructuredOutputs) { + throw new Error( + `Batch model "${candidate}" does not advertise structured-output support ` + + `(supported_parameters: ${entry.supportedParameters.join(", ") || "unknown"}), but every ` + + "Broad-Side lens requires json_schema response_format. Choose another batch model " + + "(codecarto_broadside action 'models') or pass a pricing override only if you know it works.", + ); + } + resolved.set(candidate, { + entry, + supportsStructuredOutputs, + pricing: { inputPerM: entry.inputPerM, outputPerM: entry.outputPerM, source: catalog.source }, + // Respect the provider's completion ceiling: a request asking for more + // output than the model can produce fails the whole batch. + ...(entry.maxCompletionTokens !== undefined && { outputCap: entry.maxCompletionTokens }), + }); } - // Respect the provider's completion ceiling: a request asking for more - // output than the model can produce fails the whole batch. - const outputCap = entry.maxCompletionTokens; + const pricing = resolved.get(model)!.pricing; + const outputCap = resolved.get(model)!.outputCap; + const defaultEntry = resolved.get(model)!.entry; + const limit = opts.maxCost ?? config.maxCost; // Incremental re-scouting (#142): diff against the previous run's HEAD // and scan only the modules whose files changed. Falls back to a full @@ -1998,7 +2041,14 @@ export async function runBroadsideSubmit( let estimatedInputTokens = 0; let estimatedOutputTokens = 0; let estimatedTotalCost = 0; - const perLensEstimate: Array<{ lens: LensDefinition; cost: number; maxTokens: number }> = []; + const perLensEstimate: Array<{ + lens: LensDefinition; + cost: number; + maxTokens: number; + lensModel: string; + lensPricing: ModelPricing; + lensOutputCap?: number; + }> = []; for (const lensId of lensIds) { const lens = getLens(lensId); let slices = await gatherSlices(cwd, lens, info); @@ -2008,12 +2058,21 @@ export async function runBroadsideSubmit( slices = slices.filter((s) => s.files.length === 0 || s.files.some((f) => changed!.has(f))); } slicesByLens.set(lensId, slices); - const maxTokens = outputCap ? Math.min(lens.maxTokens, outputCap) : lens.maxTokens; - const estimate = estimateCost(lens, slices, pricing, maxTokens); + const lensModel = modelForLens(lensId); + const { pricing: lensPricing, outputCap: lensOutputCap } = resolved.get(lensModel)!; + const maxTokens = lensOutputCap ? Math.min(lens.maxTokens, lensOutputCap) : lens.maxTokens; + const estimate = estimateCost(lens, slices, lensPricing, maxTokens); estimatedInputTokens += estimate.inputTokens; estimatedOutputTokens += estimate.outputTokens; estimatedTotalCost += estimate.cost; - perLensEstimate.push({ lens, cost: estimate.cost, maxTokens }); + perLensEstimate.push({ + lens, + cost: estimate.cost, + maxTokens, + lensModel, + lensPricing, + ...(lensOutputCap !== undefined && { lensOutputCap }), + }); } const exceedsLimit = limit > 0 && estimatedTotalCost > limit; @@ -2022,13 +2081,16 @@ export async function runBroadsideSubmit( const approved = await opts.confirm({ model, pricing, - lenses: perLensEstimate.map(({ lens, cost, maxTokens }) => ({ + lenses: perLensEstimate.map(({ lens, cost, maxTokens, lensModel, lensPricing }) => ({ lensId: lens.id, name: lens.name, slices: (slicesByLens.get(lens.id) ?? []).length, maxTokens, cost, + model: lensModel, + pricing: lensPricing, })), + mixedModels: perLensEstimate.some(({ lensModel }) => lensModel !== model), totalCost: estimatedTotalCost, inputTokens: estimatedInputTokens, outputTokens: estimatedOutputTokens, @@ -2041,7 +2103,9 @@ export async function runBroadsideSubmit( if (!approved) throw new BroadsideCancelledError(); } else if (exceedsLimit && !opts.force) { const breakdown = perLensEstimate - .map(({ lens, cost }) => ` ${lens.name}: ~$${cost.toFixed(4)}`) + .map(({ lens, cost, lensModel }) => + ` ${lens.name}: ~$${cost.toFixed(4)}${lensModel === model ? "" : ` (${lensModel})`}`, + ) .join("\n"); throw new Error( `Estimated Broad-Side cost ~$${estimatedTotalCost.toFixed(4)} exceeds the run limit ` + @@ -2074,20 +2138,25 @@ export async function runBroadsideSubmit( const requestsByCustomId: Record = {}; const submissions: Promise[] = []; - for (const lensId of lensIds) { - const lens = getLens(lensId); + // Submit from the estimate rather than recomputing: the user approved that + // breakdown, so the request that fires must be the one that was priced. + for (const priced of perLensEstimate) { + const { lens, maxTokens, lensModel, lensOutputCap } = priced; + const lensId = lens.id; const slices = slicesByLens.get(lensId) ?? []; - const maxTokens = outputCap ? Math.min(lens.maxTokens, outputCap) : lens.maxTokens; - const requests = slices.map((s, i) => buildBatchRequest(lens, info, s, i, slices.length, model, maxTokens)); + const requests = slices.map((sl, i) => buildBatchRequest(lens, info, sl, i, slices.length, lensModel, maxTokens)); for (const request of requests) requestsByCustomId[request.custom_id] = request; - const estimate = estimateCost(lens, slices, pricing, maxTokens); const entry: BroadsideBatchEntry = { batchId: "", requests: requests.length, status: "submitting", submittedAt: new Date().toISOString(), - estimatedCost: estimate.cost, + estimatedCost: priced.cost, + // Recorded per lens so collect's truncation retry re-submits against + // the model and ceiling this lens actually used, not the run default. + ...(lensModel !== model && { model: lensModel }), + ...(lensOutputCap !== undefined && { outputCap: lensOutputCap }), }; run.batches[lensId] = entry; @@ -2104,7 +2173,7 @@ export async function runBroadsideSubmit( // entry in "submitting" forever — allSettled would swallow the // rejection and collect would never see a terminal status. try { - const { batchId, status, error } = await submitBatch(requests, apiKey, opts.fetcher, model); + const { batchId, status, error } = await submitBatch(requests, apiKey, opts.fetcher, lensModel); entry.batchId = batchId; entry.status = status; if (error) entry.error = error; @@ -2135,11 +2204,15 @@ export async function runBroadsideSubmit( estimatedOutputTokens, pricing, maxCost: limit > 0 ? limit : undefined, + // modelInfo describes the run's default model. Per-lens overrides are + // recorded on their own batch entries. modelInfo: { - contextLength: entry.contextLength, - maxCompletionTokens: entry.maxCompletionTokens, - supportsStructuredOutputs: entry.supportedParameters.length === 0 ? undefined : supportsStructuredOutputs, - expirationDate: entry.expirationDate ?? null, + contextLength: defaultEntry.contextLength, + maxCompletionTokens: defaultEntry.maxCompletionTokens, + supportsStructuredOutputs: defaultEntry.supportedParameters.length === 0 + ? undefined + : resolved.get(model)!.supportsStructuredOutputs, + expirationDate: defaultEntry.expirationDate ?? null, }, }; } @@ -2422,8 +2495,14 @@ export async function runBroadsideCollect( if (!stored.truncated) continue; const original = requestsByCustomId[stored.customId]; if (!original) continue; + const lensEntry = run.batches[stored.lensId]; + // A lens may have run on its own model (config `lens_models`), with its + // own completion ceiling. Re-submitting against the run default would + // change the model mid-run and could exceed that lens's real ceiling. + const lensModel = lensEntry?.model ?? run.model; + const lensCap = lensEntry?.outputCap ?? run.outputCap; const previousMax = original.body.max_tokens ?? getLens(stored.lensId).maxTokens; - const bumpedMax = run.outputCap ? Math.min(previousMax * 2, run.outputCap) : previousMax * 2; + const bumpedMax = lensCap ? Math.min(previousMax * 2, lensCap) : previousMax * 2; if (bumpedMax <= previousMax) continue; // already at the ceiling const bumped: BatchRequest = { @@ -2431,7 +2510,7 @@ export async function runBroadsideCollect( body: { ...original.body, max_tokens: bumpedMax }, }; try { - const { batchId, error } = await submitBatch([bumped], apiKey, opts.fetcher, run.model); + const { batchId, error } = await submitBatch([bumped], apiKey, opts.fetcher, lensModel); if (error) continue; const batch = await pollBatchUntilTerminal(batchId, apiKey, { deadlineMs: BROADSIDE_DEFAULT_POLL_BUDGET_MS, @@ -2591,6 +2670,13 @@ export async function runBroadsideCollect( synthesis: run.synthesis, triage: run.triage, lenses: run.lenses, + // Which lens ran on which model. Absent means the run default — + // a reader comparing two runs needs to know a lens changed model. + lens_models: Object.fromEntries( + Object.entries(run.batches) + .filter(([, batch]) => batch?.model) + .map(([lensId, batch]) => [lensId, batch!.model]), + ), disclaimer: "Findings are unverified scouting signals from a batch model, not validated claims. " + "Re-verify every file:line lead with the interactive pipeline or by hand.", @@ -2692,7 +2778,8 @@ export function estimateSubmitText(result: BroadsideSubmitResult, lenses: LensDe const entry = result.batches[lens.id]; if (!entry) continue; const status = entry.batchId ? `batch ${entry.batchId}` : entry.status; - lines.push(` ${lens.name}: ${status} (${entry.requests} request(s), ~$${entry.estimatedCost.toFixed(4)})`); + const override = entry.model ? ` on ${entry.model}` : ""; + lines.push(` ${lens.name}: ${status} (${entry.requests} request(s), ~$${entry.estimatedCost.toFixed(4)})${override}`); } lines.push( `Estimated total: ~$${result.estimatedTotalCost.toFixed(4)}`, diff --git a/extensions/codecarto/index.ts b/extensions/codecarto/index.ts index 312313a..181fa1d 100644 --- a/extensions/codecarto/index.ts +++ b/extensions/codecarto/index.ts @@ -176,13 +176,16 @@ function resolveBroadsideKey(configuredKey: string): string | null { /** The spend decision, rendered for a human about to approve it. */ function describeBroadsideEstimate(estimate: BroadsideEstimate): string { const lines = [ - `Model: ${estimate.model} (pricing: ${estimate.pricing.source})`, + `Model: ${estimate.model}${estimate.mixedModels ? " (some lenses overridden — see below)" : ""} (pricing: ${estimate.pricing.source})`, `Rates: $${estimate.pricing.inputPerM.toFixed(4)}/M in · $${estimate.pricing.outputPerM.toFixed(4)}/M out`, "", "Per lens:", - ...estimate.lenses.map( - ({ name, slices, cost }) => ` ${name}: ${slices} slice${slices === 1 ? "" : "s"} — ~$${cost.toFixed(4)}`, - ), + ...estimate.lenses.map(({ name, slices, cost, model }) => { + // Naming the model only when it differs keeps the common case quiet + // and makes a mixed-model run impossible to approve without noticing. + const override = estimate.mixedModels && model !== estimate.model ? ` on ${model}` : ""; + return ` ${name}: ${slices} slice${slices === 1 ? "" : "s"} — ~$${cost.toFixed(4)}${override}`; + }), "", `Estimated total: ~$${estimate.totalCost.toFixed(4)} ` + `(~${Math.round(estimate.inputTokens / 1000)}k in, ~${Math.round(estimate.outputTokens / 1000)}k out)`, diff --git a/tests/broadside.test.mjs b/tests/broadside.test.mjs index 531891a..28db0a2 100644 --- a/tests/broadside.test.mjs +++ b/tests/broadside.test.mjs @@ -352,27 +352,61 @@ test("config carries repo defaults for every per-call run knob", async () => { test("every documented config key is one loadBroadsideConfig actually reads", async () => { // The commented-out keys in the shipped config.yaml are the only // documentation a user gets. A key documented but not parsed reads as a - // working setting that silently does nothing. + // working setting that silently does nothing; a key parsed but not + // documented is a feature nobody can find. const shipped = await readFile(join(REPO_ROOT, ".codecarto", "broadside", "config.yaml"), "utf8"); - const documented = [...shipped.matchAll(/^#\s{0,3}([a-z_]+):/gm)].map((match) => match[1]); - const parsed = new Set([ + // Commented YAML keeps its indentation after the "# ", so nesting depth + // separates top-level keys from the examples under pricing / lens_models. + const documentedTop = [...shipped.matchAll(/^# ([a-z_]+):/gm)].map((match) => match[1]); + const documentedNested = [...shipped.matchAll(/^#\s{2,}([a-z_]+):/gm)].map((match) => match[1]); + + const parsedTop = new Set([ "model", "api_key", "default_lenses", "max_cost", "pricing", - "input_per_m", - "output_per_m", + "lens_models", "incremental", "retry_truncated", "include_synthesis", "include_triage", "wait_seconds", ]); - const undocumented = [...parsed].filter((key) => !documented.includes(key)); + const undocumented = [...parsedTop].filter((key) => !documentedTop.includes(key)); assert.deepEqual(undocumented, [], `config.yaml does not document: ${undocumented.join(", ")}`); - for (const key of documented) { - assert.ok(parsed.has(key), `config.yaml documents ${key}, which loadBroadsideConfig does not read`); + for (const key of documentedTop) { + assert.ok(parsedTop.has(key), `config.yaml documents ${key}, which loadBroadsideConfig does not read`); + } + + // Nested examples must be real too: pricing's two fields, or a lens id. + const parsedNested = new Set(["input_per_m", "output_per_m", ...BROADSIDE_LENS_IDS]); + for (const key of documentedNested) { + assert.ok(parsedNested.has(key), `config.yaml shows a nested ${key} key that nothing reads`); + } + assert.ok( + documentedNested.some((key) => BROADSIDE_LENS_IDS.includes(key)), + "lens_models must be documented with at least one real lens id as an example", + ); +}); + +test("per-lens model overrides parse, and unknown lens ids are dropped", async () => { + const dir = await mkdtemp(join(tmpdir(), "broadside-lens-models-")); + try { + assert.deepEqual((await loadBroadsideConfig(dir)).lensModels, {}, "no overrides by default"); + + await writeFile( + join(dir, "config.yaml"), + "lens_models:\n security: vendor/strong:batch\n nonsense: vendor/whatever:batch\n defect: \n", + ); + const config = await loadBroadsideConfig(dir); + assert.deepEqual( + config.lensModels, + { security: "vendor/strong:batch" }, + "an unknown lens id and an empty value are both dropped rather than carried", + ); + } finally { + await rm(dir, { recursive: true, force: true }); } }); @@ -575,6 +609,79 @@ test("a confirm hook decides the run, and declining submits nothing", async () = } }); +test("a per-lens override reaches the wire, the estimate, and the retry", async () => { + const dir = await mkdtemp(join(tmpdir(), "broadside-per-lens-model-")); + try { + await writeFile(join(dir, "go.mod"), "module x\n"); + await mkdir(join(dir, "big"), { recursive: true }); + for (let i = 0; i < 10; i++) { + await writeFile(join(dir, "big", `file${i}.go`), `package big\n// ${"y".repeat(1500)}\n`); + } + await mkdir(join(dir, ".codecarto", "broadside"), { recursive: true }); + await writeFile( + join(dir, ".codecarto", "broadside", "config.yaml"), + "lens_models:\n defect: vendor/strong:batch\n", + ); + + const posted = []; + const fetcher = async (url, init) => { + if (init.method === "POST") { + posted.push(JSON.parse(init.body)); + return fakeResponse(202, { id: `batch-${posted.length}`, status: "validating" }); + } + if (String(url).includes("/models")) { + // Both models must price: the override is pre-flighted like the default. + return fakeResponse(200, modelsCatalog([ + { + id: "vendor/strong:batch", + name: "Strong", + pricing: { prompt: "0.000005", completion: "0.000025" }, + context_length: 200000, + top_provider: { max_completion_tokens: 32000 }, + supported_parameters: ["structured_outputs"], + }, + ])); + } + return fakeResponse(200, { id: "x", status: "in_progress" }); + }; + + const seen = []; + const result = await runBroadsideSubmit(dir, "sk-fake", { + lenses: ["architecture", "defect"], + fetcher, + confirm: (estimate) => { seen.push(estimate); return true; }, + }); + + // The estimate must price each lens against its own model, or the number + // the user approves is not the number they will be billed. + const [estimate] = seen; + assert.equal(estimate.mixedModels, true); + const defectRow = estimate.lenses.find((l) => l.lensId === "defect"); + const archRow = estimate.lenses.find((l) => l.lensId === "architecture"); + assert.equal(defectRow.model, "vendor/strong:batch"); + assert.equal(archRow.model, BROADSIDE_MODEL); + assert.equal(defectRow.pricing.outputPerM, 25, "the override's own rates must price its lens"); + assert.notEqual(archRow.pricing.outputPerM, defectRow.pricing.outputPerM); + + // And the batch that fires must carry it, at both payload levels. + const defectBatch = posted.find((p) => p.model === "vendor/strong:batch"); + assert.ok(defectBatch, "the defect batch must be submitted on the override model"); + assert.equal(defectBatch.requests[0].body.model, "vendor/strong:batch"); + assert.ok( + posted.some((p) => p.model === BROADSIDE_MODEL), + "the architecture batch must stay on the run default", + ); + + // Recorded per lens, so collect's truncation retry re-submits on the same + // model and against that model's ceiling rather than the run default's. + assert.equal(result.batches.defect.model, "vendor/strong:batch"); + assert.equal(result.batches.defect.outputCap, 32000); + assert.equal(result.batches.architecture.model, undefined, "the default model is not restated per lens"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + test("submit passes the configured model into batch payloads", async () => { const dir = await mkdtemp(join(tmpdir(), "broadside-model-")); try {