diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 00000000..13ea9756 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,122 @@ +name: docs + +# The witan-context documentation site. Two independent questions: +# +# freshness — do the committed generated/mirrored pages still match the code? +# build — does the site actually render, with no broken links? +# +# Path-filter rule (same as witan-tests.yml): trigger on every input the +# generator reads, not just docs/. The reference pages are derived from the tool +# objects, the cyclopts command tree, and the .pg schemas, so a change to any of +# those can make a committed page stale without touching docs/ at all. +# +# The two path lists below are duplicated rather than shared via a YAML anchor: +# GitHub Actions does not support anchors or aliases in workflow files. Keep +# them in sync by hand. +on: + push: + branches: [main] + paths: + - "docs/**" + - "mcp/servers/witan/**" + - "mcp/servers/witan-code/**" + - "packages/witan-core/**" + - "docker/**" + - "bin/gen_docs.py" + - "zensical.toml" + - ".readthedocs.yaml" + - "uv.lock" + - "pyproject.toml" + - "justfile" + - ".github/workflows/docs.yml" + pull_request: + paths: + - "docs/**" + - "mcp/servers/witan/**" + - "mcp/servers/witan-code/**" + - "packages/witan-core/**" + - "docker/**" + - "bin/gen_docs.py" + - "zensical.toml" + - ".readthedocs.yaml" + - "uv.lock" + - "pyproject.toml" + - "justfile" + - ".github/workflows/docs.yml" + # Manual trigger. Useful on its own (rebuild the site after an upstream + # Zensical release without touching the repo), and an escape hatch for the + # case that prompted adding it: a force-push GitHub did not turn into a + # `synchronize` event, leaving a PR with no docs checks attached at all. + workflow_dispatch: + +permissions: + contents: read + +jobs: + freshness: + name: docs (generated pages match the code) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + enable-cache: true + cache-dependency-glob: "uv.lock" + + - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4.0.0 + + - name: Install dependencies + # --all-packages: the generator imports BOTH servers to read their + # registered tool objects, so a single-package sync cannot run it. + # + # --python 3.12 matches the pin in bin/gen_docs.py's shebang. FastMCP's + # schema derivation orders a Literal's `enum` differently on different + # Pythons, so an unpinned sync here would let CI disagree with a + # contributor's machine about pages neither of them edited. + run: uv sync --frozen --all-packages --python 3.12 + + - name: Check generated documentation is up to date + # Fails with a diff naming every stale page. The fix is always the same + # — run `just docs-gen` and commit the result. + run: just docs-check + + build: + name: docs (site builds, no broken links) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + + - name: Build the site and fail on unresolved links + # Version-pinned to match .readthedocs.yaml. Zensical is pre-1.0 and + # ships behaviour changes between patch releases, so an unpinned install + # would let CI and Read the Docs disagree silently — CI green, published + # site broken. Renovate bumps both together. + # + # `zensical build` reports missing pages and anchors as warnings and + # still exits 0, so the summary line is what decides this job. A + # silently-broken link is precisely what a docs CI job exists to catch. + # + # ★ MATCH THE COUNT, NOT THE WORDS. Zensical prints "N issues found" on + # failure and "No issues found" on success — and the success message + # CONTAINS the failure substring, so a bare `grep "issues found"` fails + # every green build. It did exactly that on this job's first run. + # Requiring a leading digit is what separates the two. + # + # The build is also required to have finished: `zensical build` exiting + # non-zero is caught by `set -e`, but a build that dies without printing + # its summary would otherwise pass the digit test by saying nothing. + run: | + set -o pipefail + uvx zensical@0.0.56 build 2>&1 | tee build.log + if grep -qE '[0-9]+ issues? found' build.log; then + echo "::error::Zensical reported unresolved links or anchors." + grep -B2 -A4 'Warning:' build.log | head -60 + exit 1 + fi + if ! grep -q 'Build finished' build.log; then + echo "::error::Zensical did not report a completed build." + exit 1 + fi diff --git a/.gitignore b/.gitignore index dfcab06c..9a1f13fe 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,6 @@ mcp-server.log # agent-config-kit remote skill/hook fetch cache .agent-config-kit-cache/ + +# Zensical build output for the witan-context docs site. +site/ diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 00000000..7b4e1729 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,31 @@ +# Read the Docs build for the witan-context documentation site. +# +# Zensical is not one of Read the Docs' built-in builders, so the build is +# spelled out as explicit jobs: install the generator, build, then copy the +# output into the directory RTD publishes from. +# +# NOTE: this installs `zensical` ONLY — not the witan packages. The reference +# pages are generated by `bin/gen_docs.py` and committed, so RTD renders +# Markdown and never has to resolve tree-sitter, fastmcp, or an omnigraph +# binary. `just docs-check` in CI is what guarantees the committed output +# matches the code. + +version: 2 + +build: + os: ubuntu-24.04 + tools: + python: "3.12" + jobs: + install: + # Pinned to match .github/workflows/docs.yml. Zensical is pre-1.0 and + # ships behaviour changes between patch releases; an unpinned install here + # would let the published site diverge from what CI verified. Renovate + # bumps both together. + - pip install "zensical==0.0.56" + build: + html: + - zensical build + post_build: + - mkdir -p $READTHEDOCS_OUTPUT/html/ + - cp --recursive site/* $READTHEDOCS_OUTPUT/html/ diff --git a/AGENTS.md b/AGENTS.md index b061b35e..59396834 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -28,7 +28,12 @@ packages/ # Standalone, independently-versioned Python libraries agent-config-kit/ # Cross-agent MCP/skill/hook registration library agent-kit/ # PyPI meta-package (ol-agent-kit): agent-config-kit[cli] + witan + witan-code configs/ # Sample / reference agent configurations -docs/ # Design docs and implementation specs +docs/ # The witan-context documentation site (Zensical -> Read the Docs) + getting-started/ # Tutorials (handwritten) + guides/ # How-to (mostly MIRRORED from the packages by bin/gen_docs.py) + reference/ # GENERATED from live code by bin/gen_docs.py -- never edit + explanation/ # Architecture, memory model, coordination, ADRs + internals/ # Historical design docs and implementation specs ``` ## Dev Setup @@ -100,4 +105,4 @@ See [`skills/workflow/creating-skills/SKILL.md`](./skills/workflow/creating-skil - [`mcp/README.md`](./mcp/README.md) — MCP server structure and available servers - [`mcp/servers/witan/README.md`](./mcp/servers/witan/README.md) — witan graph-memory server - [`custom-agents/README.md`](./custom-agents/README.md) — agent definitions for Claude/Copilot -- [`docs/`](./docs/) — design docs and implementation specs +- [`docs/`](./docs/) — the **witan-context** documentation site (https://witan-context.readthedocs.io). `docs/reference/` is GENERATED and `docs/guides/` is mostly MIRRORED from the packages — do not hand-edit either; run `just docs-gen` and commit. `just docs-check` gates this in CI, and `just docs-serve` previews locally. Historical specs live in `docs/internals/`. diff --git a/bin/gen_docs.py b/bin/gen_docs.py new file mode 100755 index 00000000..5138976d --- /dev/null +++ b/bin/gen_docs.py @@ -0,0 +1,964 @@ +#!/usr/bin/env -S uv run --quiet --all-packages --python 3.12 python +"""Generate the reference half of the witan-context docs site from live sources. + +WHY. A reference page that is written by hand is a reference page that is wrong +by the second release. Every page this script emits is derived from the thing it +documents — the registered FastMCP tool objects, the cyclopts command tree, the +``.pg`` schema files — so the only way for it to drift is for someone to change +the code and not re-run the generator. ``--check`` is what makes that a CI +failure rather than a slow rot. + +WHAT IS *NOT* GENERATED. Prose. Tutorials, how-to guides, and explanation are +written by hand under ``docs/guides/`` and ``docs/explanation/`` and this script +never touches them. The line is deliberate: a generator can state that +``memory_store`` takes a ``kind`` parameter, but only a person can say when you +should reach for it. + +THE ONE HYBRID IS THE ENVIRONMENT REFERENCE. Env var *names* are discoverable +from source; what they mean is not. So the names are discovered here and the +descriptions live in ``docs/_data/environment.toml``, and a name with no entry +there is a hard error. That way a newly-added env var cannot ship undocumented, +but the description is still written by someone who knows what it does. + +THE INTERPRETER IS PINNED IN THE SHEBANG, AND HAS TO BE. The JSON Schema +FastMCP derives from a ``Literal`` does not order its ``enum`` the same way on +every Python: ``BindingKind`` comes out as +``env_var, package, service, endpoint`` on 3.14 and +``env_var, endpoint, package, service`` on 3.12. Nothing here is +hash-dependent — the order is stable within a version and differs between them +— so without a pin, generating on one Python and checking on another reports +pages as stale that nobody edited. That is exactly what happened: CI resolved +3.12, the author's machine had 3.14, and ``--check`` failed on two pages with +no change behind them. Pinning makes every contributor and CI agree; the +version itself is arbitrary, it only has to be fixed. + +Usage: + ./bin/gen_docs.py # regenerate everything + ./bin/gen_docs.py --check # fail if anything is stale (CI) + ./bin/gen_docs.py mcp-tools # regenerate one section +""" + +from __future__ import annotations + +import ast +import asyncio +import difflib +import itertools +import os +import re +import shutil +import sys +import tempfile +import tomllib +from pathlib import Path, PurePosixPath +from typing import Any + +import cyclopts + +REPO_ROOT = Path(__file__).resolve().parent.parent +DOCS = REPO_ROOT / "docs" +REFERENCE = DOCS / "reference" +DATA = DOCS / "_data" + +# Importing the servers constructs an OmnigraphClient at module scope and +# *creates the store directory* on the way. Point that at a throwaway rather +# than at the real graph: generation only reads registered tool metadata, and it +# must never be able to touch — or lazily initialise — someone's actual store. +# A non-writable path is not an option; `_ensure_graph` would raise on the mkdir. +_DOC_STORE = Path(tempfile.gettempdir()) / "witan-docs-generation" / "graph.omni" +os.environ["WITAN_MEMORY_URI"] = str(_DOC_STORE) + +# ★ AND THE STORE HAS TO ALREADY EXIST, OR THIS NEEDS THE OMNIGRAPH BINARY. +# `witan.server._ensure_graph` branches on it: an existing store tolerates a +# missing binary (it catches the RuntimeError and returns), while a missing one +# must be `omnigraph init`-ed and therefore cannot. Generating documentation has +# no business requiring a downloaded binary — it reads tool metadata and never +# opens a graph — and requiring one coupled the docs CI job to the binary's +# availability. That bill came due immediately: the upstream `edge` tag moved, +# the pinned checksum stopped matching, the install step failed, and a docs-only +# change went red for a reason that had nothing to do with docs. +# +# Creating the directory is enough to take the tolerant branch. Nothing ever +# reads or writes it. +_DOC_STORE.mkdir(parents=True, exist_ok=True) + + +def _ensure_omnigraph_on_path() -> None: + """Put a no-op ``omnigraph`` on PATH when no real one is installed. + + ``OmnigraphClient.__init__`` resolves the binary eagerly, and both servers + construct a module-level client, so importing them needs *something* named + ``omnigraph`` even though generation never opens a graph. On a machine with + witan set up this does nothing — the real binary is found first and + behaviour is identical to before. It only fires in a bare environment such + as the docs CI job, which has no reason to download a 12MB binary to read + docstrings. + + The stub is a no-op rather than an error: with the store pre-created above, + the only thing that runs it is ``schema_apply_if_changed``, which is + comparing a schema against a store nothing will ever read. + """ + if shutil.which("omnigraph") or (Path.home() / ".local/bin/omnigraph").exists(): + return + stub_dir = Path(tempfile.gettempdir()) / "witan-docs-generation" / "bin" + stub_dir.mkdir(parents=True, exist_ok=True) + stub = stub_dir / "omnigraph" + stub.write_text("#!/bin/sh\nexit 0\n") + stub.chmod(0o755) + os.environ["PATH"] = f"{stub_dir}{os.pathsep}{os.environ.get('PATH', '')}" + + +_ensure_omnigraph_on_path() + +BANNER = """ +""" + +app = cyclopts.App( + name="gen-docs", + help="Generate the witan-context reference documentation.", +) + +# ── Shared state ──────────────────────────────────────────────────── +# +# `--check` turns every write into a comparison. Collected rather than raised on +# the spot so one run reports every stale page, not just the first. +_check_mode = False +_stale: list[str] = [] + + +def emit(path: Path, content: str) -> None: + """Write ``content`` to ``path``, or record it as stale under ``--check``. + + ★ TRAILING WHITESPACE IS STRIPPED BECAUSE THE PRE-COMMIT HOOK STRIPS IT. + Without this the two fight forever: the generator writes a line with a + trailing space (cyclopts' Markdown does, in the CLI reference), the + `trailing-whitespace` hook removes it on commit, and the next `docs-check` + regenerates the space and declares the committed page stale. Emitting what + the hook would accept is the only stable fixed point. + """ + content = "\n".join(line.rstrip() for line in content.splitlines()) + if not content.endswith("\n"): + content += "\n" + rel = path.relative_to(REPO_ROOT) + if _check_mode: + current = path.read_text() if path.exists() else "" + if current != content: + _stale.append(str(rel)) + diff = difflib.unified_diff( + current.splitlines(keepends=True), + content.splitlines(keepends=True), + fromfile=f"{rel} (committed)", + tofile=f"{rel} (regenerated)", + n=1, + ) + sys.stderr.write("".join(list(diff)[:40])) + return + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content) + print(f" wrote {rel}") + + +# ── JSON Schema rendering ─────────────────────────────────────────── + + +def type_str(schema: dict[str, Any]) -> str: + """Render a JSON Schema fragment as a short, readable type. + + Optional parameters arrive as ``anyOf: [T, null]`` — the ``null`` branch is + dropped and the result marked with ``?``, because "string or null" is a + Pydantic implementation detail and "optional string" is what a reader wants. + """ + if "enum" in schema: + return " \\| ".join(f"`{v}`" for v in schema["enum"]) + if "anyOf" in schema: + branches = [b for b in schema["anyOf"] if b.get("type") != "null"] + nullable = len(branches) != len(schema["anyOf"]) + rendered = " \\| ".join(type_str(b) for b in branches) + return f"{rendered}?" if nullable else rendered + kind = schema.get("type") + if kind == "array": + return f"list[{type_str(schema.get('items', {}))}]" + if kind == "object": + return "object" + return { + "string": "str", + "integer": "int", + "number": "float", + "boolean": "bool", + }.get(kind, kind or "any") + + +def default_str(schema: dict[str, Any], required: bool) -> str: + if required: + return "**required**" + if "default" not in schema: + return "—" + value = schema["default"] + if value is None: + return "`null`" + if value == []: + return "`[]`" + return f"`{value!r}`" if isinstance(value, str) else f"`{value}`" + + +def clean_desc(text: str | None) -> str: + """Flatten a schema description into one Markdown table cell. + + Newlines become ``
`` so multi-line parameter docs survive a table, and + the double-backtick spans the docstrings use are already valid Markdown code + spans, so they are left alone. + + ★ PIPES MUST BE ESCAPED, AND THE FAILURE IS SILENT. A description that + enumerates its options — "``bug`` | ``feature`` | ``task``" — carries raw + ``|``, which the table parser reads as column separators. The row does not + visibly break: the cells past the fourth are simply DISCARDED, so + ``task_update``'s ``status`` rendered as the single word "open" and its + warning about closing a task unblocking its dependents vanished from the + page. ``type_str`` already escapes for the same reason. + """ + if not text: + # An empty cell reads as a broken table rather than as a missing + # docstring. `gen_mcp_tools` reports the real count separately. + return "—" + text = text.replace("|", "\\|") + return "
".join( + line.strip() for line in text.strip().splitlines() if line.strip() + ) + + +# ── MCP tool reference ────────────────────────────────────────────── + +# Which page each tool lands on, and the order the pages are listed in. Grouped +# by the concept a reader is holding in their head ("I want to record something" +# / "I want to coordinate work"), not by module or prefix — `recall` and +# `topic_get` belong with memory even though they share no prefix with it. +TOOL_GROUPS: list[tuple[str, str, str, list[str]]] = [ + ( + "memory", + "Memory & recall", + ( + "Recording durable knowledge in the shared graph, and reading it back. " + "`recall` is the default read — it composes BM25 search, graph expansion, " + "superseded-pruning, and re-ranking into one call. The narrower reads below " + "exist for when you already know exactly what you want." + ), + [ + "recall", + "memory_store", + "memory_get", + "memory_update", + "memory_delete", + "memory_list", + "memory_search", + "memory_link", + "memory_neighbors", + "memory_symbols", + "memory_for_contract", + "symbol_context", + "topic_get", + "store_merge", + # The repair half of `store_merge`: it restamps rows a migration + # already landed under a local identity, which `store_merge`'s own + # `claim_from_author` can only do on arrival. + "claim_authorship", + ], + ), + ( + "tasks", + "Tasks", + ( + "The work-coordination layer: what needs doing, what blocks what, and who " + "holds which piece of work right now. `task_claim` is a **best-effort** " + "compare-and-swap — it detects and rejects most lost races, but it is an " + "advisory lease, not a hard lock. See " + "[ADR 0003](../../explanation/decisions/0003-atomic-task-claims-cas.md) for " + "what is and is not guaranteed." + ), + [ + "task_create", + "task_get", + "task_list", + "task_ready", + "task_update", + "task_claim", + "task_release", + "task_close", + "task_link", + "task_unlink", + "task_for_branch", + ], + ), + ( + "workflow", + "Workflow projects & sessions", + ( + "Tracking an engineering objective across many agent sessions without an " + "explicit hand-off. A project spans phases and repos; each session links " + "itself to one, and a completed project leaves a trace behind for later " + "pattern-mining." + ), + [ + "workflow_project_create", + "workflow_project_get", + "workflow_project_status", + "workflow_project_list", + "workflow_project_update", + "workflow_project_advance", + "workflow_project_complete", + "workflow_project_block", + "workflow_project_unblock", + "workflow_project_get_blockers", + "workflow_project_link_memory", + "workflow_project_memories", + "workflow_session_start", + "workflow_session_end", + "workflow_session_list", + "workflow_trace_list", + "workflow_trace_get", + "workflow_trace_annotate", + "workflow_trace_mine", + ], + ), + ( + "code", + "Code graph", + ( + "Exact symbol lookups, caller graphs, change-impact analysis, and cross-repo " + "contract tracing, served from a tree-sitter index. Reach for these instead of " + "grep when you need a definition, a blast radius, or the provider of a shared " + "env var, endpoint, package, or service." + ), + [], # filled with every code_* tool, in registration order + ), +] + + +async def _collect_tools() -> dict[str, Any]: + import witan.server as witan_server + import witan_code.server as code_server + + tools = {} + for mcp in (witan_server.mcp, code_server.mcp): + for tool in await mcp._list_tools(): + tools[tool.name] = tool + return tools + + +def render_tool(tool: Any) -> str: + out = [f"## `{tool.name}`\n"] + if tool.description: + out.append(tool.description.strip() + "\n") + + params = tool.parameters or {} + props: dict[str, Any] = params.get("properties", {}) + required = set(params.get("required", [])) + if props: + out.append("| Parameter | Type | Default | Description |") + out.append("| --- | --- | --- | --- |") + # Required parameters first: that is the order you have to supply them + # in mentally, and it puts the ten optional filters below the fold. + ordered = sorted(props.items(), key=lambda kv: kv[0] not in required) + for name, schema in ordered: + out.append( + f"| `{name}` | {type_str(schema)} | {default_str(schema, name in required)} " + f"| {clean_desc(schema.get('description'))} |" + ) + out.append("") + else: + out.append("*Takes no parameters.*\n") + return "\n".join(out) + + +@app.command(name="mcp-tools") +def gen_mcp_tools() -> None: + """Generate the MCP tool reference from the registered FastMCP tools.""" + tools = asyncio.run(_collect_tools()) + + # The code group is declared empty and filled here so a new `code_*` tool is + # picked up without editing this file; the other three are listed explicitly + # because their reading order is a deliberate choice, not alphabetical. + code_tools = sorted(n for n in tools if n.startswith("code_")) + groups = [ + (slug, title, blurb, names or code_tools) + for slug, title, blurb, names in TOOL_GROUPS + ] + + assigned = {name for _, _, _, names in groups for name in names} + if unassigned := sorted(set(tools) - assigned): + raise SystemExit( + f"Tool(s) not assigned to any documentation group: {unassigned}\n" + f"Add them to TOOL_GROUPS in {Path(__file__).name}." + ) + + for slug, title, blurb, names in groups: + missing = [n for n in names if n not in tools] + if missing: + raise SystemExit(f"Group {slug!r} lists unregistered tool(s): {missing}") + body = [ + BANNER.format(source="the registered FastMCP tool objects"), + f"# {title}\n", + blurb + "\n", + ] + body.extend(render_tool(tools[n]) for n in names) + emit(REFERENCE / "mcp-tools" / f"{slug}.md", "\n".join(body)) + + _emit_tool_index(groups, tools) + _report_param_coverage(tools) + + +def _report_param_coverage(tools: dict) -> None: + """Print how many tool parameters ship with no description. + + Not a failure — several of these tools predate the convention and the + reference is still useful without them. But an undescribed parameter is not + only a hole in this page: FastMCP sends the same schema to the model, so the + agent calling the tool is working blind too. Printing the count keeps that + visible rather than letting 62 empty cells look like a rendering bug. + """ + total = missing = 0 + worst: list[tuple[int, str]] = [] + for name, tool in tools.items(): + props = (tool.parameters or {}).get("properties", {}) + gaps = sum(1 for schema in props.values() if not schema.get("description")) + total += len(props) + missing += gaps + if gaps: + worst.append((gaps, name)) + if not missing: + return + top = ", ".join(f"{n} ({c})" for c, n in sorted(worst, reverse=True)[:5]) + print( + f" note: {missing}/{total} tool parameters have no description " + f"({len(worst)}/{len(tools)} tools). Worst: {top}" + ) + + +def _emit_tool_index(groups: list, tools: dict) -> None: + body = [ + BANNER.format(source="the registered FastMCP tool objects"), + "# MCP tools\n", + ( + f"witan exposes **{len(tools)} MCP tools** across four domains. A single " + "`witan serve` mounts all of them, so one MCP entry in your agent's config " + "gets you the whole surface.\n" + ), + "| Domain | Tools | What it covers |", + "| --- | --- | --- |", + ] + for slug, title, blurb, names in groups: + # First sentence only — the full blurb is on the page itself. + summary = blurb.split(". ")[0].rstrip(".") + "." + body.append(f"| [{title}]({slug}.md) | {len(names)} | {summary} |") + body.append("") + emit(REFERENCE / "mcp-tools" / "index.md", "\n".join(body)) + + +# ── CLI reference ─────────────────────────────────────────────────── + + +@app.command(name="cli") +def gen_cli() -> None: + """Generate the CLI reference from the cyclopts command tree.""" + from witan.cli import app as witan_app + + # `witan code …` is mounted into the umbrella app when witan-code is + # installed, so a recursive render of the umbrella already contains the + # code-graph CLI. Rendering it once, from the top, is what a reader + # actually types. + markdown = witan_app.generate_docs( + output_format="markdown", + recursive=True, + heading_level=1, + ) + emit( + REFERENCE / "cli.md", + BANNER.format(source="the cyclopts command tree (`witan.cli.app`)") + markdown, + ) + + +# ── Graph schema reference ────────────────────────────────────────── + +_DIVIDER_RE = re.compile(r"[─\-=_]{3,}.*|.*[─\-=]{6,}\s*") +_NODE_RE = re.compile(r"^node\s+(\w+)\s*\{") +_EDGE_RE = re.compile(r"^edge\s+(\w+)\s*:\s*(\w+)\s*->\s*(\w+)") +_FIELD_RE = re.compile(r"^\s*(\w+)\s*:\s*(.+?)\s*(?://\s*(.*))?$") + + +def parse_pg(path: Path) -> tuple[list[dict], list[dict]]: + """Parse an omnigraph ``.pg`` schema into node and edge records. + + Comment lines immediately above a declaration are its documentation — the + convention the schema files already follow — so the prose written next to + the schema is what ends up on the page, with no second copy to maintain. + """ + nodes: list[dict] = [] + edges: list[dict] = [] + pending: list[str] = [] + current: dict | None = None + + for raw in path.read_text().splitlines(): + line = raw.rstrip() + stripped = line.strip() + + if stripped.startswith("//"): + text = stripped[2:].strip() + # Section dividers (`── Workflow Tracking ──`) are layout, not + # documentation, and would otherwise open every node's description. + if not _DIVIDER_RE.fullmatch(text): + pending.append(text) + continue + if not stripped: + # A blank line does NOT break the comment→declaration association. + # Every doc block in these schema files is separated from the thing + # it documents by exactly one blank line, so treating a blank as a + # reset silently drops the documentation for every node and edge — + # which is what the first version of this parser did. + continue + + if current is not None: + if stripped.startswith("}"): + current = None + pending = [] + continue + match = _FIELD_RE.match(line) + if match: + name, type_spec, comment = match.groups() + current["fields"].append( + { + "name": name, + "type": type_spec.rstrip(","), + "comment": comment or "", + } + ) + continue + + if match := _NODE_RE.match(stripped): + current = {"name": match.group(1), "doc": pending, "fields": []} + nodes.append(current) + pending = [] + continue + + if match := _EDGE_RE.match(stripped): + name, src, dst = match.groups() + edges.append({"name": name, "from": src, "to": dst, "doc": pending}) + pending = [] + continue + + pending = [] + + return nodes, edges + + +def _schema_doc(lines: list[str]) -> str: + """Join a declaration's comment block into renderable Markdown. + + Angle brackets are escaped: these files describe slugs as + ``wp--<6hex>``, and an unescaped ```` is + parsed as an HTML tag and vanishes from the page. + """ + text = "\n".join(lines).strip() + return text.replace("<", "<").replace(">", ">") + + +def _render_schema(title: str, intro: str, source: Path) -> str: + nodes, edges = parse_pg(source) + rel = source.relative_to(REPO_ROOT) + body = [ + BANNER.format(source=rel), + f"# {title}\n", + intro + "\n", + f"Source of truth: [`{rel}`](https://github.com/mitodl/agent-kit/blob/main/{rel}).\n", + "## Nodes\n", + ] + for node in nodes: + body.append(f"### `{node['name']}`\n") + if doc := _schema_doc(node["doc"]): + body.append(doc + "\n") + body.append("| Field | Type | Notes |") + body.append("| --- | --- | --- |") + for field in node["fields"]: + body.append( + f"| `{field['name']}` | `{field['type']}` | {field['comment']} |" + ) + body.append("") + + body.append("## Edges\n") + body.append( + "Edges are directional and typed. A traversal names the edge in lowercase " + "(`supersedes`, `blocks`), while the schema declares it in PascalCase.\n" + ) + body.append("| Edge | From | To | Meaning |") + body.append("| --- | --- | --- | --- |") + for edge in edges: + doc = _schema_doc(edge["doc"]).replace("\n", " ") + body.append(f"| `{edge['name']}` | `{edge['from']}` | `{edge['to']}` | {doc} |") + body.append("") + return "\n".join(body) + + +@app.command(name="schema") +def gen_schema() -> None: + """Generate the graph schema reference from the `.pg` files.""" + emit( + REFERENCE / "graph-schema.md", + _render_schema( + "Graph schema", + "The shape of the witan graph: what a memory, a task, a project, and a " + "session are, and how they connect. Every MCP tool is ultimately a read " + "or a write against these types.", + REPO_ROOT / "mcp/servers/witan/schema/schema.pg", + ), + ) + emit( + REFERENCE / "bridge-schema.md", + _render_schema( + "Cross-repo bridge schema", + "The bridge store links repositories to each other by shared contract " + "keys — an env var, an HTTP endpoint, a package name, a service name. It " + "is what makes `code_interface_providers` and `code_cross_repo_impact` " + "able to answer a question that spans two checkouts.", + REPO_ROOT / "mcp/servers/witan-code/witan_code/schema/bridge-schema.pg", + ), + ) + + +# ── Environment variable reference ────────────────────────────────── + +# Anything matching these is a name fragment or a test fixture, not a real +# setting. `WITAN_SCAN_` and friends appear as f-string prefixes in code that +# builds a var name dynamically; the concrete vars they build are listed +# individually in the data file. +_ENV_SKIP = re.compile(r"^WITAN_(TEST_|CODE_TEST_)|_$") +# Digits are part of a name, not a boundary: without them `WITAN_RANK_W_BM25` +# silently truncates to `WITAN_RANK_W_BM` and the docs describe a var that does +# not exist. The left boundary matters just as much — without it the private +# Python constants `_WITAN_ARGS` and `_WITAN_CODE_ARGS` are read as env vars and +# the reference grows two settings nothing has ever honoured. +_ENV_RE = re.compile(r"(? set[str]: + """``WITAN_*`` names a Python module actually *references*, not merely mentions. + + ★ SCANNING RAW TEXT HERE PRODUCES SETTINGS THAT DO NOT EXIST. Three showed up + the first time this ran: + + ``WITAN_BRANCH`` appears only in a comment saying there is *no* + such override + ``WITAN_EMBED_ENABLED`` named in two docstrings; embeddings are deferred + and nothing reads it + ``WITAN_REQUEST_TIMEOUT`` a comment's cross-reference to a budget the + deployment enforces at APISIX, not here + + A reference page that invents three knobs is worse than no page, so the AST + is the source rather than the file text: comments are absent from it + entirely, and docstrings are skipped explicitly. Ordinary string literals are + kept — plenty of real vars are declared as constants + (``HTTP_TRANSPORT_ENV_VAR = "WITAN_OMNIGRAPH_HTTP"``), and dropping those + would trade three false positives for a dozen false negatives. + """ + tree = ast.parse(source) + docstrings = set() + for node in ast.walk(tree): + if isinstance( + node, (ast.Module, ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef) + ): + body = getattr(node, "body", None) + if ( + body + and isinstance(body[0], ast.Expr) + and isinstance(body[0].value, ast.Constant) + and isinstance(body[0].value.value, str) + ): + docstrings.add(id(body[0].value)) + # PEP 257 "attribute docstrings" — a bare string after an assignment. + # These packages use them heavily to document constants, and they are + # where two of the three phantoms above came from. + for field in ("body", "orelse", "finalbody"): + stmts = getattr(node, field, None) + if not isinstance(stmts, list): + continue + for prev, cur in itertools.pairwise(stmts): + if ( + isinstance(prev, (ast.Assign, ast.AnnAssign)) + and isinstance(cur, ast.Expr) + and isinstance(cur.value, ast.Constant) + and isinstance(cur.value.value, str) + ): + docstrings.add(id(cur.value)) + + found: set[str] = set() + for node in ast.walk(tree): + if ( + isinstance(node, ast.Constant) + and isinstance(node.value, str) + and id(node) not in docstrings + ): + found.update(_ENV_RE.findall(node.value)) + return found + + +def discover_env_vars() -> set[str]: + """Every ``WITAN_*`` setting shipped (non-test) source actually reads.""" + found: set[str] = set() + roots = [ + REPO_ROOT / "packages/witan-core/witan_core", + REPO_ROOT / "mcp/servers/witan/witan", + REPO_ROOT / "mcp/servers/witan-code/witan_code", + REPO_ROOT / "docker", + ] + for root in roots: + for path in sorted(root.rglob("*")): + if not path.is_file(): + continue + if path.suffix == ".py": + found |= _py_env_names(path.read_text()) + elif path.suffix == ".sh": + # Shell has no AST to lean on; stripping `#` comments is enough, + # since the remaining `${VAR}` references are the real reads. + text = "\n".join( + line.split("#", 1)[0] for line in path.read_text().splitlines() + ) + found.update(_ENV_RE.findall(text)) + return {name for name in found if not _ENV_SKIP.search(name)} + + +@app.command(name="env") +def gen_env() -> None: + """Generate the environment variable reference. + + Names are discovered from source; descriptions come from + ``docs/_data/environment.toml``. A discovered name with no entry there is a + hard error — that is the whole point of the split. + """ + data = tomllib.loads((DATA / "environment.toml").read_text()) + documented = { + name: entry + for section in data.get("section", []) + for name, entry in section.get("vars", {}).items() + } + discovered = discover_env_vars() + + if undocumented := sorted(discovered - set(documented)): + raise SystemExit( + "Environment variable(s) used in source but not documented in " + "docs/_data/environment.toml:\n " + "\n ".join(undocumented) + ) + if stale := sorted(set(documented) - discovered): + raise SystemExit( + "Environment variable(s) documented but no longer used in source " + "(remove them from docs/_data/environment.toml):\n " + "\n ".join(stale) + ) + + body = [ + BANNER.format(source="source scan + docs/_data/environment.toml"), + "# Environment variables\n", + data["intro"].strip() + "\n", + ] + for section in data["section"]: + body.append(f"## {section['title']}\n") + if blurb := section.get("blurb"): + body.append(blurb.strip() + "\n") + body.append("| Variable | Default | Description |") + body.append("| --- | --- | --- |") + for name in sorted(section["vars"]): + entry = section["vars"][name] + default = entry.get("default", "") + default = f"`{default}`" if default else "—" + body.append(f"| `{name}` | {default} | {entry['desc'].strip()} |") + body.append("") + emit(REFERENCE / "environment.md", "\n".join(body)) + + +# ── Mirrored package documentation ────────────────────────────────── + +# Prose that already lives inside a package, and where it lands on the site. +# +# WHY MIRROR RATHER THAN MOVE. These files are shipped documentation for their +# package: `mcp/servers/witan/docs/USER_GUIDE.md` is what a PyPI reader or +# someone browsing the repo finds next to the code, and moving it into `docs/` +# to serve the site would take it away from them. Mirroring keeps one +# authoritative copy — the one next to the code — and makes the site's copy a +# build artifact, which `--check` then keeps honest. +# +# The two `CLI_REFERENCE.md` files are deliberately absent: `gen_cli` renders the +# same surface from the live command tree, so mirroring them would publish two +# CLI references that disagree the moment a flag changes. +MIRRORED: list[tuple[str, str]] = [ + ("mcp/servers/witan/docs/USER_GUIDE.md", "guides/witan-user-guide.md"), + ("mcp/servers/witan/docs/write-path-scanning.md", "guides/write-path-scanning.md"), + ("mcp/servers/witan/docs/migration-runbook.md", "guides/migration-runbook.md"), + ("mcp/servers/witan/docs/deployed-witan-onboarding.md", "guides/deployed-witan.md"), + ("mcp/servers/witan-code/docs/USER_GUIDE.md", "guides/witan-code-user-guide.md"), + ("mcp/servers/witan-code/docs/BRANCH_INDEXING.md", "guides/branch-indexing.md"), + ( + "mcp/servers/witan-code/docs/SYMBOL_FORMAT.md", + "explanation/code-graph/symbol-format.md", + ), + ( + "mcp/servers/witan-code/docs/SYMBOL_TABLE.md", + "explanation/code-graph/symbol-table.md", + ), + ( + "mcp/servers/witan-code/docs/PACKAGE_MAP.md", + "explanation/code-graph/package-map.md", + ), + ( + "mcp/servers/witan-code/docs/EDGE_PRECISION_TIERS.md", + "explanation/code-graph/edge-precision-tiers.md", + ), + ( + "mcp/servers/witan-code/docs/STAGE2_STITCHING.md", + "explanation/code-graph/stage2-stitching.md", + ), +] + +MIRROR_BANNER = """ + +!!! info "This page lives with the code" + + The authoritative copy is + [`{source}`](https://github.com/mitodl/agent-kit/blob/main/{source}). + +""" + + +GITHUB_BLOB = "https://github.com/mitodl/agent-kit/blob/main/" + +_MD_LINK = re.compile(r"\[([^\]]*)\]\(([^)\s]+)(\s+\"[^\"]*\")?\)") + + +def rewrite_links(text: str, source_rel: str, dest_rel: str, mirror_map: dict) -> str: + """Repoint a mirrored file's relative links so they still resolve on the site. + + A link like ``[CLI Reference](CLI_REFERENCE.md)`` is correct where the file + lives, next to the code, and broken once the file is served from + ``docs/guides/``. Two cases, two answers: + + * the target is **also mirrored** → rewrite to its site path, so the reader + stays on the site and the link survives + * the target is **not on the site** (a source file, a skill, a README) → + rewrite to an absolute GitHub URL, which is where the reader wanted to end + up anyway + + Doing this at mirror time rather than editing the source keeps one + authoritative copy: the file next to the code stays correct for someone + reading it in the repository. + """ + source_dir = PurePosixPath(source_rel).parent + dest_dir = PurePosixPath(dest_rel).parent + + def replace(match: re.Match) -> str: + label, target, title = match.group(1), match.group(2), match.group(3) or "" + if target.startswith(("http://", "https://", "#", "mailto:", "<")): + return match.group(0) + path, sep, fragment = target.partition("#") + if not path: + return match.group(0) + + resolved = os.path.normpath(str(source_dir / path)) + if resolved in mirror_map: + # Both ends are on the site: emit a site-relative path. + new = os.path.relpath(mirror_map[resolved], str(dest_dir)) + elif (REPO_ROOT / resolved).exists(): + new = GITHUB_BLOB + resolved + else: + # Neither a mirrored page nor a real repo path — leave it alone + # rather than inventing a URL that 404s on GitHub instead of here. + return match.group(0) + return f"[{label}]({new}{sep}{fragment}{title})" + + return _MD_LINK.sub(replace, text) + + +def _mirror_map() -> dict[str, str]: + """Repo path → site path, for every file this script mirrors.""" + mapping = {source: dest for source, dest in MIRRORED} + adr_dir = REPO_ROOT / "mcp/servers/witan/docs/adr" + for path in sorted(adr_dir.glob("*.md")): + rel = str(path.relative_to(REPO_ROOT)) + mapping[rel] = f"explanation/decisions/{path.name}" + return mapping + + +@app.command(name="mirror") +def gen_mirror() -> None: + """Copy package-resident prose into the site tree, repointing its links.""" + mapping = _mirror_map() + for source_rel, dest_rel in MIRRORED: + source = REPO_ROOT / source_rel + if not source.exists(): + raise SystemExit( + f"Mirrored source missing: {source_rel}\n" + f"It was moved or deleted — update MIRRORED in {Path(__file__).name}." + ) + body = rewrite_links(source.read_text(), source_rel, dest_rel, mapping) + emit(DOCS / dest_rel, MIRROR_BANNER.format(source=source_rel) + body) + + _mirror_adrs(mapping) + + +def _mirror_adrs(mapping: dict[str, str]) -> None: + """Mirror the ADRs, and refuse to publish a duplicated decision number. + + ★ TWO NUMBERS ARE CURRENTLY USED TWICE — 0004 is both the Keycloak actor + mapping and the optional task phase tag, and 0006 is both code-graph branch + ownership and the stateless MCP protocol era. On a site that lists decisions + by number that reads as a typo or a missing page, so it is surfaced here + rather than quietly rendered. + """ + adr_dir = REPO_ROOT / "mcp/servers/witan/docs/adr" + by_number: dict[str, list[str]] = {} + for path in sorted(adr_dir.glob("*.md")): + by_number.setdefault(path.name.split("-")[0], []).append(path.name) + source_rel = str(path.relative_to(REPO_ROOT)) + dest_rel = f"explanation/decisions/{path.name}" + body = rewrite_links(path.read_text(), source_rel, dest_rel, mapping) + emit(DOCS / dest_rel, MIRROR_BANNER.format(source=source_rel) + body) + + if dupes := {n: f for n, f in by_number.items() if len(f) > 1}: + print( + " note: duplicate ADR number(s) — " + + "; ".join(f"{n}: {', '.join(f)}" for n, f in sorted(dupes.items())) + ) + + +# ── Entry point ───────────────────────────────────────────────────── + + +@app.default +def main(*, check: bool = False) -> None: + """Regenerate every reference page. + + Parameters + ---------- + check: Do not write. Exit non-zero if any generated page is out of date, + printing a diff. This is what CI runs. + """ + global _check_mode + _check_mode = check + + for step in (gen_mcp_tools, gen_cli, gen_schema, gen_env, gen_mirror): + step() + + if check: + if _stale: + print( + f"\n{len(_stale)} generated page(s) are out of date:\n " + + "\n ".join(_stale) + + "\n\nRun `just docs-gen` and commit the result.", + file=sys.stderr, + ) + raise SystemExit(1) + print("All generated documentation is up to date.") + + +if __name__ == "__main__": + app() diff --git a/configs/pi/README.md b/configs/pi/README.md index 007ecdbe..39e099f8 100644 --- a/configs/pi/README.md +++ b/configs/pi/README.md @@ -49,7 +49,7 @@ ln -sf "$(pwd)/extensions/workflow-context.ts" ~/.pi/agent/extensions/ ``` The MCP servers themselves are configured separately in `~/.pi/agent/mcp.json` -(see [Local Development Setup](../../docs/agent-memory.md#local-development-setup)). +(see [Local Development Setup](../../docs/internals/agent-memory.md#local-development-setup)). ## Not covered diff --git a/docs/_data/environment.toml b/docs/_data/environment.toml new file mode 100644 index 00000000..5ee83f49 --- /dev/null +++ b/docs/_data/environment.toml @@ -0,0 +1,353 @@ +# Curated descriptions for the environment variable reference. +# +# The NAMES on this page are discovered from source by bin/gen_docs.py; the +# DESCRIPTIONS live here. A variable the code reads but that has no entry below +# is a hard error, and so is an entry for a variable nothing reads any more. +# That split is deliberate: a generator can prove a setting exists, but only a +# person can say what it is for. + +intro = """ +Every setting witan reads from the environment. Environment variables take +precedence over `~/.config/witan/config.toml`, which takes precedence over the +built-in default — so an env var always wins. + +Most of these have a config-file equivalent and you will never set them by hand. +The ones worth knowing on day one are +[`WITAN_MEMORY_URI`](#store-and-attribution) (where the graph lives) and +[`WITAN_AUTHOR`](#store-and-attribution) (whose name is on what you write). +Everything below that is deployment, tuning, or operations. +""" + +# ── Store and attribution ──────────────────────────────────────────────────── + +[[section]] +title = "Store and attribution" +blurb = """ +Where the graph lives and whose name goes on the nodes you create. These are the +only settings a local, single-user install normally needs. +""" + +[section.vars.WITAN_MEMORY_URI] +default = "~/.local/share/witan/graph.omni" +desc = "Graph store location: a local path, an `s3://` URI, or the base URL of a deployed `omnigraph-server`. This is the single setting that decides whether you are running against your own laptop or a shared service." + +[section.vars.WITAN_MEMORY_GRAPH] +default = "council" +desc = "Which graph to address on an `http(s)://` omnigraph-server — one server hosts many. Ignored for local paths and `s3://` stores, which name the graph in the URI itself." + +[section.vars.WITAN_MEMORY_TOKEN] +desc = "Bearer token for an `http(s)://` store. Required for a deployed server, meaningless for a local one." + +[section.vars.WITAN_AUTHOR] +desc = "Attribution written to every node you create. Falls back to `git config user.name`, then `$USER`." + +[section.vars.WITAN_CONFIG] +default = "~/.config/witan/config.toml" +desc = "Path to the config file. Both `witan` and `witan code` read the same file. An empty or whitespace-only value counts as unset, so an unexpanded `$SOME_UNSET_VAR` does not silently redirect you to a file named `$SOME_UNSET_VAR`." + +[section.vars.WITAN_OUTPUT_FORMAT] +default = "txt" +desc = "Default CLI output format: `txt`, `json`, `toml`, or `yaml`. Equivalent to passing `--output-format`." + +# ── Scoping ────────────────────────────────────────────────────────────────── + +[[section]] +title = "Repository and target scoping" +blurb = """ +Which repo a call is about, and which store answers it. witan auto-detects the +repo from `.git/config`; these override that when detection is wrong, absent, or +too slow. +""" + +[section.vars.WITAN_REPO] +desc = "Canonical repo URI for the current call, overriding git detection. Setting it also skips git entirely, which is why hooks use it. Set it to the **empty string** to suppress repo detection and operate across all repos." + +[section.vars.WITAN_TARGET] +desc = "Name of the `[targets.]` config block to use, overriding auto-detection by repo or checkout path. Lets one machine route work repos and personal repos at different stores." + +[section.vars.WITAN_AGENT] +default = "claude" +desc = "Default coding-agent CLI for `witan run`: `claude`, `pi`, `copilot`, `opencode`, or `kilo`." + +[section.vars.WITAN_MODEL] +desc = "Default `--model` passed through to the agent by `witan run`." + +# ── Reaching a deployed witan ──────────────────────────────────────────────── + +[[section]] +title = "Client: reaching a deployed witan" +blurb = """ +Set these to point the local CLI at a shared witan service instead of running +the graph in-process. They configure the *client's* view of a deployment — a CLI +user never sets the server-side identity variables in the next section. + +`witan login` performs an OIDC device grant against the issuer and caches the +token; both `witan` and `witan code` share one cache, so you log in once. +""" + +[section.vars.WITAN_REMOTE_URL] +desc = "Base URL of the deployed witan MCP endpoint. Setting it routes CLI reads and writes through the service rather than opening a store locally. A `[targets.]` block's `remote_url` overrides it." + +[section.vars.WITAN_OIDC_ISSUER] +desc = "OIDC issuer URL used for the device-authorization grant behind `witan login`." + +[section.vars.WITAN_OIDC_CLIENT_ID] +desc = "OIDC client id presented during the device grant." + +[section.vars.WITAN_OIDC_AUDIENCE] +desc = "Audience/resource to request, matching the deployment's own `WITAN_OIDC_AUDIENCE`. Sent on the device-auth and token requests so an issuer with an audience mapper stamps the right `aud` claim." + +[section.vars.WITAN_OIDC_EXPIRY_SKEW_SECONDS] +default = "90" +desc = "How long before nominal expiry a cached token is treated as already expired and refreshed. Sized so a refresh happens before a long write starts rather than partway through one." + +[section.vars.WITAN_TOKEN_CACHE] +default = "~/.config/witan/tokens.json" +desc = "Where both CLIs cache OIDC tokens. Shared on purpose, next to the shared config file." + +[section.vars.WITAN_REMOTE_CALL_BUDGET_SECONDS] +default = "0" +desc = "Deadline for a single remote graph call, used to decide whether to honour a server's retry hint or give up. `0` means no client-side deadline — obey the server's hints." + +[section.vars.WITAN_REMOTE_WRITE_MAX_INFLIGHT] +default = "4" +desc = "How many remote writes may be in flight at once. The gate that keeps a burst of concurrent writes from stranding on the data tier." + +[section.vars.WITAN_REMOTE_WRITE_QUEUE_SECONDS] +default = "10.0" +desc = "How long a write waits for a slot at the in-flight gate before failing fast rather than queueing indefinitely." + +# ── Serving ────────────────────────────────────────────────────────────────── + +[[section]] +title = "Server: running `witan serve`" +blurb = """ +Deployment and operations config for a shared, network-facing witan. A local +stdio install needs none of it. + +`WITAN_OIDC_ISSUER`, `WITAN_OIDC_AUDIENCE`, and `WITAN_ACTOR_TOKENS_FILE` must be +set **together** — witan refuses to start with a partial identity configuration +rather than serving unauthenticated. +""" + +[section.vars.WITAN_MCP_TRANSPORT] +default = "stdio" +desc = "MCP transport: `stdio` for local per-user use, or `streamable-http` (alias `http`) to bind a network listener. The legacy HTTP+SSE transport is deliberately not offered." + +[section.vars.WITAN_MCP_HOST] +default = "127.0.0.1" +desc = "Interface to bind for HTTP transports. Use `0.0.0.0` inside a container." + +[section.vars.WITAN_MCP_PORT] +default = "8000" +desc = "Port to bind for HTTP transports." + +[section.vars.WITAN_MCP_PATH] +default = "/mcp" +desc = "URL path the MCP endpoint is served on. HTTP transports only." + +[section.vars.WITAN_MCP_SHUTDOWN_GRACE_SECONDS] +default = "120.0" +desc = "How long uvicorn waits for in-flight requests after `SIGTERM`. **FastMCP's own default is 2 seconds**, which silently truncates any rollout — a witan write has been measured at 27s under load, and a severed write is an indeterminate outcome the caller cannot safely retry. Set this to the deployment's termination grace period." + +[section.vars.WITAN_ACTOR_TOKENS_FILE] +desc = "Path to a mounted `{actor_id: token}` map. The server-side half of identity: it maps an authenticated caller to the actor recorded on the nodes they write." + +[section.vars.WITAN_ACTOR] +desc = "Overrides the OIDC-derived identity. Intended for service accounts and the CI indexer, which authenticate as themselves rather than as a person." + +[section.vars.WITAN_OMNIGRAPH_HTTP] +default = "1" +desc = "Use the direct HTTP transport for reads against a deployed omnigraph-server instead of shelling out to the `omnigraph` binary. Set to `0`/`false`/`no`/`off` to revert. Kept as a one-variable revert so a transport-specific production problem is an env change rather than an image rebuild — the CLI path beneath it stays fully maintained and is still the only way to reach `load`, `branch`, and `optimize`." + +# ── Code graph ─────────────────────────────────────────────────────────────── + +[[section]] +title = "Code graph (`witan code`)" +blurb = """ +Settings for the tree-sitter code index and its cross-repo bridge. These mirror +the store settings above but address the *code* graph, which is a separate store +from the memory/task graph. +""" + +[section.vars.WITAN_CODE_DIR] +desc = "Directory holding the per-repo code-graph stores." + +[section.vars.WITAN_CODE_SERVER] +desc = "Base URL of an omnigraph-server hosting the code graphs, for a shared index." + +[section.vars.WITAN_CODE_GRAPH] +desc = "Graph id to address on `WITAN_CODE_SERVER`." + +[section.vars.WITAN_CODE_TOKEN] +desc = "Bearer token for `WITAN_CODE_SERVER`." + +[section.vars.WITAN_CODE_TRANSPORT] +default = "direct" +desc = "How the `witan code` CLI reaches the index: `direct` opens the store in-process; `mcp` proxies through a deployed witan endpoint." + +[section.vars.WITAN_CODE_STORE_TOOLS] +desc = "Force the low-level store tools on (`1`) or off (`0`), overriding the default. These expose raw graph reads and mutations alongside the curated `code_*` tools." + +[section.vars.WITAN_CODE_INDEX_ROLE] +desc = "Declares what the indexing process is entitled to write **on a shared graph**. There, only `ci` may write a repo's default (`main`) view and run the stale-file purge that goes with it, so no developer's reindex can clobber the view all readers fall back to. A local store has a single user, who is its writer — this setting does not restrict it, and a local default-branch reindex works with no role declared." + +[section.vars.WITAN_CODE_VIEW_MAX_IDLE_DAYS] +default = "14" +desc = "Reap per-branch views idle at least this long. `0` (or negative) disables reaping entirely." + +[section.vars.WITAN_CODE_OPTIMIZE_INTERVAL] +default = "86400" +desc = "Minimum seconds between throttled background `optimize` runs on the code stores. `0` disables." + +# ── CI indexer ─────────────────────────────────────────────────────────────── + +[[section]] +title = "CI code-graph indexer" +blurb = """ +Read by `witan-ci-index`, the script that keeps each repo's shared code graph +current. It runs as a Kubernetes CronJob from the same `witan` image with a +different entrypoint. Nothing else should set these. +""" + +[section.vars.WITAN_CODE_CI_REPOS] +desc = "**Required.** Whitespace-separated canonical repo URIs to sweep and index." + +[section.vars.WITAN_CODE_CI_WORKDIR] +default = "/tmp/witan-ci-index" +desc = "Scratch directory for checkouts. Rejected unless it is an absolute path at least two components deep with no `..` or empty components — the guard that keeps a misconfigured value from pointing the cleanup at something important." + +[section.vars.WITAN_CODE_CI_ALLOW_LOCAL_STORE] +desc = "Set to `1` to waive the `WITAN_CODE_SERVER`/`WITAN_CODE_TOKEN` requirement and index into local stores instead. For development runs of the indexer only." + +[section.vars.WITAN_CODE_GITHUB_APP_ID] +desc = "GitHub App id used to mint short-lived clone credentials. Set all three `_APP_` variables, or none." + +[section.vars.WITAN_CODE_GITHUB_APP_INSTALLATION_ID] +desc = "Installation id of the GitHub App, identifying which org's repos it may clone." + +[section.vars.WITAN_CODE_GITHUB_APP_KEY_FILE] +desc = "Path to the GitHub App's private key, used to sign the App JWT." + +[section.vars.WITAN_CODE_GH_TOKEN] +desc = "Clone credential. Normally minted per-repo from the GitHub App above rather than set by hand." + +[section.vars.WITAN_CODE_GITHUB_API_URL] +default = "https://api.github.com" +desc = "GitHub API base URL. Only meaningful against GitHub Enterprise." + +# ── Write-path scanning ────────────────────────────────────────────────────── + +[[section]] +title = "Write-path content scanning" +blurb = """ +witan scans everything written to the graph for secrets and PII. It ships +**enabled**, and fails closed: a scanner that raises blocks the write rather than +silently opening the gate. + +`enabled_detectors`, `disabled_detectors`, `plugins`, `allowlist`, and +`allowlist_hashes` each accept a comma-separated string here (or a TOML list in +the config file). An empty `enabled_detectors` means every registered detector is +active; naming any detector switches to an explicit allowlist. +`disabled_detectors` always wins. +""" + +[section.vars.WITAN_SCAN_ENABLED] +default = "true" +desc = "Master switch. When false the write path is not scanned at all." + +[section.vars.WITAN_SCAN_SECRET_ACTION] +default = "block" +desc = "Enforcement for `secret` findings. Fail-closed by default." + +[section.vars.WITAN_SCAN_PII_ACTION] +default = "redact" +desc = "Enforcement for `pii` findings. Mask-and-proceed by default." + +[section.vars.WITAN_SCAN_ENABLED_DETECTORS] +desc = "Explicit allowlist of detectors to run. Empty means all registered detectors." + +[section.vars.WITAN_SCAN_DISABLED_DETECTORS] +desc = "Detectors to switch off. Always wins over `WITAN_SCAN_ENABLED_DETECTORS`." + +[section.vars.WITAN_SCAN_PLUGINS] +desc = "Dotted import paths of external scanners to load, in addition to those discovered through the `witan.scanners` entry-point group." + +[section.vars.WITAN_SCAN_ALLOWLIST] +desc = "Regexes whose matches are downgraded to audit-only, for false-positive suppression. Tested against each finding's own matched span with `re.fullmatch`." + +[section.vars.WITAN_SCAN_ALLOWLIST_HASHES] +desc = "Salted SHA-256 digests (hex) of specific approved values, downgraded to audit-only without ever putting the plaintext in config. Computed as `sha256(salt + matched_span)`. Normalized to lowercase at load, so a hand-typed uppercase digest still matches." + +[section.vars.WITAN_SCAN_ALLOWLIST_SALT] +desc = "Salt for `WITAN_SCAN_ALLOWLIST_HASHES`. Empty means the hash allowlist is inert — set a deployment-specific value before relying on it." + +[section.vars.WITAN_SCAN_ON_ERROR] +default = "block" +desc = "What to do when a scanner itself raises: `block` or `warn`. Fail-closed by default so a broken detector cannot silently open the gate." + +# ── Recall ranking ─────────────────────────────────────────────────────────── + +[[section]] +title = "Recall ranking" +blurb = """ +Tuning knobs for the composite re-rank `recall` applies on top of BM25. Ranking +is always on; these change its shape, they do not switch it off. Set every `W_*` +weight to `0` to reproduce the raw BM25 order. +""" + +[section.vars.WITAN_RANK_W_BM25] +default = "1.0" +desc = "Weight of the BM25 text-relevance term." + +[section.vars.WITAN_RANK_W_RECENCY] +default = "0.3" +desc = "Weight of the recency term, decayed by `WITAN_RANK_HALFLIFE_DAYS`." + +[section.vars.WITAN_RANK_W_CORROB] +default = "0.2" +desc = "Weight of corroboration — how much other memories back this one up." + +[section.vars.WITAN_RANK_W_CONF] +default = "0.2" +desc = "Weight of the author-set confidence score." + +[section.vars.WITAN_RANK_W_HOP] +default = "0.5" +desc = "Per-hop distance penalty in graph-aware recall, so direct hits (hop 0) outrank expanded neighbours (hop ≥ 1)." + +[section.vars.WITAN_RANK_HALFLIFE_DAYS] +default = "90.0" +desc = "Half-life of the recency decay, in days. Must be greater than zero." + +[section.vars.WITAN_RANK_DEFAULT_CONF] +default = "0.6" +desc = "Confidence assumed for a memory that carries none. Must be between 0 and 1." + +[section.vars.WITAN_RANK_PEN_SUPERSEDED] +default = "1.0" +desc = "Score penalty applied to a memory something else supersedes. At the default it is effectively removed from results." + +[section.vars.WITAN_RANK_PEN_CONTRADICTED] +default = "0.25" +desc = "Score penalty for a memory another contradicts. Deliberately mild — a contradiction is surfaced for review, never hidden." + +# ── Maintenance and observability ──────────────────────────────────────────── + +[[section]] +title = "Maintenance and observability" + +[section.vars.WITAN_OPTIMIZE_INTERVAL] +default = "86400" +desc = "Minimum seconds between throttled background `optimize` runs on the memory store. `0` disables. The `Stop` hook spawns a detached run at most this often, so compaction never blocks a session." + +[section.vars.WITAN_CONTEXT_TTL] +default = "30.0" +desc = "How long the rendered session-context block is cached on disk, in seconds. Only the first prompt in the window pays to build it; the rest read one small file. `0` disables the cache. The content is advisory, so a few seconds of staleness is fine." + +[section.vars.WITAN_LOG_LEVEL] +default = "INFO" +desc = "Log level. Takes precedence over the bare `LOG_LEVEL`, which is also honoured for deployments that set it org-wide." + +[section.vars.WITAN_LOG_FORMAT] +desc = "Log rendering: `console` or `json`. Defaults to `console` when stderr is a TTY and `json` when it is not — a deployed pod gets structured logs and a developer gets colours, neither having to pass a flag." diff --git a/docs/explanation/architecture.md b/docs/explanation/architecture.md new file mode 100644 index 00000000..73888961 --- /dev/null +++ b/docs/explanation/architecture.md @@ -0,0 +1,138 @@ +# Architecture + +## Three layers, one endpoint + +witan presents three conceptually separate things through a single MCP server: + +```mermaid +flowchart TB + subgraph agent["Your coding agent"] + A["Claude Code · Pi · Copilot · OpenCode"] + end + subgraph mcp["witan MCP server — one endpoint, 60 tools"] + M["Memory & recall
memory_* · recall · topic_*"] + T["Work coordination
task_* · workflow_*"] + C["Code graph
code_*"] + end + subgraph store["Storage — omnigraph"] + G1[("Coordination graph")] + G2[("Per-repo code graphs")] + G3[("Cross-repo bridge")] + end + A -->|MCP| mcp + M --> G1 + T --> G1 + C --> G2 + C --> G3 +``` + +`witan serve` mounts witan-code's tools into its own FastMCP server **with no +prefix**, so `code_find_definition` sits beside `memory_store` as far as the +agent is concerned. One entry in your agent's config; the whole surface. + +The mount is optional and detected at import: the umbrella works standalone if +`witan-code` is not installed. That is why the packages ship separately — you +can take the coordination graph without the tree-sitter dependency tree. + +## The CLI is not a second implementation + +`witan tasks` does not reimplement task listing. It calls the very same +`witan.server` tool function the MCP server exposes, and formats the result. + +This matters more than it sounds. Repo scoping, ready-work computation, +claim semantics, scanning — all of it lives in one place, so the CLI and your +agent cannot disagree about what the graph says. The CLI is a presentation +layer, deliberately thin. + +## Two deployment shapes + +=== "Local (default)" + + ```mermaid + flowchart LR + CLI["witan CLI"] --> S["witan tools
(in-process)"] + AGT["Agent"] -->|"MCP stdio"| S + S --> F[("~/.local/share/witan/graph.omni")] + ``` + + Everything in one process, against a file on disk. Writes are serialised by + a per-store advisory `flock`, which makes claims effectively safe. No + server, no credentials, no network. + +=== "Shared (deployed)" + + ```mermaid + flowchart LR + CLI["witan CLI"] -->|"OIDC + MCP"| W + AGT["Agent"] -->|"MCP streamable-http"| W + W["witan tier
ToolHive-hosted"] -->|"http"| O + O["omnigraph-server
data tier"] --> S3[("S3")] + CI["CI indexer
CronJob"] --> O + ``` + + Two images: the MCP tier running `witan serve --transport streamable-http`, + and the data tier serving the S3-backed graph. The `flock` is gone — it is a + local-filesystem lock and cannot coordinate across pods — which is the root + of the claim-atomicity limits described in [Coordinating + work](task-coordination.md). + +The same tools work against both. Only +[`WITAN_MEMORY_URI`](../reference/environment.md) changes, plus credentials. + +## Why omnigraph + +The store is [omnigraph](https://github.com/ModernRelay/omnigraph): a +property-graph engine over Lance, addressable as a local file, an `s3://` root, +or an HTTP server. Three properties made it the right substrate: + +- **One store type covers all three deployment shapes.** A local file and a + shared cluster graph are the same engine, so nothing about the data model has + to change when a team outgrows a laptop. +- **Typed nodes and edges with BM25 indexes.** Memory search and graph + traversal are both first-class, which is precisely the combination `recall` + needs. +- **Branchable views.** The code graph leans on this: each git branch gets its + own view, so an index built on a feature branch is invisible to everyone else + until it is not. + +The costs are real and worth naming. omnigraph offers **no conditional-write +primitive**, which is why task claims are best-effort. Stores are also **not +relocatable** — a Lance store embeds absolute paths, so moving one means +export → init → load, never `mv`. And the client shells out to a pinned +`omnigraph` binary, which must be on `PATH` for the server to even start. + +## What happens on a tool call + +Taking `memory_store` as the example, because it exercises the whole path: + +1. **Repo detection.** The current repo is resolved from `.git/config`, or from + [`WITAN_REPO`](../reference/environment.md) if set — which also skips git + entirely, and is why hooks set it. +2. **Identity.** The author is resolved from config, environment, or git. On a + deployed server the actor comes from the validated OIDC token instead. +3. **Content scanning.** The write is scanned for secrets and PII *before* it + persists. Secrets block by default; PII is redacted and the call proceeds. + A scanner that itself raises also blocks — [fail + closed](decisions/0001-write-path-content-scanning.md). +4. **The write.** Batched into as few graph commits as possible; a tool call + that touches several tables is one commit, not four. +5. **The notice.** Anything the scanner rewrote is reported back in the tool's + result, so a redaction is never silent. + +Step 3 is the one people are surprised by. It is on the write path rather than +bolted on afterwards because the failure it prevents — a secret pasted into a +memory and then synced to a shared team store — is not recoverable once it has +happened. + +## Where things live + +| Concern | Package | +| --- | --- | +| Memory, tasks, workflow, the `witan` CLI | `witan-council` | +| Tree-sitter indexing, cross-repo bridge, `code_*` | `witan-code` | +| Graph client, OIDC + remote proxy, observability, repo keys | `witan-core` | +| Installing all of it | `ol-agent-kit` | + +`witan-core`'s base modules are stdlib-only on purpose, with heavier concerns +behind extras (`cli`, `mcp`, `remote`), so neither server drags in weight it does +not use. diff --git a/docs/explanation/code-graph/edge-precision-tiers.md b/docs/explanation/code-graph/edge-precision-tiers.md new file mode 100644 index 00000000..5762dd82 --- /dev/null +++ b/docs/explanation/code-graph/edge-precision-tiers.md @@ -0,0 +1,114 @@ + + +!!! info "This page lives with the code" + + The authoritative copy is + [`mcp/servers/witan-code/docs/EDGE_PRECISION_TIERS.md`](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan-code/docs/EDGE_PRECISION_TIERS.md). + +# Typed cross-repo edge precision tiers + +Status: accepted (implementation phase, 2026-07-06) +Related: [STAGE2_STITCHING.md](stage2-stitching.md), [SYMBOL_TABLE.md](symbol-table.md) + +Kythe distinguishes `ref/call/direct` from the weaker `ref/call` edge kind so +consumers can choose how much they trust a cross-repo link. This project +does the same thing: replace the previous binary consumer/provider model +with **typed edges that carry precision**, so callers filter by a minimum +trust floor (`min_precision`) instead of picking a confidence threshold +themselves. + +## Tiers + +| Tier | Source | Trust | +|-------------|------------------------------------------------------------------|-------| +| `precise` | Stage 2: canonical symbol string join (`witan_code.stitch`) | high | +| `heuristic` | Stage 3: `(kind, key_norm)` binding grouping, confidence-scored | medium| +| `fuzzy` | future: embedding/BM25 route similarity (separately tracked research task) | low | + +No `fuzzy` edges exist yet — that tier is currently identical to +`heuristic` everywhere it's accepted as a parameter. + +## `min_precision` is a floor, not an exact match + +`min_precision="precise"` returns only the precise tier. `min_precision= +"heuristic"` (the default everywhere — **preserves every tool's prior +behavior** when the parameter is omitted) returns precise + heuristic. +`min_precision="fuzzy"` returns everything. A heuristic edge is suppressed +whenever a precise edge already covers the same `(consumer_repo, +provider_repo, kind, key_norm)` triple, so the same logical link never shows +up twice at two different trust levels. + +## The merged edge (`witan_code.edges`) + +```python +TypedEdge( + precision, # "precise" | "heuristic" | "fuzzy" + consumer_repo, provider_repo, + kind, key_norm, + canonical_symbol, # consumer's symbol string; precise tier only, else None + confidence, # 1.0 for precise; the heuristic tier's score otherwise + evidence, # tuple[dict] of {repo, file, line} — see below +) +``` + +`edges.cross_repo_edges(repo_symbol_rows, binding_rows, *, min_precision= +"heuristic", min_confidence=0.5) -> list[TypedEdge]` computes precise edges +via `stitch.resolve()`, then (unless `min_precision == "precise"`) adds +heuristic edges grouped from raw `InterfaceBinding` rows the same way +`visualize.build_graph` already does, minus whatever the precise pass +already covered. + +**Evidence** is source-level backing for an edge: `{repo, file, line}` per +contributing occurrence. Heuristic edges can carry many (one per matching +consumer binding row); precise edges carry at most two — Stage 1's symbol +table keeps only one deterministic exemplar occurrence per symbol (see +SYMBOL_TABLE.md), not every occurrence, so a precise edge's evidence is +`(consumer exemplar, provider exemplar)`. + +`edges.precise_pairs(repo_symbol_rows) -> frozenset[(consumer_repo, +provider_repo, kind, key_norm)]` is a cheaper membership-test helper for +tools that only need to know whether a *specific* binding participates in a +precise edge, without building the full merged list. + +## Surface + +Every place that already produced (or filtered) cross-repo links now takes +`min_precision`, defaulting to `"heuristic"` so nothing changes unless a +caller opts in: + +* CLI: `witan code deps --min-precision precise` (fetches `all_repo_symbols` + only when asked — `heuristic`, the default, has zero extra query cost). +* `visualize.build_graph(rows, *, min_precision="heuristic", + repo_symbol_rows=None)` — pass `repo_symbol_rows` when requesting + `"precise"`. The special "repo depends on what it deploys" `service` edge + is unaffected by `min_precision`; it isn't a symbol-joined consumer/ + provider relationship. +* MCP: `code_interface_providers`, `code_interface_consumers`, + `code_interface_search`, `code_cross_repo_impact` all accept + `min_precision`. For the first three, a row is kept if its + `(kind, key_norm)` is covered by a precise edge *anywhere* in the store. + `code_cross_repo_impact` is repo-pair-specific: an `other` binding is kept + only if a precise edge links `own_repo` and that binding's repo + specifically, not merely because some unrelated repo pair resolves the + same key_norm precisely. +* `code_precise_edges` / `code_unresolved_symbols` (Stage 2's own tools, + added earlier) are unaffected — they only ever produce the `precise` tier + by definition. + +## Not in scope here + +* A `c4gen` extractor filtering to `:CALLS/precise` by default — no such + extractor exists in this repository; if one is added elsewhere, it should + call `edges.cross_repo_edges(..., min_precision="precise")` and fall back + to `"heuristic"` with a warning, per the original design. +* Repositioning the existing confidence-scoring heuristic stack + (`bridge_extractors.adjust_confidence`) as the formal `:CALLS/heuristic` + tier's scoring — that stack already exists (PR #10) and is reused as-is + here; migrating its framing/naming is a separate, still-blocked task. +* Visual retagging of the HTML/Rich `deps` graph by precision (e.g. distinct + edge colors per tier) — `min_precision` filters which edges are shown; + making the surviving edges visually distinguishable by tier is a follow-on + UX improvement, not required by this task. diff --git a/docs/explanation/code-graph/package-map.md b/docs/explanation/code-graph/package-map.md new file mode 100644 index 00000000..269bd452 --- /dev/null +++ b/docs/explanation/code-graph/package-map.md @@ -0,0 +1,115 @@ + + +!!! info "This page lives with the code" + + The authoritative copy is + [`mcp/servers/witan-code/docs/PACKAGE_MAP.md`](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan-code/docs/PACKAGE_MAP.md). + +# Package map — declaring repo → package identity + +Status: accepted (discovery phase, 2026-07-05) +Related: [SYMBOL_FORMAT.md](symbol-format.md) + +The package map declares which canonical package identity a repository +provides. It is the input Stage-2 cross-repo stitching uses to qualify +provider [symbol strings](symbol-format.md) and the source of truth for the +`known_provider_package` boost in the heuristic confidence scorer. Every +production system that does precise cross-repo linking requires an explicit +map of this kind (scip-clang, LLVM, Chromium); witan-code is no exception. + +## Decision: per-repo `witan-code.toml` (Option A) + +Two options were evaluated: + +* **Option A — per-repo `witan-code.toml`** at the repo root, read during + indexing, accumulated into the bridge store. +* **Option B — central bridge config** mapping repo URLs to identities. + +Option A wins: + +1. **Identity ownership belongs to the repo** — the same reasoning that puts + `name` in `package.json`/`pyproject.toml`. The team that owns the repo owns + its declared identity; changes ride normal PR review. +2. **The bridge store already is the central registry.** Each repo's map is + written to the shared `_bridge.omni` store at index time, so read-time + stitching sees the union without any out-of-band config distribution. A + central file would duplicate that role and have to be synced to every + machine that runs the indexer. +3. **scip-clang needs a central map only because C++ has no in-repo package + identity.** Our repos have one; they just need to state which is canonical. +4. **Incremental adoption.** A repo without the file still indexes — it gets a + fallback identity derived from the repo URI (below). Nothing blocks on a + 40-repo rollout. + +## File format + +`witan-code.toml`, repo root: + +```toml +[package] +name = "mit-learn" # required: canonical package name +manager = "pypi" # optional: package-manager namespace (default ".") +version = "main" # optional: trunk-tracking default "main" + +# Optional: additional package identities this repo publishes, as +# "manager:name" strings. Version defaults to [package].version. +provides = [ + "npm:@mitodl/course-search-utils", +] +``` + +* `name` — used as the `{package}` field of every provider symbol the repo + emits (endpoints, env vars, services). +* `manager` — `pypi` / `npm` / `.`; the `{manager}` field for the primary + identity. +* `version` — see SYMBOL_FORMAT.md § design decision 1. Leave at `main` + unless the repo genuinely maintains parallel release lines. +* `provides` — extra published identities (a service repo that also publishes + a client library). These feed the `known_provider_package` heuristic and + let package-consumer symbols in other repos resolve precisely. + +## Fallback identity + +When the file is absent (or `[package].name` is missing / the TOML is +malformed), the identity is derived from the canonical repo URI: + +* `name` = last path segment of the repo URI (`…/mitodl/mit-learn` → + `mit-learn`) +* `manager` = `.` +* `version` = `main` +* `provides` = empty + +Provider symbols therefore always carry *some* package qualifier; the TOML +only makes it authoritative. + +## Storage + +One `PackageMap` node per repo in the bridge store, keyed on the repo URI and +overwritten on every full-repo index (merge-by-slug): + +``` +node PackageMap { + slug: String @key // canonical repo URI + repo: String @index + name: String @index + manager: String + version: String + provides: String? // JSON array of "manager:name" strings + indexed_at: DateTime +} +``` + +## Consumers of the map + +* `bridge.write_bindings` — qualifies provider symbols with the repo's + identity before writing bindings. +* `known_provider_package` heuristic — a package-consumer binding whose + key matches another repo's declared `name`/`provides` boosts co-located + endpoint-consumer confidence (+0.3), replacing reliance on incidentally + indexed `package.json` provider rows. +* Stage-2 stitching (future) — resolves consumer `.`-package symbols against + provider symbols, using the map to disambiguate when two repos export the + same descriptor. diff --git a/docs/explanation/code-graph/stage2-stitching.md b/docs/explanation/code-graph/stage2-stitching.md new file mode 100644 index 00000000..a4cf2fc2 --- /dev/null +++ b/docs/explanation/code-graph/stage2-stitching.md @@ -0,0 +1,90 @@ + + +!!! info "This page lives with the code" + + The authoritative copy is + [`mcp/servers/witan-code/docs/STAGE2_STITCHING.md`](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan-code/docs/STAGE2_STITCHING.md). + +# Stage 2: cross-repo symbol stitching (read time) + +Status: accepted (implementation phase, 2026-07-06) +Related: [SYMBOL_TABLE.md](symbol-table.md), [SYMBOL_FORMAT.md](symbol-format.md), +[EDGE_PRECISION_TIERS.md](edge-precision-tiers.md) + +Stage 2 joins the per-repo symbol tables Stage 1 emits into precise +cross-repo edges — entirely at read time, entirely in Python, never written +back to the store. This is the RANGER/SCIP pattern this project is built +around: a repo's indexing pass never has to know about any other repo; the +join only happens when someone asks a cross-repo question. + +## Algorithm (`witan_code.stitch.resolve`) + +Input is every `RepoSymbol` row in the bridge store (`all_repo_symbols`). +Rows split into two sets: + +* `exported` rows, grouped by their join key (see + [SYMBOL_TABLE.md § Stage-2 join contract](symbol-table.md#stage-2-join-contract)). +* `external` rows, each resolved independently against that grouping. + +For one `external` row: + +1. Look up its join key in the `exported` grouping, excluding rows from the + same repo (a repo doesn't cross-repo-link to itself) and, for `http`, + excluding provider rows whose method doesn't match (`*` matches + anything). +2. **Zero candidates** → the row goes to `unresolved` (Stage-3 fallback + territory — see below). +3. **One or more candidates** → one edge per candidate. `match_count` is the + candidate count, so a caller can tell a clean single match from a + fan-out. Version disambiguation (SYMBOL_FORMAT.md decision 1) picks which + candidate(s) are `preferred`: exact version match, else `main`, else every + remaining candidate is preferred and the edge is flagged + `ambiguous_version` — every candidate is still returned as its own edge + rather than silently dropped, matching this project's pattern of + surfacing all cross-repo data and letting the caller filter (see + `code_cross_repo_impact`). + +No edge is ever stored: `resolve()` is pure and its output is recomputed on +every call from the current `RepoSymbol` rows, so it can never go stale the +way a written edge could. + +## Output shape + +```python +PreciseEdge( + consumer_repo, consumer_symbol, + provider_repo, provider_symbol, + kind, scheme, + match_count, preferred, ambiguous_version, +) +``` + +`resolve(rows) -> (list[PreciseEdge], list[dict])` — the second element is +the raw `RepoSymbol` rows that had no candidate (`unresolved`). + +## Stage-3 fallback + +An `external` row landing in `unresolved` isn't necessarily a dead end: the +existing heuristic tier (`visualize.cross_repo_edges`, grouping raw +`InterfaceBinding` occurrences on the coarser `(kind, key_norm)` key with +confidence scoring) still has a chance to surface it — e.g. the provider +repo hasn't been indexed yet, or its extractor doesn't understand the +provider's framework. `code_unresolved_symbols` exists specifically to find +these gaps; `code_interface_consumers`/`code_interface_providers` remain the +way to check the heuristic tier for the same reference. + +Typed edge kinds (`:CALLS/precise`, `:CALLS/heuristic`, `:CALLS/fuzzy`) that +formally merge these two tiers into one filterable result now exist — +`witan_code.edges.cross_repo_edges()`, see +[EDGE_PRECISION_TIERS.md](edge-precision-tiers.md). + +## Surface + +* CLI: `witan code stitch [--repo URI] [--unresolved]` +* MCP: `code_precise_edges(repo=None)`, `code_unresolved_symbols(repo=None)` + +Both accept an optional `repo` filter that keeps edges/gaps touching that +repo (either side, for edges). diff --git a/docs/explanation/code-graph/symbol-format.md b/docs/explanation/code-graph/symbol-format.md new file mode 100644 index 00000000..2fecf31e --- /dev/null +++ b/docs/explanation/code-graph/symbol-format.md @@ -0,0 +1,152 @@ + + +!!! info "This page lives with the code" + + The authoritative copy is + [`mcp/servers/witan-code/docs/SYMBOL_FORMAT.md`](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan-code/docs/SYMBOL_FORMAT.md). + +# Canonical symbol strings for cross-repo bindings + +Status: accepted (discovery phase, 2026-07-05) +Related: [PACKAGE_MAP.md](package-map.md) + +Every interface binding written to the bridge store carries a **canonical +symbol string** — a stable, self-contained identifier modeled on +[SCIP symbols](https://github.com/sourcegraph/scip/blob/main/scip.proto). +Symbols are the Stage-2 join key for precise cross-repo linking: a consumer +binding whose unresolved symbol matches a provider binding's exported symbol is +a `:CALLS/precise` edge; the existing `(kind, key_norm)` grouping remains as +the `:CALLS/heuristic` fallback. + +## Format + +``` +{scheme}:{manager}:{package}:{version}:{descriptor} +``` + +Five colon-separated fields. The descriptor is the final field and may itself +contain colons — parsers must split with `maxsplit=4`. An empty/unknown field +is written as `.` (SCIP's empty-field convention). A literal `:` or `%` inside +any of the first four fields is percent-encoded (`%3A`, `%25`); descriptors are +never encoded (they are terminal). + +| Field | Meaning | +|------------|----------------------------------------------------------------| +| scheme | Contract transport/kind: `http`, `env`, `pkg`, `svc` (future: `grpc`, `ws`) | +| manager | Package-manager namespace: `pypi`, `npm`, `.` | +| package | Canonical package name from the repo's [package map](package-map.md) | +| version | Package version; `main` = trunk-tracking, `.` = unknown | +| descriptor | Kind-specific identifier (see below) | + +## Descriptors by binding kind + +### `endpoint` → scheme `http` + +``` +{METHOD} {normalized-path} +``` + +* Method is upper-case; a consumer whose method cannot be determined + statically uses `*` (wildcard). +* The path is normalized exactly as `key_norm` today + (`bridge_extractors.normalize_endpoint`): path parameters + (`{id}`, `${x}`, `:id`) collapse to `{}`, duplicate slashes collapse, one + trailing slash is stripped, and any `scheme://host` prefix is dropped. +* Query parameters are **not** part of the descriptor — they select a + representation, not the resource, and no consumer/provider pair would agree + on them syntactically. + +``` +provider: http:pypi:mit-learn:main:GET /api/v0/users/me +consumer: http:.:.:.:* /api/v0/users/me +``` + +### `env_var` → scheme `env` + +Descriptor is the variable name, verbatim. + +``` +provider: env:.:mit-learn:main:MITOL_APP_BASE_URL +consumer: env:.:.:.:MITOL_APP_BASE_URL +``` + +### `package` → scheme `pkg` + +The symbol identifies the package itself; the descriptor is `.`. Manager and +package name come from the import site (consumer) or the publishing repo's +package map / `package.json` (provider). + +``` +provider: pkg:npm:@mitodl/course-search-utils:main:. +consumer: pkg:npm:@mitodl/course-search-utils:.:. +``` + +### `service` → scheme `svc` + +Descriptor is `{sub_kind}/{key_norm}` (sub_kind: `repo` | `image` | `name`). + +``` +provider: svc:.:ol-infrastructure:main:repo/https://github.com/mitodl/mit-learn +``` + +## Provider vs consumer symbols + +**Providers** get a fully-qualified symbol: package/manager/version come from +the repo's package map (or its fallback identity — see PACKAGE_MAP.md). +Exported symbols are the repo's public contract surface. + +**Consumers** emit *unresolved external symbols* (the SCIP pattern for +dependencies indexed separately): package, manager, and version are `.` unless +the reference site names the package explicitly (package imports do; endpoint +path literals do not). Stage 2 resolves them at read time by matching the +scheme + descriptor against other repos' provider symbols; the package map +disambiguates when more than one repo exports the same descriptor. + +This is deliberate: per-repo indexing stays self-contained (no repo needs any +other repo checked out or indexed first), and re-indexing repo B never +invalidates repo A's rows. + +## Design decisions (task open questions) + +1. **Version is `main`, not a package version or git SHA.** These services + deploy continuously from trunk; a git SHA would churn the join key on every + commit and version pinning across SOA repos is not practiced here. + Published libraries may override `version` in their package map when + parallel release lines actually exist. Read-time matching rule: a consumer + version of `.` matches any provider version; on multiple provider versions + prefer exact match, then `main`, else flag the edge `ambiguous_version` + rather than guessing. +2. **Path parameters** normalize to `{}` — same rule as `key_norm`, so precise + and heuristic tiers agree on path shape. +3. **Query parameters** are stripped. +4. **Transport lives in the scheme** (`http` now; `grpc`, `ws` reserved), not + in the descriptor — a gRPC method descriptor (`package.Service/Method`) has + nothing in common with an HTTP path, so overloading one scheme would push + transport dispatch into every descriptor parser. +5. **env-var and package symbols** use the same 5-field frame with + kind-appropriate schemes (`env`, `pkg`) so Stage-2 joining is a single + mechanism keyed on `(scheme, descriptor)` — not one bespoke joiner per kind. + +## Relationship to `key_norm` + +`symbol` does not replace `key_norm`. `key_norm` remains the heuristic-tier +join key and the FTS target for `search_bindings`. `symbol` adds the +precision tier on top: identical descriptors with compatible package identity +⇒ `:CALLS/precise`; `key_norm` match without symbol agreement stays +`:CALLS/heuristic` with its confidence score. + +## Storage + +`InterfaceBinding.symbol` (indexed, nullable) in `bridge-schema.pg`. Symbols +are computed at bridge-write time (`bridge.write_bindings`), not extraction +time, because provider identity comes from the package map which is loaded +once per repo. The pure construction function is +`bridge_extractors.canonical_symbol`; its inverse is +`bridge_extractors.parse_symbol`. + +Bindings are per-occurrence. The deduplicated per-repo aggregate — one +`RepoSymbol` row per (repo, role, symbol), the artifact Stage 2 actually joins +against — is specified in [SYMBOL_TABLE.md](symbol-table.md). diff --git a/docs/explanation/code-graph/symbol-table.md b/docs/explanation/code-graph/symbol-table.md new file mode 100644 index 00000000..467d05a9 --- /dev/null +++ b/docs/explanation/code-graph/symbol-table.md @@ -0,0 +1,115 @@ + + +!!! info "This page lives with the code" + + The authoritative copy is + [`mcp/servers/witan-code/docs/SYMBOL_TABLE.md`](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan-code/docs/SYMBOL_TABLE.md). + +# Per-repo symbol tables (Stage 1) + +Status: accepted (implementation phase, 2026-07-05) +Related: [SYMBOL_FORMAT.md](symbol-format.md), [PACKAGE_MAP.md](package-map.md), +[STAGE2_STITCHING.md](stage2-stitching.md) + +Stage 1 of the two-stage cross-repo model: every indexed repo emits a +**self-contained symbol table** — the stable, deduplicated artifact that +Stage 2 joins across repos at read time. No extraction pass ever writes a +cross-repo edge; linking is deferred entirely to read time (the SCIP model). + +## The artifact + +One `RepoSymbol` row per `(repo, role, symbol)` in the shared bridge store +(`_bridge.omni`), alongside the per-occurrence `InterfaceBinding` rows it is +aggregated from: + +| role | meaning | +|------------|----------------------------------------------------------------| +| `exported` | The repo provides this contract — its public surface (from provider bindings, fully qualified by the [package map](package-map.md)) | +| `external` | The repo references this contract but cannot resolve it locally (from consumer bindings; package/manager/version are `.` unless the reference site names them) | + +`external` rows are the RANGER-style *import placeholder nodes*: unresolved +stand-ins written at index time, redirected to another repo's `exported` row +at read time by Stage 2. They are rows, not edges — grouping at read time is +what makes per-repo indexing order-independent (re-indexing repo B never +invalidates repo A). + +Each row carries the parsed symbol fields (`scheme`, `manager`, `package`, +`version`, `descriptor` — see [SYMBOL_FORMAT.md](symbol-format.md)) plus: + +* `key_norm` — the coarse join key (for `http` the method-less normalized + path; consumer methods are usually the `*` wildcard, so exact descriptor + equality under-joins endpoints). +* `n_refs` — occurrence count in the repo. +* `confidence` — max over occurrences (`exported` rows are always 1.0). + Stage 2's precision tiers filter on this. +* `file` / `line` — one deterministic exemplar occurrence (min by file, line). + +## Rebuild semantics + +The table is **exactly rebuilt** on every bridge write (`bridge.write_bindings`), +not incrementally patched: delete the repo's `RepoSymbol` rows, re-aggregate +from the binding occurrences that survive that write (stored rows outside the +per-file purge set, plus the fresh batch). This keeps the table consistent +with the bindings even on narrow single-file reindexes, with no +tombstone/refcount bookkeeping. + +Rows whose stored bindings predate symbol emission (no `symbol` value) are +skipped; they regain table coverage when their file is next reindexed. + +## Stage-2 join contract + +Stage 2 matches `external` rows against other repos' `exported` rows: + +1. `env` / `svc` — exact `(scheme, descriptor)` match (`symbols_by_descriptor`). +2. `http` / `pkg` — `(scheme, key_norm)` match (`symbols_by_key`). `http` + descriptors embed the method, which consumers usually can't determine + statically (`*`), so the coarse key_norm (method-less path) is the join + key, followed by method compatibility: a consumer method of `*` matches + any provider method. `pkg` canonical descriptors are always `.` + ([SYMBOL_FORMAT.md](symbol-format.md) — identity lives in the + manager/package fields, not the descriptor); `key_norm` carries the + package name for both `exported` and `external` rows instead, so it is the + only usable join key for packages. +3. Package identity (`manager`/`package` fields) disambiguates when several + repos export the same descriptor or key_norm; version matching follows + SYMBOL_FORMAT.md decision 1 (`.` matches anything; prefer exact, then + `main`, else flag `ambiguous_version`). + +A successful join is a `:CALLS/precise` edge (computed, never stored); the +`(kind, key_norm)` binding grouping remains the `:CALLS/heuristic` fallback. +The concrete join implementation, its edge shape, and the unresolved-symbol +gap report are specified in [STAGE2_STITCHING.md](stage2-stitching.md). + +## Second consumer: the heuristic tier's confidence signals + +The table has a second reader beyond Stage 2's join: `bridge.write_bindings` +sources the cross-repo half of two confidence heuristics +(`bridge_extractors.adjust_confidence`) from other repos' `exported` rows +rather than re-deriving the same information from raw `InterfaceBinding` +rows — + +* `self_provided_key` (−0.5): the consuming repo also exports the same + `key_norm` — checked against other repos' `exported` rows plus this + repo's own surviving/fresh provider bindings (its own table hasn't been + rebuilt yet at this point in the write). +* `known_provider_package` (+0.3): a co-located package import matches an + `exported` package row from a different repo. + +Both signals degrade to their pre-Stage-1 baseline (no boost/penalty) if the +bridge store predates `RepoSymbol` — the write is never blocked on it. + +## Inspecting + +``` +witan code symbols [--repo URI] [--role exported|external] [--scheme http] +witan code stitch [--repo URI] [--unresolved] +``` + +## Write contention + +`RepoSymbol` rows are keyed by repo (slug prefix `repo|`), so concurrent index +runs from different repos never touch the same row — the same flat-node +argument that shaped `InterfaceBinding` (see bridge-schema.pg header). diff --git a/docs/explanation/decisions/0001-write-path-content-scanning.md b/docs/explanation/decisions/0001-write-path-content-scanning.md new file mode 100644 index 00000000..cabac72e --- /dev/null +++ b/docs/explanation/decisions/0001-write-path-content-scanning.md @@ -0,0 +1,320 @@ + + +!!! info "This page lives with the code" + + The authoritative copy is + [`mcp/servers/witan/docs/adr/0001-write-path-content-scanning.md`](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan/docs/adr/0001-write-path-content-scanning.md). + +# 1. Write-path content scanning & pluggable data governance + +- Status: Accepted +- Date: 2026-07-07 +- Deciders: witan platform owners +- Tracking: project `wp-witan-write-path-content-scanning-pluggable-data-d2932a`, epic `tk-epic-write-path-content-scanning-pluggable-data--554db3` +- Supersedes: — +- Related: `wp-witan-multi-user-service-deployment-dcf6ee` (multi-user deployment; this ADR is a precondition) + +## Context + +witan persists free-text authored by agents and users — memory bodies, task +and project descriptions, session summaries, trace outcomes. Agents routinely +paste command output, config fragments, and stack traces into these fields, any +of which may contain secrets (API keys, tokens, private keys) or PII (emails, +phone numbers, SSNs, card numbers). Today there is **no content validation on +the write path** (confirmed: only shape coercion in `_store_memory`; branch +names are deliberately not sanitized per `repo.py:65`). A secret written into a +Memory is embedded, indexed for BM25 search, and shared with every teammate and +future session. + +The risk is currently bounded by witan being local-per-user. It stops being +bounded the moment witan becomes a **shared, multi-user, deployed service** (the +sibling project): one user's accidental paste becomes another tenant's data +breach, and the store becomes a compliance liability (secret sprawl, PII at rest +with no deletion story, no audit trail). We want to prevent ingestion at write +time rather than scrub after the fact, and we want other organizations adopting +witan to enforce *their own* detection rules without forking. + +### Forces + +- **Single interception point exists.** Every persist funnels through + `OmnigraphClient.change(query_file, query_name, params)` at `witan/graph.py:90` + (schema DDL goes through `.apply_schema` at `:106`). The CLI does not write + independently — `cli/_common.py:_fn()` unwraps and calls the same `@mcp.tool` + server functions. Embedding + persistence happen inside the downstream + `omnigraph mutate` subprocess (`graph.py:140`), gated by `WITAN_EMBED_ENABLED`. + Anything run in Python before `change()` returns is strictly upstream of both. +- **Params are field-addressable.** The `params` dict carries every field by + name (`content`, `description`, `outcome`, `summary`, `resolution`, `title`, + `name`), so field-level context survives at the choke point. +- **A config-extension pattern is already established.** `RankConfig` + + `load_rank_config()` (`config.py:39-122`): a frozen Pydantic model sourced from + `WITAN_RANK_*` env vars and a `[rank]` TOML table with source-attributed + validation errors, instantiated once at `server.py:59`. +- **No plugin mechanism exists yet.** There is no entry-point group, registry, + or importlib discovery to hook into. +- **False positives are unacceptable friction.** If the guard blocks legitimate + writes (author emails, file paths, code snippets that look like secrets), + agents will disable it. Detection quality and suppression matter as much as + coverage. +- **Detection rulesets are a maintenance burden** best borrowed, but witan today + has only four runtime deps (`fastmcp`, `cyclopts`, `rich`, `pydantic`) and + values staying lean and offline-capable. + +## Decision + +Add a content-scanning layer on the write path, structured as four decisions. + +### D1 — Intercept at the `change()` choke point, not per-tool + +Scanning is invoked inside `OmnigraphClient.change()`. For any `query_name` +starting `insert_`/`update_`, a **static `query_name → [free-text field]` +classification map** selects which `params` values to scan; the rest (enums, +slugs, timestamps, edge endpoints) are skipped. `apply_schema` is never scanned. + +This guarantees **100% write coverage across every node type** — Memory, Topic, +WorkflowProject, WorkflowSession, WorkflowTrace, **Task**, CodeBranch — with one +integration point. Task and project writes are guarded by exactly the same code +path as memories; there is no separate guard to build or forget. New node types +are covered by adding a map entry, not by wiring a new call site. The map lives +next to the schema so field additions are a single-file change. + +### D2 — A `Scanner` protocol + registry as the extension surface + +A new `witan/scan/` package defines: + +- **`Finding`** — `detector` (id), `category` (`secret` | `pii`), `span` + (offsets), `severity`, `action`, and a **secret-free preview** (a masked or + hashed fragment; never the raw match). +- **`Scanner`** — a `Protocol` with `scan(text: str, field: str, node_type: str) + -> list[Finding]`. `field`/`node_type` let a scanner be context-aware (e.g. + ignore the `author` field for email detection). +- **`ScannerRegistry`** — assembles active scanners from three sources: + 1. built-in scanners shipped with witan, + 2. `importlib.metadata` entry-points in group **`witan.scanners`**, + 3. config-referenced dotted import paths (`[scan].plugins`). + + The registry honors per-scanner enable/disable and per-scanner mode overrides + from config. + +Entry-points are the primary third-party extension mechanism: another +organization ships a package exposing `witan.scanners` entry-points, installs it +alongside witan, and enables it in config — no fork. This is the core +"adoptable under their own governance policies" requirement. + +### D3 — Three enforcement modes, per-category defaults, fail-closed + +Each finding resolves to one of: + +- **block** — raise a `RuntimeError` from `change()` before the omnigraph + subprocess runs, with a message naming the field + detector + secret-free + preview. The write never happens; the agent sees the error and removes the + value. +- **redact** — replace each finding span with a stable, non-reversible + placeholder (e.g. `«redacted:aws_key»`), optionally flag the node with a + redaction-count property, then proceed. +- **warn** — emit an audit event and proceed unchanged. + +Defaults: **secrets → block** (fail-closed; a leaked credential must not land), +**PII → redact** (mask the span, keep the surrounding prose useful). `warn` is an +opt-in low-friction rollout mode. If a scanner itself raises, the default is +**fail-closed** (treat as block) so a broken detector cannot silently open the +gate — overridable for availability-sensitive deployments. + +**Error and audit messages never echo the matched value.** This is a hard +invariant: the whole point is to *not* propagate the secret, so surfacing it in +an exception or log would defeat the control. + +### D4 — Config via `ScanConfig`, policy admin-owned in server mode + +A frozen Pydantic **`ScanConfig`** + `load_scan_config()` mirrors +`RankConfig`/`load_rank_config()`: resolve `WITAN_SCAN_*` env > `[scan]` TOML > +defaults, with source-attributed errors, instantiated once at server import. +Surface: `enabled` (default **off** initially, matching the `WITAN_EMBED_ENABLED` +convention, so the feature ships dark and is enabled deliberately), per-category +mode, detector allow/deny, plugin dotted-paths, allowlist config, and the +scanner-error policy. + +In a **deployed multi-tenant server**, `WITAN_SCAN_*` env and user-supplied TOML +are client-controllable and therefore untrusted. Policy must be sourced +**authoritatively server-side** and must not be weakenable by a caller; +per-tenant/per-repo overlays compose with the CEDAR authz work in the multi-user +project. (Detailed mechanism deferred to task +`tk-multi-tenant-policy-control-admin-owned-scan-pol-1338d2`.) + +### Built-in detectors (default, zero-dependency) + +- **Secrets:** high-signal regex (AWS access/secret keys, GitHub + `ghp_`/`gho_`/`ghs_`/`github_pat_`, Slack `xox[baprs]-`, Google API keys, + `-----BEGIN … PRIVATE KEY-----` PEM blocks, JWTs, generic + `password=`/`api_key=`/`secret=`/`token=` assignments) plus a Shannon-entropy + heuristic for long high-entropy base64/hex strings. +- **PII:** email, phone (E.164/US), US SSN, and credit-card numbers validated + with the **Luhn checksum** to cut false positives, with field-context + suppression (skip `author`) and an allowlist. + +A vendored engine (`detect-secrets`, `gitleaks` ruleset) is **not** a core +dependency; it is exposed as an optional plugin (decision spike +`tk-decision-spike-built-in-ruleset-vs-vendored-scan-eb8adb`). + +### False-positive management + +Three mechanisms, all downgrading a finding to audit-only (never block/redact, +still exactly one audit event, tagged `suppressed`/`suppressed_by`): allowlist +regexes in `[scan]` matched against the finding's own span (not the whole +field, so a known-good value can't hide an unrelated real secret); an inline +pragma to permit a specific value (`witan: allow-secret`, or +`witan: allow-secret:` scoped to one detector); and a salted +value-hash allowlist (`allowlist_hashes` + `allowlist_salt`, never plaintext). +Implemented in `witan/scan/allowlist.py`; see `docs/write-path-scanning.md` +for usage. + +## Options considered + +### Where to scan + +1. **Per-tool, in each `insert_*`/`update_*` builder in `server.py`.** Richest + semantic context. Rejected: ~35 call sites, every new tool must remember to + scan, high drift risk, easy to bypass by writing a new mutation. +2. **At the `change()` choke point (chosen).** One integration point, total + coverage, future-proof. Slightly less context, recovered via the static field + map. Small per-write regex cost over short text — acceptable. +3. **Inside omnigraph / a Rust engine hook.** Truly unbypassable and language- + agnostic. Rejected for now: cross-repo change to the engine, slower iteration, + no plugin story in Python where our detectors and adopters live. The choke + point is the pragmatic 95% at a fraction of the cost. +4. **Post-write async sweep + delete.** Rejected: the secret is already embedded, + indexed, and possibly read before the sweep runs; deletion from a Lance store + is not a clean unwind. Prevention beats remediation here. + +### Detector source + +- **Built-in ruleset (chosen default)** — zero deps, offline, we own quality. +- **Vendored (`detect-secrets`/`gitleaks`)** — better coverage, community- + maintained rules, but a heavier dep and license/packaging questions. Adopted as + an *optional plugin* so the default stays lean and air-gap-friendly. + +### Enforcement default + +- **Block everything** — safest, highest friction; risks agents disabling the + feature. **Warn everything** — lowest friction, fails the core goal. **Chosen: + per-category (block secrets / redact PII), warn as opt-in** — matches the + differing cost of a false positive per category. + +## Consequences + +**Positive** + +- Secrets/PII are stopped before embedding + persistence, across all node types, + through one auditable code path. +- Task and project writes are covered by construction — no separate guard. +- Other orgs extend detection via a documented `witan.scanners` entry-point + without forking; witan ships a lean, offline default. +- Provides the audit trail and policy-enforcement point the multi-user + deployment needs. + +**Negative / costs** + +- Per-write latency for scanning (bounded: regex + entropy over short text). +- False positives can block legitimate writes; mitigated by allowlists/pragmas + and the `warn` rollout mode, but requires tuning. +- A new `witan/scan/` subsystem, config surface, and plugin contract to + maintain and version. +- Detection is best-effort: novel secret formats and obfuscated values will slip + through. This reduces accidental ingestion; it is not a guarantee against a + determined writer, and must not be sold as one. +- Redaction mutates user content; the placeholder must be unambiguous and the + behavior documented so it is not mistaken for data loss. + +**Neutral** + +- Feature ships **enabled** (opt-out); see the 2026-07-07 amendment below. +- `witan-code` has a **separate** write path (`witan_code/store.py`, + `indexer.py`) that does not share this choke point; whether indexed-source + secrets are in scope is a separate decision + (`tk-evaluate-witan-code-write-path-scanning-indexed--150422`). + +## Implementation + +Sequenced in the epic backlog. Spine (p0): `ScanConfig` + `load_scan_config()`, +the `Scanner` protocol + registry, and the `change()` interception with the field +map and enforcement. Then built-in secret + PII detectors, entry-point plugin +discovery, redaction, allowlisting, audit logging, the `witan scan` CLI, the +multi-tenant policy control, tests, and docs. + +## Amendment (2026-07-07): enabled by default + +D4 originally shipped the feature **disabled**, following the +`WITAN_EMBED_ENABLED` opt-in precedent. Revised: `ScanConfig.enabled` now +defaults to **`true`** — scanning is opt-out (`WITAN_SCAN_ENABLED=false` or +`[scan] enabled = false` to turn it off), not opt-in. Rationale: an opt-in +default means most installs run unscanned unless an operator deliberately +turns it on — the exact accidental-ingestion risk this ADR exists to close. +The redact-by-default PII path and fail-closed secret blocking make an +enabled default low-friction; false positives are handled via the allow/deny +detector lists, per-category mode, and the eventual allowlist engine, not by +leaving the feature off. Everything else in D1–D4 is unchanged. + +## Amendment (2026-07-09): multi-tenant policy mechanism (resolves task `tk-multi-tenant-policy-control-admin-owned-scan-pol-1338d2`) + +D4 deferred "the detailed mechanism" for keeping `ScanConfig` admin-owned in a +deployed multi-tenant server. Resolved as follows, grounded in the actual +multi-user deployment design (ADR-0002 Cedar bundle; ADR-0004 Keycloak +JWT/per-actor mapping, `witan-per-actor-client-wiring` branch — both in the +sibling `wp-witan-multi-user-service-deployment-dcf6ee` project). + +### The base case is already authoritative — no client ever influences `ScanConfig` + +`server.py` loads `scan_cfg = cfg_module.load_scan_config()` once, at process +import, from the deployment's own environment/`config.toml`. No MCP tool +accepts a parameter that reaches `ScanConfig`, and per-request actor +resolution (ADR-0004 D2/D3 — `derive_actor_id`, `ActorTokenResolver`, +`_ActorScopedClient`) is a completely separate axis: it decides which +omnigraph bearer token a write is proxied through, never which scan policy +applies. So in the sanctioned deployed topology — one shared +`streamable-http` witan-service process, `JWTVerifier`-authenticated, +resolving per-actor omnigraph credentials server-side (ADR-0004 D1/D3) — +`WITAN_SCAN_*` is already operator-controlled (Vault/K8s env for the one +process) and structurally unreachable by any authenticated client. This holds +**today**, with no new code: the "client controls `WITAN_SCAN_*`" risk D4 +flagged only materializes in a topology this project does not build — a +per-user local witan process pointed directly at a shared remote store, +bypassing the witan-service entirely. That bypass isn't a scan-policy gap to +patch; it's a deployment-topology invariant to state and hold: **every write +to a shared store must pass through the witan-service process**, because +scanning (like everything else in `witan/scan/`) runs client-side in +whichever process calls `OmnigraphClient.change()` — omnigraph itself has no +content-scanning hook, and Cedar cannot express one (ADR-0002 D1: "no +per-node-type / per-row authority... it is the query-layer's job and +unimplemented" — content scanning *is* that query-layer job). Cedar's +`witan-users` `change` grant on the memory graph (ADR-0002 D2) is safe under +this invariant only because those users' requests are proxied through +witan-service, not handed a standalone omnigraph credential to use directly. + +### Per-tenant/per-repo overlay: keyed by repo, not by actor + +Scan policy is a property of the content's governance domain (which +org/repo/project it belongs to), not of who is writing — the whole point is +that a policy applies uniformly regardless of which user triggers a write. +`ScanConfig` gains a `for_repo(repo: str | None) -> ScanConfig` resolution: +an optional `[scan.overlay.""]` TOML table (operator-authored +server config only — there is no env-var form, deliberately, since a +per-repo table doesn't fit the flat `WITAN_SCAN_*` shape and env vars are +exactly the surface D4 already excludes from client influence) overrides any +subset of the base `ScanConfig` fields for writes tagged with that repo. +`WriteGuard` resolves the effective config from `params.get("repo")` (or the +first of `params.get("repos")` for `WorkflowProject`) before scanning — Layer +1 (Memory/Task/WorkflowProject/…) is one flat shared graph (ADR-0002 D2), so +"per-repo" is a property carried on each write's params, not a store +boundary; no new graph state or admin API is needed for v1. + +### Non-goals for now + +Per-user or per-role policy overlays (as opposed to per-repo) are explicitly +out of scope: they would let a user weaken policy for their own writes, +which is exactly what D4 rules out. A future admin API to edit overlay policy +at runtime (rather than redeploying `config.toml`) is deferred — filed as a +follow-up once the multi-user project reaches a phase where an admin surface +exists to hang it off. diff --git a/docs/explanation/decisions/0002-witan-cedar-authorization-bundle.md b/docs/explanation/decisions/0002-witan-cedar-authorization-bundle.md new file mode 100644 index 00000000..81f0ee64 --- /dev/null +++ b/docs/explanation/decisions/0002-witan-cedar-authorization-bundle.md @@ -0,0 +1,187 @@ + + +!!! info "This page lives with the code" + + The authoritative copy is + [`mcp/servers/witan/docs/adr/0002-witan-cedar-authorization-bundle.md`](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan/docs/adr/0002-witan-cedar-authorization-bundle.md). + +# 2. witan v1 Cedar authorization bundle + +- Status: Accepted, amended by [0006](0006-code-graph-branch-ownership-and-reaping.md) +- Date: 2026-07-07 +- Deciders: witan platform owners +- Tracking: task `tk-witan-v1-cedar-policy-bundle-team-repo-scoped-re-77655e`, project `wp-witan-multi-user-service-deployment-dcf6ee` +- Supersedes: — +- Related: ol-infrastructure `docs/adr/0009-deploy-witan-as-shared-multi-tenant-mcp-service.md`; `docs/adr/0004-keycloak-jwt-per-user-actor-mapping.md` (how a request's `act-` and omnigraph bearer token are actually resolved); `tk-spike-validate-omnigraph-server-remote-write-ser-1a8058` (spike that mapped the omnigraph policy surface) + +## Context + +Deployed multi-user witan (the sibling project) replaces local-per-user stores +with one shared omnigraph store served by `omnigraph-server`. A shared store +needs authorization: who may read, who may write, and to which part of the +graph. omnigraph ships a Cedar policy layer; this ADR fixes the **v1 bundle** +that agent-kit authors and CI validates, and the boundary with what +ol-infrastructure templates at deploy time. + +### Forces — what omnigraph's Cedar layer can and cannot express + +Confirmed against omnigraph `docs/user/operations/policy.md`, the +`omnigraph-policy` crate, and the 0.8.1 binary (`omnigraph policy validate/test`): + +- **Allow-only.** Every rule is a `permit`; there is no `deny` key; ungranted ⇒ + denied. Restriction is expressed by omission. +- **Finest scope is graph + branch.** No per-node-type / per-row authority — + it is "the query-layer's job" and unimplemented. Within one graph, `change` + is all-or-nothing across node kinds. +- **Actor identity is server-resolved from the bearer token**, never from + client-supplied fields. +- **Branch scoping** exists: `branch_scope` (source) for `read`/`export`/ + `change`; `target_branch_scope` (destination) for `schema_apply`/`branch_*`; + values `any | protected | unprotected`; `protected_branches:` names branches. +- **Maintenance ops (`repair`/`optimize`/`cleanup`) are direct-storage-only** + and cannot be Cedar-gated. +- **The offline CLI validates only per-graph bundles.** `policy validate/test` + load bundles under the per-graph engine, reject the server-scoped `graph_list` + action in a graph bundle, and fan a `[cluster]` bundle onto every graph + (tripping "one bundle per graph scope"). So a server bundle cannot be + exercised by the same offline harness as the per-graph bundles. +- **Identity is per-user, not per-team.** omnigraph's own `groups:` are built + from individual `act-` actors; role/team is an aggregation *over* + per-user actors. Keycloak already issues each user a JWT with role/group + claims. So per-user actors are the natural v1, grouped by Keycloak claim. + +## Decision + +### D1 — Three groups: per-user humans + two distinct service accounts + +*(Four as of the 2026-08-05 amendment at the end of this section.)* + +- `witan-users` — one `act-` per authenticated human, from the Keycloak claim. +- `witan-ci` (`act-svc-witan-ci`) — the code-graph **data** pipeline + (reindex-on-merge + WIP-branch lifecycle). +- `witan-service` (`act-svc-witan`) — the witan MCP service's **own** account. + Schema definition/migration is a default, service-owned operation: the service + applies the appropriate schema on every graph (`schema_apply`) as part of its + boot/ownership duties. This is deliberately **not** the reindex pipeline and + **not** a per-user permission — separating it keeps the pipeline unable to + redefine schema, and keeps schema ownership in one place (the service) rather + than smeared across a "CI" catch-all. + +Group **names** are stable and identical across bundles; **membership** is +templated by ol-infrastructure (Keycloak claims for `witan-users`, +Vault-provisioned tokens for the service accounts), not committed. + +**Amended 2026-08-05 — a fourth group, `witan-admin`.** See the amendment to D4 +below: the break-glass maintenance principal of +[ADR 0005](0005-secure-cli-path-into-deployed-witan.md) path (b), +`act-svc-witan-admin`, needs Cedar rules after all, because the operations it +exists for (`witan migrate topics` / `repo-keys` / `merge`, a forced schema +apply) go through the **server**, not direct storage. + +### D2 — One bundle per graph scope, mapped to the layer topology + +- **`memory.policy.yaml` → `[memory]`** (Layer 1, flat shared work graph): + users read/export/change/invoke_query on the single main branch; `witan-service` + owns read + `schema_apply`. `witan-ci` has **no** role here (it is a code-graph + actor). No per-node-type rules — not expressible. +- **`code-graph.policy.yaml` → every per-repo code-graph id** (Layer 2): the + repo's default git branch maps to store `main` = `protected`; all other git + branches are `unprotected` WIP. Users read/invoke anywhere and change / create / + delete only `unprotected` branches; `witan-ci` owns `read`/`change` and + `branch_merge` into protected `main` plus the unprotected WIP-branch lifecycle + (but **not** `branch_delete` on `main` and **not** `schema_apply`); + `witan-service` owns read + `schema_apply` on `main`. WIP reindexes are + isolated; promotion into `main` is deliberate and CI-owned; schema stays + service-owned. + **Amended by [ADR 0006](0006-code-graph-branch-ownership-and-reaping.md) D3:** + users keep `branch_create` but lose `branch_delete` — Cedar cannot scope a + delete to the view's owner, so deletion on a shared graph is CI's alone. +- **`bridge.policy.yaml` → `[bridge]`** (Layer 2.5, derived cross-repo bridge): + read-only for users; `witan-ci` writes the content; `witan-service` owns the + schema. + **Amended by [ADR 0006](0006-code-graph-branch-ownership-and-reaping.md) D4:** + the bridge is neither flat nor read-only for humans — indexing a WIP git branch + writes its cross-repo bindings there too, on a per-user view. Users get + `change` + `branch_create` on unprotected branches; `main` stays CI's. + +### D3 — Server-level `graph_list` is a deploy-time bundle, structurally linted + +`server.policy.yaml` (`graph_list`, `applies_to: [cluster]`) grants graph +enumeration to all four groups (`witan-admin` added 2026-08-05 — see D1). +Because the 0.8.1 offline CLI has no server-scope +*semantic*-validation path (`policy validate`/`test` load under the per-graph +engine), it is applied and enforced by `omnigraph-server` at boot/runtime rather +than exercised by `policy test`. It is still gated in CI: `lint_bundles.py` +structurally checks it every run (group references, action names, scope/action +compatibility, allow-only), so a group-name typo or YAML error — which would +otherwise deny all users `graph_list` at runtime — fails the build. +`tests/server.tests.yaml` records the intended decisions for when upstream adds a +server-scope semantic harness. + +### D4 — Maintenance is gated by IAM, not Cedar + +`repair`/`optimize`/`cleanup` cannot be Cedar-gated. Access is restricted with +AWS IAM on the backing bucket. There is no `svc-witan-admin` Cedar principal. +(Schema application — `schema_apply` — *is* Cedar-gateable and is owned by +`witan-service`; only the storage-maintenance ops fall to IAM.) + +**Amended 2026-08-05 — there IS a `svc-witan-admin` Cedar principal, for a +different class of maintenance.** The original wording conflated two things that +happen to share the word "maintenance": + +- **Storage** maintenance (`repair`/`optimize`/`cleanup`) — still IAM-only, still + correct as written. These commands reject `--server` outright and reach the S3 + store behind the running server's back, so no policy engine is in the path. + They run as the omnigraph stack's CronJobs + (ol-infrastructure `applications/omnigraph/maintenance.py`). +- **Data and schema** maintenance (`witan migrate topics` / `migrate repo-keys` / + `migrate merge`, and a forced `witan migrate schema`) — goes **through the + server**, so Cedar *is* in the path and default-deny applies. Something has to + be granted, and the only alternatives to a purpose-made principal were both + worse: run these as `svc-witan-ci` (the code-graph pipeline, which has no + business on the memory graph, and which the in-cluster migration Job was in + fact borrowing) or as `svc-witan-service` (which would have to gain `change` on + the memory graph, widening the credential the whole serving tier holds). + +So `witan-admin` (`act-svc-witan-admin`) is added to all four bundles, with the +narrowest grant the operations need: on the memory graph +`read`/`export`/`invoke_query`/`change` + `schema_apply` (the backfills rewrite +rows in place, and this graph is writable by every human user anyway, so the only +thing it adds over a user actor is schema); on the code and bridge graphs +`read`/`export`/`invoke_query` + `schema_apply` **only** — no `change`, +`branch_merge`, or `branch_delete`, because those graphs are re-derivable, their +promotion into `main` is CI's, and Cedar cannot tell whose WIP view a delete +targets. `witan-ci` remains the only holder of `branch_delete`. + +The token is provisioned in ol-infrastructure's omnigraph stack and mounted only +into in-cluster Jobs; no human ever holds it (`witan login` gets an operator +their own `act-`). See `policy/README.md` § "Non-human actors" and +ol-infrastructure `docs/witan-admin-break-glass-runbook.md`. + +### D5 — CI validates and unit-tests the bundle against the real binary + +`policy/check.sh` (1) runs `lint_bundles.py` — a structural lint of all four +bundles including the server bundle; (2) converges a fixture cluster +(`policy/cluster.yaml`, stub graphs `memory`/`code_example`/`bridge`) and runs +`omnigraph policy validate`; (3) runs 71 declarative `policy test` cases across +the three per-graph bundles. It runs as the `witan (Cedar policy bundle)` job in +`witan-tests.yml`. The fixture is a test harness; the deployed cluster.yaml is +templated by ol-infrastructure. + +## Consequences + +- Read/write scoping is enforced at the graph+branch grain that omnigraph + actually supports; the bundle makes no promise it cannot keep (no per-type + rules, no Cedar-gated maintenance). +- WIP code-graph reindexes are isolated on unprotected branches per user; + `main` is protected and CI-owned — the isolation goal of the branching-strategy + work is satisfied by policy, not just convention. +- The agent-kit ↔ ol-infrastructure boundary is explicit: agent-kit owns the + rule *shape* and its tests; ol-infrastructure owns graph enumeration, group + membership, and IAM. +- Coarser-than-ideal Layer-1 authority (no per-node-type) is accepted for v1; + finer control waits on an omnigraph query-layer capability, not a bundle + change here. diff --git a/docs/explanation/decisions/0003-atomic-task-claims-cas.md b/docs/explanation/decisions/0003-atomic-task-claims-cas.md new file mode 100644 index 00000000..68c2a042 --- /dev/null +++ b/docs/explanation/decisions/0003-atomic-task-claims-cas.md @@ -0,0 +1,105 @@ + + +!!! info "This page lives with the code" + + The authoritative copy is + [`mcp/servers/witan/docs/adr/0003-atomic-task-claims-cas.md`](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan/docs/adr/0003-atomic-task-claims-cas.md). + +# 3. Best-effort compare-and-swap for multi-user task claims + +- Status: Accepted +- Date: 2026-07-08 +- Deciders: witan platform owners +- Tracking: task `tk-atomic-task-claims-113c26`, project `wp-witan-multi-user-service-deployment-dcf6ee` +- Supersedes: — +- Related: `docs/adr/0002-witan-cedar-authorization-bundle.md`; `tk-spike-validate-omnigraph-server-remote-write-ser-1a8058` (remote-write serialization spike); `pf-witan-multi-user-service-deployment-rfc-split-da-c48753` + +## Context + +`task_claim` is the coordination primitive that lets parallel agents and +multiple humans share one work-coordination graph without double-working a +task. The shipped implementation (Option A) is a read-check-write: read the +task, reject if held, else write `status=in_progress, assignee, claimed_at`. + +On a **local** `.omni` store this is safe enough — `OmnigraphClient` serializes +writes with a per-store advisory `flock` (`graph.py:_acquire_write_lock`). But +the multi-user deployment (the parent project) replaces the local store with a +shared `omnigraph-server` over `http(s)`/`s3`, and there the flock is skipped +(it is a local-filesystem lock and cannot coordinate across pods). Two agents +that both read a task as `open` will both write their own claim; last write +wins and silently clobbers the first claimant. This ADR records what atomicity +is actually achievable and what we shipped. + +### Forces — omnigraph's concurrency surface (0.8.0, verified against the binary) + +- **An optimistic-concurrency token exists.** `omnigraph commit list --json` + returns per-commit `graph_commit_id` (a ULID) and a monotonic + `manifest_version`; `omnigraph snapshot --json` returns the branch + `manifest_version` and per-table `table_version`. Each write advances these. +- **But there is no conditional-write primitive.** `omnigraph mutate` accepts + no `--if-version` / `--expected-commit` / precondition flag, and the query + engine cannot express a compound `... WHERE status = 'open'` guard inside the + mutation itself. So a **single-statement store-level CAS is impossible** + through the 0.8.0 client surface — you cannot ask the store to "set the claim + only if it is still unclaimed" and have the store reject a lost race. +- **Lance OCC conflicts do surface, but were being masked.** When two writers + race the same manifest version with no serializing flock (the shared case), + one commit fails with `stale view` / `manifest table version`. + `OmnigraphClient._execute` treated those as transient and **blindly retried + the same mutation** — correct for an idempotent upsert, but for a claim the + retry re-reads the now-updated state and re-applies the claim *over* whoever + won. The masking turned a should-fail claim into a clobbering success. + +## Decision + +Ship a **best-effort CAS** claim — the strongest guarantee available without an +upstream conditional-write feature — built from three parts: + +1. **Conflict-surfacing writes.** `OmnigraphClient.change(..., + surface_conflict=True)` raises a typed `OmnigraphConflict` on a Lance OCC + conflict instead of retrying it. Only `task_claim` opts in; every other + write keeps the transparent-retry behaviour idempotent upserts rely on. + +2. **Conflict-aware claim.** On `OmnigraphConflict`, `task_claim` re-reads the + task rather than re-applying its write. If a different actor now holds a + live (non-lease-expired) claim, it returns + `{"claimed": false, "reason": "lost_race", "held_by": ...}`. If the + conflicting write was unrelated (or the rival's lease has since lapsed), it + retries the claim — in a bounded loop that keeps `surface_conflict=True` for + every attempt, so a *consecutive* conflict is handled the same way and never + falls back to the blind-retry path that would clobber a new winner. + +3. **Post-write ownership verification.** Because the last writer still wins + with no store CAS, after writing the claim `task_claim` re-reads and confirms + `assignee == holder` before reporting success. A claim that was overwritten + by a rival landing last is reported as `lost_race`, not a false success. + +Together these make the common double-claim race resolve to **at most one +`claimed: true`**, and never a silent clobber. The lease (`claimed_at`, +`_CLAIM_LEASE_SECONDS`) remains the backstop for the residual window where two +callers both read *after* all racing writes have settled. + +## Consequences + +- **Not truly atomic.** A vanishingly small window remains: if both callers run + their post-write verification read after both writes commit and before either + observes the other, both could see the last writer and one is wrong. In + practice the write ordering + verification collapses this to near-zero, and + the lease recovers any task that ends up mis-owned. Callers must still treat + `claimed: true` as "you almost certainly hold it", not a hard mutex — the tool + docstring says so. +- **Depends on the server serializing writes.** The conflict-surfacing path + assumes `omnigraph-server` either serializes branch writes or lets Lance OCC + reject the loser. That assumption is validated by + `tk-spike-validate-omnigraph-server-remote-write-ser-1a8058`; if the server + instead silently accepts both writes with no conflict, only the post-write + verification (part 3) protects us — which it still does. +- **True atomic CAS is an upstream ask.** A single-round-trip guarantee needs + omnigraph to accept a manifest-version / commit-id precondition on `mutate` + (compare-and-swap) or a conditional mutation guard. Tracked as a follow-up + task against the omnigraph project; until then this ADR is the ceiling. +- **No schema or API change.** `task_claim`'s return shape gains a `lost_race` + reason; existing `claimed`/`held`/`blocked`/`closed` paths are unchanged. diff --git a/docs/explanation/decisions/0004-keycloak-jwt-per-user-actor-mapping.md b/docs/explanation/decisions/0004-keycloak-jwt-per-user-actor-mapping.md new file mode 100644 index 00000000..379dac9d --- /dev/null +++ b/docs/explanation/decisions/0004-keycloak-jwt-per-user-actor-mapping.md @@ -0,0 +1,402 @@ + + +!!! info "This page lives with the code" + + The authoritative copy is + [`mcp/servers/witan/docs/adr/0004-keycloak-jwt-per-user-actor-mapping.md`](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan/docs/adr/0004-keycloak-jwt-per-user-actor-mapping.md). + +# 4. Keycloak JWT → omnigraph per-user actor/token mapping + +- Status: Accepted +- Date: 2026-07-08 +- Deciders: witan platform owners +- Tracking: task `tk-design-keycloak-jwt-omnigraph-per-user-actor-tok-728f0c`, project `wp-witan-multi-user-service-deployment-dcf6ee` +- Supersedes: ADR-0009 (ol-infrastructure) D3's per-team-token assumption +- Related: `docs/adr/0002-witan-cedar-authorization-bundle.md` (D1 — per-user + `witan-users` group); ol-infrastructure + `docs/adr/0009-deploy-witan-as-shared-multi-tenant-mcp-service.md`; + `tk-ol-infrastructure-toolhive-witan-pulumi-stack-e843b3`; + `docs/adr/0009-stateless-mcp-protocol-era.md` (the JWT→actor mapping below is + unchanged by the stateless era — it reads the token on each request and never + depended on session state — but the `streamable-http` connection it describes + no longer carries a handshake or a session id) + +## Context + +ADR-0002 D1 already decided Cedar identity is per-user (`act-`), not +per-team — Keycloak issues each authenticated human their own JWT, and +omnigraph's own `groups:` are built from individual actors. What that ADR left +open is the actual **mapping mechanism**: given an inbound request to the +deployed witan MCP service, how do we get from "a Keycloak-authenticated +human" to (a) the `act-` id Cedar rules reference and (b) the omnigraph +bearer token that makes the server resolve the request as that actor. + +ADR-0009 (ol-infrastructure) assumed ToolHive's embedded OAuth broker would +be the vehicle for this and flagged direct OIDC/JWT validation against +Keycloak as a fallback if the broker "proves limiting." + +### Forces + +**ToolHive's embedded broker does not propagate end-user identity to the +backend container.** Confirmed by reading the `toolhive_swe` stack +(ol-infrastructure `src/ol_infrastructure/applications/toolhive_swe/`): the +`VirtualMCPServer`'s `authServerConfig` is ToolHive's *own* embedded +authorization server — it brokers login against Keycloak upstream +(`upstreamProviders[0].oidcConfig`) but then issues **its own** JWT for +`incomingAuth`, scoped to `audience: VMCP_RESOURCE_ID` (the vMCP itself, not +Keycloak). Backend `MCPServer` specs (fetch/grafana/sentry) carry only static +injected secrets/env — no per-request header, no forwarded bearer token, no +end-user claim. Today, "any Keycloak-authenticated user gets full access to +the aggregated tool set" (ADR-0009's own words) because the backend container +cannot tell users apart at all. There is no header to read here; the fallback +is the only option, not a hedge. + +**omnigraph-server's bearer-token auth is static, not mintable at +request time.** Per omnigraph `docs/user/operations/server.md` § "Auth model": +tokens are SHA-256-hashed **once at server startup** from one of three +sources — AWS Secrets Manager, a JSON file/env var (`{actor_id: token}`), or +a single legacy token. There is no HTTP endpoint or CLI subcommand to register +a new (actor, token) pair at runtime; the only way to add one is to update the +token source and restart. So witan cannot mint a fresh per-user token on the +fly the way it can derive an actor id — the token for `act-` has to +already exist in whatever source omnigraph-server was booted against. + +**Cedar actor identity is signed-claim-only and matched server-side** +(`docs/user/operations/policy.md` § "Actor identity"): "the server resolves +the token at the auth middleware boundary, looks up the actor it was minted +for" — client-supplied actor ids are never trusted. This is the same +boundary witan's own token resolution must respect: witan cannot tell +omnigraph-server "treat this request as actor X" via anything but the bearer +token itself. + +## Decision + +### D1 — witan performs its own direct OIDC/JWT validation against Keycloak + +Configure witan's FastMCP server with `auth=JWTVerifier(jwks_uri=f"{issuer}/protocol/openid-connect/certs", issuer=issuer, audience=...)` +(fastmcp 3.4's built-in verifier — already a dependency, no new package). +ToolHive still hosts the container (lifecycle, networking, registry) but is +**not** the identity boundary for witan; the fallback ADR-0009 flagged is the +decision, since the broker path was confirmed closed rather than merely +suspect. This only activates for the deployed `streamable-http` transport — +local `stdio` usage (the existing single-user mode) is unaffected and +requires no Keycloak reachability. + +`JWTVerifier` populates `AccessToken.claims` with the full JWT claim set, +retrievable per-request via fastmcp's `get_access_token()` dependency — this +is how a request-scoped `sub` reaches witan's tool handlers. + +### D2 — actor id is a deterministic, pure function of `sub` + +`act-` — lowercase, non-`[a-z0-9-]` characters collapsed to +`-`. Keycloak's `sub` is already a UUID in practice, so this is close to +identity; sanitizing defensively costs nothing and avoids ever shelling out +an unsanitized claim into a CLI arg or file path. Implemented as +`witan.identity.derive_actor_id` — pure, no I/O, unit-testable without a +running server or Keycloak. + +### D3 — per-user tokens are pre-provisioned out-of-band; witan looks up, never mints + +Because omnigraph-server only reads its token source at boot, the (actor, +token) pairs for every `witan-users` member must already exist there before a +user's first request. The provisioning pipeline (ol-infrastructure, +`tk-ol-infrastructure-toolhive-witan-pulumi-stack-e843b3`) is responsible for +walking the Keycloak `witan-users` group/role membership and writing a +generated token per user into the same source +`omnigraph-server` boots from (`OMNIGRAPH_SERVER_BEARER_TOKENS_FILE`/ +`_AWS_SECRET`), keyed by `act-`. This is the "per-user, not per-team" +shift called out in ADR-0002 D1 and in the pulumi-stack task: instead of one +shared credential per team, each user gets an individually-provisioned token +— but "individually-provisioned" still means *provisioned ahead of the +request*, not synthesized by witan out of thin air. + +witan's role is **lookup, not mint**: `witan.identity.ActorTokenResolver` +reads the *same* JSON map (`WITAN_ACTOR_TOKENS_FILE`, matching +`OMNIGRAPH_SERVER_BEARER_TOKENS_FILE`'s shape so both processes can be +pointed at one generated file with no format translation), and resolves +`act- → token` per request. "Looked-up... at request time" (the +originating task's phrasing) describes this lookup, not dynamic minting. + +The resolver reloads the file when a requested actor id is missing from its +current in-memory cache (rather than on a fixed TTL), so a newly-provisioned +user succeeds on their first request without waiting for witan's own process +to restart — as long as the provisioning pipeline has already written their +entry before that request. A `sub` with genuinely no entry (provisioning +hasn't run yet, or the user isn't in `witan-users`) fails closed with a +message naming the missing actor id, not a silent fallback to some default +identity. + +### D4 — scope boundary: this ADR ships the mapping primitives, not the full per-request tool wiring + +`witan/server.py` currently constructs one process-lifetime +`OmnigraphClient` at import time (`client = OmnigraphClient(cfg.graph_uri, +cfg.graph_token, ...)`) and every one of the ~30 MCP tool functions closes +over that single module-level `client`. Threading a per-request, per-actor +`OmnigraphClient` through every tool handler is a mechanical but wide-surface +refactor (129 call sites at last count) that deserves its own review and test +pass rather than riding on a design ADR. This ADR ships and unit-tests the +two pieces that are independently correct and independently useful — +`derive_actor_id` and `ActorTokenResolver` — and the `JWTVerifier` wiring +into the `FastMCP(...)` constructor (additive, gated on Keycloak config being +present, inert otherwise). Threading a per-actor client through every tool +function is tracked as an explicit follow-up +(`tk-witan-wire-per-actor-omnigraphclient-into-every--f1f787`), not silently +deferred. + +**Resolved** by that follow-up: rather than editing all 129 call sites (or +threading a `client` parameter through every handler and helper), the +module-level `client` name is now bound to a small proxy +(`_ActorScopedClient`) whose `__getattr__` calls `_resolve_client()` on every +access. `_resolve_client()` returns the single `_default_client` unchanged +when `identity_cfg.oidc_issuer` is unset (byte-identical local/stdio +behavior), and otherwise reads the validated JWT via fastmcp's +`get_access_token()`, derives the actor id from its `sub` claim, and +returns a per-actor `OmnigraphClient` built once and cached by actor id. A +request with no access token in scope (an admin/migration CLI command run +inside the deployed container, not an MCP tool call — FastMCP's own auth +already rejects unauthenticated tool requests) also falls back to +`_default_client`. See `witan/server.py` (`_resolve_client`, +`_ActorScopedClient`) and `tests/test_actor_client.py`. + +## Consequences + +- Closes the open question ADR-0009 left as a fallback: witan does its own + OIDC validation; ToolHive is hosting/lifecycle only for this service. Any + future ToolHive release that *does* propagate per-user identity to backends + would let us drop `JWTVerifier` in favor of trusting a forwarded header, but + nothing in this design depends on that landing. +- New non-negotiable cross-repo contract: ol-infrastructure's Keycloak→token + provisioning pipeline and omnigraph-server's token source must be the + *same* generated artifact witan reads — a drift between "who Keycloak says + is in `witan-users`" and "who has a token in the file" surfaces as a hard + lookup failure for the affected user, not a silent downgrade. +- Local single-user `stdio` mode is untouched — `JWTVerifier` is only + constructed when Keycloak issuer/audience config is present, which it never + is for local use. +- Does not solve self-service/on-demand provisioning for a brand-new Keycloak + user before the sync pipeline has run — out of scope for v1, same + limitation the per-team model had, just at finer grain. + +### Addendum (2026-07-08) — the "Forces" premise above was about `toolhive_swe`'s config, not a ToolHive platform limitation + +A capability audit of upstream `stacklok/toolhive` at `v0.33.0` — the exact +version already pinned as `TOOLHIVE_OPERATOR_CHART_VERSION` in +ol-infrastructure — found that ToolHive natively supports an "External OIDC +provider" auth scenario (`docs/middleware.md`) where **the client's JWT is +forwarded to the backend MCP container unmodified**, plus a pluggable +authorization framework including a Cedar-based authorizer (`docs/authz.md`) +and a real, tested OAuth 2.0 Token Exchange (RFC 8693) implementation +(`pkg/oauthproto/tokenexchange/`). + +The "Forces" section above is accurate about what it checked — `toolhive_swe` +specifically uses ToolHive's *other* scenario ("Embedded auth server" → +upstream-token-swap, vMCP-scoped JWT only) — but generalizes that +configuration choice into "ToolHive's embedded broker does not propagate +end-user identity to the backend container," which overstates it: a +*different* `witan`-stack configuration could plausibly get per-user JWT +forwarding, Cedar authz, or RFC 8693 token exchange from ToolHive itself, +narrowing or removing the need for D1's own `JWTVerifier` path. This wasn't +a "future ToolHive release" scenario as line 139 speculated — the capability +was already present in the pinned version at the time this ADR was written. + +Whether to actually change course (and where the authz source of truth +should live if witan's own Cedar bundle and ToolHive's authz framework would +otherwise overlap) is tracked as a separate decision, not resolved here: +`tk-revisit-adr-0004-adr-0009-per-user-identity-desi-e9005a` +(project `wp-witan-multi-user-service-deployment-dcf6ee`). + +### Resolution (2026-07-10) — keep D1–D4 as designed; fix the ToolHive scenario, not the code + +`tk-revisit-adr-0004-adr-0009-per-user-identity-desi-e9005a` is resolved as: +**adopt ToolHive's "External OIDC provider" scenario for the `witan` stack's +auth config; do not adopt ToolHive's Cedar authorizer or RFC 8693 token +exchange; make no code change to D1–D4.** + +- **Identity propagation.** The "Forces" section's mistake was inferring a + platform limitation from `toolhive_swe`'s specific scenario + ("Embedded auth server" → upstream-token-swap). The `witan` stack doesn't + have to use that scenario. Configuring it instead with ToolHive's + "External OIDC provider" scenario makes ToolHive forward the client's + genuine Keycloak-issued JWT to the backend container unmodified — which is + exactly the input D1's `JWTVerifier(jwks_uri=..., issuer=..., audience=...)` + was already written to validate. This isn't an alternative to D1, it's the + ToolHive-side configuration that makes D1 deliverable through ToolHive + instead of requiring witan to somehow sit outside ToolHive's proxy path. + `derive_actor_id` and `ActorTokenResolver` (D2/D3) are unaffected — they + operate on the validated `sub` claim regardless of which scenario delivered + the JWT. **No changes needed to PR #84 or PR #90.** +- **RFC 8693 Token Exchange — rejected for witan.** Token exchange re-mints a + token signed by ToolHive's own exchange service, which would make ToolHive + the identity boundary witan trusts instead of Keycloak directly — the + opposite of D1's explicit choice. It's a better fit for `toolhive_swe`-style + fan-out to third-party backends that need scope narrowing per tool, not for + witan, which wants the original per-user identity intact. +- **ToolHive's Cedar authorizer (`cedarv1`) — not adopted.** It authorizes at + the MCP transport layer ("can this JWT call tool X at all") with no + knowledge of witan's domain model (repos, teams, node types). Witan's own + Cedar bundle (ADR-0002) already does finer-grained, data-aware authorization + and stays the single source of truth; running a second, coarser Cedar + policy alongside it would add a policy surface to keep in sync for no + proven benefit at v1. Revisit only if a concrete need for transport-layer + pre-filtering (e.g. rate-limiting a tool before it reaches witan at all) + shows up. +- **Follow-up.** The ol-infrastructure side of this decision — configuring + the `witan` stack's `MCPServer`/`VirtualMCPServer` with the "External OIDC + provider" scenario instead of copying `toolhive_swe`'s "Embedded auth + server" pattern — is recorded in ADR-0009's own resolution addendum and in + `tk-ol-infrastructure-toolhive-witan-pulumi-stack-e843b3`. + +### Addendum (2026-07-31) — D5: the `author` field is resolved per request, not from config + +D1–D4 decided *which omnigraph client* performs a write. They said nothing +about the `author` value the write carries, and the gap showed up once the +per-request wiring landed: `cfg.author` is module-level, evaluated once at +process startup, so under a deployment every `Memory`, `WorkflowProject`, +`WorkflowTrace`, `WorkflowSession`, and `Task` was attributed to the server +container's own identity. Writes were routed per user while `author` stayed a +single constant deployment-wide — enough to make `workflow_trace_list(author=…)` +an inert filter, flatten the ranking layer's author-trust signal, and strip +mined corpus traces of the provenance that is the point of their being a shared +team artifact. + +**Decision.** A `_current_author()` helper, sibling to `_resolve_client()` and +gated by the same `_is_local_stdio()` discriminator, resolves the identity at +call time from the request's validated JWT. + +- **Local stdio keeps `cfg.author`.** `WITAN_AUTHOR` / `git config user.name` / + `$USER` is already the right answer when the server is the user's own + process. Unchanged behaviour, not a fallback that happens to work. +- **Deployed calls without a token also keep `cfg.author`.** Same reasoning as + `_resolve_client`: FastMCP rejects unauthenticated tool requests, so a + missing token means an admin/migration CLI call inside the container, which + has no caller identity to attribute. +- **Otherwise, prefer `preferred_username`, then `email`, then `act-`.** + Readability is the deciding factor: `author` is consumed by human-facing + author filters and by the author-trust ranking config, and neither has a + name-resolution step that could turn a UUID back into a person. The derived + actor id — the same one D2 defines — is the last resort, so attribution + degrades to opaque-but-correct rather than to the wrong user. + +**Rejected: a separate `actor_id` field alongside `author`.** It would survive +a Keycloak username change, which reusing `author` does not, but it costs a +schema change across five node types plus the insert mutations, the read +projections, and the read models — a large surface for a failure mode +(a renamed user's older nodes keep the old name) that is cosmetic in an +internal team corpus. Revisit if attribution ever needs to be authoritative +rather than descriptive. + +`task_claim` / `task_release` default their holder to the same helper, so the +multi-user default is the calling user rather than one shared identity that +every parallel agent would collide on. + +No backfill: this lands before the service carries production traffic, so +there are no nodes written under the old uniform-author behaviour. + +### Addendum (2026-08-05) — D3's "Keycloak `witan-users` group/role membership" was a misnomer; there is no such Keycloak group + +D3 above tells the provisioning pipeline to walk "the Keycloak `witan-users` +group/role membership". Read literally that is an instruction to create a +Keycloak group named `witan-users`, and when the pipeline was finally built +(ol-infrastructure PR #5253) it was read exactly that way and one was created. +It has since been removed. Recording why, because the wording will keep +producing the same mistake otherwise. + +**`witan-users` is a Cedar group, not a Keycloak one.** ADR-0002 D1 defines it +as one of three groups in *witan's own policy bundles*, holding one `act-` +per authenticated human, with membership "templated by ol-infrastructure +(Keycloak claims for `witan-users`)". That is the accurate statement of the +contract: the Cedar group is populated *from* Keycloak, by whatever query +identifies witan's users. D3 then narrowed "Keycloak claims" to "the Keycloak +`witan-users` group/role membership" — inventing a same-named Keycloak object +that ADR-0002 never called for. The name collision is what makes the invention +look mandatory. + +**What the pipeline actually walks: every enabled, non-service-account user of +the `ol-platform-engineering` realm.** That realm has +`registration_allowed=False`, no identity-provider brokering and no federation, +so its membership is hand-managed and already exactly the intended audience. A +Keycloak group inside it would be a second gate on an already-gated population, +and its failure mode is the bad one — somebody joins the realm, nobody adds +them to the group, and they hit D3's own fail-closed path with an error that +reads like a provisioning lag rather than a missing group membership. + +**Trade accepted, not overlooked:** realm access is now witan access. There is +no way to revoke witan while leaving that realm's other applications +(jupyterhub, superset, opik) intact. If that requirement ever appears, a +Keycloak group is the right answer — but it should be added deliberately, for +that reason, rather than because this sentence implied one already existed. + +**One consequence worth carrying into any reimplementation:** enumerating a +realm returns each confidential client's own service account as an ordinary +user (`service-account-`). Minting a human's interactive read/write +token for them would hand every such client the Cedar rights a person has under +`witan-users`, so they must be filtered — `serviceAccountClientId` is the +authoritative signal. The non-human actors that *should* hold tokens +(`witan-ci`, `witan-service`, per ADR-0002 D1) are declared in SOPS and merged +in, never discovered from Keycloak. + +Only D3's description of the provisioning *source* is corrected here. The +decision itself — tokens pre-provisioned out-of-band, witan looks up and never +mints, fail closed on a missing actor id — is unchanged, as is the Consequences +section's cross-repo contract: a drift between who Keycloak says is a witan +user and who has a token in the file still surfaces as a hard lookup failure +for that user, not a silent downgrade. + +### Addendum (2026-08-21) — D5's "no backfill" assumption did not survive store migration, and `author` turns out to be doing authorization work + +D5 above closes with: + +> No backfill: this lands before the service carries production traffic, so +> there are no nodes written under the old uniform-author behaviour. + +True of the *deployment's own* history, and false of everything merged into it. +`store_merge` (ADR-0007 D5) preserves each row's `author`, so a user migrating +a local store imports rows named by `cfg.author` — `WITAN_AUTHOR` / git +`user.name` / `$USER` — into a graph whose live identities are Keycloak +`preferred_username`. The two namespaces cannot converge, and there is no +client-side escape: once `remote_url` is set, `_is_local_stdio()` is false and +`cfg.author` is ignored for any comparison. + +That surfaced as agent-kit#267: `memory_delete` refuses everyone but the +author, so **every memory a user migrated became permanently undeletable by the +person who wrote it** — and migrated history is exactly the history most likely +to contain something that should be pruned, having been written before the +graph was shared. + +**The deeper point this exposes.** D5 frames `author` as descriptive, and +rejects a separate `actor_id` on that basis ("Revisit if attribution ever needs +to be authoritative rather than descriptive"). But `memory_delete` uses it as +an authorization control, and it is the *only* row-level ownership control +there is: ADR-0002 records that Cedar cannot scope a delete to a row's owner, +so ownership has to be enforced in Python or not at all. A field the ADR calls +descriptive is load-bearing for authorization. + +**Decision.** Fix the mismatch, not the check. + +- **`store_merge` gains `claim_from_author`.** The client sends the identity + its local store wrote; rows matching it are restamped to the calling actor + before the write, and everything else is untouched. Matching rather than + stamping unconditionally is what makes it safe as a default: on your own + store every row matches, so the repair needs no flag anyone has to discover; + on a teammate's export nothing matches, so merging their store through your + credential cannot quietly reattribute their work. +- **`claim_authorship` repairs stores already merged.** A re-merge cannot: + reconciliation is newest-record-wins and a re-sent row loses to its own + applied copy. It rewrites in place across all five authored types. + +**Not decided here: whether attribution should become authoritative.** +`claim_authorship` does not verify that the name you are claiming was ever +yours. That capability already existed — `store_merge` accepts whatever +`author` a row carries, which is what makes the hand-edited-export workaround +in #267 work — so this makes an existing capability usable rather than creating +one. Constraining it means constraining `store_merge` too, and that is the +revisit D5 anticipated, to be taken as its own decision rather than smuggled in +with a bug fix. + +The read side is unfixed and tracked separately: +`workflow_trace_list(author=…)` exact-matches, so migrated traces stay +invisible to their own author's filter, and the ranking layer's author-trust +signal keys on the same string. diff --git a/docs/explanation/decisions/0005-secure-cli-path-into-deployed-witan.md b/docs/explanation/decisions/0005-secure-cli-path-into-deployed-witan.md new file mode 100644 index 00000000..b06b30ca --- /dev/null +++ b/docs/explanation/decisions/0005-secure-cli-path-into-deployed-witan.md @@ -0,0 +1,262 @@ + + +!!! info "This page lives with the code" + + The authoritative copy is + [`mcp/servers/witan/docs/adr/0005-secure-cli-path-into-deployed-witan.md`](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan/docs/adr/0005-secure-cli-path-into-deployed-witan.md). + +# 5. Secure CLI path into the deployed witan/omnigraph store + +- Status: Accepted +- Date: 2026-07-14 +- Deciders: witan platform owners +- Tracking: task `tk-design-a-secure-cli-path-into-the-deployed-witan-4ce2b2`, + project `wp-witan-multi-user-service-deployment-dcf6ee` +- Related: `docs/adr/0004-keycloak-jwt-per-user-actor-mapping.md` (the + server-side JWT→actor→token mapping this reuses); ol-infrastructure + `docs/adr/0009-deploy-witan-as-shared-multi-tenant-mcp-service.md` + (ClusterIP-only omnigraph-server, the `svc-witan-admin` sketch); + `docs/adr/0009-stateless-mcp-protocol-era.md` (the 2026-07-28 era this path + now runs on — read it alongside every `streamable-http` reference below, + which describes the handshake-era shape) + +## Context + +Once witan is deployed as a shared multi-user service, the only +authenticated, actor-scoped way to touch the graph is an MCP client speaking +the MCP protocol over `streamable-http` — that path runs through +`_resolve_client()` in `witan/server.py`, which validates the caller's +Keycloak JWT, maps its `sub` to an `act-`, and looks up that actor's +pre-provisioned omnigraph bearer token (ADR-0004). Every agent gets its own +Cedar scoping and audit trail. + +The `witan` umbrella CLI's non-serve commands (`witan tasks`, +`witan memory search`, `witan projects …`) never take that path. They import +`witan.server` and call its tool functions **in-process**, so +`_resolve_client()` sees no MCP request (`get_access_token()` is `None`) and +falls back to the single static-token `_default_client` built from +`WITAN_GRAPH_TOKEN`. That is exactly the coarse-per-token shape ADR-0004 moved +the MCP path away from — no per-user identity, no per-user audit. And even +that fallback can't reach a deployed graph: omnigraph-server's Service is +deliberately ClusterIP-only (ADR-0009), never exposed outside the cluster, and +the CLI would need `WITAN_GRAPH_URI` pointed straight at it. + +So operators and users have **no supported, secure way** to run ad hoc CLI +commands against the deployed store — routine queries (`witan tasks`, +`witan memory search` for a specific actor) *and* maintenance (`witan migrate +topics`, `witan apply-schema`, cross-actor debugging) are both stuck. + +These are two different needs with two different identity requirements, so +they get two different answers. + +## Decision + +Adopt **both** paths, each scoped to the use case whose identity model it fits: + +### (a) Primary: `witan` CLI gains a real remote MCP-client mode — `agent-kit` + +For routine, per-user ad hoc access the CLI becomes an MCP client of the +deployed endpoint, inheriting the *exact* identity path the agent traffic +already uses: + +1. **Auth — OIDC device authorization grant (RFC 8628).** `witan login` + discovers the Keycloak realm's endpoints from + `{WITAN_OIDC_ISSUER}/.well-known/openid-configuration`, requests a device + code, prints the verification URL + user code, and polls the token endpoint + until the user approves in a browser. The resulting access/refresh tokens + are cached at `~/.config/witan/tokens.json` (mode `0600`), keyed by + `(issuer, client_id)` so multiple deployments don't clobber each other. + `witan whoami` decodes the cached token for display; `witan logout` clears + it. The device-code grant is the standard flow for CLI tools — no client + secret, no local redirect listener, works over SSH. + +2. **Transport — MCP over `streamable-http`.** When `WITAN_REMOTE_URL` is set, + `witan.cli._common._srv()` returns a `RemoteServerProxy` instead of the + in-process `witan.server` module. The proxy mirrors the server-module tool + interface via `__getattr__`, so **none of the ~40 existing CLI call sites + change** — each `_fn(s.task_ready)(…)` transparently becomes an MCP + `call_tool` against `WITAN_REMOTE_URL`, authenticated with a `BearerAuth` + carrying the cached JWT. The deployed server's `_resolve_client()` then does + the ADR-0004 JWT→actor→token mapping exactly as it does for agents. One + identity model for every remote access path; one audit trail; one Cedar + policy surface (omnigraph's own bundle, ADR-0002). + + Result-shape parity is free: FastMCP's `CallToolResult.data` already + unwraps the `{"result": …}` output-schema envelope back to the raw + `list`/`dict` an in-process call returns, so the CLI's existing rendering + code is untouched. + + Repo scoping is resolved **client-side**: the deployed server has no git + checkout, so a `repo=None` ("detect current repo") argument is rewritten to + the *client's* detected repo before the call is sent. `repo=""` (all repos) + is preserved. + +### (b) Break-glass: in-cluster `svc-witan-admin` for maintenance — `ol-infrastructure` + +Schema migration, `witan migrate topics`/`migrate storage-format`, +`merge-store`, and cross-actor debugging have **no per-user identity to +scope** — they operate on the store as a whole. Forcing them through a human's +per-user actor would be both wrong (a user actor must not have blanket +read/write over every other actor's data) and impossible (those commands are +plain in-process module functions, deliberately *not* `@mcp.tool`, so they are +unreachable over the MCP path). They keep the in-process path ADR-0004 already +documented, run **inside the cluster** where ClusterIP is reachable, and +authenticate as a narrow, separate `svc-witan-admin` principal: + +- Provisioned in the `omnigraph` Pulumi stack, sole writer of the shared + actor-token Vault source (ADR-0009 sketched it + alongside `svc-witan-ci` but never provisioned it). Its omnigraph bearer + token lives in the same actor-token source, and its Cedar policy grants only + the maintenance verbs it needs — **not** blanket read/write to every actor's + nodes. +- Invoked via a one-off Kubernetes `Job` (or `kubectl exec` into a bastion + pod) that runs `witan apply-schema` / `witan migrate …` with + `WITAN_GRAPH_URI` pointed at the in-cluster omnigraph-server and + `WITAN_GRAPH_TOKEN=`. This is the deliberate + `_default_client` fallback, now with a purpose-provisioned admin credential + instead of an accidental one. + +The remote MCP proxy from (a) **refuses** these commands: they are not MCP +tools, so `RemoteServerProxy` raises a clear "run in-cluster as +`svc-witan-admin`" error rather than silently doing the wrong thing. + +### (c) Writes: witan-code indexes cluster code graphs through the MCP tier — `agent-kit` (2026-08-01) + +Path (a) moved witan-code's *reads* onto the deployment and left indexing +local, on the reasoning that indexing needs a git checkout. That is still +true, and it is exactly why (a) was not enough: the indexer is always a local +process, but on the cluster the graph it writes is on the ClusterIP-only +omnigraph-server. A developer's checkout could reach neither. Verified against +the live CI cluster on 2026-08-01: `service/omnigraph-server` has no +HTTPRoute, Ingress, or Gateway; only `witan..ol.mit.edu` (the MCP tier) +is externally reachable, and the data tier was verified only through +`kubectl port-forward`. + +Four options were weighed; **route the writes through the witan MCP tier** +won. Exposing the raw omnigraph endpoint through APISIX was rejected as a +second, policy-unmediated boundary; in-cluster-only indexing was rejected +because it gives up the per-developer branch views ADR-0006 built; and +`port-forward` was rejected as a supported path (cluster credentials per +developer, poor ergonomics). One exposed boundary, already authenticated, +already actor-resolving. + +- **Surface.** Seven machine-facing `code_store_*` tools mirroring the store + operations the write path performs — `read`, `mutate`, `mutate_many`, `load`, + `open` (fork a branch view), `views`, `graphs`. `mutate_many` was added after + the rest, because a reindex emits two deletes per changed file and one call + apiece made both the round trips and the Lance versions scale with the repo; + it takes the same `(query, name, params)` steps `mutate` takes one at a time + and splices them server-side, so the surface stays named queries and params. + Deliberately *not* one bulk-ingest tool: + a repo index is a hash read, a per-file purge, a bulk load, and then the + same again against the bridge graph. Modelling each phase as its own tool + would move indexing policy server-side, where it would have to stay in step + with clients that can be a release behind. Mediated rather than arbitrary: + `query` may only name a query file bundled with the server, and the graph is + resolved from a repo URI against the *server's* configuration — a client + never sends a store address. +- **Identity and authorization move server-side.** `witan_code/ingest.py` + resolves the actor from the validated JWT per request (ADR-0004, the same + mapping witan uses for memory), looks that actor's omnigraph bearer token up + in the same provisioned map, and runs `check_writable` against it before any + mutation reaches the store. The client-side guard in `indexer.index_path` + stays as a fast-fail courtesy check; it is no longer the authority. An + actor with no provisioned token is refused rather than served under the + service account. +- **A consequence worth stating plainly:** a write through this boundary can + never claim the shared default-branch view. That view's single writer is the + CI indexer, which runs in-cluster over the direct transport (b's network + position, not its credential). Everything through the tier is a branch view + owned by the actor whose JWT carried it. +- **Client side.** `code_transport = "mcp"` (env `WITAN_CODE_TRANSPORT`, or + per `[targets.]`) makes the deployed endpoint the store's address: + `StoreRef.via_mcp` resolves to a `RemoteStoreClient` that stands in for an + `OmnigraphClient`, so `indexer`/`bridge` are unchanged. Unlike (a)'s proxy + it holds one connection open for the process — an index is thousands of + store calls, not one — and reconnects once on a dropped one. +- **The tools are registered only on a deployment** (`WITAN_OIDC_ISSUER` set, + overridable with `WITAN_CODE_STORE_TOOLS`). A local stdio server writes its + own stores directly, so serving them there would add six machine-facing + tools — one of which runs named mutations — to every agent's tool list to + serve a caller that cannot exist. +- **`code_server` keeps its meaning** as the in-cluster/direct transport: the + CI indexer and maintenance jobs share the cluster network and have no reason + to pay for an extra hop. + +## Consequences + +- **agent-kit (this repo):** implements (a) in full — `witan/remote/oidc.py` + (device flow + token cache), `witan/remote/proxy.py` + (`RemoteServerProxy`), `witan login`/`logout`/`whoami` commands, + `RemoteConfig` (since 2026-07-31 in `witan_core.remote.config`), and the + `_srv()` switch. No change to the existing in-process path: with + `WITAN_REMOTE_URL` unset the CLI behaves exactly as before. witan-code's CLI + mirrors all of this — see the 2026-07-31 amendment below. +- **ol-infrastructure (follow-up):** provision `svc-witan-admin` (token + + Cedar policy) and the maintenance-Job/bastion pattern in the + `witan` stack, plus register `witan-cli` as a public OIDC client + with the device grant enabled in the `ol-platform-engineering` Keycloak + realm. Tracked as a spun-off task. +- **ol-infrastructure (follow-up for (c)):** the witan MCP tier's Deployment + must set `WITAN_CODE_SERVER` (and nothing else new — the store tools + register themselves off the `WITAN_OIDC_ISSUER` the tier already has, and + resolve tokens from the `WITAN_ACTOR_TOKENS_FILE` it already mounts). Until + it does, the tier serves code-graph *reads* from whatever `code_dir` its + container has and can serve no cluster writes at all. Each repo's + `code-` graph must also be declared by the data-tier stack, as today. +- **Config surface:** the CLI's remote mode is opt-in via `WITAN_REMOTE_URL` + (+ `WITAN_OIDC_ISSUER`, `WITAN_OIDC_CLIENT_ID`, optional + `WITAN_OIDC_AUDIENCE`). These name the *client's* view of the deployment and + are distinct from the server-side `WITAN_ACTOR_TOKENS_FILE` / + `load_identity_config()` triple. + - **Amendment (2026-07-20):** these four fields are also resolvable per + named `[targets.]` block in `config.toml` (`remote_url`/ + `oidc_issuer`/`oidc_client_id`/`oidc_audience`), matched the same way as + the omnigraph `server`/`graph`/`token` fields — env var still wins, then + the matched target, then a global config.toml value. This lets + different orgs/repos/checkouts point at different deployed witan + services, and a single target block can route both the omnigraph store + and the deployed MCP endpoint together. See `RemoteConfig`/ + `load_remote_config()` in `witan/config.py`. + - **Amendment (2026-07-31):** witan-code's standalone CLI now takes the same + path. `witan serve` mounts its `code_*` tools onto this deployment with no + prefix, so the endpoint already served them — only the client side was + missing. `RemoteConfig` and the resolution above moved to + `witan_core.remote.config` (both servers keep just their own target + selection), and `witan_code/remote/proxy.py` binds the same + `RemoteMCPProxy` with witan-code's policy. Both CLIs therefore read the + same four keys off the same target block, and — since the token cache is + keyed by `(issuer, client_id)` and both default to the `witan-cli` client + id — one `witan login` authenticates both. witan-code's read commands + (`symbols`, `deps`, `stitch`, `repos`, `branches`) move; indexing and store + maintenance stay local, since they need a checkout and the store files that + a deployed replica does not share. One divergence worth noting: + witan-code's proxy deliberately does **not** resolve `repo=None` + client-side. On witan's tools that means "detect the current repo", but on + witan-code's bridge-wide tools it means "every indexed repo", so injecting + a detected repo would silently narrow the result. + + **The coupling is deliberate: there is no `WITAN_CODE_REMOTE_URL`.** Both + CLIs read the one set of keys, so configuring a deployment sends *both* + remote — you cannot point witan-council at a deployment while keeping + witan-code's reads local. That follows from the topology (one endpoint + serving both tool surfaces) and keeps one precedence chain instead of two. + A `[targets.]` block still discriminates by repo or checkout path, + just not by tool surface. Decided 2026-07-31 for the joint case, which is + how these are deployed and run today; revisit if a real "remote memory, + local code graph" need appears, since a per-server override would be + purely additive and break no existing config. +- **Known v1 limitation:** `RemoteServerProxy` opens a fresh MCP connection per + tool call, so a single CLI command that fans out to several tools pays + several MCP handshakes. Acceptable for interactive CLI use; a persistent + per-process session is deferred and tracked with the subprocess-overhead + spike (`tk-spike-subprocess-per-call-overhead-for-remote-om-d6ceac`). + - **Amendment (2026-07-30):** largely moot against a 2026-07-28 deployment. + That era has no `initialize` handshake and no session id, so a fresh + connection per call costs a connection, not a negotiation — see + `docs/adr/0009-stateless-mcp-protocol-era.md`. The proxy also gained an + elicitation handler, so a prompt the deployment raises now reaches the + human at the terminal instead of degrading to the tool's default. diff --git a/docs/explanation/decisions/0006-code-graph-branch-ownership-and-reaping.md b/docs/explanation/decisions/0006-code-graph-branch-ownership-and-reaping.md new file mode 100644 index 00000000..3a9a65b6 --- /dev/null +++ b/docs/explanation/decisions/0006-code-graph-branch-ownership-and-reaping.md @@ -0,0 +1,154 @@ + + +!!! info "This page lives with the code" + + The authoritative copy is + [`mcp/servers/witan/docs/adr/0006-code-graph-branch-ownership-and-reaping.md`](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan/docs/adr/0006-code-graph-branch-ownership-and-reaping.md). + +# 6. Code-graph branch-view ownership and reaping + +- Status: Accepted +- Date: 2026-08-01 +- Deciders: witan platform owners +- Tracking: task `tk-branch-cedar-gating-stale-code-graph-branch-reap-0c621c`, project `wp-witan-multi-user-service-deployment-dcf6ee` +- Supersedes: — +- Amends: `0002-witan-cedar-authorization-bundle.md` (D2's code-graph and bridge rule shapes) +- Related: `0004-keycloak-jwt-per-user-actor-mapping.md`; witan-code `docs/BRANCH_INDEXING.md`; task `tk-decide-where-a-developer-s-in-flight-code-graph--1cfbfa` (the decision this implements); ol-infrastructure `docs/adr/0009-deploy-witan-as-shared-multi-tenant-mcp-service.md` + +## Context + +Per-user code-graph branch views live **on the shared cluster graph**, not in +private stores (decided 2026-07-31): isolated agents seeing each other's +in-flight work as it happens is much of what the shared service is for. witan-code +therefore names every view for its writer — `act-/` on a per-repo +graph, `act-//` on the bridge (`witan_code/views.py`, +PR #164) — so two checkouts of `feature-x` no longer overwrite each other. + +That decision leaves two questions this ADR settles: **who the authorization +layer can actually hold to that naming**, and **what bounds the resulting branch +sprawl**, since every developer's every git branch now gets a view on one graph +and nothing about indexing a branch ever unindexes it. + +### Force — omnigraph 0.8.1 cannot express ownership + +The naming scheme was designed on the expectation that a Cedar rule could gate +writes with `startsWith(branch, principal.actor + "/")`. **It cannot.** Verified +against the 0.8.1 binary: + +- A bundle rule compiles to a bare + `permit(principal in Omnigraph::Group::"…", action == Omnigraph::Action::"…", + resource == …)`. There is no `when {}` clause in the generated Cedar at all. +- The bundle schema is exactly + `{version, groups, protected_branches, rules{id, allow{actors{group}, actions, + branch_scope, target_branch_scope}}}`. The only branch predicate is the + three-valued `any | protected | unprotected` scope; there is no branch-name + pattern, no principal attribute, and no raw-Cedar escape hatch. + +So "write only views prefixed with your own actor id" is inexpressible at the +policy layer, in this version, by any arrangement of the bundle. + +### Force — staleness is the only signal a shared graph has + +`omnigraph branch list --json` returns **bare names**: no creation date, no +owner, no size. The only per-branch timestamp anywhere is in the commit log — +`commit list --branch --json` returns each commit tagged with the +`manifest_branch` it landed on and a microsecond `created_at`. A branch that has +only inherited its fork point's commits has no commit tagged with itself, and +there is nothing else to date it by. + +## Decision + +### D1 — Cedar enforces main-vs-WIP; the client enforces writer-vs-writer + +Two layers, with the split stated rather than implied: + +- **Cedar** (`policy/code-graph.policy.yaml`, `policy/bridge.policy.yaml`): + `main` is protected and writable only by `witan-ci`; every other branch is + unprotected and writable by any authenticated `witan-users` member. This is + the half that survives a lying client. +- **witan-code** (`graph.py:owns_view`): a process writes only views prefixed + with its own actor id, and refuses otherwise with a message naming the owner. + This is the half Cedar cannot express. + +A client that ignores the write guard can still overwrite a colleague's view. +That is a **known, accepted v1 gap**, not an oversight: the alternative is +abandoning shared-graph visibility, which is the feature. It is pinned by +`tests/code-graph.tests.yaml:cedar-cannot-scope-wip-writes-to-owner`, which +asserts `expect: allow` for one user writing another's view — so an omnigraph +release that adds a branch-name or principal-attribute predicate **fails the +build** and forces the bundle to be tightened rather than letting the gap +persist unnoticed. + +### D2 — Reads stay open across principals, deliberately + +Any principal may read any other's branch view, on both the per-repo graphs and +the bridge. This is the point of putting views on the shared graph, so it is +asserted explicitly (`user-reads-another-users-view`, +`user-reads-another-users-bridge-view`) rather than left to follow from +`branch_scope: any` — a later tightening that scoped reads to the caller's own +views would defeat the decision, and should fail a test when it is attempted. + +### D3 — `branch_delete` on a shared graph is CI's alone + +Users get `branch_create` on unprotected branches and **not** `branch_delete`. +Cedar cannot scope a delete to the view's owner any more than it can scope a +write, and the two are not equally survivable: an overwrite is repaired by the +owner reindexing, a delete of the wrong view is the same repair plus the owner +having no idea why. No client path needs it — `witan-code branches --prune` +already refuses against a shared graph — so the grant is pure exposure. + +This amends ADR 0002 D2, which gave users `branch_create`/`branch_delete`. + +### D4 — The bridge is branched and user-writable on WIP branches + +ADR 0002 D2 described the bridge as flat and read-only for humans. It is +neither: indexing a WIP git branch writes that branch's cross-repo bindings to +the bridge in the same pass (`witan_code/bridge.py`), on a repo-qualified view +of its own. Under the previous bundle every developer's WIP index would have +half-succeeded on the cluster — per-repo graph written, bridge bindings denied. +The bridge bundle now mirrors the code-graph one: `main` is the CI-owned +committed projection, every other branch is an unprotected per-user view. + +### D5 — Reaping is server-side, idleness-based, and CI-owned + +A stale-view reaper (`witan_code/reaper.py`, `witan-code reap-views`) deletes +views nobody has written in `WITAN_CODE_VIEW_MAX_IDLE_DAYS` (default 14). It is +deliberately **not** `branches --prune` with a wider scope: that command asks +whether *this checkout* still has the git branch, which is a sound question +about a store one machine writes and a meaningless one about a store every user +of the cluster writes — from a client, "I don't have that branch" and "that +branch is gone" are the same observation. It keeps refusing on shared graphs. + +Two rules follow from the force above, and both are asserted: + +- **`main` is never reaped**, however idle. It is the committed index every + reader falls back to and is idle by design between merges. +- **A view with no commits of its own is never reaped.** It holds nothing that + isn't already on its fork point, so deleting it reclaims nothing — and with no + creation timestamp, one created ten seconds ago is indistinguishable from one + created a year ago, so reaping it would race the indexer that just made it. + +The reaper reports by default and deletes only under `--apply`, and refuses to +delete from a shared graph unless `WITAN_CODE_INDEX_ROLE=ci` — the client-side +mirror of D3, so an operator gets a clear local error instead of a Cedar denial. + +## Consequences + +- Per-writer isolation is real but **client-enforced**; the deployment's threat + model must say so. Against a hostile client, the guarantee is main-vs-WIP. +- Branch sprawl is bounded by a scheduled job, not by any client action. If the + reaper does not run, nothing else removes a view — ol-infrastructure owns + scheduling it (`tk-omnigraph-maintenance-cronjob-scheduled-optimize-4321f8`). +- Idleness is not abandonment: a branch parked past the window and picked back + up loses its view, not its work — the next index rebuilds it from the + checkout. This is only acceptable because views are re-derivable caches, which + is also why their lifecycle is deletion rather than merge. +- Never-written views accumulate unbounded, since nothing ages them. They cost + a name in `branch list` and no storage. Revisit if omnigraph adds a branch + creation timestamp. +- The reaper's staleness signal depends on `commit list --branch` reporting + `manifest_branch` and `created_at`. That is an unversioned CLI shape, so it is + asserted against the real binary in `tests/test_reaper.py` rather than mocked. diff --git a/docs/explanation/decisions/0007-local-to-shared-store-migration-transport.md b/docs/explanation/decisions/0007-local-to-shared-store-migration-transport.md new file mode 100644 index 00000000..4a35bd2c --- /dev/null +++ b/docs/explanation/decisions/0007-local-to-shared-store-migration-transport.md @@ -0,0 +1,209 @@ + + +!!! info "This page lives with the code" + + The authoritative copy is + [`mcp/servers/witan/docs/adr/0007-local-to-shared-store-migration-transport.md`](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan/docs/adr/0007-local-to-shared-store-migration-transport.md). + +# 7. Transport for a local → shared store migration + +- Status: Accepted +- Date: 2026-08-06 +- Deciders: witan platform owners +- Tracking: task `tk-no-usable-transport-for-a-local-shared-store-mig-afbf18`, + task `tk-remote-server-registration-local-remote-user-dat-dc753c`, + task `tk-un-defer-adr-0007-d5-merge-through-the-witan-mcp-f1e5a1` (D5), + project `wp-witan-multi-user-service-deployment-dcf6ee` +- Supersedes: — +- Amends: `0005-secure-cli-path-into-deployed-witan.md` (path (b) gains a + concrete data-movement procedure; the ADR provisioned the principal but never + said how bytes reach it) +- Related: `0002-witan-cedar-authorization-bundle.md`; + `0004-keycloak-jwt-per-user-actor-mapping.md`; ol-infrastructure + `docs/adr/0009-deploy-witan-as-shared-multi-tenant-mcp-service.md` (the + ClusterIP-only data tier this works within) and + `docs/witan-admin-break-glass-runbook.md`; `docs/migration-runbook.md` + +## Context + +The migration *procedure* was built and tested; the *transport* to the deployed +graph was not. `witan migrate merge --target ` exports both +sides, reconciles node collisions on `updated_at` rather than last-write-wins, +and is idempotent — the hard part, and it works. But it addressed every store +as `--store `, and none of the reachable spellings of "the deployed graph" +survive that: + +1. **`--target s3://ol-data-witan-`** — the bucket's only IAM grant is the + `omnigraph-server` ServiceAccount's IRSA role, in the `omnigraph` namespace + (ol-infrastructure `applications/omnigraph/data_tier.py`). No human IAM path + to it is declared anywhere, and minting one would route writes around the + bearer-token/Cedar model ADR-0009 exists to establish. +2. **`--target http://omnigraph-server.omnigraph.svc.cluster.local:8080`** — + ClusterIP only, never exposed (ADR-0009 decision point 2). *And*, even from + inside the cluster, this failed: omnigraph 0.8.1 rejects an http(s) + `--store` outright. A remote graph is addressable only as + `--server --graph `. +3. **Through the public MCP endpoint** (`witan.ol.mit.edu`) — `merge_store` is + in `_ADMIN_ONLY` and is not an `@mcp.tool` at all, deliberately: a bulk + store merge has no per-user identity to scope (ADR-0005 path b). + +Point 2 is the one that mattered most, because it was not a policy limit but a +plain defect. The in-cluster maintenance pods that *do* have the right network +position and credential — `witan-break-glass` and the pre-deploy migration Job +— are configured with `WITAN_MEMORY_URI` pointed at the ClusterIP server +(ol-infrastructure `applications/witan/break_glass.py`). So the sanctioned +break-glass path already ran with a remote-addressed store, and +`witan migrate merge` was the one command in the image that could not use it. +`OmnigraphClient._store_args()` had encoded the correct rule since the 0.8.1 +upgrade; `merge_store` simply bypassed it. + +The second gap is the one nobody had written down: **a store cannot travel.** +Lance embeds absolute paths, so a user's `~/.local/share/witan/graph.omni` +cannot be copied to another machine, streamed into a pod, or staged in a +bucket. Only its `omnigraph export` output can. The break-glass runbook +acknowledged this and told operators to "copy it in or export/load it through +S3 first" — but the break-glass pod declares no volume and no ServiceAccount, +so it holds neither S3 credentials nor an `aws` binary. The advice was not +executable, and `witan migrate merge` had no way to consume an export file +even once one arrived. + +## Decision + +**Make the existing in-cluster path work, rather than building a new one.** +Two changes to `merge_store`, no new infrastructure, no new server surface: + +### D1 — Address each store the way the omnigraph CLI requires + +`merge_store` resolves the source and the target independently through +`witan_core.omnigraph.store_cli_args()` / `store_subprocess_env()` — the +free-function forms of the client's own `_store_args`/`_subprocess_env`, which +now delegate to them so there is one implementation of the rule. A local path +or `s3://` root stays `--store `; an `http(s)://` store becomes +`--server --graph `, with the graph id taken from the configured +`WITAN_MEMORY_GRAPH` or written inline as `http://host:8080/graphs/`. + +The bearer token travels with it: a remote store gets the configured token when +the URI names the configured store (the in-cluster case, where +`WITAN_MEMORY_TOKEN` is the pod's only credential) and otherwise inherits the +ambient `OMNIGRAPH_BEARER_TOKEN`. A local store has an ambient token *stripped* +rather than merely unset — it has no server to present one to, and a token +exported for cluster use should not ride into an unrelated subprocess. + +### D2 — Accept an `omnigraph export` JSONL as the merge source + +Any `source` ending `.jsonl` is read as an export rather than re-exported. The +suffix is unambiguous — an omnigraph store is a Lance *directory*, never a file +— so this needs no flag. This is what makes the export, the only transportable +form of a store, a first-class input to the merge. + +It also makes the established file-ingress idiom sufficient. There is no +volume, PVC, or bucket path into the maintenance pods, but there is +`kubectl exec -i`, already proven for exactly this in ol-infrastructure's +storage-format upgrade runbook: + +```bash +kubectl -n witan exec -i job/witan-bg- -- sh -c 'cat > /tmp/alice.jsonl' < alice.jsonl +``` + +### D3 — The resulting supported route + +A user exports locally and hands the file over; an operator streams it into a +break-glass pod and merges. Full procedure in `docs/migration-runbook.md` +§ "Local → shared". Every write lands as `svc-witan-admin`, which is correct: +a bulk store merge is an administrative act, and witan's own `author` field on +each row preserves who actually wrote it. + +For a user with cluster credentials, the same two commands work over a +`kubectl port-forward` to the data tier, with a `--target +http://127.0.0.1:8080/graphs/council`. That is a convenience, not a second +supported path — it needs the actor's own bearer token out of the +`actor-tokens` Secret, which most users cannot read. + +### D4 — Rejected: exposing the data tier + +Putting omnigraph-server behind an authenticated ingress so `--server` works +from a laptop would make this fully self-service. It reverses ADR-0009's +explicit "the data tier is never exposed" and adds a second, +policy-unmediated boundary next to the MCP tier — the same reasoning that +rejected it for witan-code's writes in ADR-0005 (c). Listed here so it is +rejected deliberately rather than forgotten. + +### D5 — The default path: merge through the MCP tier + +**Implemented 2026-08-06, in this change.** The self-service shape mirrors +ADR-0005 (c): mediated store operations through the MCP tier, authorized +per-actor server-side, exactly as `code_store_load` does for the code graph. + +The argument is not convenience. Under D1–D3 every merge lands in the omnigraph +audit trail as `svc-witan-admin`; routing through the MCP tier is what puts +Cedar and the per-request actor in the path, which is the premise of the shared +deployment. It also removes the operator from a step a user should be able to +do alone, and makes "safe to run on a schedule" actionable — under D1–D3 a +scheduled merge has nobody to run it as except the admin principal. + +The split of work is what keeps this small: + +- **`store_merge(rows, dry_run)`** — a real `@mcp.tool`, so every store call in + it goes through the module-level `client`, which re-resolves to *this + request's* actor on each access (`_ActorScopedClient` → `_resolve_client`, + ADR-0004). There is no service account behind it. The server reconciles the + batch against the graph it already holds a client on, and writes the winners. +- **`RemoteServerProxy.merge_store`** — an explicit method, not a + `__getattr__` dispatch, because this is not one tool call: the source is + exported *client-side* (the deployment shares no filesystem with the caller) + and shipped in `chunk_records` batches. The CLI call site is unchanged, so + `witan migrate merge` reads identically in both modes. +- **One reconciliation rule.** Both transports call `_reconcile_nodes`; a row's + fate must not depend on which one carried it. + +The cost quoted before this was built was wrong in a useful way. "A reconciling +client (the merge must *export the target* too, so `load` alone is not enough)" +is true of a bare `load` tool, not of a merge-shaped one: the deployed witan +already holds an `OmnigraphClient` on the target, so the **server** does both +halves and only source rows cross the wire. Batching was the real work, and it +was reused rather than reinvented — `chunk_records` moved from witan-code to +`witan_core.chunking` (same 413 ceiling, same node-before-edge rule), and +`load_batch` moved to the base `OmnigraphClient`. + +`merge_store` therefore leaves `_ADMIN_ONLY`. The in-process function of the +same name stays for D1–D3 and is still not a tool: two transports for one +operation, which is why they share a name and a call site. + +**What this does not do.** `--target` is refused over a deployment — the target +is that deployment's own graph, resolved server-side, since a client never +names a store address (ADR-0005 c). And batches commit independently, so a +failure part-way leaves earlier batches applied; that is recoverable by +re-running rather than atomic, because reconciliation makes a re-sent row lose +to its own already-applied copy. + +## Consequences + +- **`witan migrate merge` reaches a deployed graph.** The in-cluster + break-glass path (ADR-0005 b) is now executable end to end, which it was not + before, and the `witan-break-glass` pod needs no redefinition to support it. +- **The runbook's headline example changed.** `--target s3://witan-shared/…` + described a target nobody outside the cluster can write to; it is replaced + with the export → `kubectl exec -i` → merge procedure. +- **`store_cli_args`/`store_subprocess_env` are public witan-core API.** The + addressing rule was previously private to `OmnigraphClient` and re-implemented + inline in six places; the two that mattered now share one function. The + remaining inline `("http://", "https://", "s3://")` checks are a *different* + predicate (is this store lockable / is it a local path) and were left alone — + note that `OmnigraphClient.is_remote` treats `s3://` as **not** remote while + `maintenance.REMOTE_PREFIXES` treats it as remote, and both are correct for + their own question. +- **Attribution is per-actor on the D5 path, admin-level on the D1–D3 one.** + Through the MCP tier the omnigraph audit trail records the merge as the + calling user's `act-`, and Cedar evaluates it as them. The in-cluster + path still records `svc-witan-admin` — correct for a bulk administrative + act, and the reason D1–D3 is now the fallback rather than the default. +- **`witan_core.chunking` and `OmnigraphClient.load_batch` are shared API.** + Both moved up from witan-code, which now imports them: the merge hits the + same buffered-body ceiling an index does, so the split rule and the + node-before-edge ordering live in one place rather than two. +- **No change to the default path.** With a local store configured, every + command behaves exactly as before — the addressing helper returns the same + `--store ` it always did. diff --git a/docs/explanation/decisions/0008-optional-task-phase-tag.md b/docs/explanation/decisions/0008-optional-task-phase-tag.md new file mode 100644 index 00000000..b611bbaa --- /dev/null +++ b/docs/explanation/decisions/0008-optional-task-phase-tag.md @@ -0,0 +1,168 @@ + + +!!! info "This page lives with the code" + + The authoritative copy is + [`mcp/servers/witan/docs/adr/0008-optional-task-phase-tag.md`](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan/docs/adr/0008-optional-task-phase-tag.md). + +# 8. Optional task `phase` field + per-phase ready-work rollup + +> "Phase tag" is used loosely in the tracking task title; the decision below is a +> typed, optional `phase` **enum field** on `Task` (not a free-form tag) — see +> the Alternatives section for why a field beats a tag value. + +- Status: Proposed (design; implementation deferred) +- Date: 2026-07-08 +- Deciders: witan platform owners +- Tracking: task `tk-optional-task-phase-tag-per-phase-task-rollup-de-c95687`, project `wp-witan-hooks-workflow-ux-progression-improvements-852aaf` +- Supersedes: — +- Related: eval `docs/internals/design/witan-workflow-hooks-elicitation-evaluation.md` §B8; `docs/adr/0003-atomic-task-claims-cas.md` + +## Context + +A `WorkflowProject` moves through phases (`discovery → spec → implementation → +delivery`), and a `WorkflowSession` records the phase it worked in. **Tasks are +phase-agnostic**: `node Task` (`schema/schema.pg:171`) has no `phase` field, and +`insert_task` never sets one. Consequences: + +- Advancing a project's phase (`workflow_project_advance`) has **no effect** on + its tasks — nothing connects "this task belongs to the spec phase" to the + project being in spec. +- There is **no per-phase ready-work rollup**: `task_ready` / the context hook + surface every ready task under a project regardless of which phase it belongs + to, so an agent resuming a project in `implementation` still sees leftover + `discovery`-phase tasks mixed in with no way to scope to "the current phase's + work". + +This is a real ergonomic gap but a **low-severity, additive** one — nothing is +broken, tasks simply can't be sliced by phase. This ADR designs the smallest +change that closes it and records the schema/query/tool implications so the +implementation is a mechanical follow-up. + +### Forces + +- **Additive-only.** Existing tasks (thousands, unscoped-by-phase) must keep + working unchanged; `phase` has to be optional with a null-tolerant default. +- **Consistency with existing enums.** Project/session already use + `enum(discovery, spec, implementation, delivery)` (`schema.pg:85,107`); a task + phase should reuse the exact same vocabulary, not invent a parallel one. +- **Readiness must not regress.** `task_ready`'s core contract (open + all + blockers closed, lease-aware) is shared with the context hook via + `readiness.filter_ready`. A phase filter must be a *narrowing* applied on top, + never a change to the readiness rule itself. +- **omnigraph field addition is a schema migration.** Adding a node field means + `witan migrate schema` must run; reads of old rows must tolerate a missing + `phase` (treated as `None`). + +## Decision + +Add an **optional `phase` field on `Task`**, reusing the project/session enum, +and thread it through create/update/read as a *filter*, not a gate. + +### 1. Schema (`schema/schema.pg`) + +``` +node Task { + ... + priority: enum(p0, p1, p2, p3) @index + phase: enum(discovery, spec, implementation, delivery)? @index // NEW — optional + ... +} +``` + +Nullable (`?`) and `@index` (phase filters are equality scans, matching how +`status`/`priority` are indexed). No new edge — phase is an attribute of the +task, not a relationship; the task→project link (`TaskBelongsTo`) already +carries project membership, and the project already owns the *current* phase. + +### 2. Mutations (`queries/mutations.gq`) + +- `insert_task` gains a `phase` param (nullable), written alongside the other + fields. Old callers that omit it persist `null` — today's behavior. +- A dedicated `update_task_phase` is unnecessary: the generic task update path + (`_update_task`) already writes arbitrary fields, so `task_update` gains a + `phase` argument for free. + +### 3. Server tools (`server.py`) + +- `task_create(..., phase: WorkflowPhase | None = None)` — passes through to + `insert_task`. +- `task_update(..., phase: WorkflowPhase | None = None)` — re-phase a task. +- `task_ready(..., phase: WorkflowPhase | None = None)` — when given, filter the + ready set to tasks whose `phase == phase` **after** `readiness.is_ready` + (narrowing only; the readiness rule is untouched). Tasks with a null phase are + **excluded** from a phase-scoped query but always included in an unscoped one, + so the default surface is unchanged. +- Optional convenience: `phase="__current__"` sentinel, or a separate + `task_ready_for_current_phase(project_slug)` that resolves the project's phase + and filters to it — the "surface this phase's ready work" one-liner. Recommend + deferring this until the plain filter proves useful, to avoid API surface we + might not need. + +### 4. Context hook (`context.py`) + +`inject_context` already knows each active project's current `phase` +(`p['phase']`). Once tasks carry a phase, the "Ready Tasks" section can *prefer* +the current phase: show current-phase ready tasks first (or exclusively, with a +"+N in other phases" note), while null-phase tasks stay visible so nothing is +hidden from the injected block. This is presentation-only and can ship after the +schema/tool change. + +### 5. CLI + +- `witan task create --phase …`, `witan task update --phase …`. +- `witan tasks --ready --phase …` (the `tasks` command's existing `--ready` + path gains a phase filter). +- `witan project tasks --phase …` already filters by `--status`; add `--phase` + symmetrically (see the `project tasks` subcommand added in this project). + +### 6. Migration + +- Bump the bundled schema and require `witan migrate schema` (the existing + path — CodeBranch was added the same way; `context.py` already tolerates a + store that predates a field). Old tasks read back with `phase = None`. +- No data backfill: a null phase is a valid, intended "unphased" state. Agents + can set a phase on new tasks going forward; historical tasks stay unphased. + +## Consequences + +- **Purely additive.** A null `phase` means "unphased", the default for every + existing task and every caller that omits the argument. No existing query, + tool, or CLI behavior changes until a caller opts into `phase`. +- **Readiness stays single-sourced.** The phase filter is a post-`is_ready` + narrowing shared nowhere else, so `task_ready` and the context hook cannot + diverge on *readiness* (the bug B7 fixed); they only differ in whether they + choose to narrow by phase. +- **Phase is advisory, not a state machine.** A task's phase does not gate its + readiness and is not auto-advanced when the project advances — mirroring the + project's own "phases are flexible, not enforced" stance (ADR-less, but see + `_advance_advisory`). Coupling task phase to project phase transitions is + explicitly out of scope. +- **Cost is one migration + a handful of optional params.** No new node or edge + type, no readiness-rule change, no backfill. + +## Alternatives considered + +- **A `phase` *tag* in the existing `tags: [String]?` list** (no schema change). + Rejected: tags are free-form and unindexed for this purpose; a typed enum + field gives validation, an index, and parity with project/session, and avoids + overloading `tags` with a semantically special value. +- **A `PhaseContains: WorkflowProject -> Task` edge** instead of a field. + Rejected: phase is an attribute of the task in the context of its project, not + an independent relationship; an edge adds traversal cost to every ready-work + query for no expressive gain, and a task belongs to exactly one phase at a + time (an attribute, not a many-to-many). +- **Auto-assign a task's phase from the project's phase at creation.** Rejected + as a default: it silently backdates tasks filed for future phases and couples + task creation to project state; leave phase explicit and optional. + +## Rollout + +Design **proposed** here (Status: Proposed — acceptance pending review); +implementation is a follow-up task (schema field + +`insert_task`/`task_create`/`task_update`/`task_ready` params + CLI flags + +migration note + tests for the null-phase-default and phase-narrowing paths). +The context-hook "prefer current phase" presentation is a separate, later slice. diff --git a/docs/explanation/decisions/0009-stateless-mcp-protocol-era.md b/docs/explanation/decisions/0009-stateless-mcp-protocol-era.md new file mode 100644 index 00000000..cc15c5b1 --- /dev/null +++ b/docs/explanation/decisions/0009-stateless-mcp-protocol-era.md @@ -0,0 +1,149 @@ + + +!!! info "This page lives with the code" + + The authoritative copy is + [`mcp/servers/witan/docs/adr/0009-stateless-mcp-protocol-era.md`](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan/docs/adr/0009-stateless-mcp-protocol-era.md). + +# 9. Serving the stateless MCP protocol era (2026-07-28) + +- Status: Accepted +- Date: 2026-07-30 +- Deciders: witan platform owners +- Tracking: project + `wp-mcp-2026-07-28-spec-adoption-across-witan-packag-723f64` +- Related: `docs/adr/0004-keycloak-jwt-per-user-actor-mapping.md` (the per-user + actor mapping this leaves intact); `docs/adr/0005-secure-cli-path-into-deployed-witan.md` + (the CLI-into-deployment path, whose per-connection handshake cost this + removes); + +## Context + +ADR-0004 and ADR-0005 both describe the deployment in terms of the MCP +`streamable-http` transport as it stood in 2025: a client opens a connection, +performs an `initialize`/`initialized` handshake, and is handed an +`Mcp-Session-Id` it must return on every subsequent request. Everything after +that is scoped to the session. + +The 2026-07-28 revision removes that. There is no handshake and no session id: +each request carries its own protocol version and client capabilities in +`params._meta`, and a server answers it in full without reference to anything +that came before. FastMCP 4 negotiates the era per request, so both shapes are +served side by side from one process — a client that sends the 2026-07-28 +envelope is answered statelessly, and one that opens with `initialize` still +gets the handshake era. + +Three consequences land on this deployment specifically. + +**Load balancing.** A session id is affinity: every request in a session has to +reach the replica holding it. Without one, any replica can answer any request, +so the `witan` stack can scale past a single pod behind plain round-robin with +no sticky sessions and no shared session store. + +**Server-initiated requests are gone.** The back-channel that carried +`elicitation/create`, `sampling/createMessage` and `roots/list` from server to +client does not exist in a stateless request/response model. Witan uses +elicitation (confirm a claim steal, confirm a supersede, offer to index, ask for +a repo URI); sampling and roots it never used. Anything that needs input now +returns an `input_required` result and the client retries the same call with the +answer — multi round-trip requests, SEP-2322. + +**Session state has to travel in the request.** Witan's own notion of a +workflow session is unrelated to the protocol's, but it was previously inferred +from the connection. A stateless replica cannot infer it, so the handle travels +as a tool argument. + +### Forces + +- FastMCP 4.0 is still a beta (`4.0.0b1`) at the time of writing; the `mcp` SDK + underneath it is 2.0.0 stable. The pins were widened to + `fastmcp>=3.4.2,<5` rather than moved to 4.x, so the packages resolve to + 3.4.5 for anyone who has not opted into prereleases, while our own lock — and + therefore the container image — runs the beta. Requiring 4.x outright was + tried and backed out; see the last entry under Consequences for why. +- Both witan servers are also used locally over `stdio`, where none of the + above matters. Nothing here may make the local path worse. +- The elicitation contract established when it was added is *additive*: a + client that cannot answer gets the caller's default and the tool proceeds as + it did before elicitation existed. Changing wire mechanism must not change + that. + +## Decision + +**D1. Serve both eras from one process; do not pin one.** FastMCP 4 negotiates +per request, and there is no flag to set — the "stateless mode" this was +originally scoped as does not exist as a switch. Verified by hand against +`witan serve --transport streamable-http`: a `tools/list` carrying +`MCP-Protocol-Version: 2026-07-28` plus the `_meta` envelope returns the full +tool list with no handshake and no `Mcp-Session-Id`, while the same request +without that header is served as handshake-era. + +**D2. Elicitation picks its mechanism per request.** `witan_core.elicit` asks +over MRTR on a 2026-07-28 connection whose client advertises elicitation, over +`ctx.elicit` on the handshake eras, and not at all — returning the caller's +default — when neither is possible. The third arm is load-bearing: under MRTR an +ask a client cannot dispatch fails the *whole tool call*, so the additive +contract only holds if the capability is checked before asking. + +**D3. Anything that must survive a retry travels in the request.** Two things +do. The workflow-session handle is a tool argument, supplied by whichever +process is client-side (`RemoteMCPProxy._resolve_session_slug`). Answers already +collected by a multi-round-trip ask ride in the protocol's `request_state`, and +only once there is more than one — a single-ask tool emits none, which keeps it +independent of the replica that minted it. + +**D4. List results declare a cache TTL.** `tools/list` and friends carry +`ttlMs`/`cacheScope` from 2026-07-28. Both servers declare 300s at `private` +scope (`witan_core.caching`), and the CLI proxy holds its cached tool list for +exactly that long instead of for the process lifetime. + +## Consequences + +- **Multi-replica is unblocked, not enabled.** Nothing here scales the + deployment on its own; it removes the protocol reason it could not. The + remaining per-replica state is the per-actor `OmnigraphClient` cache + (`witan/server.py`), which is keyed by JWT `sub` and rebuilt on a miss — a + cost, not a correctness problem, when requests spread across pods. +- **`request_state` is sealed per-process by default.** The SDK seals it under + an ephemeral key unless the server is constructed with a shared-key + `RequestStateSecurity`. No witan tool asks twice in one call today, so none + emits the field; the first one that does will need that key configured before + it can be served by more than one replica. +- **Background tasks are per-process too.** `code_reindex` accepts + task-augmented execution via the optional `witan-code[tasks]` extra, whose + Docket backend defaults to in-process `memory://` — a task created on one + replica cannot be polled from another. Moot while indexing needs a git + checkout the deployment does not have; a shared `FASTMCP_DOCKET_URL` is the + fix if that changes. +- **The deprecation offramp is 12 months.** `roots`, `sampling`, MCP `logging` + and the legacy HTTP+SSE transport are deprecated as of 2026-07-28. Witan uses + none of the first three. HTTP+SSE was removed from what + `agent-config-kit` advertises; anything still speaking it has until + 2027-07-28. +- **CLI latency improves incidentally.** ADR-0005 records that + `RemoteServerProxy` opens a fresh MCP connection per tool call, so a command + fanning out to several tools pays several handshakes. On a 2026-07-28 + connection there is no handshake to pay for. The deferred persistent-session + spike is correspondingly less urgent. +- **Revisit when FastMCP 4.0 goes GA.** The beta is what the lock and image + currently resolve; CI therefore only exercises the 4.x end of the published + pin range, and the 3.4.5 end has been verified locally only. +- **The straddle stays until GA, and the reason is distribution, not code.** + Everything above needs FastMCP 4, so supporting 3.4.x costs real + version-sniffing shims — `inputSchema` vs `input_schema`, `nextCursor` vs + `next_cursor`, a conditional `mcp_types` import, a signature check before + passing `cache_ttl` — guarding a path CI never exercises, since resolution + only ever installs one major. Requiring `fastmcp>=4.0.0b1` was implemented and + reverted anyway: `uv tool install` and `uvx --from` both refuse to resolve a + pre-release pulled in transitively (fastmcp pins `fastmcp-slim` to its own + exact version) without `--prerelease=allow`, and those are the documented + install paths. `ol-agent-kit` is caught too without being touched, since it + floors `witan-council`/`witan-code` open-ended so new releases are picked up + automatically — publishing would have broken a fresh + `uv tool install ol-agent-kit`. Nor can it be fixed from the publishing side: + `[tool.uv] prerelease` is project-local and never travels in wheel metadata. + `pip install` is unaffected. Tracked in + `tk-move-the-fastmcp-floor-to-4-when-4-0-goes-ga-454f78`. diff --git a/docs/explanation/index.md b/docs/explanation/index.md new file mode 100644 index 00000000..6ef83d06 --- /dev/null +++ b/docs/explanation/index.md @@ -0,0 +1,67 @@ +# Explanation + +Why witan is built the way it is. These pages are for understanding rather than +doing — read them when a design decision seems arbitrary, or when you are about +to work against the grain of one. + +
+ +- **[Architecture](architecture.md)** + + The three layers, the two tiers, and what actually happens when a tool is + called. + +- **[The memory model](memory-model.md)** + + Why memories are typed, why they link, and why superseding is not deleting. + +- **[Coordinating work](task-coordination.md)** + + What a claim guarantees, what it does not, and why the honest answer is + "best effort". + +- **[Code graph](code-graph/symbol-format.md)** + + Symbol identity, edge precision tiers, and how the cross-repo bridge is + stitched together. + +
+ +## Decisions + +The [ADR index](decisions/0001-write-path-content-scanning.md) records +architectural decisions with their context and consequences — including the ones +that turned out to constrain everything after them. + +The ones worth reading first: + +| ADR | Why it matters | +| --- | --- | +| [0001 Write-path content scanning](decisions/0001-write-path-content-scanning.md) | Why every write is scanned, and why it fails closed | +| [0003 Atomic task claims](decisions/0003-atomic-task-claims-cas.md) | The limits of coordination on a store with no conditional write | +| [0005 Secure CLI path into a deployed witan](decisions/0005-secure-cli-path-into-deployed-witan.md) | How the local CLI reaches a shared service | +| [0009 Stateless MCP protocol era](decisions/0009-stateless-mcp-protocol-era.md) | Why the server holds no session state | + +## The idea underneath + +Everything here follows from one observation: **a coding agent's context dies +with its session, and nothing about that is inevitable.** + +The knowledge an agent builds up — why this approach and not that one, which +invariant is load-bearing, what already failed — is exactly the knowledge that +would make the *next* session good. Left in a transcript, it is gone. Left in a +per-agent memory file, it is invisible to your teammates and to the other agent +running in the next terminal. + +So witan makes it a shared, typed, linked graph instead: + +- **Shared**, because the unit that benefits is the team, not the session. +- **Typed**, because "pattern" and "lesson" get read at different moments, and + an untyped note gets read at none of them. +- **Linked**, because knowledge changes, and a store that cannot say *this + replaced that* forces you to choose between losing history and serving stale + facts. + +The work-coordination layer exists for the same reason one step out: once +several agents can act at once, they need a shared answer to "what is being +worked on" — and that answer has to live somewhere neither of them owns. diff --git a/docs/explanation/memory-model.md b/docs/explanation/memory-model.md new file mode 100644 index 00000000..437269ba --- /dev/null +++ b/docs/explanation/memory-model.md @@ -0,0 +1,144 @@ +# The memory model + +## One node type, four kinds + +Every memory is the same node type with a `kind` discriminator: `pattern`, +`project_fact`, `lesson`, `agent_context`. A few fields are populated only for +the relevant kind — `language` for patterns, `category` for project facts, +`severity` for lessons. + +One node type rather than four keeps cross-kind search simple: a single BM25 +index over `content` serves every query, and a read that wants only lessons adds +a filter rather than a different code path. + +The kinds themselves are not decoration. They encode *when* something should +resurface: + +- A **pattern** is read when you are about to write similar code. +- A **project fact** is read when you are orienting in an unfamiliar repo. +- A **lesson** is read when something has gone wrong, or is about to. +- **Agent context** is read by whoever picks up this specific task next, and is + the only kind with a natural expiry. + +An untyped note is read at none of those moments, which is the practical +argument for making the author choose. + +## Slugs are readable on purpose + +``` +pat-always-use-uv +pf-ol-django-vault-secrets +les-no-raw-sql-in-views +ctx-ticket-1234-approach +``` + +A slug is derived from the title with a kind prefix. It is stable, human-typable +in a CLI, and identifies the kind at a glance — which matters because slugs +appear in edges, in task links, and in hook output where there is no room for +more. + +## Edges are the point + +A pile of notes is a search index. What makes this a graph is that memories +relate to each other, and five edge kinds carry those relationships: + +| Edge | Meaning | Effect on reads | +| --- | --- | --- | +| `supersedes` | This replaces that | The superseded one is hidden by default | +| `refines` | This sharpens that, without replacing it | Both surface; this one ranks higher | +| `applies_to` | This pattern/lesson applies in that context | Expansion follows it | +| `contradicts` | These two disagree | **Both** surface, flagged for a human | +| `tagged` | This memory is about that topic | Topic siblings expand together | + +### Superseding is not deleting + +This is the design decision that everything else about reads follows from. + +When knowledge changes, you store the new memory and link +`new --supersedes--> old`. The old memory stops appearing in default reads. It +is **not removed** — `include_superseded=True` still returns it. + +The alternative — editing the old memory in place — destroys the record that the +knowledge ever changed, and with it the answer to "why did we think that?" A +store that cannot distinguish *wrong* from *no longer true* forces a choice +between losing history and serving stale facts. Superseding refuses the choice. + +So the rule is: + +- **The knowledge changed** → store new, link `supersedes`. +- **The record was wrong** — typo, wrong repo, bad tag → `memory_update`. +- **It should never have existed** → `memory_delete`. + +### Contradictions are surfaced, never resolved + +Two memories that disagree both keep appearing, flagged. witan does not pick a +winner and does not hide either. + +That is deliberate. A contradiction usually means a genuine disagreement between +two people, or a fact that changed without anyone superseding the old one — both +of which need a human, and neither of which is improved by an automatic +heuristic silently choosing. The mild ranking penalty +([`WITAN_RANK_PEN_CONTRADICTED`](../reference/environment.md#recall-ranking), +0.25) nudges them down without burying them. + +## Topics: a join surface + +Free-string tags do not connect anything — two memories tagged `vault` share a +string, not an edge. So tags are promoted to `Topic` nodes, and `tagged` is a +real traversable edge. + +Topics come in kinds: `topic` (promoted from a tag), `contract` (whose name is a +bridge key — an env var, endpoint, package, or service), `entity` (a named +service, library, or concept), and `symbol` (reserved; symbols stay soft refs +for now). + +The `contract` kind is the interesting one: it is the join between the memory +graph and the code graph. `memory_for_contract("DATABASE_URL")` returns both the +memories tagged to that contract and the code that provides or consumes it. + +## How `recall` composes all of it + +`recall` exists because doing this well requires five steps, and no agent should +have to remember to run them in order. + +1. **Seed** from any combination of `query` (BM25), `symbol_id`, `task`, or + `topic`. Multiple seeds are a union, which is what makes "what do we know + about this task" a single call. +2. **Expand** one hop — capped at two — across `applies_to` / `related_to` + edges, topic siblings, and provenance siblings (memories produced by the same + session or project). +3. **Prune** superseded memories. +4. **Flag** contradiction pairs. +5. **Re-rank** by a composite score, minus a per-hop distance penalty so seeds + outrank the neighbours they pulled in. + +The composite is BM25 relevance, recency (90-day half-life), corroboration, and +author confidence — each separately weighted and all +[tunable](../reference/environment.md#recall-ranking). Setting every weight to +zero reproduces raw BM25 order, which is the useful thing to do when you suspect +ranking is hiding something. + +**With no edges in the graph, `recall` returns exactly what `memory_search` +would.** Expansion is additive, never lossy. That property is what makes it +safe as the default read from day one, on an empty graph, before anyone has +linked anything. + +## Provenance + +Memories are attributed — an author, a timestamp, and edges back to the session +and project that produced them. `workflow_project_memories` asks what a project +learned; `SessionProduced` edges make "what came out of this session" a +one-hop query. + +Provenance is also a ranking input: memories that emerged from the same piece of +work are treated as siblings during expansion, on the theory that things learned +together are usually relevant together. + +## Repo scoping + +Almost everything is scoped by repo, detected from `.git/config`. Pass `repo` +explicitly to override, or `repo=""` to operate across every repo in the store. + +Scoping is a default rather than a boundary. A pattern learned in one repo is +often exactly what another needs, so cross-repo reads are one flag away — and +topics, contracts, and the bridge are cross-repo by construction. diff --git a/docs/explanation/task-coordination.md b/docs/explanation/task-coordination.md new file mode 100644 index 00000000..d11aa1d0 --- /dev/null +++ b/docs/explanation/task-coordination.md @@ -0,0 +1,143 @@ +# Coordinating work + +Once more than one agent can act at the same time, they need a shared answer to +"what is being worked on". This page is about how witan provides that answer, +and — more importantly — how strong the answer actually is. + +## Ready work is computed + +A task carries `blocked_by`: the slugs of tasks that must close before it can +start. "Ready" is derived from that, not stored: + +> A task is ready when every task blocking it has closed **and** its status +> makes it claimable. + +Claimable is broader than "open". `readiness.status_pickable` treats `open` and +`blocked` alike — a `blocked` task whose blockers have all closed is exactly +the case that should become pickable — and it also returns an `in_progress` +task once its **lease has lapsed**, on the assumption the holder crashed +without releasing it. Only `closed` is never pickable. + +Closing a blocker unblocks its dependents automatically, so the ready list stays +correct without anyone maintaining it. `task_ready` and `witan tasks --ready` +both answer the same computed question. + +The dependency edges are worth using properly: + +| Edge | Meaning | +| --- | --- | +| `blocks` | This must close before that can start | +| `parent_of` | Epic → sub-issue | +| `discovered_from` | This was found while working on that | +| `addresses` | This task acts on that memory | +| `task_belongs_to` | This rolls up to that workflow project | + +`discovered_from` is the one people skip and later wish they had not. Follow-up +work found mid-task is the easiest thing to lose, and the edge preserves the +reason the task exists at all. + +## What a claim guarantees + +Here is the honest version, because a confident wrong answer here causes real +double-work. + +**A claim is an advisory lease with a best-effort compare-and-swap. It is not a +mutex.** + +### Why it cannot be more + +On a local `.omni` store, `OmnigraphClient` serialises writes with a per-store +advisory `flock`, and claiming is effectively safe. + +A shared deployment removes that. A `flock` is a local-filesystem lock; it +cannot coordinate across pods. And omnigraph — this is the crux — offers **no +conditional-write primitive**. There is no `--if-version`, no +`--expected-commit`, and the query engine cannot express a +`... WHERE status = 'open'` guard inside the mutation. You cannot ask the store +to "set this claim only if it is still unclaimed" and have the store reject the +loser. + +So a naive read-check-write lets two agents both read a task as `open`, both +write their own claim, and the last write silently clobbers the first. + +### What witan does instead + +Three mechanisms, reconstructing as much of a CAS as the storage layer permits: + +1. **Conflict-surfacing writes.** `task_claim` opts into raising a typed + `OmnigraphConflict` on a Lance optimistic-concurrency conflict, rather than + retrying. Every other write keeps transparent retry, which idempotent upserts + rely on — but for a claim, a blind retry re-reads the updated state and + re-applies the claim *over* whoever won, turning a should-fail claim into a + clobbering success. + +2. **Conflict-aware retry.** On conflict, `task_claim` re-reads rather than + re-applying. If someone else now holds a live claim, it returns + `{"claimed": false, "reason": "lost_race", "held_by": …}`. If the conflicting + write was unrelated, or the rival's lease has lapsed, it retries — in a + bounded loop that keeps surfacing conflicts, so a consecutive conflict never + falls back to the clobbering path. + +3. **Post-write verification.** Because the last writer still wins, `task_claim` + re-reads after writing and confirms it is still the assignee. A claim that + was overwritten by a rival landing later is reported as `lost_race`, not as a + false success. + +Together these collapse the common double-claim race to at most one +`claimed: true`, and never a silent clobber. + +### The residual window + +A small window remains: if both callers run their verification read after both +writes commit and before either observes the other, both can see the last writer +and one of them is wrong. + +This is not merely theoretical — it has been observed under concurrent write +load. Treat `claimed: true` as **"you almost certainly hold this"**, not as a +hard mutex, and design work so that a brief overlap between two agents is +wasteful rather than destructive. + +A true single-round-trip guarantee needs omnigraph to accept a manifest-version +or commit-id precondition on `mutate`. Until that exists upstream, this is the +ceiling. [ADR 0003](decisions/0003-atomic-task-claims-cas.md) has the full +analysis. + +### Leases + +Claims carry `claimed_at` and expire. A task held by a session that crashed does +not stay held forever, and the lease is also the backstop that recovers any task +that ends up mis-owned through the window above. `task_release` hands one back +deliberately. + +## Projects and sessions + +A `WorkflowProject` tracks an objective across many sessions, through four +phases: `discovery` → `spec` → `implementation` → `delivery`. A project may span +several repos, or none. + +The mechanism that makes this useful is the **session**. `workflow_session_start` +registers a session against a project; `workflow_session_end` records a summary. +That summary is what a *different* session — possibly a different agent, on a +different machine, days later — reads to pick the thread up. + +This is why witan needs no hand-off document. The hand-off is a graph edge. + +`workflow_session_start` is **re-entrant**: calling it again for a still-open +`(project, session_id)` returns the same handle rather than minting a second +node, so a hook retry or a transport reconnect cannot silently duplicate a +session. Two genuinely simultaneous starts can still both insert, and are +de-duplicated immediately afterwards rather than left for a migration to find. + +When a project completes, `workflow_project_complete` assembles a +`WorkflowTrace` — a corpus record built from every contributing session, kept so +that how work actually got done can be mined later. + +## Branch tracking + +`task_claim` and `workflow_session_start` both upsert a `CodeBranch` node linking +the current repo and branch to the task or project in flight. Best-effort, no +command, silent no-op outside a checkout. + +The payoff is that "which branch carries task X" is a one-hop query, and the +session-start hook can tell you the branch you just checked out already has work +in progress against it. diff --git a/docs/getting-started/code-graph.md b/docs/getting-started/code-graph.md new file mode 100644 index 00000000..c6ca98d4 --- /dev/null +++ b/docs/getting-started/code-graph.md @@ -0,0 +1,134 @@ +# Indexing a repository + +The code graph is a tree-sitter index of your repository: every symbol, where it +is defined, and what refers to it. It exists so an agent can ask *"who calls +this?"* or *"what breaks if I change this?"* and get an answer that understands +syntax rather than matching text. + +Understanding syntax is not the same as being right. Call and reference edges +come from heuristic name resolution, not a resolved call graph, so they can miss +a caller or report one that isn't — see [edge precision +tiers](../explanation/code-graph/edge-precision-tiers.md). They beat grep +because they know a definition from a mention, not because they are ground +truth. + +## Build the index + +From inside a checkout: + +```bash +witan code index +``` + +The first run walks the repository, parses every supported file, and writes a +per-repo store. Later runs are incremental — `witan code reindex` forces a full +rebuild when you need one. + +```bash +witan code repos # which repos are indexed, and how fresh +witan code branches # which branch views exist for this repo +``` + +!!! note "Indexing is a CLI job; querying is not" + + The `witan code` CLI **builds and operates** the index — `index`, `reindex`, + `optimize`, `branches`, `cleanup`. The questions you actually want answered + are [MCP tools](../reference/mcp-tools/code.md), called by your agent. There + is deliberately no `witan code find-definition` for you to type: these + queries return graph rows meant to be reasoned over, not read in a terminal. + +## Ask it something + +In an agent session: + +> Where is `resolve_target` defined, and who calls it? + +The agent calls +[`code_find_definition`](../reference/mcp-tools/code.md#code_find_definition) to +get a `symbol_id`, then +[`code_callers`](../reference/mcp-tools/code.md#code_callers) with that id. A +symbol id looks like: + +``` +https://github.com/mitodl/agent-kit#packages/witan-core/witan_core/target_config.py::resolve_target +``` + +— repo, path, and symbol name, which is why it stays meaningful across +checkouts and machines. + +The question worth learning is **blast radius**, before you edit something: + +> What would be affected if I change the signature of `resolve_target`? + +[`code_impact`](../reference/mcp-tools/code.md#code_impact) walks the caller +graph transitively and reports what sits downstream. Doing this *before* an edit +is the single highest-value use of the code graph. + +## Across repositories + +A service-oriented codebase has contracts that no single repository contains: an +env var one service sets and another reads, an HTTP endpoint one serves and +another calls, a package one publishes and another depends on. + +The **bridge store** links repositories by those shared keys, so the graph can +answer questions that span two checkouts: + +``` +code_interface_providers(key="DATABASE_URL") # who defines it +code_interface_consumers(key="DATABASE_URL") # who reads it +code_cross_repo_impact(symbol_id=...) # blast radius across repos +``` + +These only work for repositories that are actually indexed — the bridge joins +what it has. `code_indexed_repos` tells you what that is. + +Indexing already writes the bridge bindings, so there is no extra build step. +Two read-only commands let you inspect what it produced: + +```bash +witan code stitch # print the precise cross-repo edges +witan code stitch --unresolved # external refs with no match yet +witan code symbols --role exported # this repo's public contract surface +``` + +`stitch` computes and prints the join; it stores nothing. +[Stage-2 stitching](../explanation/code-graph/stage2-stitching.md) explains what +that join does and why it is a second pass. + +## Branches + +Each branch gets its own view, so an index built on a feature branch does not +disturb what everyone else reads. + +One rule is worth knowing before you meet it on a shared graph: **there, only +a CI indexer may write a repo's default (`main`) view.** A process that has not +declared `WITAN_CODE_INDEX_ROLE=ci` is refused that write and the stale-file +purge that goes with it, so nobody's reindex can clobber the view every reader +falls back to. + +It does not apply to the local store this tutorial uses. A local store has one +user, who is its writer, so indexing your checkout on its default branch just +works. + +Idle branch views are reaped after 14 days by default. See [Branch +indexing](../guides/branch-indexing.md) and [ADR +0006](../explanation/decisions/0006-code-graph-branch-ownership-and-reaping.md). + +## Keeping it current + +An index is only as good as its freshness. Two mechanisms keep it current +without you thinking about it: + +- **Session hooks** — `witan code session-init` and `reindex-hook` refresh the + index around agent sessions. +- **CI indexing** — a scheduled job sweeps the repos in + `WITAN_CODE_CI_REPOS` and writes the shared `main` view for everyone. That is + the copy your teammates and any deployed witan actually read. + +--- + +That is the tour. From here: + +- [Guides](../guides/index.md) — specific tasks, in depth +- [Reference](../reference/index.md) — every tool, flag, and setting +- [Explanation](../explanation/index.md) — why it is built this way diff --git a/docs/getting-started/first-memory.md b/docs/getting-started/first-memory.md new file mode 100644 index 00000000..6ff317b4 --- /dev/null +++ b/docs/getting-started/first-memory.md @@ -0,0 +1,128 @@ +# Your first memory + +A memory is one durable fact worth keeping past the end of a session. Not a +summary of what you did — a thing that will still be true, and still useful, in +three months. + +## The four kinds + +Every memory has a `kind`, and picking the right one is most of what makes +recall useful later: + +| Kind | What it holds | Example | +| --- | --- | --- | +| `pattern` | A reusable technique or convention | "Use `uv`, never system `pip`, for tooling" | +| `project_fact` | A structural fact about a repo or service | "`ol-django` reads Vault secrets through `hvac`, role from env" | +| `lesson` | A correction, or something that bit you | "`perl -pi` with `\x{…}` re-encodes the whole file's non-ASCII" | +| `agent_context` | What a future agent on *this* task should know | "The retry test is flaky above 8 workers; run it serially" | + +The split matters because reads filter on it. Someone asking "how do we do X +here" wants patterns; someone debugging wants lessons. + +## Write one + +**Writes go through the MCP tools, not the CLI.** The `witan` CLI reads the +graph — it deliberately has no `memory store` command, because the intended +author of a memory is the agent that just learned the thing. + +So ask your agent, in whatever session you are already in: + +> Store a memory: the pattern that this repo's tests must run through +> `just test-`, because the uv workspace shares one `.venv` and running +> pytest directly cross-contaminates sibling packages. + +It will call: + +```python +memory_store( + kind="pattern", + title="Run package tests through `just test-`", + content="The uv workspace shares one .venv, so running pytest directly ...", + tags=["testing", "uv"], +) +``` + +and hand back a slug like `pat-run-package-tests-through-just-a1b2c3`. The +`repo` is filled in automatically from your checkout. + +!!! tip "What makes a memory worth storing" + + Ask whether it would have saved you the last hour. Facts the repository + already records — its structure, its git history, what a function does — + are not worth storing; anyone can read those. What is worth storing is the + thing that was *non-obvious*: why an approach was rejected, which invariant + is load-bearing, what looked correct and was not. + +## Read it back + +```bash +witan memory "test isolation" +``` + +``` + Memory search: 'test isolation' +┏━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃ kind ┃ slug ┃ title ┃ +┡━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ pattern │ pat-run-package-tes... │ Run package tests thr...│ +└─────────┴────────────────────────┴─────────────────────────┘ +``` + +With no query, `witan memory` lists instead of searching, and `--kind` filters +either mode: + +```bash +witan memory --kind lesson # every lesson in this repo +witan memory "vault" --all-repos # search across all repos +``` + +## Why `recall` beats search + +`witan memory` runs a BM25 text search. That finds documents containing your +words. It does not know that one memory replaced another, that two memories +contradict each other, or that the *really* relevant fact is one hop away and +shares none of your vocabulary. + +[`recall`](../reference/mcp-tools/memory.md#recall) is the tool your agent +should reach for instead. In one call it: + +1. **Seeds** from any combination of a text query, a code symbol, a task, or a + topic. +2. **Expands** one hop (up to two) across `applies_to` / `related_to` edges, + topic siblings, and provenance siblings. +3. **Prunes** memories that something else supersedes, so you get the current + version rather than the history. +4. **Flags** contradictions rather than hiding them — a disagreement is + surfaced for a human to resolve. +5. **Re-ranks** everything by a composite of text relevance, recency, + corroboration, and author confidence, with a per-hop penalty so direct hits + still outrank neighbours. + +With no edges in the graph, `recall` returns exactly what `memory_search` would +— expansion is additive, never lossy. So it is always the right default; it just +gets better as the graph fills in. + +## Linking memories + +Edges are what turn a pile of notes into a graph. The one to learn first is +`supersedes`: + +```python +memory_link(from_slug="", to_slug="", kind="supersedes") +``` + +After that link, the old memory stops appearing in default reads — but it is +**not deleted**. It stays in the graph, retrievable with +`include_superseded=True`, so the history of a decision survives. + +This is the correct way to change knowledge that has *changed*. Use +`memory_update` only when a memory was simply *wrong* — a typo'd title, the +wrong repo. The distinction matters: updating destroys the old version, +superseding keeps it. + +The other edge kinds — `refines`, `applies_to`, `contradicts`, `related_to` — +are covered in [The memory model](../explanation/memory-model.md). + +--- + +**Next:** [Tasks and projects →](tasks-and-projects.md) diff --git a/docs/getting-started/index.md b/docs/getting-started/index.md new file mode 100644 index 00000000..71763a6e --- /dev/null +++ b/docs/getting-started/index.md @@ -0,0 +1,68 @@ +# Get started + +Four short pages. By the end you will have witan installed and wired into your +agent, one memory stored and recalled, one task claimed and closed, and one +repository indexed into the code graph. + +They are meant to be read in order — each builds on the last — and the whole +sequence takes about twenty minutes. + +
+ +- **1. [Installation](installation.md)** + + Install `ol-agent-kit`, run `witan setup`, and confirm your agent can see + the tools. + +- **2. [Your first memory](first-memory.md)** + + Store something worth keeping, then get it back — and see why `recall` + returns more than a text search would. + +- **3. [Tasks and projects](tasks-and-projects.md)** + + File work, claim it, and understand what a claim actually guarantees when + two agents want the same task. + +- **4. [Indexing a repository](code-graph.md)** + + Build the code graph and ask it questions grep cannot answer. + +
+ +## Before you begin + +You will need: + +- **Python 3.11 or newer**, and [`uv`](https://docs.astral.sh/uv/). Every + install path here uses `uv`; nothing is installed with system `pip`. +- **A git repository to work in.** witan scopes almost everything by repo, + detected from `origin` in `.git/config`. It works outside a repo, but the + defaults make much less sense. +- **A coding agent** — Claude Code, Pi, GitHub Copilot, OpenCode, or Kilo. + `witan setup` registers itself with whichever it finds. + +You do **not** need a server, a database, or any credentials. The default store +is a single file at `~/.local/share/witan/graph.omni`, and everything in this +tutorial runs against it locally. Pointing witan at a shared, deployed service +is a later, separate step — see [Using a deployed +witan](../guides/deployed-witan.md). + +## A note on where things run + +witan has two faces over the same graph, and both appear throughout these pages: + +- **The `witan` CLI** — what *you* type. Good for browsing, triage, and + operations. +- **The MCP tools** — what your *agent* calls. Where both offer an operation + they share an implementation: the CLI calls the very same functions the MCP + server exposes, so they cannot disagree about what the graph says. + +When a page shows `witan tasks --ready` and then mentions `task_ready`, those +are the same operation from the two sides. + +**The surfaces are not equivalent, though.** Some things are deliberately +MCP-only, because the intended caller is an agent rather than a person: +`memory_store` has no CLI equivalent (the CLI reads memory, it does not write +it), and the code-graph queries are tools only — the `witan code` CLI builds +and operates the index rather than querying it. diff --git a/docs/getting-started/installation.md b/docs/getting-started/installation.md new file mode 100644 index 00000000..e2c3e312 --- /dev/null +++ b/docs/getting-started/installation.md @@ -0,0 +1,107 @@ +# Installation + +## Pick an install shape + +Which one you want depends on whether your agent platform needs `witan` on +`PATH`. + +=== "Claude Code / Pi" + + These shell out to `witan` directly from hooks and extensions, so it has to + stay installed: + + ```bash + uv tool install ol-agent-kit + witan setup --agent claude # or: pi + ``` + + `ol-agent-kit` is a meta-package — it pulls in `witan-council`, + `witan-code`, and the `agent-kit` CLI together. To install only the + coordination graph without the code index, use `uv tool install + witan-council` instead. + +=== "Copilot / OpenCode / Kilo" + + These launch the MCP server via `uvx` on demand, so nothing needs to remain + installed: + + ```bash + uvx --from ol-agent-kit witan setup --agent copilot # or: opencode | kilo + ``` + +=== "Tracking unreleased code" + + To run ahead of the latest PyPI release, install from the repository: + + ```bash + uv tool install \ + "git+https://github.com/mitodl/agent-kit#subdirectory=mcp/servers/witan" + witan setup --agent claude + ``` + +!!! tip "`witan-council`, not `witan`" + + The PyPI project is named `witan-council` — `witan` was already taken. The + import path, the console command, and every tool and CLI name are still + `witan`. Only the install artifact's name differs. + +## What `witan setup` does + +Four things, in order: + +1. Downloads the pinned `omnigraph` binary to `~/.local/bin/omnigraph`, unless + it is already there. witan shells out to this binary for every graph + operation, so it must be present. +2. Writes a starter `~/.config/witan/config.toml`, with every optional setting + commented out at its real default — unless a config already exists, which is + never overwritten. +3. Copies the bundled skills and hooks into the agent's config directories + (`~/.claude/skills/`, `~/.claude/hooks/`, and equivalents). +4. Merges the witan MCP server entry into that agent's config file. + +Useful flags: + +```bash +witan setup --dry-run # show what would change, write nothing +witan setup --author "Your Name" # set graph attribution up front +witan setup --agent all # register with every detected platform +``` + +!!! warning "Re-run it after every upgrade" + + Steps 3 and 4 copy files rather than linking them, and step 1 pins a + specific `omnigraph` version. Upgrading the package does not refresh any of + that on its own — re-run `witan setup` so the installed skills, hooks, and + binary match the version you just installed. + +## Verify + +```bash +witan --version +witan tasks +``` + +From inside a git repository, `witan tasks` should print an empty ready-work +list rather than an error. If it complains about the store, check +[`WITAN_MEMORY_URI`](../reference/environment.md) — by default the graph lives +at `~/.local/share/witan/graph.omni` and is created on first use. + +To confirm your *agent* can see the tools, start a session and ask it to call +`recall`. In Claude Code, `/witan-task` also lists the task tools if the skills +installed correctly. + +## Attribution + +Every node you create records an author. It resolves in this order: + +1. [`WITAN_AUTHOR`](../reference/environment.md) +2. `author` in `~/.config/witan/config.toml` +3. `git config user.name` +4. `$USER` + +Worth setting deliberately if you share a store with a team — it is how anyone +later works out who recorded a lesson, and who is holding a task. + +--- + +**Next:** [Your first memory →](first-memory.md) diff --git a/docs/getting-started/tasks-and-projects.md b/docs/getting-started/tasks-and-projects.md new file mode 100644 index 00000000..814e255b --- /dev/null +++ b/docs/getting-started/tasks-and-projects.md @@ -0,0 +1,143 @@ +# Tasks and projects + +Two things live at this layer, and they answer different questions: + +- **Tasks** — *what needs doing.* Discrete units of work, with dependencies, + priorities, and an owner while someone is on them. +- **Workflow projects** — *what we are trying to achieve.* An objective that + spans many sessions and possibly several repos, moving through phases. + +You will use tasks constantly and projects occasionally. + +## File a task + +```bash +witan task create "Retry logic drops the last attempt's error" \ + --type bug --priority p1 \ + --description "The final exception is swallowed, so a permanent failure looks like a timeout." +``` + +``` +Created task: tk-retry-logic-drops-the-last-attempt-s-e-4f9c21 + status: open + repo: mitodl/agent-kit +``` + +The `repo` is auto-detected. Useful flags: + +| Flag | Effect | +| --- | --- | +| `--type` | `bug`, `feature`, `task`, `chore`, `epic` | +| `--priority` | `p0` (highest) through `p3` | +| `--parent` | Roll this up under an epic | +| `--blocked-by` | `tk-` slugs that must close before this is ready | +| `--discovered-from` | The task you were on when you found this | +| `--project` | The `wp-` project this belongs to | +| `--symbol-refs` | Code-graph symbols (`repo#path::Name`) this concerns | + +`--discovered-from` is worth the habit. Follow-up work found mid-task is the +easiest thing to lose, and the edge records *why* the task exists. + +## Find work + +```bash +witan tasks # open tasks in this repo, by priority +witan tasks --ready # only those with no open blockers +witan tasks --all-repos # across every repo in the store +``` + +"Ready" is computed, not stored: a task is ready when every task it is +`blocked_by` has closed *and* its status still makes it claimable. Closing a +blocker automatically unblocks its dependents, so the ready list stays correct +without anyone maintaining it. + +Claimable covers more than `open`: a `blocked` task counts once its blockers +close, and an `in_progress` task comes back when its claim lease lapses — that +is how work abandoned by a crashed session returns to the list rather than +being held forever. + +## Claim, work, close + +```bash +witan run tk-retry-logic-drops-the-last-attempt-s-e-4f9c21 +``` + +This claims the task under your author name and launches your configured agent +with a prompt seeded from the task's title, description, and symbol refs. To see +that prompt without doing anything: `--dry-run`. To launch without claiming: +`--claim=false`. + +From inside an agent session, use the tools directly — `task_claim`, then +`task_close` with a resolution: + +```bash +witan task close tk-retry-logic-... --resolution "Re-raise the final exception; test added" +``` + +### What a claim actually guarantees + +This is worth being precise about, because the answer is "less than you might +assume". + +A claim is an **advisory lease with a best-effort compare-and-swap**, not a +lock. On a local store, writes are serialised by a file lock and a claim is +effectively safe. Against a **shared, deployed** store there is no such lock, +and omnigraph offers no conditional-write primitive — you cannot ask the store +to "set this claim only if it is still unclaimed" and have it reject the loser. + +witan reconstructs as much of that as it can: it detects the lost-race conflict +and surfaces it rather than retrying over the winner. But under genuine +concurrent write load, mutual exclusion has been observed to fail. Treat a claim +as *coordination* — a strong signal that someone is on this — rather than as a +correctness guarantee. Design work so that two agents briefly overlapping is +wasteful, not destructive. + +[ADR 0003](../explanation/decisions/0003-atomic-task-claims-cas.md) records the +full reasoning and the exact limits. + +Claims also carry a **lease expiry**, so a task held by a session that died does +not stay held forever. `witan task release` hands one back deliberately. + +## Multi-session projects + +When work will not finish in one sitting, create a project instead of holding +the thread in your head: + +```bash +witan project create "Migrate auth to OAuth2" --phase discovery +``` + +Projects move through four phases — `discovery` → `spec` → `implementation` → +`delivery` — via `witan project advance`. + +The part that makes them worth using is **session linking**. At the top of each +session, `workflow_session_start` registers that session against the project; at +the end, `workflow_session_end` records a summary. That summary is what a +*different* session, or a different agent, reads to pick up the thread — which +is why the loop works without an explicit hand-off document. + +```bash +witan projects # active projects for this repo +witan project status wp-... # phase, sessions, last summary +witan project tasks wp-... # the tasks rolling up to it +``` + +`witan project complete` closes a project out and assembles a `WorkflowTrace` — +a corpus record built from every session that contributed, kept for later +pattern mining. + +!!! tip "Let the skills drive this" + + `/witan-task` and `/witan-workflow` automate the picking and linking. The + CLI shown here is what they call underneath, and what you want for triage. + +## Branch tracking, for free + +`task_claim` and `workflow_session_start` both quietly upsert a `CodeBranch` +node linking your current repo and branch to the task or project in flight. No +command, no configuration — it just means "which branch carries task X" is a +one-hop query later. It no-ops silently outside a git checkout. + +--- + +**Next:** [Indexing a repository →](code-graph.md) diff --git a/docs/guides/branch-indexing.md b/docs/guides/branch-indexing.md new file mode 100644 index 00000000..0f615cb1 --- /dev/null +++ b/docs/guides/branch-indexing.md @@ -0,0 +1,347 @@ + + +!!! info "This page lives with the code" + + The authoritative copy is + [`mcp/servers/witan-code/docs/BRANCH_INDEXING.md`](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan-code/docs/BRANCH_INDEXING.md). + +# Branch-aware indexing — omnigraph branches mirror git branches + +Status: per-repo branching implemented (2026-07-05); bridge overlay +implemented (2026-07-06); CodeBranch↔task linking (witan, Layer 1) +implemented (2026-07-06); per-writer view namespacing implemented +(2026-07-31) +Related: [SYMBOL_FORMAT.md](../explanation/code-graph/symbol-format.md), [PACKAGE_MAP.md](../explanation/code-graph/package-map.md) + +Today every index write lands on the store's default branch regardless of the +git branch checked out: an agent working on a feature branch overwrites the +`main` view of the repo, and a second agent (or the same user in another +worktree) sees half-in-flight symbols with no way to tell. Omnigraph has +native branching (`branch create/list/delete/merge`, `--branch` on +query/mutate, `load --branch --from main` fork-on-first-write), so the fix +is structural: **index a git branch onto the omnigraph branch of the same +name**. + +## Per-writer branch views + +A git branch is not a unique key for an index. Two checkouts on `feature-x` +— two developers, one developer in two worktrees, an agent and its human — +are two working trees, and on a shared cluster graph a view named for the +branch alone means the second writer overwrites the first with a different +uncommitted state. The symptom is the class of confidently-wrong answer PR +\#157 fixed for nested checkouts: a symbol resolving to somebody else's WIP. + +So the writer is part of the name. One scheme covers both stores +(`witan_code/views.py`): + +```text +per-repo graph: [/] +bridge graph: [/]/ +``` + +The actor comes first in both, so **ownership is a prefix**: "may I write +this view" is one string comparison, whichever store it is looking at, and +the stale-view reaper can sweep by owner. + +That prefix is enforced **client-side only**, which was not the original +intent. omnigraph 0.8.1 compiles a policy-bundle rule to a bare +`permit(principal in Omnigraph::Group::…)` with no `when {}` clause, and its +only branch predicate is the three-valued protected/unprotected scope — there +is no branch-name pattern and no `principal.actor` to compare against, so the +hoped-for `startsWith(branch, principal.actor + "/")` rule cannot be written. +Cedar therefore enforces main-vs-WIP and `graph.owns_view` enforces +writer-vs-writer; a client that ignores the guard gets past the second and +not the first. See witan `docs/adr/0006-code-graph-branch-ownership-and-reaping.md`. + +`` is the ADR-0004 `act-` id — the same derivation the deployed +server uses (`witan_core.identity.derive_actor_id`), resolved client-side +from the `witan login` session (`witan_code/identity.py`), never from +`$USER`. It is absent when this process has no identity to name, which is the +normal case for purely local use: existing local stores keep the names they +have, and indexing offline needs no login. + +**Isolation and visibility are not in tension.** Each view has exactly one +writer; every view is readable by everyone. `code_indexed_branches(branch=…)` +(CLI: `witan code branches --branch `) lists every writer's view of a git +branch, and any listed view name can be passed straight back as `branch=` to +`code_find_definition` / `code_search_symbol` / `code_symbols_in_file`. That +cross-agent visibility is why branch views live on the shared graph at all +(DECIDED, 2026-07-31) rather than in per-user local stores. + +**Ownership gates both destructive operations.** `graph.owns_view` is the one +predicate behind both `check_writable` (may I write this view) and +`indexer._may_purge` (may I delete rows from it): a local store has one user +who is its writer; CI owns the shared default view +(`WITAN_CODE_INDEX_ROLE=ci`); every actor owns its own branch views. The +earlier rule — "remote and not the designated writer" — got the first two +right and the third wrong, refusing a developer the purge of their own view, +where files they had deleted therefore lingered. + +## Per-repo stores + +* Git default branch (`main`/`master`) → omnigraph `main` (unchanged). +* Any other git branch → the writer's view `[/]sanitize_branch()` on that repo's store, forked from `main` on first write + (`load --branch --from main`). The fork means the branch starts as a + full copy of the `main` view; the incremental indexer then rewrites only + the files that differ — exactly the delta the git branch carries. +* Branch detection: `git rev-parse --abbrev-ref HEAD` in `repo.py` + (worktrees resolve per-worktree, which is precisely what parallel agents + need). Detached HEAD indexes to `main` behavior? No — detached HEAD writes + to a `_detached` scratch branch so it can never corrupt `main`. +* Reads (`code_*` MCP tools, CLI) default to the current checkout's branch — + to *this* actor's view of it first, then any other writer's, then `main`. + The fallback to a colleague's view is deliberate: before you have indexed a + branch yourself, the closest thing to "the code on feature-x" is whatever + view of it exists, and reading is not the operation that needs an owner. + Tools take an optional `branch`, which accepts either a git branch name + (resolved the same way) or a full view name like `act-bob/feature_x` to + inspect one specific agent's in-flight view — that is the cross-agent + visibility payoff. + +### Lifecycle + +Branch stores are re-derivable caches, so lifecycle is deletion, not merge: + +* When the git branch merges to the default branch, the post-merge index of + `main` already reflects the result — the omnigraph branch is simply + deleted. `omnigraph branch merge` is not used for index data (re-indexing + is cheaper and always consistent; merging stale Lance rows is neither). +* `witan-code branches [--prune]` lists omnigraph branches per store and + deletes those whose git branch no longer exists (checked against + `git for-each-ref`), plus `_detached`. Pruning is a **local-store** + operation on both counts: it is refused in remote MCP-client mode (ADR + 0005) and refused against a remote store (below). On a shared graph one + machine's missing git ref is not evidence a branch is dead — it would + delete another user's in-flight view. + +## Who may write the shared default-branch view + +On the deployed omnigraph cluster a per-repo code graph is **one graph for +the whole team**, so `main` — the view with no branch scoping, which every +reader falls back to — has no natural owner. It gets one explicitly: **CI +indexes the default branch, everyone else reads it.** + +Writer authority is a **role, not a transport**. "Refuse writes when the +store is remote" cannot be applied unconditionally, because the CI indexer is +remote too and is the one actor that must write. So the role is declared: + +```shell +WITAN_CODE_INDEX_ROLE=ci # or index_role = "ci" on a [targets.] block +``` + +The role only bites once the store actually *is* a shared graph, which is what +`code_server` (env `WITAN_CODE_SERVER`) or `code_transport = "mcp"` makes it — +see [Shared cluster graphs](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan-code/README.md#shared-cluster-graphs). Without +either, every graph is a local directory with one user, and every row of the +table below is the "Local store" column. + +A write that travels through the deployed MCP endpoint +([Writing through the MCP tier](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan-code/README.md#writing-through-the-mcp-tier)) is +checked twice: once locally, as a fast-fail, and once by the deployment +against the actor in the caller's JWT — which is the check that counts, since +it is the only one the client cannot influence. There the role is the +*deployment's*, not the caller's, so the `ci` column below is unreachable +through that route by design: the CI indexer writes in-cluster. + +Values are `client` (the default — reads the shared view, never writes it) +and `ci`. An unrecognized value is an error rather than a silent +demotion to `client`, which would leave the shared view frozen with nothing +to explain it. + +What the role gates (`witan_code.graph.check_writable`, +`witan_code.indexer._may_purge`): + +| Write | Local store | Shared graph, `client` | Shared graph, `ci` | +| --- | --- | --- | --- | +| default-branch (`main`) view | ✅ | ❌ refused | ✅ | +| own branch view (`act-me/`) | ✅ | ✅ | ✅ | +| another actor's branch view | ✅ | ❌ refused | ❌ refused | +| stale-file purge | ✅ | ✅ own view only | ✅ | +| `branches --prune` | ✅ | ❌ | ❌ | +| `reap-views --apply` | ✅ | ❌ refused | ✅ | + +Local stores are unaffected by the role: they have one user, who is their +writer. + +Note the role governs only the *default* view. Branch views are governed by +their name (§ Per-writer branch views): nobody, including CI, may write a +view prefixed with someone else's actor, and a branch view with no owner at +all is refused on a shared graph — that un-owned name is exactly the +collision this replaced. + +### Per-user branch views live on the shared graph + +**Decided (2026-07-31).** In-flight branch views go on the shared cluster +graph, not in a local store queried alongside it. Isolated agents being able +to see each other's work *as it is being made* — rather than after a merge — +is a large part of what the shared service is for, and that only works if +the views are somewhere every reader can reach. + +Branch views are therefore exempt from the default-view *role* gate above: a +branch-scoped write cannot reach the view everyone falls back to, so it needs +no role. Anyone may write their own — and only their own. + +Both things this decision required are **implemented** (2026-07-31): + +1. **Branch views are namespaced per writer** — `[/]`, see + § Per-writer branch views. Reads can enumerate every view of a git branch + (`code_indexed_branches(branch=…)`) and query any of them by name, so + isolation did not cost visibility. + +2. **Stale-file purging follows view ownership.** `_may_purge` and + `check_writable` share one predicate, `graph.owns_view`: "CI owns `main`" + and "I own my own branch view" are its two shared-graph cases. A developer + purging their own view is now allowed, which it was not while views had no + single authoritative writer. + +Reaping is consequently **server-side and mandatory**, not a client +convenience: branch sprawl is real under this decision, and no client can +tell whose branch views are still live — which is why `branches --prune` is +refused against a remote store above and stays that way. + +### Reaping stale views + +`witan-code reap-views` is that sweeper (witan `docs/adr/0006`). It is a +different question from `branches --prune`, not a wider-scoped version of it: + +| | `branches --prune` | `reap-views` | +| --- | --- | --- | +| asks | does *this checkout* still have the git branch? | how long since anyone wrote this view? | +| authority | this machine's git refs | the store's own commit log | +| shared graph | refused, always | the only place it makes sense | +| runs as | the user | `WITAN_CODE_INDEX_ROLE=ci` | + +```bash +witan-code reap-views # report every local store +witan-code reap-views --store https://… --graph code-x --apply # the scheduled job +``` + +Idleness comes from `omnigraph commit list --branch`, filtered to commits whose +`manifest_branch` is the view itself — `branch list` returns bare names, with no +date, owner, or size. Two rules follow: + +* **`main` is never reaped**, however idle. It is the view every reader falls + back to and is idle by design between merges. +* **A view with no commits of its own is never reaped.** It holds nothing that + isn't already on its fork point, and with no branch-creation timestamp + anywhere, one made ten seconds ago is indistinguishable from one made a year + ago — reaping it would race the indexer that just created it. + +The window is `WITAN_CODE_VIEW_MAX_IDLE_DAYS` (default 14; `0` disables). +Idleness is not abandonment: a branch parked past the window and picked back up +loses its view, not its work, and the next index rebuilds it. Reporting is the +default and `--apply` deletes, because the window is the one input nothing +inside the store can validate. + +## Bridge store + +Bridge bindings from an in-flight branch must not pollute the shared `main` +cross-repo view, but a branch view should still see every *other* repo's +`main` bindings. Omnigraph branch forking gives this overlay for free — the +subtlety is naming: branch names collide across repos (`feature-x` in two +repos), so bridge branches are **repo-qualified**: + +```text +bridge branch = [/]/ +``` + +forked from the bridge `main`. The repo qualifier keeps `feature-x` in two +repos apart; the actor qualifier keeps `feature-x` in two *checkouts of one +repo* apart. Both are needed and they compose in that order — actor first, so +ownership stays a prefix of the name in both stores. + +Writes for repo R on git branch B go to that branch +(`bridge.write_bindings`'s `branch`/`actor` parameters, composed internally by +`views.bridge_view`); `code_cross_repo_impact`/`code_interface_*` auto-detect +the current checkout's repo+branch and read this actor's overlay of `R/B` +when it exists, then any writer's, else `main` — +so an agent working on branch B sees its own in-flight bindings overlaid on +everyone else's `main`. Because the branch is forked once (on first write) +rather than kept continuously in sync, an overlay's view of *other* repos +can go stale relative to their current `main` if they're reindexed while +this branch is still open — the same re-derivable-cache tradeoff already +accepted for per-repo `main` (see Lifecycle above); prune/re-fork on the +next index resolves it. Bridge branch pruning rides the same +`branches --prune` sweep locally, and the same `reap-views` sweep on the +cluster — the reaper sweeps the bridge store alongside the per-repo ones, and +the bridge's Cedar bundle grants users `change`/`branch_create` on unprotected +branches for exactly this write path (witan `docs/adr/0006` D4). The CLI +(`witan code deps`/`stitch`/`symbols`) does not yet follow this — it always +reads bridge `main`. + +## Linking code branches to projects and tasks (witan graph) + +Branch-aware stores answer "what does branch B look like"; the +work-coordination graph should answer "*why* does branch B exist and who is +on it". This linkage lives in **witan, not witan-code**: it is coordination +state that must be shared and durable, while witan-code stores are local +re-derivable caches that `branches --prune` may destroy at any time. The +coupling stays one-way via soft references — the same pattern as +`Task.symbol_refs` — with git as the shared vocabulary: `CodeBranch` +references the **raw git branch name** (`feature/new-api`), never +witan-code's sanitized omnigraph branch name (`feature_new-api`), which is a +storage detail that must not leak into the witan schema. Consumers sanitize +at the edge before calling `code_*` tools with `branch=…`. New node + edges +in the witan (Layer 1) schema: + +```graphql +node CodeBranch { + slug: String @key // "|" + repo: String @index + branch: String @index + status: enum(active, merged, abandoned) @index + created_at: DateTime + updated_at: DateTime +} + +edge WorksOn: CodeBranch -> Task +edge ForProject: CodeBranch -> WorkflowProject +``` + +* `workflow_session_start` records the session's git branch and upserts the + CodeBranch + `ForProject` edge automatically — no manual bookkeeping. +* `task_claim` on a repo checkout upserts `WorksOn` from the current branch, + so "which branch carries task X" and "which tasks are in flight on branch + B" are single-hop queries. +* The context-injection hook can then surface, at session start: *"branch + feature-x is linked to task tk-… (claimed by session s-…)"* — in-flight + work becomes visible before an agent duplicates it. +* Status transitions ride the prune sweep: git branch gone + task closed → + `merged`; git branch gone + task open → `abandoned` (a signal, not a + cleanup). + +### From a task to the code on its branch + +The two layers agree on the branch, and only on the branch — deliberately. +`CodeBranch` keys on `(repo, raw git branch)`; a code-graph view keys on +`(actor, sanitized branch)`. Neither holds the other's key, so "show me the +code as it stands on the branch carrying task X" is a mechanical two-step +rather than an edge across stores: + +1. `task_get` → `CodeBranch.branch` (the raw name, e.g. `feature/new-api`). +2. `code_indexed_branches(branch="feature/new-api")` → every writer's view of + it (`act-alice/feature_new-api`, …). Pass one back as `branch=` to any + name-routed `code_*` tool. + +Sanitizing is one-way, so the hop only works in this direction — which is the +direction that has an authority for the raw name. A `CodeBranch` may have +several views (one per writer on that branch, plus a possible un-owned local +one); when the task names a claimant, prefer that actor's view. + +## Implementation order + +1. ✅ `OmnigraphClient` grows a `branch: str | None` (adds `--branch`, and + `--from main` on `load`); `repo.py` grows `current_branch()`. +2. ✅ Per-repo indexer + `code_*` read tools honor the branch with + `main` fallback; `witan-code branches --prune`. +3. ✅ Bridge writes/reads use repo-qualified branch overlay. +4. ✅ witan-graph `CodeBranch` node + `WorksOn`/`ForProject` edges, wired into + `workflow_session_start` / `task_claim` / the context hook (lives in + witan, not witan-code — see witan's README § Code Branch Tracking). +5. ✅ Views namespaced per writer (`views.py`, `identity.py`); one ownership + predicate behind `check_writable` and `_may_purge`; + `code_indexed_branches(branch=…)` enumerates every writer's view. diff --git a/docs/guides/deployed-witan.md b/docs/guides/deployed-witan.md new file mode 100644 index 00000000..80c77f74 --- /dev/null +++ b/docs/guides/deployed-witan.md @@ -0,0 +1,302 @@ + + +!!! info "This page lives with the code" + + The authoritative copy is + [`mcp/servers/witan/docs/deployed-witan-onboarding.md`](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan/docs/deployed-witan-onboarding.md). + +# Pointing your CLI and agent at the deployed witan + +How to stop using your local `~/.local/share/witan/graph.omni` and start using +the shared, deployed service — so your agent sessions read and write the same +graph as everyone else's. + +This is one half of the cutover. The other half is +[migrating the history you already have](migration-runbook.md#local-shared-the-cutover); +they should be sequenced together, because a store you keep writing to after +its export was taken is a store whose tail nobody will merge. + +## What you are switching to + +| | Local (default) | Deployed | +|---|---|---| +| Store | `~/.local/share/witan/graph.omni` on your disk | one shared graph in the cluster | +| Reached via | the `omnigraph` binary, directly | an MCP call to `witan[.].ol.mit.edu` | +| Identity | your `author` string, honour-system | a Keycloak JWT → `act-` → your own omnigraph token | +| Authorization | none | Cedar policy bundles, per actor | +| Who else sees it | nobody | the team | + +The switch is opt-in and per-config: with `remote_url` unset the CLI runs +exactly as it does today. See +[ADR-0005](../explanation/decisions/0005-secure-cli-path-into-deployed-witan.md) for the design. + +## Prerequisites + +- `witan` on `PATH` (`uv tool install witan-council`), version new enough to + have `witan target` — check with `witan target --help`. That is the newest + of the commands below (witan-council 0.11.0), so a CLI that passes this + check has all of them; checking `witan login` instead would let an older + CLI through, to fail at step 1 with an unknown command. +- A Keycloak account in the `ol-platform-engineering` realm, enabled. The + hourly `witan-token-sync` job mints an actor entry for every enabled realm + user, so if you can log in to other OL services you almost certainly already + have one. +- A browser you can reach from wherever you run the CLI. The device-code flow + is designed for this to work over SSH — you approve on any device. + +## 1. Register the target + +`witan target add` writes the config for you. Start against CI, which is the +recommended way in: + +```bash +witan target add ol \ + --remote-url https://witan.ci.ol.mit.edu/mcp \ + --oidc-issuer https://sso-ci.ol.mit.edu/realms/ol-platform-engineering \ + --oidc-audience witan \ + --match-orgs mitodl +``` + +Drop the `ci`/`-ci` from the two hostnames for production. `oidc_client_id` +defaults to `witan-cli`, the public client registered for the device grant; you +only pass `--oidc-client-id` if that changes. + +**The issuer is checked before anything is written.** `target add` fetches the +issuer's `.well-known/openid-configuration` and confirms the document advertises +the issuer you gave it, so a typo is an error about the issuer, right here — +rather than a confusing auth failure at step 2, which is where it used to +surface. Pass `--no-verify` if you are registering offline and want to skip the +check. + +A named target beats exporting env vars: it scopes the deployment to the repos +it actually covers, survives the shell, and routes **both** `witan` and +`witan-code` at once (they share one endpoint and one token cache — there is no +separate `WITAN_CODE_REMOTE_URL`). + +`witan target list` shows what is configured and marks with `*` the target in +effect for the current checkout; `witan target remove ol` deletes the block +again. Re-running `target add` with an existing name refuses rather than +overwriting — pass `--force` to replace it in place. + +
+What that writes, if you would rather edit the TOML by hand + +```toml +[targets.ol] +remote_url = "https://witan.ci.ol.mit.edu/mcp" +oidc_issuer = "https://sso-ci.ol.mit.edu/realms/ol-platform-engineering" +oidc_audience = "witan" +match_orgs = ["mitodl"] +``` + +…and for production, the same block with `witan.ol.mit.edu` / +`sso.ol.mit.edu`. Hand-editing `~/.config/witan/config.toml` still works +exactly as before; the command is a convenience, not a new format. +
+ +`match_orgs` is what makes this safe to leave in place: outside a `mitodl` +checkout the target doesn't match, and the CLI keeps using your local store. +`match_paths` (checkout prefixes), `match_repos`, and `match_hosts` are the +other selectors — see the `load()` docstring in `witan/config.py` for the +precedence order. To force a target regardless, `WITAN_TARGET=ol witan …`. + +Note the corollary: a target with **no** `match_*` selectors never selects +itself, so it is only ever reached explicitly. **Export `WITAN_TARGET=ol` for +that case** — the read and write commands (`witan tasks`, `witan memory`, …) +resolve their target from the environment and the checkout, and take no +by-name flag. `WITAN_TARGET` covers every command; the by-name flags cover +`login`/`logout`/`whoami` (`--target `) and `witan migrate merge` +(`--from `/`--to `, which name the two ends of a merge). + +Selector precedence is by **specificity, not file order**: every target's +`match_paths` is checked before any `match_repos`, then `match_hosts`, then +`match_orgs` (`witan_core.target_config.match_target`). A `match_paths` target +at the bottom of the file therefore beats a `match_orgs` target at the top; +position only breaks ties *within* one tier. + +## 2. Log in + +```bash +witan login --target ol +``` + +This runs the OIDC device authorization grant: it prints a URL and a user code, +you approve in a browser, and the resulting token is cached at +`~/.config/witan/tokens.json` (mode `0600`), keyed by `(issuer, client_id)` so +several deployments don't clobber each other. It refreshes automatically; you +should not need to run this again until the refresh token expires. + +`--target` is accepted by `login`, `logout`, and `whoami`. Inside a `mitodl` +checkout the `match_orgs` above already selects the target, so you can leave it +off — but it is always correct, and it is *required* for a target with no +`match_*` selectors. (`witan target add --login` runs this step for you +immediately after registering.) + +```bash +witan whoami --target ol +``` + +Confirm the endpoint, your username, and — the part worth actually reading — +your `actor`. It is `act-`, **not** `act-`. That +uuid is what appears in the Cedar policy logs and in the `actor-tokens` map, so +it is the string to quote when asking why a write was refused. + +`witan logout` clears the cached token. + +## 3. Verify reads, then a write + +```bash +witan tasks --all-repos | head # a read through the deployment +witan memory "cedar" --all-repos # BM25 search, server-side +``` + +If these return the team's data rather than yours alone, the read path works. +Then check a write actually lands — this is the step that exercises the whole +ADR-0004 chain (JWT → actor → that actor's own omnigraph bearer token → Cedar): + +``` +memory_store(kind="lesson", title="onboarding probe", content="delete me") +``` + +…from an agent session, then `witan memory --kind ` to confirm, and +`memory_delete` to clean up. A write that returns a Cedar denial rather than a +slug means your actor has a token but no policy grant — quote the `act-…` from +`witan whoami` when reporting it. + +## 4. Point your agent at it + +The MCP server your agent launches reads the same `config.toml`, so once step 1 +is in place, `witan setup --agent claude` (or `pi`/`copilot`/`opencode`/`kilo`) +is all that is needed — no separate MCP-level configuration. Re-run `witan +setup` after upgrading. + +Verify from inside a session: the context hook's output should show tasks and +projects that other people created. + +## What happens when the deployment is unreachable + +**It hard-fails. There is no fallback to your local store, by design.** + +A configured-but-unreachable remote fails the command you ran, naming the +endpoint and saying so out loud: + +``` +The deployed service at https://witan.qa.ol.mit.edu/mcp could not be reached: +Client failed to connect: All connection attempts failed. witan does not fall +back to your local store — falling back silently would split your memory across +two graphs with no signal that it happened, leaving a merge nobody knew to run. +Check the endpoint is reachable and that your session is still valid (`witan +whoami`, then `witan login`), or unset `remote_url` on target [qa] to work +against your local store on purpose. +``` + +The CLI does not quietly serve you a different graph. This is deliberate — a +silent fallback would split the corpus in two, writing some sessions' work to +the shared graph and some to a local one with no signal that it happened, and +the two would then have to be reconciled by a merge that nobody knew to run. + +`witan-code` prints the same shape for its own reads, with its own reason: an +answer with no hits from a stale or absent local index is indistinguishable +from a true "nothing calls this". + +So an outage means witan commands fail while it lasts, and your agent's context +hook comes back empty rather than stale. If you need to keep working offline, +that is a config change you make deliberately: comment out `remote_url` (or +`WITAN_TARGET=` a local target) and know that anything you write then lives in +a separate store, to be merged later with +[`witan migrate merge`](migration-runbook.md). + +Related: `witan login` failing with an expired refresh token looks similar but +is not an outage — re-run `witan login`. + +## What happens when the deployment is busy + +Not the same thing, and the difference matters: the service is up, and it is +writes — not reads — that are scarce. The shared graph serialises them at +roughly one every 3-4 seconds, so a burst of concurrent writers queues. + +You may see either of two answers, and they say different things: + +``` +omnigraph mutate was refused before it was sent: 4 writes are already in flight +against https://.../council and no slot freed within 10s. … NOTHING WAS +WRITTEN — retry once the burst clears. +``` + +That one is clean. It happened before anything left the client, so the graph is +untouched and retrying is unambiguous. + +``` +The deployed service at https://… answered HTTP 502 for `memory_store`: the +request reached it and was cut off before a reply came back. `memory_store` +writes, so ITS OUTCOME IS INDETERMINATE — the write may or may not have been +applied … Re-read before retrying; retrying blind writes it twice if it did land. +``` + +That one is not. The call was cut at the deployment's 30-second deadline with +the write already in flight, and nothing in the reply says whether it committed +— measured live, most such writes had committed and some had not. **Re-read +before you retry.** `witan migrate merge` is the exception: it reconciles +newest-record-wins, so re-running it is safe and its message says so. + +Reads are unaffected by all of this and stay fast under the same load. + +## Troubleshooting + +- **"Remote mode is not configured."** No `remote_url` resolved: your target + didn't match this repo. Check with `witan target list` — if no row is marked + `*`, nothing matches here. Pass `--target ol`, or force it with + `WITAN_TARGET=ol`. +- **"Could not verify OIDC issuer …" from `target add`.** The issuer URL is + wrong, or unreachable from where you ran it. Nothing was written, so just fix + it and re-run. Note the check also fails if the discovery document advertises + a *different* issuer than the one you passed — that mismatch is refused + deliberately (RFC 8414 §3.3), not worked around. +- **A remote URL is configured but no OIDC issuer.** The CLI refuses to fall + through to the unauthenticated in-process path — set `oidc_issuer` on the + same target, or unset `remote_url`. `target add` rejects this combination up + front, so this only comes from a hand-edited config. +- **`target add` says the target already exists.** Deliberate: it will not + silently overwrite. `--force` replaces the block in place, keeping its + position — which matters for ties, since within one selector tier the first + matching target wins. (Across tiers, specificity decides; see step 1.) Or + pick another name. +- **"could not be reached" but the endpoint is definitely up.** The same + message covers a token the *server* rejects, because both fail while the + connection is being opened and the client cannot tell them apart from + outside. Check `witan whoami` first — an expired session, or a missing `aud` + claim (below), reads identically to an outage. +- **401 / token rejected.** The deployment validates the `aud` claim. If your + realm's audience mapper is not stamping `aud: witan`, set `oidc_audience` to + match the deployment's `WITAN_OIDC_AUDIENCE`. +- **Your writes are refused but reads work.** Cedar. Human actors get `change` + on the memory graph and on their own code-graph branch views, but *not* on a + code graph's protected `main` — that one is CI's, and the refusal is + deliberate. +- **`witan migrate schema`/`topics`/`repo-keys` are refused.** Correct: those + have no per-user identity to scope, so they run in-cluster as + `svc-witan-admin` (ADR-0005 path b). **`witan migrate merge` is the + exception** — it has a per-actor form and is how you bring your own history + across. See the + [migration runbook](migration-runbook.md#local-shared-the-cutover). +- **`witan migrate merge --target …` is refused.** Against a deployment the + target is the deployment's own graph, resolved server-side. Name the + deployment with `--to ` instead, or unset `remote_url` to merge + between stores you address yourself. +- **witan-code went remote and you didn't want it to.** Expected — one endpoint + serves both tool surfaces, so the four `remote_*`/`oidc_*` keys route both + CLIs. Indexing stays local either way (it needs your checkout); only reads + move. ADR-0005's 2026-07-31 amendment explains the coupling. + +## References + +- [ADR-0005](../explanation/decisions/0005-secure-cli-path-into-deployed-witan.md) — the CLI's + remote MCP-client mode (path a) and the in-cluster admin path (path b). +- [ADR-0004](../explanation/decisions/0004-keycloak-jwt-per-user-actor-mapping.md) — JWT → actor → + token mapping, i.e. what `witan whoami`'s `actor` line is showing you. +- [ADR-0007](../explanation/decisions/0007-local-to-shared-store-migration-transport.md) / + [migration runbook](migration-runbook.md) — the data half of the cutover. +- ol-infrastructure `docs/adr/0009-…` — the deployment this connects to. diff --git a/docs/guides/index.md b/docs/guides/index.md new file mode 100644 index 00000000..3c3d1096 --- /dev/null +++ b/docs/guides/index.md @@ -0,0 +1,64 @@ +# Guides + +Task-oriented pages: you already know what you want to do, and you want the +steps. If you are still orienting, start with [Get +started](../getting-started/index.md) instead. + +## Using witan + +
+ +- **[witan user guide](witan-user-guide.md)** + + The full day-to-day loop for the coordination graph — memory, tasks, + projects, sessions, and the operating modes the store can run in. + +- **[witan-code user guide](witan-code-user-guide.md)** + + Indexing, querying, and the cross-repo bridge, in depth. + +- **[Branch indexing](branch-indexing.md)** + + How per-branch views work, who is allowed to write the default view, and + how idle views are reaped. + +
+ +## Operating it + +
+ +- **[Using a deployed witan](deployed-witan.md)** + + Point your local CLI and agent at a shared service: OIDC login, target + configuration, and what changes when the store is no longer on your disk. + +- **[Write-path scanning](write-path-scanning.md)** + + Every write is scanned for secrets and PII before it persists. How + enforcement is configured, how to suppress a false positive, and how to add + a detector. + +- **[Migration runbook](migration-runbook.md)** + + Moving a store: local to shared, format upgrades, and reconciling two stores + that both have writes. + +
+ +!!! info "These pages live with the code" + + Every guide here is mirrored from the package it documents, so it stays in + step with the release rather than drifting into a second, slowly-wrong copy. + Each page links to its authoritative source at the top — edit there. + +## Things people commonly need + +| I want to… | Where | +| --- | --- | +| Change where the graph is stored | [`WITAN_MEMORY_URI`](../reference/environment.md#store-and-attribution) | +| Route work repos and personal repos at different stores | [Named targets](witan-user-guide.md) | +| Stop a detector flagging a false positive | [Write-path scanning](write-path-scanning.md) | +| Run the code indexer in CI | [`WITAN_CODE_CI_REPOS`](../reference/environment.md#ci-code-graph-indexer) | +| Understand why a claim was rejected | [Coordinating work](../explanation/task-coordination.md) | +| Tune what `recall` returns | [`WITAN_RANK_*`](../reference/environment.md#recall-ranking) | diff --git a/docs/guides/migration-runbook.md b/docs/guides/migration-runbook.md new file mode 100644 index 00000000..2b8e5d86 --- /dev/null +++ b/docs/guides/migration-runbook.md @@ -0,0 +1,328 @@ + + +!!! info "This page lives with the code" + + The authoritative copy is + [`mcp/servers/witan/docs/migration-runbook.md`](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan/docs/migration-runbook.md). + +# Store migration runbook: local → shared, and cross-machine merges + +How to move a witan store onto the shared deployment +([ADR-0009](https://github.com/mitodl/ol-infrastructure/blob/main/docs/adr/0009-deploy-witan-as-shared-multi-tenant-mcp-service.md)), +between two of your own machines, or merged with a teammate's. All three are +`witan migrate merge` with a different destination. + +> **Never `mv`, `cp`, rsync, or tar a `.omni` store.** Lance embeds absolute +> paths, so a copied store fails to open or reads the wrong files. Move data +> with `witan migrate merge` (or an `omnigraph export` file), never with the +> filesystem. + +## Local → shared: the cutover + +Moving your own store onto the deployment. No kubectl, no port-forward, no AWS +credentials. + +**1. Register the deployment and log in** (once — hostnames are in +[`deployed-witan-onboarding.md`](deployed-witan.md)): + +```bash +witan target add ol --remote-url … --oidc-issuer … +witan login --target ol +``` + +**2. Take stock of what is in the store.** The shared graph is org-wide, and a +local store accumulates whatever you worked on — personal repos included. Merge +is all-or-nothing, so decide what goes *before* the dry run: + +```bash +omnigraph export --store ~/.local/share/witan/graph.omni > witan-export.jsonl +jq -r 'select(.type) | .data.repo // (.data.repos // [] | join(", ")) // "" + | if . == "" then "(no repo)" else . end' witan-export.jsonl | + sort | uniq -c | sort -rn +``` + +That is every repo represented, by row count. `(no repo)` is mostly general +engineering lessons that belong to no checkout — usually the ones most worth +sharing, so don't drop them by reflex. + +If anything on that list should not go, merge a filtered export instead of the +store. Both passes below are needed: `from`/`to` on an edge are slugs, so +dropping a node without dropping its edges leaves the edges dangling. + +```bash +# 1. the slugs to leave behind — adjust the predicate to your own list +jq -r 'select(.type) + | select([.data.repo // empty, (.data.repos // [])[]] + | any(startswith("https://github.com/alice/"))) + | .data.slug' witan-export.jsonl | sort -u > drop-slugs.txt + +# 2. drop those nodes and every edge that touches one +jq -c --rawfile drop drop-slugs.txt ' + ($drop | split("\n") | map(select(length > 0)) | INDEX(.)) as $d + | select(if .type then ($d[.data.slug] | not) + else ($d[.from] | not) and ($d[.to] | not) end) + ' witan-export.jsonl > witan-work-only.jsonl +``` + +Then use `witan-work-only.jsonl` as the source everywhere below, in place of +the store path. Nothing is removed from your local store by any of this. + +**3. Preview the merge:** + +```bash +witan migrate merge ~/.local/share/witan/graph.omni --to ol --dry-run +``` + +`--to ol` names the target block, so the destination is on the command line +rather than in your environment. Read the decisions: `added` should be roughly +the row count of your store, and `updated` should be small. A large `updated` +on a first migration means slugs are colliding that shouldn't — stop there. + +**4. Run it:** + +```bash +witan migrate merge ~/.local/share/witan/graph.omni --to ol +``` + +Your store is exported locally and the rows ship through the deployment's +`store_merge` tool in batches, written **as you**, under your own credential. +Batches commit independently, so a failure part-way leaves earlier batches +applied — just re-run, the merge is idempotent. + +"As you" now covers attribution as well as authorization. A local store writes +`author` from `WITAN_AUTHOR` / git `user.name` / `$USER`, while the deployment +resolves it from your token's `preferred_username` — two namespaces that never +converge. Rows carrying your local name are restamped to your deployed identity +as they arrive, so the history you migrate is owned by the same identity that +owns everything you write afterwards, and `memory_delete` (author-only) still +works on it. + +Rows authored by anyone else are left exactly as they are. That matters for the +two merges below: bringing in a teammate's export through your credential does +not reattribute their work to you. + +> **Merged before witan-council 0.23.0?** Those rows kept your local name, and +> `memory_delete` refuses them — permanently, since your deployed identity can +> never match. Re-merging will not fix it: reconciliation is +> newest-record-wins, so a re-sent row loses to its own already-applied copy. +> Repair them in place instead: +> +> ```bash +> witan migrate claim-authorship # dry by default; --was defaults +> # to your local author +> witan migrate claim-authorship --apply +> ``` + +**5. Verify by slug, not by search:** + +```bash +witan whoami +witan memory --kind lesson --all-repos | head # a listing, not a search +witan task +witan tasks --all-repos | head +``` + +`witan memory ""` will very likely return *No memories.* on a +freshly-populated graph even when every row is present — that is a BM25 +property of a small corpus, not a failed merge +([why](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan/docs/store-merge-findings.md#search-looks-broken-on-a-near-empty-graph--verify-by-slug)). +Listings and `witan task ` read the graph directly and do not go through +BM25, which is what makes them usable here. + +**6. Keep your local store until you have verified.** It is the backup. + +Once everyone has merged, an operator runs `witan migrate repo-keys` and +`witan migrate topics` **once**, in-cluster (see the fallback below). + +### Or hand the cutover to an agent + +Same steps, run by Claude/pi instead of by you. Paste this, replacing `ol` +with your target name if it differs: + +```text +Run my witan local-to-shared cutover, following +mcp/servers/witan/docs/migration-runbook.md's "Local → shared: the cutover". + +1. `witan whoami --target ol`. If it says I am not logged in, stop and tell me + to run `witan login --target ol` — do not attempt the login yourself. +2. Take stock before sending anything. The destination is an org-wide shared + graph and the merge is all-or-nothing, so inventory the local store first + with the export + `jq` repo-count command in step 2 of that section. Show me + the full list of repos and their row counts, and flag anything that looks + personal rather than work — a non-org repo, a side project, a personal + checkout path. Do NOT decide this yourself and do NOT assume `(no repo)` + rows are personal (they are mostly general engineering lessons). Ask me + which, if any, to leave behind. +3. If I name anything to exclude, build the filtered export with the two-pass + `jq` recipe in that same step (nodes AND the edges touching them) and use + the filtered `.jsonl` as the source from here on. Tell me the before/after + row counts. +4. `witan migrate merge --to ol --dry-run`, where `` is + `~/.local/share/witan/graph.omni` or the filtered export. Report the + added/updated/kept counts. STOP AND ASK before going further if `updated` is + more than a handful — on a first migration that means slugs are colliding + that should not, and each one silently drops a record. +5. Once I approve, run the same command without --dry-run. +6. Verify with `witan memory --kind --all-repos` listings and + `witan task ` on two or three slugs from the dry-run decision list. + Do NOT verify with `witan memory ""` — that is a BM25 search, and it + returns nothing on a small corpus even when every row landed, so an empty + result is not evidence of anything. +7. Report what landed. Do not delete, move, or clean up my local store or any + export file — they are the backup until I say otherwise. + +If any step fails, stop and show me the error rather than retrying or working +around it. +``` + +The guardrails are the point: the merge is idempotent, so re-running is safe, +but what gets sent to a shared graph is not undoable by re-running, and a large +`updated` count and a "search finds nothing" reading are both things an agent +will otherwise sail past. + +## Cross-machine merge and machine migration + +Same command, a store or a named target on each end: + +```bash +# two named local targets — neither end is a path anyone types +witan migrate merge --from personal --to work + +# by path: merge each machine's store into a shared third one +witan migrate merge machine-a.omni --target combined.omni +witan migrate merge machine-b.omni --target combined.omni + +# machine migration: the target starts empty and is created automatically +witan migrate merge old-machine.omni --target new-machine.omni + +# from a store that can't travel — hand over its export instead +omnigraph export --store ~/.local/share/witan/graph.omni > alice.jsonl +witan migrate merge alice.jsonl --target combined.omni +``` + +Preview with `--dry-run` first, then verify: + +```bash +omnigraph export --store | jq -r .type | sort | uniq -c +``` + +Type counts in the target should equal the union of the sources' counts, minus +collisions resolved in the target's favour. + +## Flags + +``` +witan migrate merge [SOURCE] [--from ] [--to ] [--target ] [--dry-run] +``` + +| Flag | Means | +|---|---| +| `SOURCE` | Store URI to merge **from**: local path, `s3://`, `file://`, `http(s)://`, or a local `omnigraph export` `.jsonl`. | +| `--from ` | A `[targets.]` block's `server`, in place of `SOURCE`. A target with only a `remote_url` is refused — nothing local to export. | +| `--to ` | A `[targets.]` block as the destination: through its deployment if it has a `remote_url`, into its `server` store if not. | +| `--target ` | A destination store URI. Defaults to your configured store. Mutually exclusive with `--to`; `.jsonl` is refused (a target is a graph, not a snapshot). | +| `--dry-run` | Print the per-slug decisions, write nothing. | + +Notes: + +- A `.jsonl` **source** must be a readable local file — witan fetches no remote + exports. Download it first (`aws s3 cp …`) and pass the path. +- A missing local destination is created and schema-applied; a missing remote + one is assumed to exist. +- A deployed graph addressed by URI is + `http(s)://:/graphs/` — the `/graphs/` part is + required. +- Against a deployment, `--target` is refused: the destination is that + deployment's own graph, resolved server-side. Use `--to `. +- Merging is **repeatable**. A re-run against an already-merged target loads + nothing, because every source row loses reconciliation to its own applied + copy. Safe on a schedule. +- Reconciliation covers nodes only. Edge rows (`Tagged`, `ParentOf`, …) have no + slug and pass through unreconciled, same as raw `--mode merge`. + +## Fallback: in-cluster merge (operator) + +Use when the MCP tier is unavailable, or to merge on someone else's behalf. +Every write lands as `svc-witan-admin` rather than as the user, which is why +this is the fallback. The data tier is ClusterIP-only, so the merge runs inside +the cluster; the user's store cannot travel, so they hand over its export. + +**1. User exports and stops writing locally:** + +```bash +omnigraph export --store ~/.local/share/witan/graph.omni > "$USER-witan.jsonl" +wc -l "$USER-witan.jsonl" # thousands of rows, not zero +``` + +**2. Operator opens a break-glass pod and streams the file in:** + +```bash +kubectl -n witan create job witan-bg-$(date +%s) --from=cronjob/witan-break-glass +JOB=witan-bg- # from the output above + +kubectl -n witan exec -i job/$JOB -- sh -c 'cat > /tmp/alice.jsonl' < alice-witan.jsonl +kubectl -n witan exec -it job/$JOB -- wc -l /tmp/alice.jsonl +``` + +`kubectl exec -i`, not `kubectl cp` and not S3: the pod declares no volume and +no ServiceAccount, so it holds no bucket credentials and no `aws` binary. Check +an unusually large export against the pod's 1Gi memory limit. + +**3. Operator dry-runs, reviews, merges.** The pod's `WITAN_MEMORY_URI` and +`WITAN_MEMORY_TOKEN` already address the in-cluster graph, so no destination +flag is needed: + +```bash +kubectl -n witan exec -it job/$JOB -- witan migrate merge /tmp/alice.jsonl --dry-run +kubectl -n witan exec -it job/$JOB -- witan migrate merge /tmp/alice.jsonl +``` + +**4. Repeat steps 1–3 per user**, then run the post-merge migrations once at +the end and clean up: + +```bash +kubectl -n witan exec -it job/$JOB -- witan migrate repo-keys +kubectl -n witan exec -it job/$JOB -- witan migrate topics +kubectl -n witan delete job $JOB +``` + +**5. User verifies** through the deployment, as in step 4 of the cutover above. + +## Fallback: no `witan` CLI, only `omnigraph` + +```bash +omnigraph export --store > data.jsonl +omnigraph init --schema schema/schema.pg # skip if the target exists +omnigraph load --store --data data.jsonl --mode merge --as +``` + +Run the `load` once per source when merging several. This has **no** +reconciliation: a slug present in both stores is silently overwritten by +whichever file loads last. Diff the slug sets first and resolve any hit by hand: + +```bash +omnigraph export --store | jq -r 'select(.data.slug) | .data.slug' | sort > target-slugs.txt +jq -r 'select(.data.slug) | .data.slug' machine-a.jsonl | sort | comm -12 target-slugs.txt - +``` + +A `Topic` hit is harmless (deterministic, content-equivalent by design). A +`Memory`/`Task`/`WorkflowProject`/`WorkflowSession` hit is two different +records, one about to disappear. + +## Why it works this way + +[`store-merge-findings.md`](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan/docs/store-merge-findings.md) — the verified `--mode +merge` collision behaviour, the slug-collision probability, and the BM25 +measurement behind "verify by slug, not by search". + +## References + +- [`deployed-witan-onboarding.md`](deployed-witan.md) — the other + half of the cutover: pointing your CLI and agent at the deployment. +- [ADR-0007](../explanation/decisions/0007-local-to-shared-store-migration-transport.md) — why the + local → shared path is a client-side export merged through the MCP tier. +- ol-infrastructure `docs/witan-admin-break-glass-runbook.md` — the + `witan-break-glass` pod and its `svc-witan-admin` token. diff --git a/docs/guides/witan-code-user-guide.md b/docs/guides/witan-code-user-guide.md new file mode 100644 index 00000000..bf7f679e --- /dev/null +++ b/docs/guides/witan-code-user-guide.md @@ -0,0 +1,254 @@ + + +!!! info "This page lives with the code" + + The authoritative copy is + [`mcp/servers/witan-code/docs/USER_GUIDE.md`](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan-code/docs/USER_GUIDE.md). + +# User guide + +## What it is + +witan-code is a tree-sitter-based code graph for a single repository. It +parses source files into symbols (functions, methods, classes, modules) and +their relationships (defines, contains, calls, references, imports, +inherits), stores them in a local [Omnigraph](https://github.com/ModernRelay/omnigraph) +graph, and exposes definition / reference / caller / impact queries to a +coding agent through MCP tools and a CLI. The problem it solves: an agent +working in a large repo needs "where is this defined", "who calls this", and +"what breaks if I change this" answered from the actual syntax tree instead +of grep-and-guess. + +A second layer, the **cross-repo bridge**, extends this past the repo +boundary: it extracts interface contracts (env vars, HTTP endpoints, +published packages, deployed services) that couple repos in a +service-oriented architecture, and answers "what depends on this repo" / +"who provides this contract" across every repo you've indexed. + +witan-code is Layer 2 of a two-layer stack. Layer 1, `witan`, holds +team-synced memory (patterns, project facts, lessons, workflow traces) in a +shared store. The two compose through **soft symbol-id references**: a +Layer-1 node can record a symbol id of the form +`repo#relative/path.py::QualifiedName`, which resolves in the code graph via +`code_find_definition` — there is no hard cross-store edge, so the +team-synced memory graph stays independent of any one machine's local code +index. See the main [README](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan-code/README.md) for the full two-layer table. + +## Feature set + +- **Symbol indexing** across Python, TypeScript/JS/JSX/TSX, Bash/Zsh, and + YAML, with signatures, docstrings, and decorators attached to each symbol + — see the [supported languages table](#supported-languages) below. +- **Definition / reference / caller / impact queries** — + `code_find_definition`, `code_find_references`, `code_callers`, + `code_impact` (transitive BFS), `code_symbols_in_file`, + `code_search_symbol` (BM25). +- **Cross-repo interface bindings** — `env_var` / `endpoint` / `package` / + `service` contracts linking repos in an SOA; see [README § + Cross-repo context bridge](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan-code/README.md#cross-repo-context-bridge-layer-25). +- **Precision-tiered edges** — every cross-repo tool accepts `min_precision` + (`precise` / `heuristic` / `fuzzy`) to filter by trust level; see + [EDGE_PRECISION_TIERS.md](../explanation/code-graph/edge-precision-tiers.md). +- **Precise (Stage 2) symbol stitching** — canonical-symbol-string joins + computed at read time, no cross-repo edge ever stored; see + [STAGE2_STITCHING.md](../explanation/code-graph/stage2-stitching.md) and + [SYMBOL_TABLE.md](../explanation/code-graph/symbol-table.md). +- **Branch-aware indexing** — a non-default git branch indexes onto its own + view, named for its writer as well as the branch, so in-flight work never + overwrites the shared `main` view nor another checkout of the same branch; + see [BRANCH_INDEXING.md](branch-indexing.md). +- **Dependency visualization** — `witan-code deps` prints a text summary and + can emit an interactive HTML force-directed graph of cross-repo links. + +## Installation + +witan-code is usually installed as part of the `witan` umbrella package — +`witan setup` wires both servers together and installs the shared +`omnigraph` binary in one step. See the [witan README](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan/README.md) +for that one-step setup. + +To use witan-code **standalone** (code graph only, no memory/task tools): + +```bash +# One-shot: +uvx --from witan-code witan-code index + +# Persistent CLI install: +uv tool install witan-code +witan-code index +``` + +To track pre-release/unreleased code instead of the latest PyPI release, +install from the git repo directly: + +```bash +# One-shot: +uvx --from git+https://github.com/mitodl/agent-kit#subdirectory=mcp/servers/witan-code \ + witan-code index + +# Persistent CLI install: +uv tool install git+https://github.com/mitodl/agent-kit#subdirectory=mcp/servers/witan-code +witan-code index +``` + +Standalone use also needs the `omnigraph` binary on `PATH`; if `witan` isn't +installed to provide it, run `witan-code setup` once (see [Troubleshooting](#troubleshooting)). +To wire witan-code as a standalone MCP server (no witan memory/task tools), +copy the matching snippet from `config/` into your agent's config — see +[README § Install](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan-code/README.md#install). + +## First-run walkthrough + +1. **Install the omnigraph binary** (skip if `witan setup` already ran): + + ```bash + witan-code setup + ``` + +2. **Index the repo** from its root: + + ```bash + cd ~/code/my-repo + witan-code index + ``` + + This creates the repo's store lazily on first run and prints a summary + (`scanned=… indexed=… skipped=… symbols=… edges=… bindings=… + errors=…`). Re-running `index` is incremental — unchanged files are + skipped by content hash; use `reindex` to force a full rebuild. + + A run that fails while writing prints the same counts under `partial …`, + followed by the phase that failed, how long it ran, and the batch sizes it + was working with. Those numbers are the diagnosis: they say whether the + write was large, how far it got, and which of the deletes or the two loads + died. The exit status is unchanged, so a sweep still counts the repo as + failed. + +3. **Ask "where is this defined"** — via the CLI or, more commonly, through + an agent calling the MCP tool directly: + + ```bash + # MCP tool (what an agent calls): + code_find_definition(name="ServiceClient.run") + ``` + + Returns matching symbols with their file, line, signature, and docstring. + +4. **Ask "who calls this"**: + + ```bash + code_callers(symbol_id="https://github.com/org/repo#app/client.py::ServiceClient.run") + code_impact(symbol_id="...", max_depth=5, max_nodes=200) # transitive callers + ``` + + `code_impact` walks the caller graph breadth-first up to `max_depth` + hops or `max_nodes` results — use it before changing a function's + signature to see the blast radius. Remember `Calls`/`References` are + **heuristic** (syntactic name matching, not verified dispatch) — treat + the result as a high-recall starting point, not ground truth. + +5. **Ask "what depends on this repo"** once more than one repo is indexed: + + ```bash + witan-code deps --repo my-repo + # or, for the precise tier only: + witan-code deps --repo my-repo --min-precision precise + ``` + + Cross-repo tools only see something once at least two repos that share a + contract (an env var, an endpoint, a package, a deployed service) have + both been indexed — the bridge is populated incrementally as you index + each repo, with no separate registration step. + +## Supported languages + +| Language | Extensions | Symbols extracted | +|---|---|---| +| Python | `.py` `.pyi` | functions, classes, methods, module | +| TypeScript / JS / JSX / TSX | `.ts` `.tsx` `.mts` `.cts` `.js` `.jsx` `.mjs` `.cjs` | functions, arrow consts, classes, methods (incl. arrow fields), interfaces, types, enums | +| Bash / Zsh | `.sh` `.bash` `.zsh` | functions | +| YAML | `.yaml` `.yml` | mapping keys as dotted paths (e.g. `jobs.build.steps`) | + +Each symbol also carries a full `signature`, a `docstring` (Python +docstrings, TS/JS JSDoc), and `decorators` — returned by +`code_find_definition` / `code_search_symbol` / `code_symbols_in_file`. See +[README § Supported languages](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan-code/README.md#supported-languages) for what's +not indexed yet (HCL/Terraform is the leading candidate). + +## Cross-repo bridge in brief + +Per-repo indexing stops at the repo boundary; the bridge is a separate, +shared, local-only store (`_bridge.omni`) that links repos through contracts +they share — an env var one repo's infra sets and another reads, an HTTP +endpoint one serves and another calls, a package one publishes and others +import, a service one repo deploys. It is **zero-config**: every `index` / +`reindex` of any repo also extracts that repo's bindings into the bridge +store automatically — there's no registry of repos to maintain. + +Two tiers of cross-repo linking exist and are merged into one +precision-filterable result: + +- **Heuristic** (Stage 3) — bindings grouped by `(kind, key_norm)` with a + confidence score. This is the original, always-on behavior. +- **Precise** (Stage 2) — a canonical-symbol-string join computed at read + time, never stored. Higher trust, narrower coverage. + +Don't re-derive the mechanics here — see: + +- [SYMBOL_FORMAT.md](../explanation/code-graph/symbol-format.md) — the canonical symbol string format + bindings are keyed on. +- [STAGE2_STITCHING.md](../explanation/code-graph/stage2-stitching.md) — the precise join algorithm. +- [EDGE_PRECISION_TIERS.md](../explanation/code-graph/edge-precision-tiers.md) — the `min_precision` + parameter every cross-repo tool and the `deps`/`stitch` CLI commands + accept. + +## Troubleshooting + +- **`omnigraph: command not found` / commands fail silently.** witan-code + needs the `omnigraph` binary on `PATH`. If `witan` is installed, its + `witan setup` already placed it; standalone, run `witan-code setup` (or + `witan-code setup --dry-run` to preview). Re-run after an omnigraph + version bump — `setup` always re-downloads, it doesn't skip an existing + binary. +- **A tool returns `[]` / "No code graph yet."** Per-repo stores are created + **lazily** on first `index` — there is no separate "create store" step. + If you see the `No code graph yet. Run \`witan code index\` to build it.` + hint, you haven't indexed this repo yet (or you're not in a directory + witan-code recognizes as the repo — see `WITAN_REPO` in [README § + Environment variables](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan-code/README.md#environment-variables)). +- **Bridge tools (`deps`, `code_interface_*`) return nothing.** The bridge + store is also created lazily, on the first index that yields any bindings, + and cross-repo links only appear once **both** sides of a contract have + been indexed (e.g. the repo that reads an env var and the repo whose + Pulumi config sets it). Indexing one repo alone won't show a link. + Generic env names (`DEBUG`, `PORT`, `SECRET_KEY`, …) are deliberately + excluded from cross-repo impact fan-out. +- **A feature-branch checkout doesn't see the same symbols as `main`.** Git + branches other than the default index onto their own view, forked from + `main` on first write — reads from that checkout follow the branch + automatically. Branch names are sanitized (`[^A-Za-z0-9._-]` → `_`); a + branch literally named `main` in a `master`-default repo maps to `_main`, + and a detached HEAD checkout writes to a `_detached` scratch branch rather + than ever touching `main`. `witan-code branches` lists what exists per + store; `--prune` deletes the current repo's views whose git branch is gone. + See [BRANCH_INDEXING.md](branch-indexing.md). +- **A teammate's in-flight work isn't in my results.** Each writer gets their + own view of a shared branch (`act-/feature-x`), so you see yours, not + theirs. `witan-code branches --branch feature-x` lists every writer's view + of that branch; pass one back as `--branch act-/feature_x` (or as the + `branch` argument of `code_find_definition` / `code_search_symbol` / + `code_symbols_in_file`) to read it. +- **Caller/impact results look wrong or incomplete.** `Calls`, `References`, + `Imports`, and `Inherits` are **heuristic** — syntactic name resolution + that prefers same-file definitions. Dynamic dispatch, re-exports, + shadowing, and cross-file resolution beyond name matching aren't modeled + precisely. Treat them as a high-recall starting point for investigation, + not a verified call graph. `Defines`/`Contains` (and Stage-2 precise + cross-repo edges) are exact by contrast. +- **No pre-built `omnigraph` binary for your platform.** Only + `linux/x86_64` and `darwin/arm64` have pinned release assets; `setup` + prints a message and does nothing on other platforms — install the + binary manually and put it on `PATH`. diff --git a/docs/guides/witan-user-guide.md b/docs/guides/witan-user-guide.md new file mode 100644 index 00000000..914b4712 --- /dev/null +++ b/docs/guides/witan-user-guide.md @@ -0,0 +1,268 @@ + + +!!! info "This page lives with the code" + + The authoritative copy is + [`mcp/servers/witan/docs/USER_GUIDE.md`](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan/docs/USER_GUIDE.md). + +# witan User Guide + +witan is a team-wide shared knowledge and coordination graph for coding +agents. It solves two related problems: (1) engineering knowledge — +patterns, project facts, lessons — discovered by one agent session is +normally lost the moment that session ends, and (2) multiple agents (or +people) working on the same repo have no shared view of what work exists, +what's claimed, and what's blocked. witan stores both in one graph, backed +by [Omnigraph](https://github.com/ModernRelay/omnigraph), and exposes it over +MCP so any agent platform (Claude Code, Pi, GitHub Copilot, OpenCode) reads +and writes the same store without platform-specific glue. + +Distributed on PyPI as `witan-council` (the `witan` project name was already +taken); the import path, console command, and every tool/CLI name are still +`witan`. Only the install artifact's name changed. + +## Feature set + +- **Memory search & store** — full-text (BM25) search over patterns, project + facts, lessons, and agent context, with graph-aware re-ranking. See + [Day-to-day loop](#day-to-day-loop) below and the `memory` command in the + [CLI Reference](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan/docs/CLI_REFERENCE.md#memory). +- **Workflow project tracking** — track an engineering objective across + multiple agent sessions (discovery → spec → implementation → delivery + phases), with session hand-off state. See + [`witan-project-tracker`](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan/witan/skills/witan-project-tracker/SKILL.md) + and the `project`/`projects` commands in the + [CLI Reference](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan/docs/CLI_REFERENCE.md#projects). +- **Task tracking with dependencies** — a hierarchical, dependency-aware task + tracker (epics → sub-issues, `blocked_by`, advisory claims with lease + expiry) shared across agents/users. See + [`witan-task`](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan/witan/skills/witan-task/SKILL.md) and the + [CLI Reference](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan/docs/CLI_REFERENCE.md#tasks). +- **Code-branch tracking** — links a git branch to the task/project it + carries, wired in automatically by `task_claim` and + `workflow_session_start`. See [Code branch tracking](#code-branch-tracking). +- **Write-path secret/PII scanning** — every memory/task/project/session + write is scanned for secrets and PII before it's persisted, with + block/redact/warn enforcement and a plugin mechanism. See + [`docs/write-path-scanning.md`](write-path-scanning.md) and the + [`scan` command](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan/docs/CLI_REFERENCE.md#scan). +- **Code graph integration (`witan-code`)** — when the sibling `witan-code` + package is installed, `witan code …` mounts tree-sitter-derived symbol + search, references, and cross-repo impact analysis into the same CLI/MCP + server. See the [CLI Reference](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan/docs/CLI_REFERENCE.md#code-witan-code-only) and + [`../witan-code/README.md`](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan-code/README.md). + +## Installation + +Two install shapes, depending on whether your agent platform needs the +`witan` binary on `PATH`: + +- **Persistent CLI** — required for **Claude Code** and **Pi** (their + hooks/extensions shell out to `witan` directly): + + ```bash + uv tool install witan-council + witan setup --agent claude # or: pi | all + ``` + + To track pre-release/unreleased code instead of the latest PyPI release, + install from the git repo directly: + + ```bash + uv tool install git+https://github.com/mitodl/agent-kit#subdirectory=mcp/servers/witan + witan setup --agent claude # or: pi | all + ``` + +- **MCP-only** — sufficient for **Copilot**, **OpenCode**, and **Kilo**, + whose MCP server launches via `uvx` on demand, so nothing needs to stay + installed: + + ```bash + uvx --from witan-council witan setup --agent copilot # or: opencode | kilo + ``` + + Same pre-release option here — swap in the `git+…` source: + + ```bash + uvx --from git+https://github.com/mitodl/agent-kit#subdirectory=mcp/servers/witan \ + witan setup --agent copilot # or: opencode | kilo + ``` + +See the main [README](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan/README.md#quick-start) for the manual-wiring +fallback (`./install.sh` + hand-editing agent MCP config). + +## First-run setup + +`witan setup --agent ` does four things, in order: + +1. Downloads the pinned `omnigraph` binary release to `~/.local/bin/omnigraph` + (skipped if already present). +2. Writes a starter `~/.config/witan/config.toml` — every optional setting + commented out at its actual default — unless one already exists. +3. Copies bundled skills and hooks/extensions into the target agent's config + directories (e.g. `~/.claude/skills/`, `~/.claude/hooks/`). +4. Merges the witan MCP server entry into that agent's config file. + +Re-run it after every witan upgrade to refresh installed files and pick up an +`omnigraph` version bump. Pass `--dry-run` to preview without writing +anything, and `--author "Your Name"` to set graph attribution up front +(otherwise it falls back to `git config user.name`, then `$USER`). + +The graph itself lives at `~/.local/share/witan/graph.omni` by default (a +local Omnigraph store) — nothing else to provision for local-disk mode. See +[Operating modes](#operating-modes) for RustFS and remote-server setups. + +## Day-to-day loop + +A typical session: + +1. **Check ready work.** + + ```bash + witan tasks --ready + ``` + + Lists open tasks with no open blockers, ordered by priority, scoped to + the repo you're standing in (detected from `origin` in `.git/config`). + Add `--project wp-` to scope to one workflow project, or + `--all-repos` to see everything. + +2. **Claim one and start working.** + + ```bash + witan run tk-fix-flaky-retry-abc123 + ``` + + Sets the task `in_progress` under your author name (an advisory lease, + not a hard lock — see `task_claim` in the + [CLI Reference](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan/docs/CLI_REFERENCE.md#run)), then launches your configured + agent CLI with a prompt seeded from the task's title/description/symbol + refs. Pass `--dry-run` to print that prompt without claiming or + launching, or `--claim=false` to launch without claiming. + + If you're inside an already-running agent session (rather than the + `witan run` launcher), use the MCP tools directly: `task_claim`, and + `task_close` with a `resolution` when you're done. Filing follow-up work + discovered mid-task: `task_create(discovered_from=["tk-…"], …)`. + +3. **Store what you learned.** + + Any durable, shareable fact — a coding pattern, a project-specific quirk, + a lesson from a bug you just fixed — belongs in witan, not your agent's + private session memory, so other sessions and teammates can find it: + + ``` + memory_store(kind="pattern", title="...", content="...", repo=, tags=[...]) + ``` + + Search before you start new work so you don't rediscover something + already recorded: + + ```bash + witan memory "flaky retry" --kind pattern + ``` + +4. **Track a multi-session project.** For work that spans more than one + session, create (or resume) a `WorkflowProject` rather than tracking + state in your head: + + ```bash + witan project create "Migrate auth to OAuth2" --phase discovery + ``` + + Then `workflow_session_start` at the top of each session (the + [`witan-workflow`](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan/witan/skills/witan-workflow/SKILL.md) skill + automates the picker), and `workflow_session_end` with a summary before + you stop — this is what lets a *different* session or agent pick the + thread back up later. `workflow_project_advance` moves the project to + its next phase; `workflow_project_complete` closes it out and assembles a + `WorkflowTrace` corpus record from every session for later pattern + mining. + +### Code-branch tracking + +No dedicated command — this rides along automatically, best-effort: +`workflow_session_start` and `task_claim` both upsert a `CodeBranch` node +linking the current checkout's repo+branch to the project/task in flight, so +"which branch carries task X" is a one-hop graph query. It silently no-ops +outside a git repo, on a detached HEAD, or against a store that predates the +feature — never a hard requirement for the tool call it's attached to. + +## Operating modes + +Three ways to point witan at a store, in increasing order of shared-ness: + +- **Local disk (default)** — no extra infrastructure; the graph lives at + `~/.local/share/witan/graph.omni`. What you get out of the box. +- **Local RustFS** — an S3-compatible store running in Docker on your + machine, for exercising the remote-server code path without standing up + real infrastructure: `RUSTFS=1 ./install.sh`. +- **Deployed service (shared, multi-user)** — the team mode. Your CLI and + agent become MCP clients of a deployed endpoint, authenticated with your own + Keycloak identity, with per-actor Cedar authorization over one shared graph. + Configured with a `[targets.*]` block plus `witan login`. + +To join a deployed service, follow +[**Pointing your CLI and agent at the deployed witan**](deployed-witan.md), +and migrate the history you already have with the +[migration runbook](migration-runbook.md#local-shared-the-cutover). Sequence +the two together — a store you keep writing to after its export was taken has +a tail nobody will merge. + +Note that pointing `WITAN_MEMORY_URI` straight at an omnigraph-server is a +*different*, lower-level mode: it addresses the data tier directly with a +shared bearer token and no per-user identity. That is how a self-hosted or +in-cluster maintenance process connects, not how a person does. See +[`docs/internals/agent-memory.md` § Operating Modes](https://github.com/mitodl/agent-kit/blob/main/docs/internals/agent-memory.md#6-operating-modes) +for the graph schema and server-deployment mechanics. + +`config.toml` can also define named `[targets.*]` sections that route +different repos/orgs at different stores (e.g. work vs. personal) — see the +`load()` docstring in `witan/config.py` and the commented example block that +`witan setup` writes into your starter config file. + +## Troubleshooting + +- **`witan: command not found` in a hook.** Claude Code/Pi hooks and + extensions call the `witan` binary directly — it must be on `PATH` for the + user those hooks run as. `witan setup` warns explicitly if it can't find + `witan` on `PATH` when you run it; install with `uv tool install + witan-council` (or, for pre-release code, `uv tool install + git+https://github.com/mitodl/agent-kit#subdirectory=mcp/servers/witan`). +- **`omnigraph` binary missing.** `witan setup` downloads it to + `~/.local/bin/omnigraph`; if that directory isn't on `PATH`, both the CLI + and MCP server will fail to reach the graph. Re-run `witan setup` after an + `omnigraph` version bump (tracked via the `omnigraph-version` Renovate + customManager) to refresh the pinned binary. +- **`witan migrate storage` needed after an omnigraph upgrade.** omnigraph + uses strict single-version on-disk storage — a release that bumps the + internal schema refuses to open a store an older binary wrote. If your + local graph suddenly won't open, run `witan migrate storage`; it detects + the old binary, exports with it, and reloads into the new format, + preserving nodes/edges/vectors (commit history and branches are not + preserved; the original is kept as `.pre-migrate`). +- **No tasks/projects showing up for this repo.** Repo scoping is detected + from the `origin` remote in `.git/config` (or `WITAN_REPO` if set). Work + created without repo context, or from a different remote URL form (SSH vs. + HTTPS — both normalize to the same canonical URI, but a mismatch elsewhere + won't), won't show up under `--repo`; pass `--all-repos` to check. +- **Code-branch tracking silently absent.** `workflow_session_start` and + `task_claim` only upsert a `CodeBranch` when they can detect a git repo and + a named branch. Detached HEAD, a directory outside any git repo, or a + store that predates the `CodeBranch` schema (run `witan migrate schema`) + all cause a silent no-op — this is metadata riding along, not a + requirement for the underlying task/workflow call to succeed. +- **A write got blocked or redacted unexpectedly.** That's the write-path + content scanner (secrets block by default, PII redacts by default). Run + `witan scan test ""` to see exactly which detector fired and why, + and `witan scan rules` to see what's active. See + [`docs/write-path-scanning.md`](write-path-scanning.md) for the full + config surface if you need to tune or disable it. +- **`witan run`/`task run`/`project run` can't find your agent CLI.** It + shells out to whatever `--agent` (or `WITAN_AGENT`/config default) + resolves to, verbatim, on `PATH`; a `FileNotFoundError` prints + `Agent '' not found on PATH.` — install or alias that CLI, or pass + `--agent` explicitly. diff --git a/docs/guides/write-path-scanning.md b/docs/guides/write-path-scanning.md new file mode 100644 index 00000000..0e2318f6 --- /dev/null +++ b/docs/guides/write-path-scanning.md @@ -0,0 +1,325 @@ + + +!!! info "This page lives with the code" + + The authoritative copy is + [`mcp/servers/witan/docs/write-path-scanning.md`](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan/docs/write-path-scanning.md). + +# Write-path content scanning + +witan can scan every free-text value written to the graph — memory bodies, +task/project descriptions, session summaries, trace outcomes — for secrets +and PII, before it is embedded or persisted. Design rationale and the +alternatives considered live in [ADR 0001](../explanation/decisions/0001-write-path-content-scanning.md); +this doc is the operator + developer guide for using it. + +Ships **enabled by default** — it's opt-out, not opt-in. Turn it off with: + +```bash +export WITAN_SCAN_ENABLED=false +``` + +or in `config.toml`: + +```toml +[scan] +enabled = false +``` + +By default, `OmnigraphClient.change()` scans the free-text fields of every +`insert_*`/`update_*` mutation (see `FIELD_MAP` in `witan/scan/enforce.py`) — +this covers memories, topics, workflow projects/sessions/traces, and tasks by +construction; no per-tool wiring is needed for new node types beyond adding a +`FIELD_MAP` entry. + +## Config surface + +All settings resolve `WITAN_SCAN_*` env > `[scan]` in `config.toml` > the +defaults below (see `ScanConfig` in `witan/config.py`). + +| Setting | Env var | Default | Description | +|---|---|---|---| +| `enabled` | `WITAN_SCAN_ENABLED` | `true` | Master switch — opt-out | +| `secret_action` | `WITAN_SCAN_SECRET_ACTION` | `block` | Enforcement for `secret` findings | +| `pii_action` | `WITAN_SCAN_PII_ACTION` | `redact` | Enforcement for `pii` findings | +| `enabled_detectors` | `WITAN_SCAN_ENABLED_DETECTORS` | `[]` (all) | Explicit allowlist of detector names | +| `disabled_detectors` | `WITAN_SCAN_DISABLED_DETECTORS` | `[]` | Detector names to switch off; always wins over `enabled_detectors` | +| `plugins` | `WITAN_SCAN_PLUGINS` | `[]` | Dotted `module:Attr` paths to extra scanners (see below) | +| `allowlist` | `WITAN_SCAN_ALLOWLIST` | `[]` | Regexes tested against each finding's own matched span (`re.fullmatch`) — a hit downgrades that finding to audit-only | +| `allowlist_hashes` | `WITAN_SCAN_ALLOWLIST_HASHES` | `[]` | Salted SHA-256 digests of specific approved values — a hit downgrades to audit-only, same as `allowlist`, without the plaintext ever appearing in config | +| `allowlist_salt` | `WITAN_SCAN_ALLOWLIST_SALT` | `""` | Salt for `allowlist_hashes`. Empty means the hash allowlist is inert | +| `on_scanner_error` | `WITAN_SCAN_ON_ERROR` | `block` | `block` (fail-closed) or `warn` if a scanner itself raises | + +List-shaped env vars accept a comma-separated string +(`WITAN_SCAN_DISABLED_DETECTORS=phone,high_entropy_string`); list-shaped TOML +values accept a TOML array or a bare string. + +```toml +[scan] +enabled = true +secret_action = "block" +pii_action = "redact" +disabled_detectors = ["phone"] # noisy for this org +plugins = ["acme_scanners:EmployeeIdScanner"] +``` + +## Enforcement modes + +Every finding resolves to one of three actions — a `Finding` can also carry +an explicit `action` that overrides its category's configured default: + +- **`block`** — the write is rejected with a `WriteBlocked` error before it + reaches the store. The error names the field and detector and includes a + secret-free preview; it never includes the matched text. +- **`redact`** — the matched span is replaced in place with + `«redacted:»` and the write proceeds. The node is also tagged + `scan:redacted` (via its `tags` list, where the mutation has one) so + redacted content is discoverable later, **and the tool result tells the + caller** — see [Redaction is reported back](#redaction-is-reported-back). +- **`warn`** — an audit event is emitted and the write proceeds unchanged. + Useful for rolling out a new detector or policy change without blocking + anyone yet. + +Defaults are asymmetric on purpose: **secrets block** (a leaked credential +must never land), **PII redacts** (mask the span, keep the surrounding text +useful). If a scanner itself raises, the default is fail-closed +(`on_scanner_error = "block"`) so a broken detector can't silently let +everything through. + +## Redaction is reported back + +A redaction is an **unrecoverable edit to the caller's data**: the original +span is kept nowhere, so there is nothing to restore from once it is gone. +It used to be invisible from the outside — the tool returned success and the +caller only found out by reading the row back. That cost a real measurement +(`tk-write-path-redaction-silently-rewrites-content-a-aec2b6`). + +Every tool now reports what it altered. When (and only when) something was +rewritten, the result grows two keys: + +```json +{ + "slug": "tk-…", + "redactions": [ + {"query_name": "update_task", "slug": "tk-…", "field": "description", + "detector": "credit_card", "category": "pii", "start": 41, "end": 60} + ], + "redaction_note": "⚠ CONTENT WAS ALTERED BEFORE STORAGE: tk-….description[41:60] matched credit_card. …" +} +``` + +The report is attached by `witan.server._tool`, which wraps **every** tool +rather than an enumerated list of write paths — so it necessarily runs after +the tool's last write, no intermediate caller can discard it, and a newly +added write tool is covered without being remembered. `slug` names the row +that lost content, which matters when one call rewrites many: `migrate_repo_keys` +walks every task and memory in the graph. + +`start`/`end` index the value **as the caller sent it**, so you can find the +span in your own input. The matched text itself is deliberately absent: a tool +result goes into the caller's transcript, which is a worse place for a +`secret`-category match than a log line (ADR 0001 §D3). + +A clean write grows no keys at all. + +**If it was a false positive**, re-send the content in a shape the detector +does not claim, then correct the stored value. For the `credit_card` rule, +separating long digit runs with commas or units (`3s, 5s, 8s`) is enough. +There is no "store it anyway" override today — see +`tk-the-cli-can-never-reach-the-server-s-steal-promp-555c64` for why an +elicitation-based one would silently do nothing for CLI users. + +## Built-in detectors + +Zero-dependency regex + entropy rules, each independently addressable by name +in `enabled_detectors`/`disabled_detectors`: + +- **Secrets:** `aws_access_key`, `github_token`, `slack_token`, + `google_api_key`, `private_key_block`, `jwt`, `secret_assignment` (generic + `password=`/`api_key=`/`token=` patterns), `high_entropy_string` (Shannon + entropy over long base64/hex-looking tokens). +- **PII:** `email`, `phone`, `us_ssn`, `credit_card` (Luhn-validated, and + additionally required to be *grouped* the way a card is printed — 4-4-4-4, + Amex's 4-6-5, Diners' 4-6-4 and 4-4-4-2, the 13-digit Visa's 4-4-4-1, or one + contiguous run. Luhn alone is a transcription checksum with a 1-in-10 hit rate + on arbitrary digits, so without the grouping rule a whitespace-separated table + of numbers was card-shaped and roughly one in ten of them was silently eaten). + +Run `witan scan rules` to see exactly what's active in your environment (see +below) rather than trusting this list to stay in sync — detectors can be +added, and third-party plugins add more. + +## False-positive suppression (allowlisting) + +Three independent mechanisms downgrade a finding to **audit-only** — the +value is written unchanged (never blocked or redacted) and the finding still +emits exactly one audit event, with `outcome = "suppressed"` and +`suppressed_by` naming which mechanism fired: + +1. **Regex allowlist** (`[scan] allowlist`) — each pattern is matched with + `re.fullmatch` against the finding's own matched span (not the whole + field), so a pattern for one known value can't suppress an unrelated, + longer secret that happens to contain it: + + ```toml + [scan] + allowlist = ["AKIA[A-Z0-9]{16}EXAMPLE"] # a documented fixture key + ``` + +2. **Inline pragma** — a trailing marker in the authored text itself: + `witan: allow-secret` suppresses every finding in that value; + `witan: allow-secret:` scopes it to one detector. Use this when + an author knows a specific write is fine and wants to say so inline rather + than editing config: + + ``` + Run with API_KEY=AKIAIOSFODNN7EXAMPLE (docs fixture) witan: allow-secret:aws_access_key + ``` + +3. **Hash allowlist** (`[scan] allowlist_hashes` + `allowlist_salt`) — + approve a specific value by its salted digest instead of its plaintext + pattern, so the approved value never appears in config: + + ```bash + python3 -c "import hashlib; print(hashlib.sha256(('mysalt' + 'the-approved-value').encode()).hexdigest())" + ``` + + ```toml + [scan] + allowlist_salt = "mysalt" + allowlist_hashes = [""] + ``` + +`witan scan test` shows a `suppressed` column and a summary line so you can +validate an allowlist entry before relying on it in production. + +## `witan scan` — dry-run and introspection + +Validate policy or debug a false positive without writing anything: + +```bash +$ witan scan test "my email is a@b.com, key AKIAIOSFODNN7EXAMPLE" # pragma: allowlist secret gitleaks:allow + Findings +┏━━━━━━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━┳━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━┓ +┃ detector ┃ category ┃ severity ┃ span ┃ action ┃ preview ┃ +┡━━━━━━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━╇━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━┩ +│ aws_access_key │ secret │ high │ 22-42 │ block │ │ +│ email │ pii │ medium │ 12-19 │ redact │ │ +└────────────────┴──────────┴──────────┴───────┴────────┴──────────────────────┘ + +Redacted preview: my email is «redacted:email», key «redacted:aws_access_key» + +1 finding(s) would block this write. + +$ witan scan rules +Scanning: enabled (on_scanner_error=block) +...detector | category | mode | source... +``` + +`witan scan test` runs the exact same `ScannerRegistry` the write path uses +(it works even when `WITAN_SCAN_ENABLED=false`, so you can validate a policy +before turning it on). `witan scan rules` lists every active detector, its +category, its resolved enforcement mode, and its source (`built-in`, +`entry-point:`, or `config:`). + +## Audit trail + +Every finding — blocked, redacted, warned, or suppressed by an allowlist — +emits one structured log line via the standard `logging` module, on the +`witan.scan.audit` logger (`witan/scan/audit.py`). Fields: `query_name`, +`node_type`, `field`, `slug` (when the mutation has one), `detector`, +`category`, `severity`, `action`, `outcome` (`blocked` | `redacted` | +`warned` | `suppressed`), `suppressed_by` (`regex` | `pragma` | `hash`, or +`None`), and `preview` — the matched value is never included, by construction +(this is a hard invariant of `Finding.preview`, not an after-the-fact scrub). +Point your log aggregator (Loki, CloudWatch, journald) at this logger to +build dashboards or alerts on scan activity — e.g. a spike in `suppressed` +events is a signal an allowlist entry may be too broad; there is deliberately +no separate graph node or metrics sink for this yet, to avoid adding new +sensitive-adjacent state to secure and retention-manage. + +## Writing a custom scanner plugin + +Other organizations extend detection without forking witan. A scanner is +anything with: + +```python +class MyScanner: + name: str = "acme_employee_id" # stable, unique detector id + category: Literal["secret", "pii"] = "pii" + + def scan(self, text: str, field: str, node_type: str) -> list[Finding]: + ... # return zero or more Finding objects; never echo the match +``` + +`witan.scan.Scanner` is a `runtime_checkable` `Protocol` (structural typing — +no base class to inherit). See `witan/scan/models.py` for `Finding`'s exact +shape and `witan/scan/detectors.py` for worked examples (`RegexScanner`, +`EntropyScanner`). + +**Never put the matched value in `Finding.preview`** — it ends up in log +lines and, for a blocked write, in the rejection error surfaced to the agent. +Use `witan.scan.masked_preview(detector, value)` to build a safe one. + +Two ways to register a plugin, both read by `ScannerRegistry`: + +1. **Entry point** — declare it in the plugin package's `pyproject.toml`: + + ```toml + [project.entry-points."witan.scanners"] + acme_employee_id = "acme_scanners:EmployeeIdScanner" + ``` + + Once the package is installed alongside witan, it's discovered + automatically — no config change needed. This is the primary mechanism for + a published, shareable plugin. + +2. **Dotted config path** — for a scanner that isn't packaged, point + `plugins` (or `WITAN_SCAN_PLUGINS`) at it directly: + + ```toml + [scan] + plugins = ["acme_scanners:EmployeeIdScanner"] + ``` + +Either way, `enabled_detectors`/`disabled_detectors` then select or silence it +like any built-in rule, and a load failure (bad import path, missing +attribute, wrong shape) raises loudly when the registry is built rather than +silently starting with a detector missing. + +A complete, runnable example package lives at +[`examples/example-scanner-plugin`](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan/examples/example-scanner-plugin) — +copy it as a starting point. + +## Multi-tenant / deployed-server mode + +`ScanConfig` is loaded once, at process import, from the deployment's own +environment/`config.toml` — no MCP tool call can influence it, so in the +sanctioned deployed topology (one shared `streamable-http` witan-service +process; see ADR 0004 in the multi-user deployment project) scan policy is +already admin-owned by construction. That invariant depends on every write to +a shared store passing through the witan-service process — omnigraph itself +has no content-scanning hook, and Cedar cannot express one (ADR 0002 §D1), so +a write that reaches omnigraph-server by any other path skips scanning +entirely. + +Per-repo policy overrides are supported via `[scan.overlay.""]` +tables in `config.toml` — deliberately **TOML-only, no `WITAN_SCAN_*` +env-var form**, since env vars are exactly the surface a write's own process +could otherwise control: + +```toml +[scan.overlay."github.com/example/legacy-repo"] +secret_action = "warn" # rolling out scanning on a noisy repo before enforcing +``` + +Any `ScanConfig` field except `overlay` itself may be overridden. `WriteGuard` +resolves the effective policy from the write's own `repo` (or the first entry +of `repos`) param — see `ScanConfig.for_repo` in `witan/config.py` and the +2026-07-09 amendment in [ADR 0001](../explanation/decisions/0001-write-path-content-scanning.md) +for the full design rationale, including why the detector set itself (as +opposed to enforcement policy) is not overlay-able in this version. Everything +else in this doc applies as-is to local, single-user witan today. diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 00000000..c8b36766 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,101 @@ +# witan-context + +**Shared memory, work coordination, and a code graph for coding agents.** + +A coding agent starts every session knowing nothing. It re-derives the same +facts, rediscovers the same constraints, and repeats the same mistakes — and +when two agents work in parallel, neither knows what the other is doing. + +witan is the missing layer: a persistent, team-wide graph that agents read from +and write to. What one session learns, the next one recalls. What one agent +is working on, the others can see. + +--- + +## The three layers + +
+ +- **Memory** + + Durable, shareable knowledge — patterns, project facts, lessons, decisions. + Memories link to each other (`supersedes`, `refines`, `contradicts`), so + recall returns what is *current*, not merely what matched. + + [Memory tools →](reference/mcp-tools/memory.md) + +- **Work coordination** + + Tasks, dependency edges, and multi-session projects. Claiming is a + best-effort compare-and-swap with a lease — enough for parallel agents to + divide work without double-doing it. + + [Task tools →](reference/mcp-tools/tasks.md) + +- **Code graph** + + A tree-sitter index for exact symbol lookups, caller graphs, and + change-impact analysis — plus a cross-repo bridge that traces a shared env + var, endpoint, or package from provider to consumer. + + [Code tools →](reference/mcp-tools/code.md) + +
+ +All three are served by **one MCP endpoint**. A single `witan serve` mounts 60 +tools; one entry in your agent's config gets you the whole surface. + +--- + +## Start here + +| If you want to… | Go to | +| --- | --- | +| Install it and store your first memory | [Get started](getting-started/index.md) | +| Do a specific thing — index a repo, run against a deployed service, migrate a store | [Guides](guides/index.md) | +| Look up a tool, flag, env var, or node type | [Reference](reference/index.md) | +| Understand *why* it works the way it does | [Explanation](explanation/index.md) | + +--- + +## Quick start + +```bash +uv tool install ol-agent-kit +witan setup +``` + +`witan setup` registers the MCP server and its skills with whichever coding-agent +platforms it finds locally — Claude Code, Pi, GitHub Copilot, OpenCode, and +others. Then, from inside any git repository: + +```bash +witan tasks # ready work in this repo +witan memory "vault auth" # what do we know about this? +witan code index # build this repo's code graph +``` + +Then ask your agent things like *"who calls `retry_with_backoff`?"* or *"what +breaks if I change this signature?"* — the `code_*` tools answer from the index +rather than from grep. + +See [Installation](getting-started/installation.md) for the other install paths +and for pointing witan at a shared, deployed service instead of a local store. + +--- + +## What makes up witan + +witan-context covers three published packages, all developed in the +[`mitodl/agent-kit`](https://github.com/mitodl/agent-kit) monorepo: + +| Package | What it is | +| --- | --- | +| [`witan-council`](https://pypi.org/project/witan-council/) | The memory, task, and workflow tools, plus the `witan` umbrella CLI | +| [`witan-code`](https://pypi.org/project/witan-code/) | The tree-sitter code graph and cross-repo bridge; mounts as `witan code` | +| [`witan-core`](https://pypi.org/project/witan-core/) | Shared internals: the graph client, OIDC/remote transport, observability | +| [`ol-agent-kit`](https://pypi.org/project/ol-agent-kit/) | Meta-package that installs all of the above in one shot | + +Storage is [omnigraph](https://github.com/ModernRelay/omnigraph) — a local file, an +`s3://` bucket, or a shared `omnigraph-server`. The same tools work against all +three; only [`WITAN_MEMORY_URI`](reference/environment.md) changes. diff --git a/docs/agent-memory.md b/docs/internals/agent-memory.md similarity index 98% rename from docs/agent-memory.md rename to docs/internals/agent-memory.md index 4bedec15..28dffd91 100644 --- a/docs/agent-memory.md +++ b/docs/internals/agent-memory.md @@ -107,12 +107,12 @@ reindex-hook, `UserPromptSubmit` → inject-context, `Stop` → checkpoint) are bare `witan-code` CLI commands — no scripts to symlink. `witan-code setup` (standalone) or `witan setup` (when witan-code is also importable) registers them for you; to register manually instead, see -[`configs/hooks/README.md`](../configs/hooks/README.md) for the exact JSON. +[`configs/hooks/README.md`](https://github.com/mitodl/agent-kit/blob/main/configs/hooks/README.md) for the exact JSON. **Pi** has no Claude-style hooks but provides the equivalent via extension events. Symlink the mirror extensions into `~/.pi/agent/extensions/` (codegraph — all four hooks in one extension — and workflow context -injection) — see [`configs/pi/README.md`](../configs/pi/README.md): +injection) — see [`configs/pi/README.md`](https://github.com/mitodl/agent-kit/blob/main/configs/pi/README.md): ```bash ln -sf "$REPO/configs/pi/extensions/codegraph.ts" ~/.pi/agent/extensions/ @@ -1212,7 +1212,7 @@ export WITAN_AUTHOR="Alice Smith" > The multi-user shape puts an MCP tier in front, so each person authenticates > as themselves (Keycloak JWT → `act-` → that actor's own omnigraph > token). That is what mitodl runs, and joining it is -> [`deployed-witan-onboarding.md`](../mcp/servers/witan/docs/deployed-witan-onboarding.md) +> [`deployed-witan-onboarding.md`](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan/docs/deployed-witan-onboarding.md) > — a `[targets.*]` block and `witan login`, not these env vars. **Deploying the server:** @@ -1270,11 +1270,11 @@ witan migrate merge ~/.local/share/witan/graph.omni --target For the **deployed** multi-tenant service the target is not reachable from a laptop at all — the data tier is ClusterIP-only, so the merge runs in-cluster from a handed-over export. That procedure is -[`mcp/servers/witan/docs/migration-runbook.md` § Local → shared](../mcp/servers/witan/docs/migration-runbook.md#local--shared-the-cutover), +[`mcp/servers/witan/docs/migration-runbook.md` § Local → shared](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan/docs/migration-runbook.md#local-shared-the-cutover), with the reasoning in -[witan ADR-0007](../mcp/servers/witan/docs/adr/0007-local-to-shared-store-migration-transport.md). +[witan ADR-0007](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan/docs/adr/0007-local-to-shared-store-migration-transport.md). Pointing your own CLI at that service is -[`deployed-witan-onboarding.md`](../mcp/servers/witan/docs/deployed-witan-onboarding.md). +[`deployed-witan-onboarding.md`](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan/docs/deployed-witan-onboarding.md). --- @@ -1546,7 +1546,7 @@ independently. See [MCP Tools](#mcp-tools) in the server README for signatures. Full usage documentation is in -[`mcp/servers/witan/witan/skills/witan-project-tracker/SKILL.md`](../mcp/servers/witan/witan/skills/witan-project-tracker/SKILL.md). +[`mcp/servers/witan/witan/skills/witan-project-tracker/SKILL.md`](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan/witan/skills/witan-project-tracker/SKILL.md). ### Session State File diff --git a/docs/design/agent-config-kit-cli-spec.md b/docs/internals/design/agent-config-kit-cli-spec.md similarity index 100% rename from docs/design/agent-config-kit-cli-spec.md rename to docs/internals/design/agent-config-kit-cli-spec.md diff --git a/docs/design/agent-config-kit-profiles-composition-spec.md b/docs/internals/design/agent-config-kit-profiles-composition-spec.md similarity index 100% rename from docs/design/agent-config-kit-profiles-composition-spec.md rename to docs/internals/design/agent-config-kit-profiles-composition-spec.md diff --git a/docs/design/agent-config-kit-spec.md b/docs/internals/design/agent-config-kit-spec.md similarity index 100% rename from docs/design/agent-config-kit-spec.md rename to docs/internals/design/agent-config-kit-spec.md diff --git a/docs/design/graph-structured-memory-spec.md b/docs/internals/design/graph-structured-memory-spec.md similarity index 100% rename from docs/design/graph-structured-memory-spec.md rename to docs/internals/design/graph-structured-memory-spec.md diff --git a/docs/design/graph-structured-memory.md b/docs/internals/design/graph-structured-memory.md similarity index 100% rename from docs/design/graph-structured-memory.md rename to docs/internals/design/graph-structured-memory.md diff --git a/docs/design/omnigraph-remote-call-overhead-spike.md b/docs/internals/design/omnigraph-remote-call-overhead-spike.md similarity index 100% rename from docs/design/omnigraph-remote-call-overhead-spike.md rename to docs/internals/design/omnigraph-remote-call-overhead-spike.md diff --git a/docs/design/witan-core-extraction-spec.md b/docs/internals/design/witan-core-extraction-spec.md similarity index 100% rename from docs/design/witan-core-extraction-spec.md rename to docs/internals/design/witan-core-extraction-spec.md diff --git a/docs/design/witan-surface-refinement-spec.md b/docs/internals/design/witan-surface-refinement-spec.md similarity index 100% rename from docs/design/witan-surface-refinement-spec.md rename to docs/internals/design/witan-surface-refinement-spec.md diff --git a/docs/design/witan-workflow-hooks-elicitation-evaluation.md b/docs/internals/design/witan-workflow-hooks-elicitation-evaluation.md similarity index 100% rename from docs/design/witan-workflow-hooks-elicitation-evaluation.md rename to docs/internals/design/witan-workflow-hooks-elicitation-evaluation.md diff --git a/docs/design/witan-workflow-ux-p1-spec.md b/docs/internals/design/witan-workflow-ux-p1-spec.md similarity index 100% rename from docs/design/witan-workflow-ux-p1-spec.md rename to docs/internals/design/witan-workflow-ux-p1-spec.md diff --git a/docs/internals/index.md b/docs/internals/index.md new file mode 100644 index 00000000..9185c0ed --- /dev/null +++ b/docs/internals/index.md @@ -0,0 +1,60 @@ +# Internals + +Design documents and implementation specs, kept for the record. + +!!! warning "These are point-in-time documents" + + Everything in this section describes what was intended *when it was + written*. Some of it shipped as specified, some of it changed during + implementation, and some describes work that was never done. None of it is + maintained against the current code. + + For what witan does **now**, use [Reference](../reference/index.md) — those + pages are generated from the source and verified in CI. For why it is shaped + the way it is, use [Explanation](../explanation/index.md), and in particular + the [ADRs](../explanation/decisions/0001-write-path-content-scanning.md), + which *are* maintained. + +They are published anyway because the reasoning in them is often the only record +of why an alternative was rejected — which is exactly what you want when you are +about to propose it again. + +## What is here + +| Document | Subject | +| --- | --- | +| [Agent memory implementation guide](agent-memory.md) | The original end-to-end build guide: schema, queries, server, install, operating modes | +| [Graph-structured memory](design/graph-structured-memory.md) | The case for typed edges over a flat note store | +| [Graph-structured memory (spec)](design/graph-structured-memory-spec.md) | The detailed specification that followed it | +| [witan-core extraction](design/witan-core-extraction-spec.md) | Splitting shared internals out of the two servers | +| [witan surface refinement](design/witan-surface-refinement-spec.md) | Consolidating the tool surface | +| [Workflow UX (P1)](design/witan-workflow-ux-p1-spec.md) | The project/session tracking experience | +| [Workflow hooks & elicitation](design/witan-workflow-hooks-elicitation-evaluation.md) | Evaluating hook-driven vs. elicited session linking | +| [Remote call overhead spike](design/omnigraph-remote-call-overhead-spike.md) | Measuring what a remote graph call actually costs | +| [agent-config-kit](design/agent-config-kit-spec.md) · [CLI](design/agent-config-kit-cli-spec.md) · [profiles](design/agent-config-kit-profiles-composition-spec.md) | The installer that registers witan with each agent platform | + +## Contributing to the docs + +The site is built with [Zensical](https://zensical.org) from `docs/` in the +[`mitodl/agent-kit`](https://github.com/mitodl/agent-kit) repository. + +Three kinds of page, with different rules: + +- **Generated** (`docs/reference/`) — produced by `bin/gen_docs.py` from the + live code. Never edit these; change the source and re-run the generator. +- **Mirrored** (most of `docs/guides/`, the ADRs, the code-graph explanation + pages) — copied from the package that owns them. Edit the file next to the + code; the banner on each page links to it. +- **Handwritten** (the tutorials, the section overviews, the rest of + `docs/explanation/`) — edit directly. + +```bash +just docs-gen # regenerate + re-mirror everything +just docs-check # fail if any generated page is stale (what CI runs) +just docs-serve # incremental local preview +``` + +`just docs-check` runs in CI, so a change to a tool signature, a CLI flag, or the +graph schema fails the build until the reference is regenerated and committed. +That is the mechanism keeping this site from becoming another point-in-time +document. diff --git a/docs/reference/bridge-schema.md b/docs/reference/bridge-schema.md new file mode 100644 index 00000000..d7b14050 --- /dev/null +++ b/docs/reference/bridge-schema.md @@ -0,0 +1,110 @@ + + +# Cross-repo bridge schema + +The bridge store links repositories to each other by shared contract keys — an env var, an HTTP endpoint, a package name, a service name. It is what makes `code_interface_providers` and `code_cross_repo_impact` able to answer a question that spans two checkouts. + +Source of truth: [`mcp/servers/witan-code/witan_code/schema/bridge-schema.pg`](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan-code/witan_code/schema/bridge-schema.pg). + +## Nodes + +### `InterfaceBinding` + +Cross-Repo Context Bridge (Layer 2.5) — one shared store across all repos. + +Deployed as the `code-bridge` graph on the shared, S3-backed omnigraph-server +cluster (config.BRIDGE_GRAPH_ID); locally it is the `_bridge.omni` store. + +One flat node per interface binding. Cross-repo linkages are computed by +GROUPING bindings on (kind, key_norm) where roles/repos differ — there are +NO link edges. An anchor-node + edge model would force every repo touching a +shared contract (e.g. env_var DATABASE_URL) to upsert the same node, maxing +write contention on a store with many concurrent writers (every repo's index +run + every PostToolUse reindex hook). Flat bindings are each scoped to their +own repo+file, so concurrent writers never touch the same row. + +Re-derivable, like the per-repo Layer-2 graphs; seed the shared copy by +re-indexing on the server or export→load, never by copying a local store to S3. + +Id convention: +InterfaceBinding.slug = repo|file|kind|key_norm|role|symbol_id +e.g. https://github.com/mitodl/mit-learn|main/settings.py|env_var| +MITOL_APP_BASE_URL|consumer|...settings.py::<module> + +| Field | Type | Notes | +| --- | --- | --- | +| `slug` | `String @key` | | +| `kind` | `enum(env_var, package, service, endpoint) @index` | | +| `key` | `String @index` | raw as written (METHOD path, pkg name, env NAME) | +| `key_norm` | `String @index` | normalized join key (collapsed path params, …) | +| `role` | `enum(provider, consumer, shared) @index` | | +| `repo` | `String @index` | canonical HTTPS repo URI (join key to Layer 2) | +| `file` | `String @index` | repo-relative source path of the binding | +| `repo_file` | `String @index` | "repo|file" — single-field key for per-file delete | +| `sub_kind` | `String?` | service anchor variant: repo | image | name | +| `symbol_id` | `String?` | enclosing Symbol id (repo#path::Qn) when applicable | +| `line` | `I32?` | | +| `language` | `String?` | | +| `framework` | `String?` | django | drf | pulumi | npm | nextjs | … | +| `generic` | `String?` | "1" for stoplisted generic keys (DEBUG, PORT, …) | +| `confidence` | `F32?` | 0.0–1.0 endpoint-consumer trust score (phantom | +| `symbol` | `String? @index` | canonical symbol string (docs/SYMBOL_FORMAT.md): | +| `indexed_at` | `DateTime` | | + +### `RepoSymbol` + +Per-repo symbol table (Stage 1 artifact — docs/SYMBOL_TABLE.md): one row per +(repo, role, symbol), aggregated from that repo's InterfaceBinding rows on +every bridge write. role=exported is the repo's public contract surface; +role=external is an unresolved reference to another repo's contract (a +RANGER-style import placeholder, redirected at read time by Stage 2 — no +cross-repo edges are ever written). Rows are repo-scoped, so concurrent +writers never touch the same row (same argument as InterfaceBinding). + +| Field | Type | Notes | +| --- | --- | --- | +| `slug` | `String @key` | repo|role|symbol | +| `repo` | `String @index` | canonical HTTPS repo URI | +| `role` | `enum(exported, external) @index` | | +| `symbol` | `String @index` | full canonical string (docs/SYMBOL_FORMAT.md) | +| `scheme` | `String @index` | http | env | pkg | svc | +| `descriptor` | `String @index` | precise Stage-2 join key (with scheme) | +| `key_norm` | `String @index` | coarse join key — descriptor minus the | +| `manager` | `String?` | "." = unknown | +| `package` | `String?` | "." = unresolved (typical for external) | +| `version` | `String?` | | +| `kind` | `String` | binding kind: env_var | package | service | endpoint | +| `n_refs` | `I32` | occurrence count in this repo | +| `confidence` | `F32?` | max occurrence confidence; 1.0 for exported | +| `file` | `String?` | exemplar occurrence (deterministic: min file/line) | +| `line` | `I32?` | | +| `indexed_at` | `DateTime` | | + +### `PackageMap` + +One row per indexed repo: its declared (or fallback) package identity from +witan-code.toml (docs/PACKAGE_MAP.md). Overwritten on each full-repo index +(merge by slug). Qualifies provider symbols and backs the +known_provider_package confidence heuristic. + +| Field | Type | Notes | +| --- | --- | --- | +| `slug` | `String @key` | canonical repo URI | +| `repo` | `String @index` | | +| `name` | `String @index` | canonical package name | +| `manager` | `String` | pypi | npm | "." | +| `version` | `String` | "main" = trunk-tracking | +| `provides` | `String?` | JSON array of extra "manager:name" identities | +| `declared` | `String?` | "1" when read from witan-code.toml (vs fallback) | +| `indexed_at` | `DateTime` | | + +## Edges + +Edges are directional and typed. A traversal names the edge in lowercase (`supersedes`, `blocks`), while the schema declares it in PascalCase. + +| Edge | From | To | Meaning | +| --- | --- | --- | --- | diff --git a/docs/reference/cli.md b/docs/reference/cli.md new file mode 100644 index 00000000..4613fe5d --- /dev/null +++ b/docs/reference/cli.md @@ -0,0 +1,1562 @@ + +# witan + +```console +witan COMMAND [OPTIONS] +``` + +witan — agent memory, planning, and collaboration graph. + +## Table of Contents + +- [`login`](#witan-login) +- [`logout`](#witan-logout) +- [`whoami`](#witan-whoami) +- [`graph`](#witan-graph) +- [`inject-context`](#witan-inject-context) +- [`session-checkpoint`](#witan-session-checkpoint) +- [`optimize`](#witan-optimize) +- [`cleanup`](#witan-cleanup) +- [`memory`](#witan-memory) +- [`projects`](#witan-projects) +- [`project`](#witan-project) + - [`status`](#witan-project-status) + - [`tasks`](#witan-project-tasks) + - [`create`](#witan-project-create) + - [`update`](#witan-project-update) + - [`advance`](#witan-project-advance) + - [`complete`](#witan-project-complete) + - [`block`](#witan-project-block) + - [`unblock`](#witan-project-unblock) + - [`run`](#witan-project-run) +- [`scan`](#witan-scan) + - [`test`](#witan-scan-test) + - [`rules`](#witan-scan-rules) +- [`session`](#witan-session) + - [`start`](#witan-session-start) + - [`end`](#witan-session-end) + - [`sweep`](#witan-session-sweep) + - [`list`](#witan-session-list) +- [`setup`](#witan-setup) +- [`target`](#witan-target) + - [`add`](#witan-target-add) + - [`list`](#witan-target-list) + - [`remove`](#witan-target-remove) +- [`tasks`](#witan-tasks) +- [`task`](#witan-task) + - [`create`](#witan-task-create) + - [`close`](#witan-task-close) + - [`claim`](#witan-task-claim) + - [`release`](#witan-task-release) + - [`update`](#witan-task-update) + - [`link`](#witan-task-link) + - [`unlink`](#witan-task-unlink) + - [`run`](#witan-task-run) +- [`traces`](#witan-traces) +- [`trace`](#witan-trace) + - [`list`](#witan-trace-list) +- [`migrate`](#witan-migrate) + - [`schema`](#witan-migrate-schema) + - [`storage`](#witan-migrate-storage) + - [`merge`](#witan-migrate-merge) + - [`topics`](#witan-migrate-topics) + - [`repo-keys`](#witan-migrate-repo-keys) + - [`dedupe-sessions`](#witan-migrate-dedupe-sessions) + - [`all`](#witan-migrate-all) + - [`claim-authorship`](#witan-migrate-claim-authorship) +- [`code`](#witan-code) + - [`index`](#witan-code-index) + - [`reindex`](#witan-code-reindex) + - [`deps`](#witan-code-deps) + - [`symbols`](#witan-code-symbols) + - [`stitch`](#witan-code-stitch) + - [`inject-context`](#witan-code-inject-context) + - [`serve`](#witan-code-serve) + - [`optimize`](#witan-code-optimize) + - [`cleanup`](#witan-code-cleanup) + - [`reap-views`](#witan-code-reap-views) + - [`checkpoint`](#witan-code-checkpoint) + - [`session-init`](#witan-code-session-init) + - [`reindex-hook`](#witan-code-reindex-hook) + - [`setup`](#witan-code-setup) + - [`branches`](#witan-code-branches) + - [`repos`](#witan-code-repos) + - [`login`](#witan-code-login) + - [`logout`](#witan-code-logout) + - [`whoami`](#witan-code-whoami) +- [`serve`](#witan-serve) +- [`run`](#witan-run) + +**Commands**: + +* [`cleanup`](#witan-cleanup): Remove old Lance versions to reclaim disk (**destructive**). +* [`code`](#witan-code): witan-code — tree-sitter code graph + cross-repo bridge. +* [`graph`](#witan-graph): Visualize the workflow project and task dependency graph. +* [`inject-context`](#witan-inject-context): Print workflow context for the UserPromptSubmit hook. +* [`login`](#witan-login): Authenticate to the deployed witan service via the OIDC device grant. +* [`logout`](#witan-logout): Forget the cached token for the configured deployment. +* [`memory`](#witan-memory): Search memory (BM25), or with no query list memories (filtered by --kind). +* [`migrate`](#witan-migrate): One-shot, idempotent schema and data migrations. +* [`optimize`](#witan-optimize): Compact the graph store's Lance fragments (non-destructive). +* [`project`](#witan-project): Manage workflow projects. +* [`projects`](#witan-projects): List workflow projects (default: active in the current repo). +* [`run`](#witan-run): Claim a task and launch an agent to execute it. +* [`scan`](#witan-scan): Introspect and dry-run write-path content scanning (ADR 0001). +* [`serve`](#witan-serve): Run the witan MCP server. +* [`session`](#witan-session): Manage workflow sessions. +* [`session-checkpoint`](#witan-session-checkpoint): Auto-close the active WorkflowSession on agent stop (Stop hook). +* [`setup`](#witan-setup): Install witan for one or all supported coding agents. +* [`target`](#witan-target): Register and inspect named [targets.*] blocks (deployed witan endpoints). +* [`task`](#witan-task): Manage tasks. +* [`tasks`](#witan-tasks): List tasks for the current repo (or filtered). +* [`trace`](#witan-trace): Inspect corpus trace records. +* [`traces`](#witan-traces): List corpus workflow traces (default: current repo). +* [`whoami`](#witan-whoami): Show the identity the CLI presents to the deployed witan service. + +**Parameters**: + +* `--output-format`: projects, memory, traces, scan, and mounted witan-code tables. Values: + txt | json | toml | yaml. Env: WITAN_OUTPUT_FORMAT. *[choices: txt, json, toml, yaml]* *[env: WITAN_OUTPUT_FORMAT]* *[default: txt]* + +## witan login + +```console +witan login [OPTIONS] +``` + +Authenticate to the deployed witan service via the OIDC device grant. + +Prints a verification URL and a user code; approve it in a browser, and the +resulting token is cached (mode 0600) and refreshed automatically for +subsequent ``witan …`` commands. + +**Parameters**: + +* `--output-format`: projects, memory, traces, scan, and mounted witan-code tables. Values: + txt | json | toml | yaml. Env: WITAN_OUTPUT_FORMAT. *[choices: txt, json, toml, yaml]* *[env: WITAN_OUTPUT_FORMAT]* *[default: txt]* +* `--target`: a target with no ``match_*`` criteria, which never selects itself. Also + settable via ``WITAN_TARGET``. + +## witan logout + +```console +witan logout [OPTIONS] +``` + +Forget the cached token for the configured deployment. + +**Parameters**: + +* `--output-format`: projects, memory, traces, scan, and mounted witan-code tables. Values: + txt | json | toml | yaml. Env: WITAN_OUTPUT_FORMAT. *[choices: txt, json, toml, yaml]* *[env: WITAN_OUTPUT_FORMAT]* *[default: txt]* +* `--target`: + +## witan whoami + +```console +witan whoami [OPTIONS] +``` + +Show the identity the CLI presents to the deployed witan service. + +**Parameters**: + +* `--output-format`: projects, memory, traces, scan, and mounted witan-code tables. Values: + txt | json | toml | yaml. Env: WITAN_OUTPUT_FORMAT. *[choices: txt, json, toml, yaml]* *[env: WITAN_OUTPUT_FORMAT]* *[default: txt]* +* `--target`: + +## witan graph + +```console +witan graph [OPTIONS] +``` + +Visualize the workflow project and task dependency graph. + +Prints a Rich summary of projects and tasks, then optionally writes an +interactive HTML graph (vis-network) or a Graphviz DOT file. + +**Parameters**: + +* `--output-format`: projects, memory, traces, scan, and mounted witan-code tables. Values: + txt | json | toml | yaml. Env: WITAN_OUTPUT_FORMAT. *[choices: txt, json, toml, yaml]* *[env: WITAN_OUTPUT_FORMAT]* *[default: txt]* +* `--repo`: Scope to a specific repo URI (default: current git repo). +* `--all-repos, --no-all-repos`: Include projects and tasks from every repo. *[default: False]* +* `--status`: Project status filter: active | completed | abandoned. + Defaults to ``active``. Pass an empty string to include all. *[default: active]* +* `--all-tasks, --no-all-tasks`: Include closed tasks (default: open + in_progress + blocked only). *[default: False]* +* `--no-belongs-to, --no-no-belongs-to`: Omit dashed task→project edges to reduce clutter. *[default: False]* +* `--html`: Write a self-contained interactive HTML graph to this path. +* `--dot`: Write a Graphviz DOT file to this path. +* `--open-browser, --no-open-browser`: Open the generated HTML in the default browser (requires --html). *[default: False]* + +## witan inject-context + +```console +witan inject-context [OPTIONS] +``` + +Print workflow context for the UserPromptSubmit hook. + +Emits active WorkflowProjects and ready Tasks for the current git repo to +stdout. Designed to be called by ``~/.claude/hooks/workflow-context-inject.sh`` +— always exits 0 and never blocks even when the graph is missing or the repo +is not in git. + +**Parameters**: + +* `--output-format`: projects, memory, traces, scan, and mounted witan-code tables. Values: + txt | json | toml | yaml. Env: WITAN_OUTPUT_FORMAT. *[choices: txt, json, toml, yaml]* *[env: WITAN_OUTPUT_FORMAT]* *[default: txt]* +* `--debug, --no-debug`: and the reason for any swallowed failure) to stderr. stdout still carries + only the injected block, so ``witan inject-context --debug`` is safe to + run by hand to see why the block is blank. *[default: False]* + +## witan session-checkpoint + +```console +witan session-checkpoint +``` + +Auto-close the active WorkflowSession on agent stop (Stop hook). + +Reads the session handle ``workflow_session_start`` returned (persisted +locally, see ``witan.session_state``) and passes its ``session_slug`` back to +``workflow_session_end``. No-op when there is no handle — the session was +already closed explicitly. Always exits 0 and never blocks. Also +opportunistically triggers a throttled background store compaction. + +The end call goes through ``_srv()``, so it reaches whichever server actually +holds the session: the in-process module locally, or the deployment over MCP. +Writing straight to a local store here is what used to leave deployed +sessions open forever. + +**Parameters**: + +* `--output-format`: projects, memory, traces, scan, and mounted witan-code tables. Values: + txt | json | toml | yaml. Env: WITAN_OUTPUT_FORMAT. *[choices: txt, json, toml, yaml]* *[env: WITAN_OUTPUT_FORMAT]* *[default: txt]* + +## witan optimize + +```console +witan optimize [OPTIONS] +``` + +Compact the graph store's Lance fragments (non-destructive). + +Collapses the many tiny fragments that accrue from every write so opening +the store stays cheap. Safe to run repeatedly; takes the store write lock. + +**Parameters**: + +* `--output-format`: projects, memory, traces, scan, and mounted witan-code tables. Values: + txt | json | toml | yaml. Env: WITAN_OUTPUT_FORMAT. *[choices: txt, json, toml, yaml]* *[env: WITAN_OUTPUT_FORMAT]* *[default: txt]* +* `--store`: + +## witan cleanup + +```console +witan cleanup [OPTIONS] +``` + +Remove old Lance versions to reclaim disk (**destructive**). + +``optimize`` compacts fragments but leaves old versions behind; this GCs +them, keeping the most recent ``keep`` versions per table (and/or those +newer than ``older_than``). Irreversible, so it requires ``--yes``. + +**Parameters**: + +* `--output-format`: projects, memory, traces, scan, and mounted witan-code tables. Values: + txt | json | toml | yaml. Env: WITAN_OUTPUT_FORMAT. *[choices: txt, json, toml, yaml]* *[env: WITAN_OUTPUT_FORMAT]* *[default: txt]* +* `--store`: +* `--keep`: *[default: 10]* +* `--older-than`: +* `--yes, --no-yes`: *[default: False]* + +## witan memory + +```console +witan memory [OPTIONS] [ARGS] +``` + +Search memory (BM25), or with no query list memories (filtered by --kind). + +**Parameters**: + +* `--output-format`: projects, memory, traces, scan, and mounted witan-code tables. Values: + txt | json | toml | yaml. Env: WITAN_OUTPUT_FORMAT. *[choices: txt, json, toml, yaml]* *[env: WITAN_OUTPUT_FORMAT]* *[default: txt]* +* `QUERY, --query`: +* `--kind`: *[choices: pattern, project_fact, lesson, agent_context]* +* `--repo`: +* `--all-repos, --no-all-repos`: *[default: False]* +* `--limit`: *[default: 20]* + +## witan projects + +```console +witan projects [OPTIONS] +``` + +List workflow projects (default: active in the current repo). + +**Parameters**: + +* `--output-format`: projects, memory, traces, scan, and mounted witan-code tables. Values: + txt | json | toml | yaml. Env: WITAN_OUTPUT_FORMAT. *[choices: txt, json, toml, yaml]* *[env: WITAN_OUTPUT_FORMAT]* *[default: txt]* +* `--repo`: +* `--status`: *[default: active]* +* `--all-repos, --no-all-repos`: *[default: False]* +* `--limit`: *[default: 50]* + +## witan project + +```console +witan project COMMAND SLUG +``` + +Manage workflow projects. + +**Commands**: + +* [`advance`](#witan-project-advance): Advance a project to a new phase. +* [`block`](#witan-project-block): Declare that ``slug`` must complete before ``blocks`` can begin. +* [`complete`](#witan-project-complete): Complete a project and seal its immutable corpus trace. +* [`create`](#witan-project-create): Create a new workflow project. +* [`run`](#witan-project-run): Launch an agent session focused on a workflow project. +* [`status`](#witan-project-status): Resume view — phase, ready tasks, last session, blockers ("what next"). +* [`tasks`](#witan-project-tasks): List a project's tasks, optionally with their dependency structure. +* [`unblock`](#witan-project-unblock): Remove a project dependency declared with ``project block``. +* [`update`](#witan-project-update): Correct a project's metadata after creation. + +**Parameters**: + +* `--output-format`: projects, memory, traces, scan, and mounted witan-code tables. Values: + txt | json | toml | yaml. Env: WITAN_OUTPUT_FORMAT. *[choices: txt, json, toml, yaml]* *[env: WITAN_OUTPUT_FORMAT]* *[default: txt]* +* `SLUG, --slug`: **[required]** + +### witan project status + +```console +witan project status [OPTIONS] SLUG +``` + +Resume view — phase, ready tasks, last session, blockers ("what next"). + +The single-call resume view for a project. Pass ``--json`` for the raw +``workflow_project_status`` payload. + +**Parameters**: + +* `SLUG, --slug`: **[required]** +* `--json, --no-json`: *[default: False]* + +### witan project tasks + +```console +witan project tasks [OPTIONS] SLUG +``` + +List a project's tasks, optionally with their dependency structure. + +``project `` already shows a flat task list; this focuses on the tasks +and, with ``--detail``, expands each task's blockers (what it waits on) and +dependents (what waits on it), resolving statuses from the project's own task +set so the dependency chain is visible without hopping between commands. + +**Parameters**: + +* `SLUG, --slug`: **[required]** +* `--status`: +* `--detail, --no-detail`: *[default: False]* + +### witan project create + +```console +witan project create [OPTIONS] TITLE +``` + +Create a new workflow project. + +**Parameters**: + +* `TITLE, --title`: **[required]** +* `--description`: *[default: ""]* +* `--phase`: *[choices: discovery, spec, implementation, delivery]* *[default: discovery]* +* `--repo`: +* `--github-issue`: +* `--tags, --empty-tags`: + +### witan project update + +```console +witan project update [OPTIONS] SLUG +``` + +Correct a project's metadata after creation. + +Only what you pass is touched, so this can never blank a field by accident. + +The common case is repos: a project's real blast radius is rarely known +during discovery, and until the set is right, the project doesn't surface +in the injected context of the repos where the work actually lands. + +Two things this deliberately can't do, matching the MCP tool. It can't set +the phase — ``project advance`` stays the only route, so a transition is +always seen by its ordering check (it allows going backwards, which is how +a phase set in error gets corrected). And it can't complete a project: +``--status`` takes ``active`` or ``abandoned``, but ``completed`` belongs to +``project complete``, which seals a corpus trace. Nothing should mint a +trace without a narrative. + +**Parameters**: + +* `SLUG, --slug`: **[required]** +* `--title`: +* `--description`: +* `--repos, --empty-repos`: +* `--add-repo, --empty-add-repo`: +* `--remove-repo, --empty-remove-repo`: after additions. +* `--tags, --empty-tags`: +* `--github-issue`: +* `--status`: + +### witan project advance + +```console +witan project advance --phase LITERAL[DISCOVERY, SPEC, IMPLEMENTATION, DELIVERY] [OPTIONS] SLUG +``` + +Advance a project to a new phase. + +A backward or skip transition is not blocked from the CLI (elicitation is +only available in an MCP session), but the resulting ``advisory`` note is +surfaced so an unusual transition is still visible. + +**Parameters**: + +* `SLUG, --slug`: **[required]** +* `--phase`: **[required]** *[choices: discovery, spec, implementation, delivery]* +* `--github-pr`: + +### witan project complete + +```console +witan project complete --outcome STR [OPTIONS] SLUG +``` + +Complete a project and seal its immutable corpus trace. + +**Parameters**: + +* `SLUG, --slug`: **[required]** +* `--outcome`: **[required]** +* `--github-pr`: + +### witan project block + +```console +witan project block SLUG BLOCKS +``` + +Declare that ``slug`` must complete before ``blocks`` can begin. + +**Parameters**: + +* `SLUG, --slug`: **[required]** +* `BLOCKS, --blocks`: **[required]** + +### witan project unblock + +```console +witan project unblock SLUG BLOCKS +``` + +Remove a project dependency declared with ``project block``. + +**Parameters**: + +* `SLUG, --slug`: **[required]** +* `BLOCKS, --blocks`: **[required]** + +### witan project run + +```console +witan project run [OPTIONS] [ARGS] +``` + +Launch an agent session focused on a workflow project. + +Without a slug, shows an interactive picker of active projects. Multiple +selections offer a choice between a consolidated single-session prompt or +running each project sequentially in separate agent invocations. + +**Parameters**: + +* `SLUG, --slug`: +* `--target`: +* `--agent`: +* `--model`: +* `--dry-run, --no-dry-run`: *[default: False]* +* `--repo`: +* `--all-repos, --no-all-repos`: *[default: False]* + +## witan scan + +Introspect and dry-run write-path content scanning (ADR 0001). + +**Parameters**: + +* `--output-format`: projects, memory, traces, scan, and mounted witan-code tables. Values: + txt | json | toml | yaml. Env: WITAN_OUTPUT_FORMAT. *[choices: txt, json, toml, yaml]* *[env: WITAN_OUTPUT_FORMAT]* *[default: txt]* + +### witan scan test + +```console +witan scan test [OPTIONS] TEXT +``` + +Dry-run active detectors against TEXT and print findings. Nothing is written. + +Runs the exact same :class:`~witan.scan.ScannerRegistry` the write path +uses, so a clean run here means the write path will accept ``text`` +unchanged. Findings are reported with their secret-free preview only — +the matched text is never printed. + +**Parameters**: + +* `TEXT, --text`: **[required]** +* `--field`: e.g. skipping ``author``). *[default: content]* +* `--node-type`: *[default: Memory]* + +### witan scan rules + +```console +witan scan rules +``` + +List active detectors: category, source, and enforcement mode. + +## witan session + +Manage workflow sessions. + +**Parameters**: + +* `--output-format`: projects, memory, traces, scan, and mounted witan-code tables. Values: + txt | json | toml | yaml. Env: WITAN_OUTPUT_FORMAT. *[choices: txt, json, toml, yaml]* *[env: WITAN_OUTPUT_FORMAT]* *[default: txt]* + +### witan session start + +```console +witan session start --phase LITERAL[DISCOVERY, SPEC, IMPLEMENTATION, DELIVERY] [OPTIONS] PROJECT-SLUG +``` + +Link a session to a workflow project. + +**Parameters**: + +* `PROJECT-SLUG, --project-slug`: **[required]** +* `--phase`: **[required]** *[choices: discovery, spec, implementation, delivery]* +* `--session-id`: generated uuid). The Stop hook keys its state file on this. +* `--repo`: +* `--tags, --empty-tags`: + +### witan session end + +```console +witan session end --summary STR [OPTIONS] SESSION-SLUG +``` + +Close a session with a handoff summary. + +**Parameters**: + +* `SESSION-SLUG, --session-slug`: **[required]** +* `--summary`: **[required]** +* `--tools-used, --empty-tools-used`: +* `--files-changed, --empty-files-changed`: + +### witan session sweep + +```console +witan session sweep [OPTIONS] +``` + +Close sessions that leaked open. + +A session with no ``ended_at`` is not cosmetic: ``project complete`` folds +every linked session into the corpus trace, so a leaked one inflates +``session_count``, contributes its phase having recorded nothing, carries no +handoff summary, and cannot extend ``duration`` (computed from +``max(ended_at)``). It also drives the context hook's "N sessions in +" staleness nag on a project whose phase is progressing fine. + +Dry-run by default — prints what it would close. Pass ``--yes`` to do it. +Closing an already-closed session just re-stamps ``ended_at``, so re-running +is harmless. + +Against a deployment the per-actor client scopes the listing to the calling +user, so a sweep cannot reach a teammate's sessions. + +**Parameters**: + +* `--older-than`: Guards against closing a session that is legitimately running right now. *[default: 6h]* +* `--project`: +* `--yes, --no-yes`: *[default: False]* + +### witan session list + +```console +witan session list PROJECT-SLUG +``` + +List a project's sessions, newest last. + +**Parameters**: + +* `PROJECT-SLUG, --project-slug`: **[required]** + +## witan setup + +```console +witan setup [OPTIONS] +``` + +Install witan for one or all supported coding agents. + +Installs the omnigraph binary to ``~/.local/bin/``, writes a starter +``config.toml`` if one doesn't exist yet, copies bundled skills and +hooks/extensions to the agent's config directories, and merges the witan +MCP server entry into the agent's config file. When witan-code is also +installed (importable in this environment — e.g. via ``--with`` in the +MCP server's uvx invocation), its skill and hooks (registered as +``witan code …``, not a separate ``witan-code`` binary) are folded into +the same install pass — no separate MCP entry, since ``witan serve`` +already mounts witan-code's tools in-process. A single ``witan setup`` +then covers both packages; otherwise install witan-code separately with +``witan-code setup`` (or the mounted ``witan code setup``). + +Re-run after every upgrade to refresh installed files. + +**Parameters**: + +* `--output-format`: projects, memory, traces, scan, and mounted witan-code tables. Values: + txt | json | toml | yaml. Env: WITAN_OUTPUT_FORMAT. *[choices: txt, json, toml, yaml]* *[env: WITAN_OUTPUT_FORMAT]* *[default: txt]* +* `--agent`: pending a config-path verification fix — tracked separately.) *[choices: claude, pi, copilot, opencode, all]* *[default: claude]* +* `--author`: +* `--dry-run, --no-dry-run`: *[default: False]* + +## witan target + +Register and inspect named [targets.*] blocks (deployed witan endpoints). + +**Parameters**: + +* `--output-format`: projects, memory, traces, scan, and mounted witan-code tables. Values: + txt | json | toml | yaml. Env: WITAN_OUTPUT_FORMAT. *[choices: txt, json, toml, yaml]* *[env: WITAN_OUTPUT_FORMAT]* *[default: txt]* + +### witan target add + +```console +witan target add [OPTIONS] NAME +``` + +Register a named target — a deployed witan endpoint, or a local store. + +Writes a ``[targets.]`` block to the config file in effect +(``WITAN_CONFIG``, else ``~/.config/witan/config.toml``), creating a +starter config first if none exists. Comments in an existing file are +preserved: the block is appended, not re-serialised. + +Joining a deployment is then:: + + witan target add hosted \ + --remote-url https://witan.example.org/mcp \ + --oidc-issuer https://sso.example.org/realms/ol-platform-engineering \ + --match-orgs my-org + witan login --target hosted + witan whoami --target hosted + +Passing ``--match-orgs``/``--match-repos``/``--match-hosts``/``--match-paths`` +lets the target select itself for matching checkouts, so ``--target`` is not +needed after the first time. Without any of them the target is only ever +reached explicitly (``--target``/``WITAN_TARGET``). + +**Parameters**: + +* `NAME, --name`: **[required]** +* `--remote-url`: +* `--oidc-issuer`: +* `--oidc-client-id`: +* `--oidc-audience`: +* `--server`: +* `--graph`: +* `--author`: +* `--agent`: +* `--match-orgs, --empty-match-orgs`: +* `--match-repos, --empty-match-repos`: +* `--match-hosts, --empty-match-hosts`: +* `--match-paths, --empty-match-paths`: +* `--force, --no-force`: *[default: False]* +* `--verify, --no-verify`: *[default: True]* +* `--login, --no-login`: *[default: False]* +* `--dry-run, --no-dry-run`: *[default: False]* + +### witan target list + +```console +witan target list +``` + +List configured targets, marking the one in effect here with ``*``. + +### witan target remove + +```console +witan target remove [OPTIONS] NAME +``` + +Delete a ``[targets.]`` block from the config file. + +**Parameters**: + +* `NAME, --name`: **[required]** +* `--dry-run, --no-dry-run`: *[default: False]* + +## witan tasks + +```console +witan tasks [OPTIONS] +``` + +List tasks for the current repo (or filtered). + +Closed tasks are elided by default — the list is a working view of live work. +Pass ``--status closed`` to see them (or any other status to filter to it). + +**Parameters**: + +* `--output-format`: projects, memory, traces, scan, and mounted witan-code tables. Values: + txt | json | toml | yaml. Env: WITAN_OUTPUT_FORMAT. *[choices: txt, json, toml, yaml]* *[env: WITAN_OUTPUT_FORMAT]* *[default: txt]* +* `--repo`: +* `--status`: non-closed statuses. +* `--project`: +* `--assignee`: +* `--ready, --no-ready`: *[default: False]* +* `--all-repos, --no-all-repos`: *[default: False]* +* `--limit`: *[default: 50]* + +## witan task + +```console +witan task COMMAND SLUG +``` + +Manage tasks. + +**Commands**: + +* [`claim`](#witan-task-claim): Claim a task for work (status in_progress, with a lease). +* [`close`](#witan-task-close): Close a task, recording an optional resolution. +* [`create`](#witan-task-create): Create a task in the work-coordination graph. +* [`link`](#witan-task-link): Link two tasks (or a task to a memory). +* [`release`](#witan-task-release): Release a claim, returning the task to ``open`` (or another status). +* [`run`](#witan-task-run): Claim one or more tasks and launch an agent to execute them. +* [`unlink`](#witan-task-unlink): Remove a link between two tasks (or a task and a memory). +* [`update`](#witan-task-update): Update a task's mutable fields (only provided fields change). + +**Parameters**: + +* `--output-format`: projects, memory, traces, scan, and mounted witan-code tables. Values: + txt | json | toml | yaml. Env: WITAN_OUTPUT_FORMAT. *[choices: txt, json, toml, yaml]* *[env: WITAN_OUTPUT_FORMAT]* *[default: txt]* +* `SLUG, --slug`: **[required]** + +### witan task create + +```console +witan task create [OPTIONS] TITLE +``` + +Create a task in the work-coordination graph. + +**Parameters**: + +* `TITLE, --title`: **[required]** +* `--description`: *[default: ""]* +* `--type`: *[choices: bug, feature, task, chore, epic]* *[default: task]* +* `--priority`: *[choices: p0, p1, p2, p3]* *[default: p2]* +* `--repo`: +* `--project`: +* `--parent`: +* `--blocked-by, --empty-blocked-by`: +* `--discovered-from, --empty-discovered-from`: +* `--external-uri`: +* `--symbol-refs, --empty-symbol-refs`: +* `--tags, --empty-tags`: + +### witan task close + +```console +witan task close [OPTIONS] SLUG +``` + +Close a task, recording an optional resolution. + +Closing a blocker unblocks its dependents. + +**Parameters**: + +* `SLUG, --slug`: **[required]** +* `--resolution`: + +### witan task claim + +```console +witan task claim [OPTIONS] SLUG +``` + +Claim a task for work (status in_progress, with a lease). + +A live claim held by someone else is refused unless ``--force`` is passed +(CLI has no interactive steal prompt). + +**Parameters**: + +* `SLUG, --slug`: **[required]** +* `--assignee`: session so parallel sessions don't share one claim). +* `--force, --no-force`: *[default: False]* + +### witan task release + +```console +witan task release [OPTIONS] SLUG +``` + +Release a claim, returning the task to ``open`` (or another status). + +**Parameters**: + +* `SLUG, --slug`: **[required]** +* `--assignee`: by this agent session — a claim taken by another of your own sessions + still matches, since the check is on identity, not session). +* `--status`: *[choices: open, in_progress, blocked, closed]* *[default: open]* +* `--force, --no-force`: *[default: False]* + +### witan task update + +```console +witan task update [OPTIONS] SLUG +``` + +Update a task's mutable fields (only provided fields change). + +To *close* a task prefer ``task close``; to *claim* it prefer ``task claim``; +to add dependencies use ``task link``. + +**Parameters**: + +* `SLUG, --slug`: **[required]** +* `--title`: +* `--description`: +* `--type`: *[choices: bug, feature, task, chore, epic]* +* `--priority`: *[choices: p0, p1, p2, p3]* +* `--status`: *[choices: open, in_progress, blocked, closed]* +* `--repo`: +* `--project`: +* `--parent`: +* `--assignee`: +* `--external-uri`: +* `--tags, --empty-tags`: + +### witan task link + +```console +witan task link FROM-SLUG TO-SLUG KIND +``` + +Link two tasks (or a task to a memory). + +``from``/``to`` meaning depends on ``kind``: +blocks — from blocks to; parent — from is parent of to; +discovered_from — from was discovered from to; addresses — from addresses +memory to. + +**Parameters**: + +* `FROM-SLUG, --from-slug`: **[required]** +* `TO-SLUG, --to-slug`: **[required]** +* `KIND, --kind`: **[required]** *[choices: blocks, parent, discovered_from, addresses]* + +### witan task unlink + +```console +witan task unlink FROM-SLUG TO-SLUG KIND +``` + +Remove a link between two tasks (or a task and a memory). + +The inverse of ``link``, with the same ``from``/``to`` meanings. Use it +when a link was recorded backwards or against the wrong slug; removing a +``blocks`` link is how a wrongly-blocked task becomes ready again. + +Reports plainly when there was no such link — that is a no-op, not an +error, so re-running is safe. + +**Parameters**: + +* `FROM-SLUG, --from-slug`: **[required]** +* `TO-SLUG, --to-slug`: **[required]** +* `KIND, --kind`: **[required]** *[choices: blocks, parent, discovered_from, addresses]* + +### witan task run + +```console +witan task run [OPTIONS] [ARGS] +``` + +Claim one or more tasks and launch an agent to execute them. + +Without a slug, shows an interactive picker of ready tasks. Multiple +selections offer a choice between a consolidated single-session prompt or +running each task sequentially in separate agent invocations. + +**Parameters**: + +* `SLUG, --slug`: +* `--target`: +* `--agent`: +* `--model`: +* `--claim, --no-claim`: *[default: True]* +* `--force, --no-force`: this the command could report a task as held and offer no way past it + from the CLI it was reported in — the interactive steal prompt is + server-side and unreachable through ``_fn``, which passes no ``ctx``. *[default: False]* +* `--dry-run, --no-dry-run`: *[default: False]* +* `--repo`: +* `--all-repos, --no-all-repos`: *[default: False]* +* `--project`: + +## witan traces + +```console +witan traces [OPTIONS] +``` + +List corpus workflow traces (default: current repo). + +**Parameters**: + +* `--output-format`: projects, memory, traces, scan, and mounted witan-code tables. Values: + txt | json | toml | yaml. Env: WITAN_OUTPUT_FORMAT. *[choices: txt, json, toml, yaml]* *[env: WITAN_OUTPUT_FORMAT]* *[default: txt]* +* `--repo`: +* `--tags, --empty-tags`: +* `--author`: +* `--all-repos, --no-all-repos`: *[default: False]* +* `--limit`: *[default: 50]* + +## witan trace + +```console +witan trace COMMAND SLUG +``` + +Inspect corpus trace records. + +**Commands**: + +* [`list`](#witan-trace-list): List corpus workflow traces (alias of ``witan traces``). + +**Parameters**: + +* `--output-format`: projects, memory, traces, scan, and mounted witan-code tables. Values: + txt | json | toml | yaml. Env: WITAN_OUTPUT_FORMAT. *[choices: txt, json, toml, yaml]* *[env: WITAN_OUTPUT_FORMAT]* *[default: txt]* +* `SLUG, --slug`: **[required]** + +### witan trace list + +```console +witan trace list [OPTIONS] +``` + +List corpus workflow traces (alias of ``witan traces``). + +**Parameters**: + +* `--repo`: +* `--tags, --empty-tags`: +* `--author`: +* `--all-repos, --no-all-repos`: *[default: False]* +* `--limit`: *[default: 50]* + +## witan migrate + +One-shot, idempotent schema and data migrations. + +**Parameters**: + +* `--output-format`: projects, memory, traces, scan, and mounted witan-code tables. Values: + txt | json | toml | yaml. Env: WITAN_OUTPUT_FORMAT. *[choices: txt, json, toml, yaml]* *[env: WITAN_OUTPUT_FORMAT]* *[default: txt]* + +### witan migrate schema + +```console +witan migrate schema +``` + +Apply the bundled schema to the configured store (idempotent). + +Reconciles an existing store with the current schema (new nodes/edges/fields). +Startup now does this on its own when ``schema.pg`` changes; this forces the +apply regardless of the mtime stamp. + +### witan migrate storage + +```console +witan migrate storage [OPTIONS] [ARGS] +``` + +Rebuild a local store stuck on an old, incompatible omnigraph format. + +omnigraph uses strict single-version storage: a release that bumps the +internal on-disk schema (e.g. 0.7 → 0.8) refuses to open graphs an older +binary wrote. This detects that refusal against your configured store +and, using a still-installed pre-upgrade ``omnigraph`` binary, replays +the documented rebuild — export with the old binary, then ``init`` + +``load`` with the new one. Node/edge data, vectors, and blobs are +preserved; commit history and branches are not. The original store is +renamed ``.pre-migrate`` rather than deleted. + +No-op if the store already opens fine with the current binary. Only +handles local on-disk stores — s3:// and http(s):// stores are managed +externally and must be rebuilt by hand per omnigraph's upgrade docs. + +**Parameters**: + +* `OLD-BINARY, --old-binary`: Path to the omnigraph binary that last wrote this store. Auto-detected + as the first ``omnigraph`` on PATH that isn't the one witan is + currently using, if omitted. +* `--yes, --no-yes`: Skip the confirmation prompt. *[default: False]* + +### witan migrate merge + +```console +witan migrate merge [OPTIONS] [ARGS] +``` + +Merge another store's data into this store, newest-record-wins on collisions. + +Implements docs/migration-runbook.md's export -> reconcile -> load +(--mode merge) path: for every node present in both stores (same type + +slug), keeps whichever has the newer timestamp instead of `omnigraph load +--mode merge`'s raw last-loaded-wins overwrite, which ignores content +entirely. Rows only in ``source`` are always added; rows only in the +target are left untouched. Repeatable — re-running against an +already-merged target loads nothing new. + +**Parameters**: + +* `SOURCE, --source`: Store URI to merge from (local path, ``s3://``, ``file://``, or an + ``http(s)://`` omnigraph-server), or the path to a *local* + ``omnigraph export`` JSONL — anything ending ``.jsonl`` is read as an + export rather than re-exported, and is never fetched remotely. Use the + export form to merge a store from another machine: Lance embeds + absolute paths, so a ``.omni`` directory cannot be copied, but its + export can. +* `--from`: Named ``[targets.]`` block to merge *from*, in place of + ``source`` — its ``server`` is the store URI. A target carrying only a + ``remote_url`` is refused: there is no remote-export path, so it has + nothing to merge from. +* `--to`: Named ``[targets.]`` block to merge *into*, in place of the + ambient destination. Spells out on the command line what setting + ``WITAN_TARGET`` does out of the environment: a target with a + ``remote_url`` is merged into through that deployment (as you, over + MCP), one with only a ``server`` into that store URI. Mutually + exclusive with ``target``, which names a store rather than a target. +* `--target`: Store URI to merge into. Defaults to the configured store. Created + automatically if it's a local path that doesn't exist yet. A deployed + graph is ``http(s)://:/graphs/`` (or just the + configured store, when running in-cluster). Unlike ``source``, a + ``.jsonl`` target is refused rather than treated as a store: merging + appends to a graph, and an export is a snapshot of one. +* `--dry-run, --no-dry-run`: Preview the reconciliation decision for every colliding slug without + writing anything. *[default: False]* + +### witan migrate topics + +```console +witan migrate topics +``` + +Backfill Topic nodes from existing memory tags. + +For every distinct memory ``tag``, upsert a ``Topic{kind:"topic"}`` and a +``Tagged`` edge. Safe to re-run — already-created topics and edges are +skipped. Fails fast if the Topic schema isn't applied yet. + +### witan migrate repo-keys + +```console +witan migrate repo-keys +``` + +Fold every stored repo key onto its canonical, case-folded form (#142). + +``normalise`` now lowercases GitHub/GitLab repo keys, so a key written +before that fix may still carry the old case and silently drop out of +every repo-scoped read. Rewrites Task/Memory/WorkflowSession ``repo`` (and +their ``symbol_refs`` repo prefixes), WorkflowProject/WorkflowTrace +``repos`` lists, and CodeBranch (recreated under the canonical slug, the +stale row marked ``abandoned``). Idempotent — safe to re-run, and safe to +run on a store with nothing to fix. Does not touch the code graph +(witan-code); prints which repos need `witan-code reindex` instead. + +### witan migrate dedupe-sessions + +```console +witan migrate dedupe-sessions [OPTIONS] +``` + +Flag WorkflowSessions a pre-upsert ``workflow_session_start`` duplicated. + +Reports overlapping sessions that share a ``session_id`` — the signature of +a hook retry or transport reconnect — and marks the ones carrying no +summary as ``superseded_by`` the surviving session, so trace assembly and +the context hook's counts stop double-counting them. Nothing is deleted. + +Dry by default: prints what it would do and changes nothing until +``--apply``. Sessions that share a ``session_id`` but ran one after another +are left alone — one session id legitimately spans several working stints. +Runs where every member wrote a real summary are reported rather than +guessed at; resolve those with ``--supersede``. + +Deliberately not part of ``migrate all``: unlike the other migrations this +one makes a judgment call about corpus content, so it should be read before +it's applied. + +**Parameters**: + +* `--apply, --no-apply`: Write the marks instead of only reporting them. *[default: False]* +* `--supersede, --empty-supersede`: ``=`` pairs to mark regardless of the + automatic rule. Repeatable. + +### witan migrate all + +```console +witan migrate all +``` + +Run the full bring-up: apply schema, backfill topics, fold repo keys. + +All three steps are idempotent, so this is safe to re-run — including as +part of every deploy, to keep a live store self-healing. + +### witan migrate claim-authorship + +```console +witan migrate claim-authorship [OPTIONS] [ARGS] +``` + +Take ownership of rows an earlier migration left under your local name. + +A local store writes ``author`` from ``WITAN_AUTHOR`` / git ``user.name`` / +``$USER``; a deployment resolves it from your token's +``preferred_username``. The two never converge, so before this was fixed +every row you migrated kept a name your deployed identity cannot match — +and ``memory_delete`` refuses anyone but the author, permanently (#267). + +``witan migrate merge`` now claims rows as they arrive, so this is only +needed for a store merged before that landed. Re-merging will not fix +those: reconciliation is newest-record-wins, and a re-sent row loses to its +own already-applied copy. + +Dry by default. Run ``witan whoami`` first if you are unsure which identity +you are claiming *to*. + +**Parameters**: + +* `WAS, --was`: The author string the rows currently carry. Defaults to this machine's + configured local author, which is the right answer when you are + repairing your own cutover from this same checkout. +* `--apply, --no-apply`: Write the change instead of only reporting it. *[default: False]* + +## witan code + +witan-code — tree-sitter code graph + cross-repo bridge. + +**Parameters**: + +* `--output-format`: projects, memory, traces, scan, and mounted witan-code tables. Values: + txt | json | toml | yaml. Env: WITAN_OUTPUT_FORMAT. *[choices: txt, json, toml, yaml]* *[env: WITAN_OUTPUT_FORMAT]* *[default: txt]* + +### witan code index + +```console +witan code index [ARGS] +``` + +Incrementally index PATH (file or directory). Unchanged files are skipped. + +**Parameters**: + +* `PATH, --path`: *[default: .]* + +### witan code reindex + +```console +witan code reindex [ARGS] +``` + +Force re-index PATH, ignoring content hashes. + +**Parameters**: + +* `PATH, --path`: *[default: .]* + +### witan code deps + +```console +witan code deps [ARGS] +``` + +Visualize cross-repo dependencies from the shared bridge store. + +Prints a Rich summary of "repo A depends on repo B" links (A consumes a +contract B provides). Pass --html PATH to also emit an interactive graph. + +**Parameters**: + +* `KIND, --kind`: Filter to one contract kind (env_var/package/service/endpoint). *[choices: env_var, endpoint, package, service]* +* `REPO, --repo`: Keep only links touching a repo whose slug contains this substring. +* `HTML, --html`: Write a self-contained interactive HTML graph to this path. +* `OPEN-BROWSER, --open-browser, --no-open-browser`: Open the generated HTML in the default browser. *[default: False]* +* `MIN-PRECISION, --min-precision`: Minimum edge precision tier (docs/EDGE_PRECISION_TIERS.md). Default + `heuristic` preserves prior behavior (every consumer/provider link + this command has always shown). `precise` keeps only edges also + covered by a Stage-2 canonical-symbol join — see `witan code stitch`. *[choices: precise, heuristic, fuzzy]* *[default: heuristic]* + +### witan code symbols + +```console +witan code symbols [ARGS] +``` + +Print a repo's symbol table from the bridge store (docs/SYMBOL_TABLE.md). + +One row per (role, symbol): `exported` rows are the repo's public contract +surface; `external` rows are unresolved references Stage 2 joins against +other repos' exports. + +**Parameters**: + +* `REPO, --repo`: Canonical repo URI. Defaults to the repo detected from the CWD. +* `ROLE, --role`: Filter to exported or external rows. *[choices: exported, external]* +* `SCHEME, --scheme`: Filter to one symbol scheme (http/env/pkg/svc). + +### witan code stitch + +```console +witan code stitch [OPTIONS] [ARGS] +``` + +Print Stage-2 precise cross-repo edges from the bridge store (docs/SYMBOL_TABLE.md). + +Joins every repo's unresolved external symbols against other repos' +exported symbols by canonical symbol string — distinct from the coarser +`witan code deps` heuristic (kind, key_norm) grouping. + +**Parameters**: + +* `REPO, --repo`: Keep only edges/gaps touching this repo. Omit to see the whole store. +* `--unresolved, --no-unresolved`: Print external references with no precise match instead of edges — + gaps in indexing coverage (a provider isn't indexed yet, or none + exists in this SOA). *[default: False]* + +### witan code inject-context + +```console +witan code inject-context +``` + +Print a short code-graph status block for the UserPromptSubmit hook. + +Registered as the bare ``UserPromptSubmit`` hook command; always exits 0 +and prints nothing when there's no store or in-flight index for the +current repo. + +### witan code serve + +```console +witan code serve +``` + +Run the code-graph MCP server standalone (code_* tools only). + +When witan-code is mounted into the umbrella ``witan serve`` instead, that +command has already configured observability; the call here is idempotent so +the standalone path gets it too without double-configuring the combined one. + +### witan code optimize + +```console +witan code optimize [OPTIONS] +``` + +Compact a code-graph store's Lance fragments (non-destructive). + +Collapses the many tiny fragments that accrue from every index/reindex so +opening the store stays cheap. Safe to run repeatedly; takes the store's +write lock. + +**Parameters**: + +* `--store`: +* `--bridge, --no-bridge`: *[default: False]* + +### witan code cleanup + +```console +witan code cleanup [OPTIONS] +``` + +Remove old Lance versions from a code-graph store (**destructive**). + +``optimize`` compacts fragments but leaves old versions behind; this GCs +them, keeping the most recent ``keep`` versions per table (and/or those +newer than ``older_than``). Irreversible, so it requires ``--yes``. + +**Parameters**: + +* `--store`: +* `--bridge, --no-bridge`: *[default: False]* +* `--keep`: *[default: 10]* +* `--older-than`: +* `--yes, --no-yes`: *[default: False]* + +### witan code reap-views + +```console +witan code reap-views [OPTIONS] +``` + +Delete branch views nobody has written in a long time (**destructive**). + +On a shared cluster graph every developer's every git branch gets a view of +its own, and nothing ever removes one — this is what bounds that. Views are +re-derivable caches, so a reaped view costs its owner a reindex, not work. + +Distinct from ``branches --prune``, which asks whether *this checkout* still +has the git branch and so only makes sense against a store this machine +alone writes. This asks how long ago a view was last written, which a shared +graph can answer for every writer. A view with no writes of its own is never +reaped: it holds nothing, and there is no creation timestamp to age it by. + +Reports by default; ``--apply`` is what deletes. On a shared graph deleting +requires ``WITAN_CODE_INDEX_ROLE=ci`` — Cedar grants ``branch_delete`` to +the CI indexer alone, and refusing here makes that a clear local error +rather than a server denial. + +**Parameters**: + +* `--store`: URL. Default: every store this config resolves to (cluster graphs when + ``code_server`` is set, else the local ones), the shared bridge + included. +* `--graph`: encode one as ``.../graphs/``. +* `--max-idle-days`: ``WITAN_CODE_VIEW_MAX_IDLE_DAYS``). ``0`` disables reaping. +* `--apply, --no-apply`: *[default: False]* + +### witan code checkpoint + +```console +witan code checkpoint +``` + +Opportunistically compact the current repo's store(s) (Stop hook). + +Spawns a throttled, detached ``witan-code optimize`` for the current +repo's store and the shared bridge store, each at most once per +``WITAN_CODE_OPTIMIZE_INTERVAL``, if either exists and is due. Best-effort +and non-blocking: always exits 0 and never raises, so a maintenance +failure can't fail the Stop hook. Registered as the bare ``Stop`` hook +command; not usually run by hand. + +A no-op against cluster graphs — ``maintenance.due()`` never fires for a +remote store, since compacting the shared storage root is the cluster's +job rather than every client's at the end of every session. + +### witan code session-init + +```console +witan code session-init +``` + +Seed/refresh the whole repo's code graph in the background (SessionStart hook). + +Detached and non-blocking — returns immediately regardless of repo size. +A per-repo lock (shared with ``inject-context``'s "indexing in progress" +check) prevents overlapping sessions from indexing at once. Registered as +the bare ``SessionStart`` hook command; not usually run by hand. + +### witan code reindex-hook + +```console +witan code reindex-hook +``` + +Incrementally reindex the file named in stdin's hook JSON (PostToolUse hook). + +Reads the Claude Code hook payload from stdin, extracts +``tool_input.file_path`` (or ``path``/``filename``), and reindexes it if +it exists and is a known source type — foreground and fast (one file), so +the agent sees the change land immediately. Best-effort: a missing or +malformed payload is a silent no-op. Registered as the bare +``PostToolUse`` (matcher ``Edit|Write``) hook command; not usually run by +hand. + +### witan code setup + +```console +witan code setup [OPTIONS] +``` + +Install witan-code for one or all supported coding agents. + +Installs the omnigraph binary to ~/.local/bin/, copies the bundled skill +and Pi extension to the agent's config directories, registers the four +hooks (bare CLI commands — no wrapper scripts to copy), and merges the +witan-code MCP server entry into the agent's config file. Independent of +`witan setup` — running both is fine (each only touches its own entries); +running just this one is enough for a witan-code-only install. + +Re-run after every upgrade to refresh installed files. + +**Parameters**: + +* `--agent`: *[choices: claude, pi, copilot, opencode, all]* *[default: claude]* +* `--author`: +* `--dry-run, --no-dry-run`: *[default: False]* + +### witan code branches + +```console +witan code branches [OPTIONS] +``` + +List the in-flight branch views per indexed repo store, and who owns each. + +A non-default git branch is indexed onto its own view, named for its +writer as well as the branch (docs/BRANCH_INDEXING.md), so two checkouts +of the same branch do not overwrite each other. Views are re-derivable +caches, so lifecycle is deletion, not merge. + +**Parameters**: + +* `--branch`: Show only views of this git branch — every writer's, which is how you + find a teammate's in-flight work. Pass a listed view name to + ``--branch`` on the read commands to query it. +* `--prune, --no-prune`: Delete the CURRENT repo's views whose git branch no longer exists + locally, plus the ``_detached`` scratch view. Other repos' stores are + only listed (their git refs aren't visible from here). Local stores + only, on both counts below: pruning reads this machine's git refs as + the authority, which is true of a store only this machine writes and + false of a shared cluster graph. *[default: False]* + +### witan code repos + +```console +witan code repos +``` + +List the repositories that have a code graph indexed. + +### witan code login + +```console +witan code login +``` + +Authenticate to the deployed witan service via the OIDC device grant. + +Prints a verification URL and a user code; approve it in a browser, and the +resulting token is cached (mode 0600) and refreshed automatically for +subsequent `witan-code …` commands. + +The cache is shared with the `witan` CLI and keyed by (issuer, client id), +so if you already ran `witan login` against the same deployment you do not +need this at all — and running it here also logs `witan` in. + +### witan code logout + +```console +witan code logout +``` + +Forget the cached token for the configured deployment. + +The cache is shared with the `witan` CLI, so this logs both out. + +### witan code whoami + +```console +witan code whoami +``` + +Show the identity the CLI presents to the deployed witan service. + +## witan serve + +```console +witan serve [OPTIONS] +``` + +Run the witan MCP server. + +Serves the work-coordination tools (memory_*, task_*, workflow_*) and, when +witan-code is installed, mounts the code-graph tools (code_*) into the same +server so a single MCP entry exposes everything. + +Defaults to ``stdio`` for local per-user use (Claude Desktop, ``uvx``). Pass +``--transport streamable-http`` (or set ``WITAN_MCP_TRANSPORT``) to expose an +HTTP endpoint for a shared, deployed service — this is what ToolHive hosts. + +The legacy HTTP+SSE transport is not offered: MCP 2026-07-28 deprecates it +with a 12-month offramp, and witan has no deployment on it to carry over. + +**Parameters**: + +* `--output-format`: projects, memory, traces, scan, and mounted witan-code tables. Values: + txt | json | toml | yaml. Env: WITAN_OUTPUT_FORMAT. *[choices: txt, json, toml, yaml]* *[env: WITAN_OUTPUT_FORMAT]* *[default: txt]* +* `--transport`: ``http`` alias) binds a network listener. Env: ``WITAN_MCP_TRANSPORT``. *[choices: stdio, http, streamable-http]* *[env: WITAN_MCP_TRANSPORT]* *[default: stdio]* +* `--host`: Env: ``WITAN_MCP_HOST``. *[env: WITAN_MCP_HOST]* *[default: 127.0.0.1]* +* `--port`: *[env: WITAN_MCP_PORT]* *[default: 8000]* +* `--path`: Env: ``WITAN_MCP_PATH``. *[env: WITAN_MCP_PATH]* *[default: /mcp]* +* `--shutdown-grace-seconds`: SIGTERM before dropping them. FastMCP's own default is **2 seconds**, + which silently truncates any deployment that expects a rollout to drain + — a witan write has been measured at 27s. Set this to the deployment's + termination grace period. Env: + ``WITAN_MCP_SHUTDOWN_GRACE_SECONDS``. *[env: WITAN_MCP_SHUTDOWN_GRACE_SECONDS]* *[default: 120.0]* + +## witan run + +```console +witan run [OPTIONS] SLUG +``` + +Claim a task and launch an agent to execute it. + +Claims the task (status in_progress, assignee = your author), then hands the +terminal to ```` seeded with a prompt describing the work. Run from +the task's repo checkout so the agent has the right working directory. + +**Parameters**: + +* `--output-format`: projects, memory, traces, scan, and mounted witan-code tables. Values: + txt | json | toml | yaml. Env: WITAN_OUTPUT_FORMAT. *[choices: txt, json, toml, yaml]* *[env: WITAN_OUTPUT_FORMAT]* *[default: txt]* +* `SLUG, --slug`: **[required]** +* `--target`: Also overridable via WITAN_TARGET env var. +* `--agent`: WITAN_AGENT env var and target/config-file default. +* `--model`: var and target/config-file default. +* `--claim, --no-claim`: *[default: True]* +* `--dry-run, --no-dry-run`: *[default: False]* diff --git a/docs/reference/environment.md b/docs/reference/environment.md new file mode 100644 index 00000000..65ce5e02 --- /dev/null +++ b/docs/reference/environment.md @@ -0,0 +1,172 @@ + + +# Environment variables + +Every setting witan reads from the environment. Environment variables take +precedence over `~/.config/witan/config.toml`, which takes precedence over the +built-in default — so an env var always wins. + +Most of these have a config-file equivalent and you will never set them by hand. +The ones worth knowing on day one are +[`WITAN_MEMORY_URI`](#store-and-attribution) (where the graph lives) and +[`WITAN_AUTHOR`](#store-and-attribution) (whose name is on what you write). +Everything below that is deployment, tuning, or operations. + +## Store and attribution + +Where the graph lives and whose name goes on the nodes you create. These are the +only settings a local, single-user install normally needs. + +| Variable | Default | Description | +| --- | --- | --- | +| `WITAN_AUTHOR` | — | Attribution written to every node you create. Falls back to `git config user.name`, then `$USER`. | +| `WITAN_CONFIG` | `~/.config/witan/config.toml` | Path to the config file. Both `witan` and `witan code` read the same file. An empty or whitespace-only value counts as unset, so an unexpanded `$SOME_UNSET_VAR` does not silently redirect you to a file named `$SOME_UNSET_VAR`. | +| `WITAN_MEMORY_GRAPH` | `council` | Which graph to address on an `http(s)://` omnigraph-server — one server hosts many. Ignored for local paths and `s3://` stores, which name the graph in the URI itself. | +| `WITAN_MEMORY_TOKEN` | — | Bearer token for an `http(s)://` store. Required for a deployed server, meaningless for a local one. | +| `WITAN_MEMORY_URI` | `~/.local/share/witan/graph.omni` | Graph store location: a local path, an `s3://` URI, or the base URL of a deployed `omnigraph-server`. This is the single setting that decides whether you are running against your own laptop or a shared service. | +| `WITAN_OUTPUT_FORMAT` | `txt` | Default CLI output format: `txt`, `json`, `toml`, or `yaml`. Equivalent to passing `--output-format`. | + +## Repository and target scoping + +Which repo a call is about, and which store answers it. witan auto-detects the +repo from `.git/config`; these override that when detection is wrong, absent, or +too slow. + +| Variable | Default | Description | +| --- | --- | --- | +| `WITAN_AGENT` | `claude` | Default coding-agent CLI for `witan run`: `claude`, `pi`, `copilot`, `opencode`, or `kilo`. | +| `WITAN_MODEL` | — | Default `--model` passed through to the agent by `witan run`. | +| `WITAN_REPO` | — | Canonical repo URI for the current call, overriding git detection. Setting it also skips git entirely, which is why hooks use it. Set it to the **empty string** to suppress repo detection and operate across all repos. | +| `WITAN_TARGET` | — | Name of the `[targets.]` config block to use, overriding auto-detection by repo or checkout path. Lets one machine route work repos and personal repos at different stores. | + +## Client: reaching a deployed witan + +Set these to point the local CLI at a shared witan service instead of running +the graph in-process. They configure the *client's* view of a deployment — a CLI +user never sets the server-side identity variables in the next section. + +`witan login` performs an OIDC device grant against the issuer and caches the +token; both `witan` and `witan code` share one cache, so you log in once. + +| Variable | Default | Description | +| --- | --- | --- | +| `WITAN_OIDC_AUDIENCE` | — | Audience/resource to request, matching the deployment's own `WITAN_OIDC_AUDIENCE`. Sent on the device-auth and token requests so an issuer with an audience mapper stamps the right `aud` claim. | +| `WITAN_OIDC_CLIENT_ID` | — | OIDC client id presented during the device grant. | +| `WITAN_OIDC_EXPIRY_SKEW_SECONDS` | `90` | How long before nominal expiry a cached token is treated as already expired and refreshed. Sized so a refresh happens before a long write starts rather than partway through one. | +| `WITAN_OIDC_ISSUER` | — | OIDC issuer URL used for the device-authorization grant behind `witan login`. | +| `WITAN_REMOTE_CALL_BUDGET_SECONDS` | `0` | Deadline for a single remote graph call, used to decide whether to honour a server's retry hint or give up. `0` means no client-side deadline — obey the server's hints. | +| `WITAN_REMOTE_URL` | — | Base URL of the deployed witan MCP endpoint. Setting it routes CLI reads and writes through the service rather than opening a store locally. A `[targets.]` block's `remote_url` overrides it. | +| `WITAN_REMOTE_WRITE_MAX_INFLIGHT` | `4` | How many remote writes may be in flight at once. The gate that keeps a burst of concurrent writes from stranding on the data tier. | +| `WITAN_REMOTE_WRITE_QUEUE_SECONDS` | `10.0` | How long a write waits for a slot at the in-flight gate before failing fast rather than queueing indefinitely. | +| `WITAN_TOKEN_CACHE` | `~/.config/witan/tokens.json` | Where both CLIs cache OIDC tokens. Shared on purpose, next to the shared config file. | + +## Server: running `witan serve` + +Deployment and operations config for a shared, network-facing witan. A local +stdio install needs none of it. + +`WITAN_OIDC_ISSUER`, `WITAN_OIDC_AUDIENCE`, and `WITAN_ACTOR_TOKENS_FILE` must be +set **together** — witan refuses to start with a partial identity configuration +rather than serving unauthenticated. + +| Variable | Default | Description | +| --- | --- | --- | +| `WITAN_ACTOR` | — | Overrides the OIDC-derived identity. Intended for service accounts and the CI indexer, which authenticate as themselves rather than as a person. | +| `WITAN_ACTOR_TOKENS_FILE` | — | Path to a mounted `{actor_id: token}` map. The server-side half of identity: it maps an authenticated caller to the actor recorded on the nodes they write. | +| `WITAN_MCP_HOST` | `127.0.0.1` | Interface to bind for HTTP transports. Use `0.0.0.0` inside a container. | +| `WITAN_MCP_PATH` | `/mcp` | URL path the MCP endpoint is served on. HTTP transports only. | +| `WITAN_MCP_PORT` | `8000` | Port to bind for HTTP transports. | +| `WITAN_MCP_SHUTDOWN_GRACE_SECONDS` | `120.0` | How long uvicorn waits for in-flight requests after `SIGTERM`. **FastMCP's own default is 2 seconds**, which silently truncates any rollout — a witan write has been measured at 27s under load, and a severed write is an indeterminate outcome the caller cannot safely retry. Set this to the deployment's termination grace period. | +| `WITAN_MCP_TRANSPORT` | `stdio` | MCP transport: `stdio` for local per-user use, or `streamable-http` (alias `http`) to bind a network listener. The legacy HTTP+SSE transport is deliberately not offered. | +| `WITAN_OMNIGRAPH_HTTP` | `1` | Use the direct HTTP transport for reads against a deployed omnigraph-server instead of shelling out to the `omnigraph` binary. Set to `0`/`false`/`no`/`off` to revert. Kept as a one-variable revert so a transport-specific production problem is an env change rather than an image rebuild — the CLI path beneath it stays fully maintained and is still the only way to reach `load`, `branch`, and `optimize`. | + +## Code graph (`witan code`) + +Settings for the tree-sitter code index and its cross-repo bridge. These mirror +the store settings above but address the *code* graph, which is a separate store +from the memory/task graph. + +| Variable | Default | Description | +| --- | --- | --- | +| `WITAN_CODE_DIR` | — | Directory holding the per-repo code-graph stores. | +| `WITAN_CODE_GRAPH` | — | Graph id to address on `WITAN_CODE_SERVER`. | +| `WITAN_CODE_INDEX_ROLE` | — | Declares what the indexing process is entitled to write **on a shared graph**. There, only `ci` may write a repo's default (`main`) view and run the stale-file purge that goes with it, so no developer's reindex can clobber the view all readers fall back to. A local store has a single user, who is its writer — this setting does not restrict it, and a local default-branch reindex works with no role declared. | +| `WITAN_CODE_OPTIMIZE_INTERVAL` | `86400` | Minimum seconds between throttled background `optimize` runs on the code stores. `0` disables. | +| `WITAN_CODE_SERVER` | — | Base URL of an omnigraph-server hosting the code graphs, for a shared index. | +| `WITAN_CODE_STORE_TOOLS` | — | Force the low-level store tools on (`1`) or off (`0`), overriding the default. These expose raw graph reads and mutations alongside the curated `code_*` tools. | +| `WITAN_CODE_TOKEN` | — | Bearer token for `WITAN_CODE_SERVER`. | +| `WITAN_CODE_TRANSPORT` | `direct` | How the `witan code` CLI reaches the index: `direct` opens the store in-process; `mcp` proxies through a deployed witan endpoint. | +| `WITAN_CODE_VIEW_MAX_IDLE_DAYS` | `14` | Reap per-branch views idle at least this long. `0` (or negative) disables reaping entirely. | + +## CI code-graph indexer + +Read by `witan-ci-index`, the script that keeps each repo's shared code graph +current. It runs as a Kubernetes CronJob from the same `witan` image with a +different entrypoint. Nothing else should set these. + +| Variable | Default | Description | +| --- | --- | --- | +| `WITAN_CODE_CI_ALLOW_LOCAL_STORE` | — | Set to `1` to waive the `WITAN_CODE_SERVER`/`WITAN_CODE_TOKEN` requirement and index into local stores instead. For development runs of the indexer only. | +| `WITAN_CODE_CI_REPOS` | — | **Required.** Whitespace-separated canonical repo URIs to sweep and index. | +| `WITAN_CODE_CI_WORKDIR` | `/tmp/witan-ci-index` | Scratch directory for checkouts. Rejected unless it is an absolute path at least two components deep with no `..` or empty components — the guard that keeps a misconfigured value from pointing the cleanup at something important. | +| `WITAN_CODE_GH_TOKEN` | — | Clone credential. Normally minted per-repo from the GitHub App above rather than set by hand. | +| `WITAN_CODE_GITHUB_API_URL` | `https://api.github.com` | GitHub API base URL. Only meaningful against GitHub Enterprise. | +| `WITAN_CODE_GITHUB_APP_ID` | — | GitHub App id used to mint short-lived clone credentials. Set all three `_APP_` variables, or none. | +| `WITAN_CODE_GITHUB_APP_INSTALLATION_ID` | — | Installation id of the GitHub App, identifying which org's repos it may clone. | +| `WITAN_CODE_GITHUB_APP_KEY_FILE` | — | Path to the GitHub App's private key, used to sign the App JWT. | + +## Write-path content scanning + +witan scans everything written to the graph for secrets and PII. It ships +**enabled**, and fails closed: a scanner that raises blocks the write rather than +silently opening the gate. + +`enabled_detectors`, `disabled_detectors`, `plugins`, `allowlist`, and +`allowlist_hashes` each accept a comma-separated string here (or a TOML list in +the config file). An empty `enabled_detectors` means every registered detector is +active; naming any detector switches to an explicit allowlist. +`disabled_detectors` always wins. + +| Variable | Default | Description | +| --- | --- | --- | +| `WITAN_SCAN_ALLOWLIST` | — | Regexes whose matches are downgraded to audit-only, for false-positive suppression. Tested against each finding's own matched span with `re.fullmatch`. | +| `WITAN_SCAN_ALLOWLIST_HASHES` | — | Salted SHA-256 digests (hex) of specific approved values, downgraded to audit-only without ever putting the plaintext in config. Computed as `sha256(salt + matched_span)`. Normalized to lowercase at load, so a hand-typed uppercase digest still matches. | +| `WITAN_SCAN_ALLOWLIST_SALT` | — | Salt for `WITAN_SCAN_ALLOWLIST_HASHES`. Empty means the hash allowlist is inert — set a deployment-specific value before relying on it. | +| `WITAN_SCAN_DISABLED_DETECTORS` | — | Detectors to switch off. Always wins over `WITAN_SCAN_ENABLED_DETECTORS`. | +| `WITAN_SCAN_ENABLED` | `true` | Master switch. When false the write path is not scanned at all. | +| `WITAN_SCAN_ENABLED_DETECTORS` | — | Explicit allowlist of detectors to run. Empty means all registered detectors. | +| `WITAN_SCAN_ON_ERROR` | `block` | What to do when a scanner itself raises: `block` or `warn`. Fail-closed by default so a broken detector cannot silently open the gate. | +| `WITAN_SCAN_PII_ACTION` | `redact` | Enforcement for `pii` findings. Mask-and-proceed by default. | +| `WITAN_SCAN_PLUGINS` | — | Dotted import paths of external scanners to load, in addition to those discovered through the `witan.scanners` entry-point group. | +| `WITAN_SCAN_SECRET_ACTION` | `block` | Enforcement for `secret` findings. Fail-closed by default. | + +## Recall ranking + +Tuning knobs for the composite re-rank `recall` applies on top of BM25. Ranking +is always on; these change its shape, they do not switch it off. Set every `W_*` +weight to `0` to reproduce the raw BM25 order. + +| Variable | Default | Description | +| --- | --- | --- | +| `WITAN_RANK_DEFAULT_CONF` | `0.6` | Confidence assumed for a memory that carries none. Must be between 0 and 1. | +| `WITAN_RANK_HALFLIFE_DAYS` | `90.0` | Half-life of the recency decay, in days. Must be greater than zero. | +| `WITAN_RANK_PEN_CONTRADICTED` | `0.25` | Score penalty for a memory another contradicts. Deliberately mild — a contradiction is surfaced for review, never hidden. | +| `WITAN_RANK_PEN_SUPERSEDED` | `1.0` | Score penalty applied to a memory something else supersedes. At the default it is effectively removed from results. | +| `WITAN_RANK_W_BM25` | `1.0` | Weight of the BM25 text-relevance term. | +| `WITAN_RANK_W_CONF` | `0.2` | Weight of the author-set confidence score. | +| `WITAN_RANK_W_CORROB` | `0.2` | Weight of corroboration — how much other memories back this one up. | +| `WITAN_RANK_W_HOP` | `0.5` | Per-hop distance penalty in graph-aware recall, so direct hits (hop 0) outrank expanded neighbours (hop ≥ 1). | +| `WITAN_RANK_W_RECENCY` | `0.3` | Weight of the recency term, decayed by `WITAN_RANK_HALFLIFE_DAYS`. | + +## Maintenance and observability + +| Variable | Default | Description | +| --- | --- | --- | +| `WITAN_CONTEXT_TTL` | `30.0` | How long the rendered session-context block is cached on disk, in seconds. Only the first prompt in the window pays to build it; the rest read one small file. `0` disables the cache. The content is advisory, so a few seconds of staleness is fine. | +| `WITAN_LOG_FORMAT` | — | Log rendering: `console` or `json`. Defaults to `console` when stderr is a TTY and `json` when it is not — a deployed pod gets structured logs and a developer gets colours, neither having to pass a flag. | +| `WITAN_LOG_LEVEL` | `INFO` | Log level. Takes precedence over the bare `LOG_LEVEL`, which is also honoured for deployments that set it org-wide. | +| `WITAN_OPTIMIZE_INTERVAL` | `86400` | Minimum seconds between throttled background `optimize` runs on the memory store. `0` disables. The `Stop` hook spawns a detached run at most this often, so compaction never blocks a session. | diff --git a/docs/reference/graph-schema.md b/docs/reference/graph-schema.md new file mode 100644 index 00000000..e8e517e8 --- /dev/null +++ b/docs/reference/graph-schema.md @@ -0,0 +1,224 @@ + + +# Graph schema + +The shape of the witan graph: what a memory, a task, a project, and a session are, and how they connect. Every MCP tool is ultimately a read or a write against these types. + +Source of truth: [`mcp/servers/witan/schema/schema.pg`](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan/schema/schema.pg). + +## Nodes + +### `Memory` + +Agent Memory — team-wide knowledge graph for coding agents. + +One node type with a kind discriminator keeps cross-kind search simple. +Optional fields are populated only for the relevant kind: +pattern → language +project_fact → category +lesson → severity +agent_context → (no additional fields) + +Slug convention: +pat- pattern e.g. pat-always-use-uv +pf- project_fact e.g. pf-ol-django-vault-secrets +les- lesson e.g. les-no-raw-sql-in-views +ctx- agent_context e.g. ctx-ticket-1234-approach + +| Field | Type | Notes | +| --- | --- | --- | +| `slug` | `String @key` | | +| `kind` | `enum(pattern, project_fact, lesson, agent_context) @index` | | +| `title` | `String @index` | | +| `content` | `String @index` | @index enables the BM25 search($m.content, …) queries | +| `repo` | `String? @index` | | +| `language` | `String? @index` | | +| `category` | `String? @index` | | +| `severity` | `enum(info, warning, critical)? @index` | | +| `author` | `String @index` | | +| `created_at` | `DateTime @index` | | +| `updated_at` | `DateTime` | | +| `tags` | `[String]?` | | +| `symbol_refs` | `[String]?` | soft refs into the code-graph store (repo#path::Name) | +| `confidence` | `F32?` | author/agent-set trust 0.0–1.0; null treated as default | + +### `Topic` + +Topic: a join-surface node memories attach to. One node type, several kinds: +topic — promoted from a free-string tag +contract — name == bridge key_norm (env_var/endpoint/package/service) +symbol — reserved; not populated in the first cut (symbols stay soft refs) +entity — a named entity (service, library, person, concept) +Slug convention: tp-<kind>-<slug(name)> + +| Field | Type | Notes | +| --- | --- | --- | +| `slug` | `String @key` | | +| `name` | `String @index` | | +| `kind` | `enum(topic, contract, symbol, entity) @index` | | +| `created_at` | `DateTime @index` | | + +### `WorkflowProject` + +WorkflowProject tracks an overarching engineering objective across +multiple sessions and phases. One project per logical unit of work, +regardless of how many Claude Code sessions contribute to it. + +Slug convention: wp-<sanitised-title>-<6hex> +e.g. wp-add-vault-k8s-auth-a3f912 + +| Field | Type | Notes | +| --- | --- | --- | +| `slug` | `String @key` | | +| `title` | `String @index` | | +| `description` | `String` | | +| `repos` | `[String]?` | | +| `status` | `enum(active, completed, abandoned) @index` | | +| `phase` | `enum(discovery, spec, implementation, delivery) @index` | | +| `author` | `String @index` | | +| `created_at` | `DateTime @index` | | +| `updated_at` | `DateTime` | | +| `completed_at` | `DateTime?` | | +| `tags` | `[String]?` | | +| `github_issue` | `String?` | | +| `github_pr` | `String?` | | +| `blocked_by` | `[String]?` | denormalized blocker project slugs (drives ready-work) | + +### `WorkflowSession` + +WorkflowSession tracks a single Claude Code session contributing to +a project. One project has many sessions; sessions may run in parallel. + +project_slug is denormalized for fast indexed lookup without graph traversal. +Slug convention: ws-<project-slug-prefix>-<6hex> + +| Field | Type | Notes | +| --- | --- | --- | +| `slug` | `String @key` | | +| `project_slug` | `String @index` | | +| `session_id` | `String @index` | | +| `repo` | `String? @index` | | +| `phase` | `enum(discovery, spec, implementation, delivery) @index` | | +| `summary` | `String` | | +| `tools_used` | `[String]?` | | +| `files_changed` | `[String]?` | | +| `author` | `String @index` | | +| `started_at` | `DateTime @index` | | +| `ended_at` | `DateTime?` | | +| `tags` | `[String]?` | | +| `superseded_by` | `String?` | | + +### `WorkflowTrace` + +WorkflowTrace is an assembled, corpus-ready record of a completed project. +Created by workflow_project_complete. Immutable after creation. +Used for downstream pattern mining to generate skills and hooks. + +Slug convention: wt-<project-slug> + +| Field | Type | Notes | +| --- | --- | --- | +| `slug` | `String @key` | | +| `project_slug` | `String @index` | | +| `repos` | `[String]?` | | +| `title` | `String @index` | | +| `description` | `String` | | +| `session_count` | `I32` | | +| `phases` | `[String]` | | +| `duration` | `I32?` | | +| `outcome` | `String` | | +| `lessons_slug` | `[String]?` | | +| `patterns_slug` | `[String]?` | | +| `author` | `String @index` | | +| `created_at` | `DateTime @index` | | +| `tags` | `[String]?` | | + +### `Task` + +A dependency-aware task tracker (beads-like) living in the same graph as +memory and workflow so tasks can hard-link to projects, sessions, and +memories. The "ready work" query (open tasks with no open blocker) is the +core multi-agent coordination primitive. + +Tasks are hierarchical: an `epic` decomposes into child tasks/sub-issues via +the ParentOf edge, with parent_slug denormalized for fast child lookup. + +Slug convention: tk-<sanitised-title>-<6hex> +e.g. tk-wire-vault-sidecar-9c1d04 + +| Field | Type | Notes | +| --- | --- | --- | +| `slug` | `String @key` | | +| `title` | `String @index` | | +| `description` | `String` | | +| `repo` | `String? @index` | | +| `type` | `enum(bug, feature, task, chore, epic) @index` | | +| `status` | `enum(open, in_progress, blocked, closed) @index` | | +| `priority` | `enum(p0, p1, p2, p3) @index` | | +| `project_slug` | `String? @index` | denormalized link to WorkflowProject | +| `parent_slug` | `String? @index` | denormalized hierarchy (epic → sub-issue) | +| `blocked_by` | `[String]?` | denormalized blocker slugs (drives ready-work) | +| `assignee` | `String? @index` | who owns it (vs author = creator) | +| `external_uri` | `String? @index` | GitHub issue/PR or any reference URI | +| `resolution` | `String?` | free-text note set when closed | +| `author` | `String @index` | | +| `created_at` | `DateTime @index` | | +| `updated_at` | `DateTime` | | +| `closed_at` | `DateTime?` | | +| `claimed_at` | `DateTime?` | advisory-claim lease start (assignee holds it) | +| `symbol_refs` | `[String]?` | soft refs into the code-graph store | +| `tags` | `[String]?` | | + +### `CodeBranch` + +Links a git branch to the task/project it is carrying, so "which branch +carries task X" and "which tasks are in flight on branch B" are one-hop +graph queries. Coordination state — lives here (shared, durable), not in +witan-code's per-repo/bridge omnigraph stores, which are local +re-derivable caches that `witan-code branches --prune` may destroy at any +time. The coupling to witan-code stays one-way, via the raw git branch +name as the shared vocabulary: `branch` is always the branch as git names +it (e.g. "feature/new-api"), never witan-code's sanitized omnigraph +branch name ("feature_new-api") — that sanitization is a witan-code +storage detail and must not leak here. + +Slug convention: "<repo URI>|<git branch>" + +| Field | Type | Notes | +| --- | --- | --- | +| `slug` | `String @key` | | +| `repo` | `String @index` | | +| `branch` | `String @index` | | +| `status` | `enum(active, merged, abandoned) @index` | | +| `created_at` | `DateTime @index` | | +| `updated_at` | `DateTime` | | + +## Edges + +Edges are directional and typed. A traversal names the edge in lowercase (`supersedes`, `blocks`), while the schema declares it in PascalCase. + +| Edge | From | To | Meaning | +| --- | --- | --- | --- | +| `Supersedes` | `Memory` | `Memory` | Supersedes: a newer memory replaces an older one. Link new → old when updating a pattern or lesson that has changed. | +| `AppliesTo` | `Memory` | `Memory` | AppliesTo: links a pattern or lesson to a project fact that provides context. e.g. a pattern "always use uv" AppliesTo project fact "ol-django uses uv". | +| `Refines` | `Memory` | `Memory` | Refines: a newer memory sharpens/extends an older one without replacing it. | +| `Contradicts` | `Memory` | `Memory` | Contradicts: two memories conflict. Symmetric in meaning; stored one direction and traversed both ways. Never hidden — surfaced for review. | +| `RelatedTo` | `Memory` | `Memory` | RelatedTo: soft associative link. Symmetric; stored one direction. | +| `Tagged` | `Memory` | `Topic` | Tagged: a Memory is about a Topic. Real Layer-1 edge (traversable). | +| `BelongsTo` | `WorkflowSession` | `WorkflowProject` | BelongsTo: links each WorkflowSession to its WorkflowProject. | +| `Produced` | `WorkflowProject` | `WorkflowTrace` | Produced: links a completed WorkflowProject to its WorkflowTrace (one-to-one). | +| `Informed` | `WorkflowProject` | `Memory` | Informed: links a WorkflowProject to Memory nodes consulted or created during the project (patterns, lessons, agent_context, project_facts). | +| `SessionProduced` | `WorkflowSession` | `Memory` | SessionProduced: a WorkflowSession created or substantively updated a Memory. Session-grain provenance (Informed is project-grain). The bare name `Produced` is taken (WorkflowProject -> WorkflowTrace), hence the qualified name. | +| `ProjectBlocks` | `WorkflowProject` | `WorkflowProject` | ProjectBlocks: a blocking project must complete before the blocked project is "ready". | +| `Blocks` | `Task` | `Task` | Blocks: a blocker task must close before the blocked task is "ready". | +| `ParentOf` | `Task` | `Task` | ParentOf: hierarchy — an epic (or parent task) contains child tasks. | +| `DiscoveredFrom` | `Task` | `Task` | DiscoveredFrom: provenance — a task surfaced while working another task. | +| `TaskBelongsTo` | `Task` | `WorkflowProject` | TaskBelongsTo: a task rolls up to a WorkflowProject. | +| `Addresses` | `Task` | `Memory` | Addresses: a task is motivated by a Memory node (lesson, project fact). | +| `Closes` | `WorkflowSession` | `Task` | Closes: a WorkflowSession executed/closed a task. | +| `WorksOn` | `CodeBranch` | `Task` | WorksOn: a CodeBranch is carrying out a Task. | +| `ForProject` | `CodeBranch` | `WorkflowProject` | ForProject: a CodeBranch belongs to a WorkflowProject. | diff --git a/docs/reference/index.md b/docs/reference/index.md new file mode 100644 index 00000000..17041cd1 --- /dev/null +++ b/docs/reference/index.md @@ -0,0 +1,71 @@ +# Reference + +Complete, precise, and generated. Every page in this section is derived from the +code it documents — the registered MCP tool objects, the cyclopts command tree, +the `.pg` schema files — and CI fails if a committed page no longer matches its +source. + +That is a deliberate trade: these pages will never be as readable as the +[guides](../guides/index.md), and they will never be out of date. + +
+ +- **[MCP tools](mcp-tools/index.md)** + + All 60 tools your agent can call, with full parameter schemas. Grouped into + [memory](mcp-tools/memory.md), [tasks](mcp-tools/tasks.md), + [workflow](mcp-tools/workflow.md), and [code](mcp-tools/code.md). + +- **[CLI](cli.md)** + + Every `witan` command and flag, including `witan code …`, rendered from the + live command tree. + +- **[Environment variables](environment.md)** + + All 67 settings, what they do, and their defaults. + +- **[Graph schema](graph-schema.md)** + + Node and edge types: `Memory`, `Task`, `WorkflowProject`, and the rest. + The [bridge schema](bridge-schema.md) covers cross-repo linking. + +
+ +## How configuration resolves + +Three sources, highest precedence first: + +1. **Environment variable** — always wins. +2. **`~/.config/witan/config.toml`** — including a matched `[targets.]` + block, which overrides the file's global settings. +3. **Built-in default.** + +Both CLIs read the same config file, so configuring a deployment once points +`witan` and `witan code` at it together. + +## Conventions in these pages + +**Slug prefixes** identify a node's type at a glance: + +| Prefix | Type | +| --- | --- | +| `pat-` | pattern memory | +| `pf-` | project fact | +| `les-` | lesson | +| `ctx-` | agent context | +| `tp-` | topic | +| `tk-` | task | +| `wp-` | workflow project | +| `ws-` | workflow session | + +**Symbol ids** are `repo#path::Name`, e.g. +`https://github.com/mitodl/agent-kit#witan_core/repo_key.py::canonical`. + +**Repo keys** are canonical HTTPS URIs — `https://github.com/mitodl/agent-kit`, +not `git@github.com:mitodl/agent-kit.git`. witan canonicalises what it detects; +pass the HTTPS form when passing one explicitly. + +**Types** in parameter tables use `str`, `int`, `bool`, `list[str]`, with `?` +marking optional. A parameter shown as `—` in the Description column has no +docstring in the source yet. diff --git a/docs/reference/mcp-tools/code.md b/docs/reference/mcp-tools/code.md new file mode 100644 index 00000000..4091cfda --- /dev/null +++ b/docs/reference/mcp-tools/code.md @@ -0,0 +1,272 @@ + + +# Code graph + +Exact symbol lookups, caller graphs, change-impact analysis, and cross-repo contract tracing, served from a tree-sitter index. Reach for these instead of grep when you need a definition, a blast radius, or the provider of a shared env var, endpoint, package, or service. + +## `code_callers` + +Find symbols that call ``symbol_id`` (incoming Calls edges). + +Heuristic name-resolution based; not a precise call graph. For calls plus +other references use code_find_references. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `symbol_id` | str | **required** | The symbol to start from, as returned in the ``symbol_id`` field of
code_find_definition or code_search_symbol. Form:
``#::``. | + +## `code_cross_repo_impact` + +Find the cross-repo surface of a symbol. + +Looks up the interface bindings attributed to ``symbol_id`` (env vars it +reads, packages it imports, endpoints it serves/calls), then returns every +binding for those same contracts in OTHER repos — i.e. who else is coupled to +this symbol across the SOA. Returns a shaped empty result when the symbol has +no cross-repo surface or the bridge store does not exist yet. + +Generic env vars (DEBUG, PORT, …) are flagged ``generic`` and excluded from +the cross-repo fan-out so a trivial edit doesn't appear to touch every repo. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `symbol_id` | str | **required** | The symbol whose cross-repo surface to trace, as returned in the
``symbol_id`` field of code_find_definition or code_search_symbol. Its
repo prefix also scopes the bridge read, so asking about another repo's
symbol does not pick up an unrelated overlay from your own checkout. | +| `min_precision` | `precise` \| `heuristic` \| `fuzzy` | `'heuristic'` | ``heuristic`` (default) \| ``precise`` — see server instructions. | + +## `code_find_definition` + +Find symbol definitions whose name or qualified name matches ``name``. + +Returns matching symbols (function, method, class, module, …) with their +``symbol_id``, file, line range, signature, docstring, and ``repo`` of +origin. Feed a returned ``symbol_id`` to code_find_references / code_callers +/ code_impact. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `name` | str | **required** | Bare name (``run``) or qualified name (``Service.run``). | +| `repo` | str? | `null` | Canonical repo URI to search. Defaults to the repo detected from the
checkout, and fans out across every indexed repo only when no repo is
detected at all. Note this differs from the witan memory/task tools:
``repo=""`` does **not** force a fan-out here — it is falsy, so it
behaves exactly like omitting the argument. | +| `branch` | str? | `null` | Git branch whose indexed view to query (e.g. another agent's in-flight
branch). Defaults to the checkout's branch when querying the current
repo; when ``repo`` names a different repo and ``branch`` is omitted,
reads that store's default (main) view. | + +## `code_find_references` + +Find symbols that reference or call ``symbol_id`` (incoming edges). + +Supersets code_callers (references includes calls); use code_callers when +you want calls only. Heuristic — may miss or over-report. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `symbol_id` | str | **required** | The symbol to start from, as returned in the ``symbol_id`` field of
code_find_definition or code_search_symbol. Form:
``#::``. | + +## `code_impact` + +Estimate change impact: the transitive set of callers of ``symbol_id``. + +Performs a breadth-first traversal over Calls edges, capping at ``max_depth`` +levels and ``max_nodes`` total. Returns the reached symbols and whether the +traversal was truncated. Heuristic — see code_callers. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `symbol_id` | str | **required** | The symbol to start from, as returned in the ``symbol_id`` field of
code_find_definition or code_search_symbol. Form:
``#::``.
This is the symbol you are about to change; the result is its blast
radius. | +| `max_depth` | int | `5` | Maximum BFS depth (default 5). | +| `max_nodes` | int | `200` | Cap on total symbols returned (default 200). | + +## `code_indexed_branches` + +The in-flight branch views each indexed repo's store carries, and who owns each. + +A non-default git branch is indexed onto its own store view, named for the +writer as well as the branch (docs/BRANCH_INDEXING.md), so several people +— or an agent and its human, or one developer in two worktrees — can each +have a view of the same branch without overwriting one another. + +Pass ``branch`` (a raw git branch name) to see every writer's view of that +one branch: this is how you find a teammate's in-flight work. Feed a +returned ``view`` straight back as the ``branch`` argument of +code_find_definition / code_search_symbol / code_symbols_in_file to read +it. Omit ``branch`` to list everything. + +Each row is ``{repo, views}``, where a view is +``{view, branch, actor}`` — ``view`` the name to pass back, ``branch`` the +sanitized git branch it is a view of, ``actor`` its owner (null on a +single-writer local store). ``views`` is null for a store that could not be +listed. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `branch` | str? | `null` | A raw git branch name (not a view name) to filter by, showing every
writer's view of that one branch — this is how you find a teammate's
in-flight work. Omit to list every view of every indexed repo. | + +## `code_indexed_repos` + +The repositories that have a code graph indexed, and how big each is. + +Use it to check coverage before trusting a negative result — a symbol +search returning nothing means something different when the repo in +question was never indexed. ``files`` is None for a store that could not be +read; ``last_indexed`` is a Unix timestamp. + +``bytes`` and ``last_indexed`` are both null for a graph on the shared +omnigraph-server: they describe a directory on this machine, and a client +of a shared graph has neither the directory nor any business reporting the +server's disk. ``files`` stays real — it is a query, not a walk. + +*Takes no parameters.* + +## `code_interface_consumers` + +Find repos that CONSUME a cross-repo contract ``key`` of ``kind``. + +The mirror of ``code_interface_providers`` — e.g. which repos read an env var +or call an API endpoint. Spans every indexed repo. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `kind` | `env_var` \| `package` \| `service` \| `endpoint` | **required** | ``env_var``, ``package``, ``service``, or ``endpoint``. | +| `key` | str | **required** | The contract value (``endpoint`` paths are normalized). | +| `min_precision` | `precise` \| `heuristic` \| `fuzzy` | `'heuristic'` | ``heuristic`` (default) \| ``precise`` — see server instructions. | + +## `code_interface_providers` + +Find repos that PROVIDE a cross-repo contract ``key`` of ``kind``. + +Spans every indexed repo via the shared bridge store. Examples: +``code_interface_providers("env_var", "MITOL_APP_BASE_URL")`` returns the +ol-infrastructure binding that sets it; ``("endpoint", "/api/v1/courses/")`` +returns the Django/OpenAPI route that serves it. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `kind` | `env_var` \| `package` \| `service` \| `endpoint` | **required** | ``env_var``, ``package``, ``service``, or ``endpoint``. | +| `key` | str | **required** | The contract value. For ``endpoint`` a raw path is accepted and
normalized (path params collapse to ``{}``). | +| `min_precision` | `precise` \| `heuristic` \| `fuzzy` | `'heuristic'` | ``heuristic`` (default) \| ``precise`` — see server instructions. | + +## `code_interface_search` + +BM25 search over cross-repo interface bindings (by normalized key). + +Use to discover contract keys when you only remember part of a name — +e.g. ``code_interface_search("BASE_URL")`` or ``("courses", kind="endpoint")``. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `query` | str | **required** | Search terms matched against the normalized key. | +| `kind` | `env_var` \| `endpoint` \| `package` \| `service`? | `null` | Optional filter to one kind. | +| `min_precision` | `precise` \| `heuristic` \| `fuzzy` | `'heuristic'` | ``heuristic`` (default) \| ``precise`` — see server instructions. | + +## `code_precise_edges` + +Precise cross-repo edges resolved by canonical symbol string. + +Joins every repo's unresolved external-symbol references against other +repos' exported symbols — a read-time join, distinct from the coarser +(kind, key_norm) heuristic grouping ``code_interface_*`` use. Each edge +carries ``match_count`` (how many providers a reference joined to) and +``ambiguous_version`` (true when more than one provider survives version +disambiguation); filter to ``preferred`` edges to narrow a fan-out to its +best candidate(s). A reference with no precise match shows up in +``code_unresolved_symbols`` instead. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `repo` | str? | `null` | Keep only edges whose consumer OR provider is this repo. Omit to see
every precise edge in the bridge store. | + +## `code_reindex` + +Index (or re-index) the current repo, or a subpath of it. + +Incremental by default — unchanged files (matching content hash) are +skipped. Lazily creates the per-repo store on first run. Returns a summary of +files scanned/indexed/skipped and symbols/edges written. + +A full rebuild runs for minutes on a large repo, so this tool accepts +task-augmented execution (`io.modelcontextprotocol/tasks`): a client that +asks for it gets a task handle back immediately and polls `tasks/get`, +instead of holding one tool call open for the whole index. Asking is the +client's choice — omit it and the call runs to completion as it always has. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `path` | str? | `null` | Optional file or directory under the repo. Defaults to the repo root
(or cwd if not in a git repo). | +| `force` | bool | `False` | Re-index every file regardless of content hash. | + +## `code_repo_dependencies` + +The repo-to-repo dependency graph over every indexed repo. + +Aggregates the bridge store's interface bindings into "repo A depends on +repo B" links (A consumes a contract B provides; for ``service`` bindings, +the deploying repo depends on what it deploys). Returns +``{"repos": [...], "edges": [{consumer, provider, weight, kinds, +contracts}]}`` — the coarse, whole-SOA view, where +``code_interface_providers``/``_consumers`` answer about one contract key. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `kind` | `env_var` \| `endpoint` \| `package` \| `service`? | `null` | Filter to one contract kind (``env_var``/``package``/``service``/``endpoint``). | +| `repo` | str? | `null` | Keep only links touching a repo whose URI contains this substring. | +| `min_precision` | `precise` \| `heuristic` \| `fuzzy` | `'heuristic'` | ``heuristic`` (default) \| ``precise`` — see server instructions. | + +## `code_repo_symbols` + +A repo's cross-repo symbol table (docs/SYMBOL_TABLE.md). + +One row per (role, symbol): ``exported`` rows are the repo's public +contract surface — what other repos can resolve against — and ``external`` +rows are the unresolved references Stage 2 joins against other repos' +exports. Use it to see what a repo publishes and what it expects from the +rest of the SOA; ``code_precise_edges`` is the resolved join over the same +table. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `repo` | str? | `null` | Canonical repo URI. Defaults to the current detected repo. | +| `role` | `exported` \| `external`? | `null` | Filter to ``exported`` or ``external`` rows. | +| `scheme` | str? | `null` | Filter to one symbol scheme (``http``/``env``/``pkg``/``svc``). | + +## `code_search_symbol` + +Full-text/substring search over symbol qualified names (BM25-ranked). + +Use to locate a symbol when you only remember part of its name. Each result +carries its ``symbol_id`` and ``repo`` of origin. BM25 ranking is per-store; +cross-repo fan-out concatenates each store's ranked results. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `query` | str | **required** | Search terms matched against ``qualified_name``. | +| `kind` | `function` \| `method` \| `class` \| `module` \| `variable` \| `interface` \| `type` \| `enum` \| `key` \| `table` \| `cte` \| `block`? | `null` | Optional filter to a single symbol kind: ``function``, ``method``,
``class``, ``module``, ``variable``, ``interface``, ``type``, ``enum``,
``key``, ``table``, ``cte``, or ``block``. Pass e.g. ``kind="function"``
to exclude the many YAML ``key`` symbols when searching for code. | +| `repo` | str? | `null` | Canonical repo URI to search. Defaults to the repo detected from the
checkout, and fans out across every indexed repo only when no repo is
detected at all — ``repo=""`` does **not** force that, unlike the witan
memory/task tools; it is falsy and behaves like omitting the argument.
BM25 ranking is per-store, so a fan-out concatenates each store's
ranked results rather than producing one global ordering. | +| `branch` | str? | `null` | Git branch whose indexed view to query. Defaults to the checkout's
branch when querying the current repo; when ``repo`` names a different
repo and ``branch`` is omitted, reads that store's default (main) view. | + +## `code_symbols_in_file` + +List symbols defined in ``path`` (repo-relative or absolute). + +A file is inherently repo-local, so this resolves a single repo (no fan-out): +the given ``repo`` or, if omitted, the current repo. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `path` | str | **required** | Source file path. | +| `repo` | str? | `null` | Canonical repo URI the file belongs to. Defaults to the current repo. | + +## `code_unresolved_symbols` + +External symbol references with no precise cross-repo match. + +Surfaces indexing-coverage gaps: a repo consumes a contract (env var, +package, endpoint, service) that no indexed repo currently exports — +either the provider repo isn't indexed yet, or the reference genuinely has +no provider in this SOA. These still get a heuristic-tier chance via +``code_interface_consumers``/``_providers``; this tool finds what's NOT +precisely resolved. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `repo` | str? | `null` | Keep only unresolved references from this consumer repo. Omit to see
every unresolved reference in the bridge store. | diff --git a/docs/reference/mcp-tools/index.md b/docs/reference/mcp-tools/index.md new file mode 100644 index 00000000..04f8aef5 --- /dev/null +++ b/docs/reference/mcp-tools/index.md @@ -0,0 +1,16 @@ + + +# MCP tools + +witan exposes **62 MCP tools** across four domains. A single `witan serve` mounts all of them, so one MCP entry in your agent's config gets you the whole surface. + +| Domain | Tools | What it covers | +| --- | --- | --- | +| [Memory & recall](memory.md) | 15 | Recording durable knowledge in the shared graph, and reading it back. | +| [Tasks](tasks.md) | 11 | The work-coordination layer: what needs doing, what blocks what, and who holds which piece of work right now. | +| [Workflow projects & sessions](workflow.md) | 19 | Tracking an engineering objective across many agent sessions without an explicit hand-off. | +| [Code graph](code.md) | 17 | Exact symbol lookups, caller graphs, change-impact analysis, and cross-repo contract tracing, served from a tree-sitter index. | diff --git a/docs/reference/mcp-tools/memory.md b/docs/reference/mcp-tools/memory.md new file mode 100644 index 00000000..6ad3738d --- /dev/null +++ b/docs/reference/mcp-tools/memory.md @@ -0,0 +1,337 @@ + + +# Memory & recall + +Recording durable knowledge in the shared graph, and reading it back. `recall` is the default read — it composes BM25 search, graph expansion, superseded-pruning, and re-ranking into one call. The narrower reads below exist for when you already know exactly what you want. + +## `recall` + +Graph-aware contextual recall — the composition of every other memory tool. + +Seeds from any combination of ``query`` (BM25), ``symbol_id`` +(``symbol_context``), ``task`` (memories it Addresses + memories sharing +its symbol_refs), and ``topic`` (memories tagged to it). Expands ``hops`` +(default 1, capped at 2) along AppliesTo/RelatedTo edges, topic siblings, and +provenance siblings; prunes superseded memories (unless +``include_superseded``); flags Contradicts pairs; and re-ranks with the +composite score minus a per-hop distance penalty so seeds outrank neighbours. + +With no edges in the graph the result equals ``memory_search`` — expansion is +additive, never lossy. Embeddings are deferred behind ``WITAN_EMBED_ENABLED`` +(default off); ``recall`` works with BM25 only and needs no embedding provider. + +Returns ``{"memories": [...ranked...], "contradictions": [...], "seeds": {...}}``. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `query` | str? | `null` | Free-text BM25 query, matched against ``content`` and ``title``. | +| `symbol_id` | str? | `null` | A code-graph symbol id (``repo#path::Name``) to seed from — the memories
and tasks whose ``symbol_refs`` include it. Use this before editing a
symbol: "what do we already know about this function?" | +| `task` | str? | `null` | A ``tk-`` slug to seed from — the memories it ``Addresses`` plus those
sharing its ``symbol_refs``. The one-call way to load context for a task
you are about to start. | +| `topic` | str? | `null` | A Topic slug (``tp-...``) or a ``name:kind`` spec (e.g. ``uv:topic``,
``DATABASE_URL:contract``) to seed from. Topics are a cross-repo join
surface, so this seed in particular can pull in other repositories. | +| `repo` | str? | `null` | Repo scoping — see instructions. Applies to the ``query`` seed only;
symbol, task, and topic seeds resolve wherever they live. | +| `kind` | `pattern` \| `project_fact` \| `lesson` \| `agent_context`? | `null` | Restrict the ``query`` seed to one memory kind: ``pattern``,
``project_fact``, ``lesson``, or ``agent_context``. | +| `hops` | int | `1` | How far to expand from the seeds along ``AppliesTo`` / ``RelatedTo``
edges, topic siblings, and provenance siblings. Clamped to 0–2. ``0``
disables expansion and returns the seeds alone; ``1`` (the default) is
almost always right, since each extra hop widens results faster than it
deepens them. | +| `limit` | int | `20` | Maximum memories to return after re-ranking. | +| `include_superseded` | bool | `False` | When ``True``, keep memories that a newer memory ``Supersedes``. Default
``False`` hides them, which is what makes recall return current
knowledge rather than its history. | + +## `memory_store` + +Store a new memory in the shared graph. + +Prefer this over your private built-in/session memory for anything durable +and team-shareable — patterns, project facts, lessons, decisions — so other +agents and future sessions can find it. Returns the slug of the created node +so callers can link to it. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `kind` | `pattern` \| `project_fact` \| `lesson` \| `agent_context` | **required** | ``pattern`` — coding convention or reusable technique
``project_fact`` — structural fact about a repo/service
``lesson`` — a correction or cautionary finding
``agent_context``— information a future agent on this task should know | +| `title` | str | **required** | Short, human-readable label. Used in listings and search. | +| `content` | str | **required** | Full text of the memory. Be specific: include the what, why, and any
examples. This is the primary search target. | +| `repo` | str? | `null` | Repo scoping — see instructions. | +| `language` | str? | `null` | Programming language (for ``pattern`` kind). e.g. ``python``, ``typescript``. | +| `category` | str? | `null` | Thematic category (for ``project_fact`` kind).
e.g. ``architecture``, ``deployment``, ``testing``, ``dependencies``. | +| `severity` | `info` \| `warning` \| `critical`? | `null` | Importance level (for ``lesson`` kind).
``info`` \| ``warning`` \| ``critical``. | +| `tags` | list[str]? | `null` | Optional list of free-form tags for grouping. | +| `symbol_refs` | list[str]? | `null` | Optional code-graph symbol ids (``#::``,
from the witan-code tools' ``symbol_id`` field) this memory concerns,
e.g. the function a lesson is about. Stored as a soft reference. | +| `confidence` | float? | `null` | Optional author/agent trust in this memory, 0.0–1.0. Feeds the search
re-rank; omitted memories use the configured default. | +| `session_slug` | str? | `null` | The ``ws-`` handle returned by ``workflow_session_start``, recording
which session produced this memory. Pass it whenever you have one: the
protocol carries no session state, so against a deployed service this is
the only way the ``SessionProduced`` provenance edge can be created.
Omit it under a local stdio server, which finds the handle itself. | + +## `memory_get` + +Retrieve a single memory by its slug. + +Returns the full node or ``null`` if not found. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `slug` | str | **required** | The ``pat-`` / ``pf-`` / ``les-`` / ``ctx-`` slug to retrieve. | +| `include_topics` | bool | `False` | When ``True``, attach a ``topics`` list of the Topic nodes this memory is
tagged with (slug/name/kind). | + +## `memory_update` + +Correct a memory's fields in place. Only non-null arguments are applied. + +This is the repair tool for a memory whose *content was always meant to be +what you are about to write* — a wrong ``repo`` (so it never showed up in +repo-scoped reads), a typo'd title, a missing tag. Returns the updated node, +or ``null`` if no memory has that slug. + +Which tool to reach for: + +- a field is wrong → ``memory_update`` (this one) +- the knowledge itself changed → ``memory_store`` the new one, then + ``memory_link(kind="supersedes")``; the old one stays readable as history +- it should never have existed (accidental duplicate, test write) → + ``memory_delete`` +- it contains secret material → **rotate the credential.** Neither this tool + nor ``memory_delete`` erases the old value from the graph's history. + +``kind`` is deliberately not updatable: a memory that turns out to be a +different kind is a different memory — store it and supersede this one. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `slug` | str | **required** | The ``pat-`` / ``pf-`` / ``les-`` / ``ctx-`` slug of the memory to
correct. | +| `title` | str? | `null` | New short, human-readable label. | +| `content` | str? | `null` | New full text. Replaces the existing content rather than appending —
and note that rewriting content here leaves no record that it changed.
If the *knowledge* changed, store a new memory and supersede this one
instead. | +| `repo` | str? | `null` | Canonical repo URI to (re)scope the memory to. Case-folded on write, as
every other repo-key path does: correcting a mis-scoped memory is this
tool's headline use, and a key that does not match what repo detection
returns would just mis-scope it again, differently. | +| `language` | str? | `null` | Programming language (``pattern`` kind). e.g. ``python``, ``typescript``. | +| `category` | str? | `null` | Thematic category (``project_fact`` kind). e.g. ``architecture``,
``deployment``, ``testing``, ``dependencies``. | +| `severity` | `info` \| `warning` \| `critical`? | `null` | Importance (``lesson`` kind): ``info`` \| ``warning`` \| ``critical``. | +| `tags` | list[str]? | `null` | Free-form tags. Replaces the existing list. Each tag is also promoted to
a ``Topic`` and linked. **Tags removed here keep their ``Tagged``
edge** — edges cannot be individually retracted, and deleting the Topic
to drop one would take out every other memory's edge to it. The string
list stays authoritative for what the memory claims to be tagged with. | +| `symbol_refs` | list[str]? | `null` | Code-graph symbol ids (``repo#path::Name``) this memory concerns.
Replaces the existing list. | +| `confidence` | float? | `null` | Author/agent trust in this memory, 0.0–1.0. Feeds the recall re-rank. | + +## `memory_delete` + +Hard-delete a memory. Graph hygiene only — NOT a way to erase secrets. + +Use when a memory should never have existed: an accidental duplicate, a test +write, a node created against the wrong graph. For anything else prefer +``memory_update`` (a field is wrong) or ``memory_store`` + +``memory_link(kind="supersedes")`` (the knowledge changed) — superseding is +the soft delete, and it keeps the history legible. + +**This does not erase content.** The row remains fully readable, content +included, from any prior commit of the graph. If a memory captured a +credential, the fix is to **rotate the credential**; scrubbing history is an +admin ``omnigraph cleanup``, which no MCP tool performs. + +Deleting a node also removes its incident edges in both directions, so a +deleted memory leaves no dangling ``Supersedes``/``RelatedTo``/``Tagged`` +behind. Topic nodes on the far end of a ``Tagged`` edge survive and may be +left with no memories. + +That is not the only way to remove an edge — edges are deletable on their +own (``task_unlink``, ``_unlink_edge``). Deleting the node is still the +only route for a *memory* edge, since no unlink tool covers those yet. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `slug` | str | **required** | The memory to delete permanently. | +| `confirm` | bool | `False` | Must be ``True``. Without it this is a no-op returning
``{"deleted": False, "reason": ...}``. | + +## `memory_list` + +List memories (no search), optionally filtered by kind, repo, and/or language. + +Browse stored memories without a search query — e.g. all ``lesson`` or +``pattern`` memories. Ordered most-recent first. To load context prefer +``recall``; use this for a plain kind-scoped listing (e.g. +``memory_list(kind="project_fact")`` at session start, or +``memory_list(kind="pattern", language="python")`` before writing code). + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `kind` | `pattern` \| `project_fact` \| `lesson` \| `agent_context`? | `null` | Optional filter: ``pattern``, ``project_fact``, ``lesson``, or
``agent_context``. Omit to list all kinds. | +| `repo` | str? | `null` | Repo scoping — see instructions. With no repo detected and none passed,
returns slim records (slug, kind, title, tags — no content) for unscoped
memories; ``memory_get`` a slug for its full content. | +| `language` | str? | `null` | Optional post-filter by ``language`` (e.g. ``python``); applies to the
full-content results, not the slim unscoped listing. | + +## `memory_search` + +Plain BM25 text search over memories (no graph expansion — for that use +``recall``). + +Returns the top-20 memories ranked by BM25 relevance. Superseded memories are +hidden unless ``include_superseded=True``. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `query` | str | **required** | Free-text search query. Searched against ``content`` and ``title``.
Content matches seed ahead of title-only matches, so they carry the
higher relevance proxy — but final order is the composite score, which
also weighs recency, corroboration and confidence. | +| `repo` | str? | `null` | Repo scoping — see instructions. | +| `kind` | `pattern` \| `project_fact` \| `lesson` \| `agent_context`? | `null` | Optional filter: ``pattern``, ``project_fact``, ``lesson``,
or ``agent_context``. | +| `include_superseded` | bool | `False` | When ``True``, keep memories that a newer memory ``Supersedes``. Default
``False`` drops them. | + +## `memory_link` + +Create a typed edge between two memories. + +- ``supersedes`` — ``from`` (newer) replaces ``to`` (older). ``to`` is hidden + from default ``memory_search`` results. +- ``refines`` — ``from`` sharpens/extends ``to`` without replacing it. +- ``applies_to`` — ``from`` (a pattern/lesson) applies in the context of ``to`` + (a project_fact). +- ``contradicts`` — ``from`` and ``to`` conflict. Symmetric; surfaced for + review, never hidden. +- ``related_to`` — soft association. Symmetric. +- ``tagged`` — ``from`` (a Memory) is about ``to`` (a Topic). ``to`` is + either an existing Topic slug (``tp-...``) or a ``name:kind`` + spec (e.g. ``cryptography:topic``, ``DATABASE_URL:contract``), + in which case the Topic is auto-created. + +For memory↔memory kinds both endpoints must already exist as ``Memory`` nodes; +the edge is not written otherwise (avoids dead off-type edges). A memory cannot +link to itself. Returns ``linked: False`` in those cases rather than raising. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `from_slug` | str | **required** | The memory the edge points **from**. Direction is load-bearing for the
asymmetric kinds — for ``supersedes`` this is the *newer* memory. | +| `to_slug` | str | **required** | The memory the edge points **to** — for ``supersedes``, the older memory
being replaced. For ``kind="tagged"`` this is a Topic instead: either an
existing ``tp-`` slug or a ``name:kind`` spec, which auto-creates it. | +| `kind` | `supersedes` \| `refines` \| `applies_to` \| `contradicts` \| `related_to` \| `tagged` | **required** | Which edge to write: ``supersedes`` \| ``refines`` \| ``applies_to`` \|
``contradicts`` \| ``related_to`` \| ``tagged``. See the descriptions
above — ``supersedes`` is the one that changes what default reads
return. | + +## `memory_neighbors` + +Return the memories directly linked to ``slug``, grouped by edge kind. + +For symmetric kinds (``contradicts``, ``related_to``) both directions are +unioned and de-duplicated. Use after ``memory_get`` to see what a memory +connects to. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `slug` | str | **required** | The memory whose neighbours to fetch. | +| `kinds` | list[`supersedes` \| `refines` \| `applies_to` \| `contradicts` \| `related_to` \| `tagged`]? | `null` | Optional subset of edge kinds to include. Omit (``None``) for all kinds;
an explicit empty list returns no kinds. | + +## `memory_symbols` + +Code symbols a memory concerns (direction: memory → symbols). + +The reverse of ``symbol_context``: returns each of the memory's ``symbol_refs``, +enriched with the live definition when witan-code is reachable, or the raw ref +strings otherwise. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `slug` | str | **required** | The memory whose ``symbol_refs`` to resolve. An unknown slug returns an
empty ``symbols`` list rather than raising. | + +## `memory_for_contract` + +What do we know about a contract (env_var / endpoint / package / service)? + +Resolves the ``Topic{kind:"contract", name:key_norm}`` anchor and walks +``Tagged`` to the memories about it (a single Layer-1 traversal, cross-repo by +nature), then — best-effort — asks witan-code for the bridge bindings that +share the same ``key_norm`` so callers can pivot to the code that produces or +consumes it. The two halves are joined in Python on the shared key, never an +edge across stores. + +Tag a memory to a contract first with +``memory_link(memory_slug, ":contract", "tagged")`` (``from`` is the +memory's slug). + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `key_norm` | str | **required** | The normalised contract key (e.g. ``DATABASE_URL`` or
``GET /api/v1/courses/``). | +| `kind` | `env_var` \| `endpoint` \| `package` \| `service`? | `null` | The bridge binding kind (``env_var`` / ``endpoint`` / ``package`` /
``service``) used to look up bindings. Omit to skip the bridge lookup. | + +## `symbol_context` + +Memories and tasks attached to a code symbol (direction: symbol → work). + +The reverse of ``memory_symbols``: given a symbol id, returns the memories and +tasks whose ``symbol_refs`` include it — "what lessons and open tasks concern +this function?". Call it after locating a symbol with the witan-code ``code_*`` +tools, before editing. + +``symbol_id`` has the form ``#::``; the +repo prefix scopes the lookup, or the current repo when the id has no ``#``. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `symbol_id` | str | **required** | The symbol to look up, as returned in the ``symbol_id`` field of the
witan-code ``code_*`` tools. | + +## `topic_get` + +Resolve a Topic and return it with the memories tagged to it. + +``topic`` is either a Topic slug (``tp-...``) or a ``name:kind`` spec +(e.g. ``uv:topic``). Because topics are a cross-repo join surface, the +returned memories may span repositories — this is the traversal-based +retrieval primitive: two memories in different repos sharing a topic are +one hop apart. + +Returns ``{"topic": {...}, "memories": [...]}`` or ``null`` if no such Topic. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `topic` | str | **required** | Either a Topic slug (``tp-...``) or a ``name:kind`` spec (``uv:topic``,
``DATABASE_URL:contract``). A spec resolves through the same
deterministic slugify used on write rather than an exact name match, so
a tag stored as ``UV`` is still found by ``uv:topic``. | + +## `store_merge` + +Merge a batch of exported rows into this deployment's graph, as you. + +The MCP-tier half of ``witan migrate merge`` (ADR-0007 D5). A client +exports its own store, splits the rows into batches, and calls this once +per batch; the server reconciles each batch against its own graph and +writes the winners. That keeps the whole cutover inside the per-actor +identity and Cedar model — the write is authorized as the calling user, +not as ``svc-witan-admin``, which is the difference between this and the +in-cluster path (ADR-0005 b). + +Every store operation here goes through the module-level ``client``, which +re-resolves to *this request's* actor on each access. There is no service +account behind it: an actor with no provisioned omnigraph token is refused +rather than served under one, and a row type the caller's Cedar grant does +not cover fails at the data tier. + +``rows`` are ``omnigraph export`` records — ``{"type": Node, "data": {…}}`` +for a node, ``{"edge": Edge, "from": …, "to": …}`` for an edge — the shape +``merge_store``'s own export parsing produces. Nodes are reconciled +newest-record-wins per ``(type, slug)`` against what this graph already +holds, by the *same* ``_reconcile_nodes`` the in-process path uses. Edges +carry no slug and pass through additively, exactly as they do there. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `rows` | list[object] | **required** | One batch of ``omnigraph export`` records — ``{"type": Node, "data":
{…}}`` for a node, ``{"edge": Edge, "from": …, "to": …}`` for an edge. | +| `dry_run` | bool | `False` | Reconcile and return the per-row ``decisions`` **without writing
anything**. Run the whole migration this way first: it is the only way
to see which side wins each ``(type, slug)`` before the graph changes. | +| `claim_from_author` | str? | `null` | The identity the *source* store wrote, when that store is your own.
Rows authored by exactly this name are restamped to the calling actor
before they are written; every other row keeps its author untouched.
Pass your local ``cfg.author`` here — the server cannot derive it,
having neither the caller's config nor their git checkout.
Without it, a migrated row keeps a name that no deployed identity can
ever match, and ``memory_delete`` refuses its own author forever
(#267). With it, the rows you migrate end up owned by the same identity
that owns everything you write afterwards. See ``_claim_authorship``
for why this matches rather than stamping unconditionally. | + +## `claim_authorship` + +Take ownership of rows a migration left under a local identity. + +The repair half of #267. ``store_merge``'s ``claim_from_author`` fixes rows +as they arrive, but does nothing for a store already merged — a re-sent row +loses reconciliation to its own applied copy, so re-running the migration +cannot rescue it. This rewrites in place instead. + +``was`` is the identity the rows currently carry: your local ``cfg.author`` +(``WITAN_AUTHOR`` / git ``user.name`` / ``$USER``) at the time you merged, +which is what ``witan whoami`` contrasts against your deployed identity. +Every matching row across all five authored node types is restamped to the +calling actor. + +Dry by default — pass ``apply=True`` to write. Idempotent: a second run +finds nothing, because the rows now carry the new identity. + +**This does not widen the trust boundary, and it is worth being explicit +about why, because it looks like it should.** Nothing here verifies that +``was`` was ever *you*, so this will hand you rows written by a colleague +if you name them. That capability already exists: ``store_merge`` accepts +whatever ``author`` a row carries, which is exactly what makes the +hand-edited-JSONL workaround in #267 work. This makes the existing +capability usable without hand-editing a JSONL against a live shared graph; +it does not create one. Constraining it means constraining ``store_merge`` +too, which is the ADR-0004 D5 revisit ("if attribution ever needs to be +authoritative rather than descriptive"), not a change to make here alone. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `was` | str | **required** | The author string the rows currently carry. | +| `apply` | bool | `False` | Write the change. Without it, only report what would change. | diff --git a/docs/reference/mcp-tools/tasks.md b/docs/reference/mcp-tools/tasks.md new file mode 100644 index 00000000..8c90f53d --- /dev/null +++ b/docs/reference/mcp-tools/tasks.md @@ -0,0 +1,237 @@ + + +# Tasks + +The work-coordination layer: what needs doing, what blocks what, and who holds which piece of work right now. `task_claim` is a **best-effort** compare-and-swap — it detects and rejects most lost races, but it is an advisory lease, not a hard lock. See [ADR 0003](../../explanation/decisions/0003-atomic-task-claims-cas.md) for what is and is not guaranteed. + +## `task_create` + +Create a task in the work-coordination graph. + +Tasks are dependency-aware and hierarchical. Use ``parent`` to attach a +sub-issue to an ``epic`` (or any parent task); use ``blocked_by`` to record +dependencies so ``task_ready`` can withhold the task until its blockers +close. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `title` | str | **required** | Short label and full text of the work. | +| `description` | str | **required** | Short label and full text of the work. | +| `type` | `bug` \| `feature` \| `task` \| `chore` \| `epic` | `'task'` | ``bug`` \| ``feature`` \| ``task`` \| ``chore`` \| ``epic``. | +| `priority` | `p0` \| `p1` \| `p2` \| `p3` | `'p2'` | ``p0`` (highest) … ``p3``. Drives ``task_ready`` ordering. | +| `repo` | str? | `null` | Repo scoping — see instructions. | +| `project_slug` | str? | `null` | ``wp-`` slug of the WorkflowProject this task rolls up to. | +| `parent` | str? | `null` | ``tk-`` slug of the parent task/epic. Sets the hierarchy edge. | +| `blocked_by` | list[str]? | `null` | ``tk-`` slugs that must close before this task is ready. | +| `discovered_from` | list[str]? | `null` | ``tk-`` slugs of tasks during which this work was discovered. | +| `external_uri` | str? | `null` | A reference URI — e.g. a GitHub issue or PR. | +| `symbol_refs` | list[str]? | `null` | Code-graph symbol ids (``repo#path::Name``) this task concerns. | +| `tags` | list[str]? | `null` | Optional free-form tags. | + +## `task_get` + +Retrieve a single task by slug. Returns the full node or ``null``. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `slug` | str | **required** | The ``tk-`` slug to retrieve. | + +## `task_list` + +List tasks, filtered by repo, status, project, parent, and/or assignee. + +``project_slug`` and ``parent`` take precedence as the primary scope; other +filters are applied on top in Python. With no filters, lists recent tasks +across all repos. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `repo` | str? | `null` | Repo scoping — see instructions. | +| `status` | `open` \| `in_progress` \| `blocked` \| `closed`? | `null` | ``open`` \| ``in_progress`` \| ``blocked`` \| ``closed``. | +| `project_slug` | str? | `null` | List the tasks of a WorkflowProject. | +| `parent` | str? | `null` | List the direct children of a parent task/epic. | +| `assignee` | str? | `null` | Filter to a single owner. | + +## `task_ready` + +Return ready-to-work tasks: pickable tasks whose blockers are all closed. + +A task is ready when its status is ``open``/``blocked`` (nobody is on it yet +and it is not closed), OR ``in_progress`` with an expired lease (the holder +likely abandoned it — see ``readiness.status_pickable``), AND every task in +its ``blocked_by`` list is closed. A returned ``in_progress`` task is +therefore a reclaim, not fresh work — check ``assignee``/``claimed_at`` +(falling back to ``updated_at`` when ``claimed_at`` is null, e.g. a legacy +row) before starting it. This is the core coordination primitive — call it +to pick the next actionable item without manual triage. Results are +ordered by priority (``p0`` first). + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `repo` | str? | `null` | Repo scoping — see instructions. | +| `project_slug` | str? | `null` | Restrict to a single WorkflowProject. | +| `assignee` | str? | `null` | Restrict to a single owner (or pass to find your own ready work). | +| `limit` | int | `20` | Maximum tasks to return. Defaults to 20. | + +## `task_update` + +Update a task's mutable fields. Only non-null arguments are applied. + +Use this to re-prioritise, re-parent (``parent``), reassign (``assignee``), +correct the repo association (``repo``), or attach an ``external_uri``. To +*claim* a task for work prefer ``task_claim`` (it sets ``in_progress`` + a +lease and refuses if someone else holds it); to close a task prefer +``task_close``; to add dependencies use ``task_link``. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `slug` | str | **required** | The ``tk-`` slug of the task to update. | +| `title` | str? | `null` | New short label for the work. | +| `description` | str? | `null` | New full description. Replaces the existing text; it is not appended to. | +| `type` | `bug` \| `feature` \| `task` \| `chore` \| `epic`? | `null` | ``bug`` \| ``feature`` \| ``task`` \| ``chore`` \| ``epic``. | +| `status` | `open` \| `in_progress` \| `blocked` \| `closed`? | `null` | ``open`` \| ``in_progress`` \| ``blocked`` \| ``closed``. Two values have
side effects: ``in_progress`` stamps a fresh ``claimed_at`` lease, and
``closed`` stamps ``closed_at`` **and unblocks this task's dependents**,
exactly as ``task_close`` does. Prefer ``task_claim`` / ``task_close``
for those two transitions — they carry the ownership checks this does
not. | +| `priority` | `p0` \| `p1` \| `p2` \| `p3`? | `null` | ``p0`` (highest) … ``p3``. Drives ``task_ready`` ordering. | +| `repo` | str? | `null` | Canonical repo URI to (re)assign this task to. Pass an explicit value to
correct tasks that were created without proper repo context. | +| `assignee` | str? | `null` | Holder identity to reassign the task to. Prefer ``task_claim`` to take a
task for yourself — it checks nobody else holds it, which this does not. | +| `project_slug` | str? | `null` | ``wp-`` slug of the WorkflowProject this task rolls up to. | +| `parent` | str? | `null` | ``tk-`` slug of the parent task/epic. Written as both the
``parent_slug`` field and the ``ParentOf`` edge in one commit, so a
concurrent reader never sees the task parented one way and not the
other. **Re-parenting does not retract the previous edge**: the field
moves to the new parent while the old ``ParentOf`` remains, so a task
re-parented this way is reachable from both. Call
``task_unlink(kind="parent")`` on the old pair first if the edge
matters to you. | +| `external_uri` | str? | `null` | Reference URI — e.g. the GitHub issue or PR this task tracks. | +| `symbol_refs` | list[str]? | `null` | Code-graph symbol ids (``repo#path::Name``) this task concerns.
Replaces the existing list. | +| `tags` | list[str]? | `null` | Free-form tags. Replaces the existing list rather than merging into it. | + +## `task_claim` + +Claim a task for work: set it ``in_progress`` under ``assignee`` with a lease. + +The coordination primitive for parallel/multi-user agents — call it before +starting a ready task so others see it is taken. Returns ``{"claimed": true, +…}`` on success, or ``{"claimed": false, "reason": …}`` when the task is +closed, still blocked, held by someone else (``"held"``/``"lost_race"``), +or the graph is too busy to complete the CAS after retrying +(``"contention"`` — safe and expected to retry). The lease (``claimed_at``) +expires if the holder never closes/releases, making the task reclaimable +(see ``task_ready``); re-calling renews it. Pass ``force`` to steal a live +claim. + +BEST-EFFORT CAS — omnigraph 0.8.x exposes no conditional-write primitive, so +the claim write cannot be made atomic at the store. Instead we surface the +Lance optimistic-concurrency conflict (rather than masking it with a retry) +and re-read + post-write-verify ownership, so a lost race reports +``{"claimed": false, "reason": "lost_race", "held_by": …}`` instead of +silently clobbering the winner. The lease is the backstop for the residual +simultaneous-post-read case. True atomic CAS awaits an upstream omnigraph +conditional-write feature — see docs/adr/0003. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `slug` | str | **required** | The ``tk-`` slug to claim. | +| `assignee` | str? | `null` | Holder identity. Defaults to the calling user (the JWT's
``preferred_username`` when deployed, the configured author locally)
qualified by ``session_id`` — see ``_claim_holder``. Two of one
person's parallel sessions must not share a holder string, or the
contention check passes and the second silently renews the first's
lease. Pass an explicit id to override (a worker name, a CI job). | +| `session_id` | str? | `null` | The calling agent session's id, which qualifies the default holder.
**Pass this when calling a deployed witan** — the server cannot infer
it (no shared environment, and MCP 2026-07-28 carries no session
state), and without it every one of your concurrent sessions claims
under the same name. The CLI's remote proxy fills it in automatically;
under local stdio the server falls back to its own
``$CLAUDE_SESSION_ID``, which it inherits from the agent. | +| `force` | bool | `False` | Steal the task even if another holder's lease is still valid. | +| `repo` | str? | `null` | Repo scoping — see instructions. Used only for the branch tracking
below, not to scope the claim itself. | +| `branch` | str? | `null` | The git branch the caller is on. Together with ``repo`` this is what
the ``WorksOn`` link is keyed on, and both have to come from the caller
because a deployed server has no checkout to read them from — see
``_code_branch_steps``. The CLI's remote proxy fills both in
automatically; under local stdio they default to the server's own
working directory, which is the agent's. | + +## `task_release` + +Release a claim: clear the assignee/lease and return the task to ``open``. + +Call when stepping away from an unfinished task so others can pick it up +(closing a finished task — use ``task_close`` — also ends the claim). Refuses +if the task is held by a different ``assignee`` unless ``force`` is set. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `slug` | str | **required** | The ``tk-`` slug to release. | +| `assignee` | str? | `null` | Holder identity releasing the task. Defaults to the calling user, same
resolution as ``task_claim``'s ``assignee``. The held-by check compares
*identities*, ignoring the ``#`` qualifier, so you can release a
claim one of your own other sessions took; another person's still needs
``force``. | +| `session_id` | str? | `null` | The calling agent session's id, same resolution and same reason as
``task_claim``'s. Only affects the holder string this call is compared
*as*; since the comparison is identity-level, omitting it against a
deployed server is harmless here in a way it is not for ``task_claim``. | +| `status` | `open` \| `in_progress` \| `blocked` \| `closed` | `'open'` | Status to return the task to (default ``open``). | +| `force` | bool | `False` | Release even if held by a different assignee. | + +## `task_close` + +Close a task: set status ``closed``, stamp ``closed_at``, record a resolution. + +Closing a blocker is what unblocks its dependents — they become visible to +``task_ready`` once every blocker is closed. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `slug` | str | **required** | The ``tk-`` slug to close. | +| `resolution` | str? | `null` | Short note on what was actually done. Worth writing: this is what a
later reader sees when asking why the task ended, and closing without
one leaves that unanswerable. | + +## `task_link` + +Link two tasks (or a task to a memory). + +The meaning of ``from``/``to`` depends on ``kind``: +- ``blocks`` — ``from`` is the blocker, ``to`` is the blocked task. + This is the only way to set a task's ``blocked_by`` after creation + (``task_update`` does not). +- ``parent`` — ``from`` is the parent/epic, ``to`` is the child. + Sets the same edge as ``task_update(parent=…)``; prefer ``task_update``. +- ``discovered_from`` — ``from`` is the new task, ``to`` is the source it came from. +- ``addresses`` — ``from`` is the task, ``to`` is a Memory slug it addresses. + +For ``blocks`` and ``parent`` the denormalized ``blocked_by`` / ``parent_slug`` +fields on the affected task are kept in sync so ``task_ready`` stays correct. + +Reversible: ``task_unlink`` removes any of these, including one recorded in +the wrong direction. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `from_slug` | str | **required** | The ``tk-`` slug the edge points **from**. Its meaning depends on
``kind`` — see above; for ``blocks`` this is the blocker, for ``parent``
the epic, for ``discovered_from`` the newly-found task. | +| `to_slug` | str | **required** | The slug the edge points **to** — the blocked task, the child, the task
the work was discovered during, or (for ``addresses``) a ``Memory``
slug. | +| `kind` | `blocks` \| `parent` \| `discovered_from` \| `addresses` | **required** | ``blocks`` \| ``parent`` \| ``discovered_from`` \| ``addresses``. Getting
the direction wrong is the common mistake here, and it matters: for
``blocks`` it decides which task ``task_ready`` withholds. | + +## `task_unlink` + +Remove a link between two tasks (or a task and a memory) — the inverse of +``task_link``, with the same ``from``/``to`` meanings. + +Use it when a link was recorded the wrong way round or against the wrong +slug. Removing a ``blocks`` link is how a task wrongly marked blocked +becomes ready again. + +For ``blocks`` and ``parent`` the denormalized ``blocked_by`` / +``parent_slug`` fields are updated to match, so ``task_ready`` stays +correct. Unblocking a task whose remaining blockers are all closed returns +it from ``blocked`` to ``open`` — the mirror of what ``task_link`` does. + +Returns ``{"from", "to", "kind", "removed"}``. ``removed`` is ``False`` +when the edge was not there, which is not an error: calling this twice, or +on a link that never existed, is a safe no-op. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `from_slug` | str | **required** | The ``tk-`` slug the edge points **from** — the same direction it was
written with. Pass the endpoints as ``task_link`` received them, not
reversed. | +| `to_slug` | str | **required** | The slug the edge points **to**, or a ``Memory`` slug for
``kind="addresses"``. | +| `kind` | `blocks` \| `parent` \| `discovered_from` \| `addresses` | **required** | Which edge to remove: ``blocks`` \| ``parent`` \| ``discovered_from`` \|
``addresses``. | + +## `task_for_branch` + +Return the tasks linked to a git branch — "what is this branch already for". + +A ``WorksOn`` link is written by a SUCCESSFUL ``task_claim`` and by nothing +else — losing a claim race does not link, and ``workflow_session_start`` +touches the same ``CodeBranch`` but links it ``ForProject``. So a non-empty +result means somebody actually holds a task on this branch. Call it before +starting fresh work on a branch you did not just create: on a shared graph +the holder may be a different actor, in which case the answer is to +continue or coordinate, not to open a second task. + +An empty result is therefore weaker than it looks: a branch someone is +working on without having claimed a task returns nothing. + +Closed tasks are included — filter on ``status`` if you only want live +work. Returns ``[]`` for an unknown branch, an unlinked one, or when no +repo can be resolved; none of those are errors. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `branch` | str | **required** | Git branch name. Required: the server has no checkout to infer it from,
unlike ``repo``. | +| `repo` | str? | `null` | Repo scoping — see instructions. | diff --git a/docs/reference/mcp-tools/workflow.md b/docs/reference/mcp-tools/workflow.md new file mode 100644 index 00000000..8e17a773 --- /dev/null +++ b/docs/reference/mcp-tools/workflow.md @@ -0,0 +1,389 @@ + + +# Workflow projects & sessions + +Tracking an engineering objective across many agent sessions without an explicit hand-off. A project spans phases and repos; each session links itself to one, and a completed project leaves a trace behind for later pattern-mining. + +## `workflow_project_create` + +Create a new workflow project to track an engineering objective. + +Call this at the start of a multi-session project before calling +``workflow_session_start``. The returned slug is used to link sessions. + +A project may span several repos (a service that touches a Django app, its +frontend, and the infra repo that deploys it) or none (a cross-cutting +objective not yet tied to any repo). The repo set also grows automatically +as sessions run in new repos — see ``workflow_session_start``. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `title` | str | **required** | Short name for the project. Used in listings and injected context. | +| `description` | str | **required** | Full description of the objective — what will be built or changed and why. | +| `phase` | `discovery` \| `spec` \| `implementation` \| `delivery` | `'discovery'` | Starting phase. One of ``discovery``, ``spec``, ``implementation``,
``delivery``. Defaults to ``discovery``. | +| `repos` | list[str]? | `null` | Canonical repo URIs this project spans. The current repo (detected from
``.git/config``) is added automatically. Omit entirely when creating a
repo-less "floating" project from outside any git repo. Guessing here is
fine — a project's real blast radius is rarely known at creation. The
set can be corrected at any time with ``workflow_project_update``
(``repos`` to replace it, ``add_repos``/``remove_repos`` to nudge it),
and it also grows on its own as sessions run in new repos. | +| `github_issue` | str? | `null` | URL of the GitHub issue tracking this work.
e.g. ``github.com/mitodl/ol-django/issues/847``. | +| `tags` | list[str]? | `null` | Optional list of tags for grouping and searching. | + +## `workflow_project_get` + +Retrieve a single workflow project by slug. + +Returns the full project node (including ``blocked_by`` and ``blocks`` +lists) or ``null`` if not found. + +``blocked_by`` lists the ``wp-`` slugs of projects that must complete +before this one is ready. ``blocks`` lists projects this project is +currently blocking (derived by scanning all active projects). + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `slug` | str | **required** | The ``wp-`` slug to retrieve. | + +## `workflow_project_status` + +One-call "what should I do next" resume view for a workflow project. + +Combines the four things an agent (or the context hook) needs to pick up a +project without re-deriving state: current **phase**, the **ready tasks** +under it (same readiness rule as ``task_ready``), the **last session's +handoff summary** (and whether it's still open), and any project-level +**blockers**. Returns ``None`` if the project doesn't exist. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `slug` | str | **required** | The ``wp-`` slug of the project to resume. | + +## `workflow_project_list` + +List workflow projects, optionally filtered by repo, status, and phase. + +Defaults to listing only ``active`` projects. Pass ``status=None`` to see +all statuses. The ``UserPromptSubmit`` hook calls this to inject project +context into new sessions. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `repo` | str? | `null` | Canonical repo URI to filter to (membership test against each
project's repo set). Auto-detected from ``.git/config`` if omitted.
Pass an empty string to list projects across all repos. | +| `status` | `active` \| `completed` \| `abandoned`? | `'active'` | ``active`` \| ``completed`` \| ``abandoned`` \| ``None`` for all.
Defaults to ``active``. | +| `phase` | `discovery` \| `spec` \| `implementation` \| `delivery`? | `null` | Optional phase filter applied after fetching. | +| `ready` | bool | `False` | When ``True``, only return active projects whose blockers are all
completed (i.e. projects that are unblocked and actionable). | + +## `workflow_project_update` + +Correct a project's metadata after creation. + +The escape hatch for everything ``workflow_project_advance`` (phase, +``github_pr``), ``workflow_project_complete`` (completion) and +``workflow_project_block``/``_unblock`` (dependencies) don't cover. Every +parameter is optional and only what you pass is touched; omitting a field +leaves it exactly as it was, so this can never blank something by accident. +Returns the updated project, or ``None`` if the slug doesn't exist. + +The common case is repos. A project's real blast radius is rarely known +during discovery, and until the set is right, repo-scoped recall from the +repos where the work actually lands won't surface the project at all. Pass +``repos`` to replace the set wholesale, or ``add_repos``/``remove_repos`` to +nudge it (both may be passed together; removals are applied after +additions). Repos are a plain list field on the project node, not edges, so +a removal really removes — unlike ``workflow_project_unblock``, which can +only update its denormalized field because omnigraph edges are append-only. + +Two things this deliberately can't do: + +- **Set the phase.** ``workflow_project_advance`` stays the only route, so + a transition is always seen by its ordering check. It does allow going + backwards (with a confirmation), which is how a phase set in error gets + corrected — this tool would just bypass the prompt. +- **Complete a project.** ``status`` accepts ``abandoned`` (for work that + stopped without an outcome) and ``active`` (to revive it), but not + ``completed``: that belongs to ``workflow_project_complete``, which seals + a corpus trace. Nothing should mint a trace without a narrative. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `slug` | str | **required** | The ``wp-`` slug of the project to update. | +| `title` | str? | `null` | Replacement short name. | +| `description` | str? | `null` | Replacement description. | +| `repos` | list[str]? | `null` | Replace the repo set wholesale with these canonical URIs. | +| `add_repos` | list[str]? | `null` | Canonical repo URIs to add to the set. | +| `remove_repos` | list[str]? | `null` | Canonical repo URIs to drop from the set. | +| `tags` | list[str]? | `null` | Replacement tag list. Pass ``[]`` to clear. | +| `github_issue` | str? | `null` | URL of the GitHub issue tracking this work. | +| `status` | `active` \| `abandoned`? | `null` | ``active`` \| ``abandoned``. | + +## `workflow_project_advance` + +Advance a workflow project to the next phase. + +Call when transitioning from e.g. spec to implementation. Optionally +record a PR URL when moving to or through the delivery phase. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `slug` | str | **required** | The ``wp-`` slug of the project to update. | +| `phase` | `discovery` \| `spec` \| `implementation` \| `delivery` | **required** | New phase: ``discovery`` \| ``spec`` \| ``implementation`` \| ``delivery``. | +| `github_pr` | str? | `null` | URL of the GitHub PR if one has been opened. | + +## `workflow_project_complete` + +Mark a workflow project as completed and assemble its corpus trace. + +This creates a ``WorkflowTrace`` node that aggregates all linked sessions +into an immutable record for later pattern mining. Idempotent: if a trace +already exists for this project, it is returned without re-inserting. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `slug` | str | **required** | The ``wp-`` slug of the project to complete. | +| `outcome` | str | **required** | Free-text narrative of what was delivered. Be specific — this is
the primary content of the corpus record. | +| `github_pr` | str? | `null` | URL of the merged PR, if applicable. | + +## `workflow_project_block` + +Declare that one project must complete before another can begin. + +Project-level sequencing — coarse ordering between whole projects. For +fine-grained ordering between individual tasks use ``task_link(kind="blocks")``; +the two are deliberately separate (Project vs Task nodes). + +Adds a ``ProjectBlocks`` graph edge (``slug`` → ``blocks_slug``) and +appends ``slug`` to ``blocks_slug.blocked_by`` so the ready-work check +in ``workflow_project_list`` can filter without traversing edges. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `slug` | str | **required** | The ``wp-`` slug of the *blocking* project (must finish first). | +| `blocks_slug` | str | **required** | The ``wp-`` slug of the project being *blocked*. | + +## `workflow_project_unblock` + +Remove a project dependency declared with ``workflow_project_block``. + +Removes ``slug`` from ``blocks_slug.blocked_by`` AND deletes the +``ProjectBlocks`` edge, so the graph and the denormalized field (which +drives the ready-work check) agree. + +Earlier versions left the edge in place, on the belief that omnigraph +edges were append-only. They are not — see ``_unlink_edge``. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `slug` | str | **required** | The ``wp-`` slug of the *blocking* project to remove. | +| `blocks_slug` | str | **required** | The ``wp-`` slug of the project to unblock. | + +## `workflow_project_get_blockers` + +Return all projects that are blocking the given project. + +Resolves each slug in the project's ``blocked_by`` list and returns the +full node for each blocker, including its current status. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `slug` | str | **required** | The ``wp-`` slug of the project to check. | + +## `workflow_project_link_memory` + +Link a memory to a workflow project (the ``Informed`` edge). + +Records that a project consulted or produced a memory — a ``pattern``, +``lesson``, ``project_fact``, or ``agent_context``. The linked memories +surface when the project's corpus trace is mined for reusable patterns. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `project_slug` | str | **required** | The ``wp-`` slug of the project. | +| `memory_slug` | str | **required** | The ``pat-`` / ``les-`` / ``pf-`` / ``ctx-`` slug returned by ``memory_store``. | + +## `workflow_project_memories` + +"What did we learn during project X" — the provenance walk. + +Assembles the memories connected to a project from two grains: +- **session-grain** (``SessionProduced``): memories the project's sessions + created, auto-recorded by ``memory_store`` when a session is active; +- **project-grain** (``Informed``): memories explicitly linked via + ``workflow_project_link_memory``. + +De-duplicated by slug. The flat ``memories`` list is assembled with two +queries regardless of session count. Pass ``group_by_session=True`` to also +get a ``by_session`` breakdown — that costs one extra query per session, so +it is opt-in. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `project_slug` | str | **required** | The ``wp-`` slug whose memories to assemble. | +| `group_by_session` | bool | `False` | Also return a ``by_session`` breakdown of which session produced what.
Costs one extra query per session, so leave it off unless you need the
attribution. | + +## `workflow_session_start` + +Link the current agent session to a workflow project. + +Call this at the start of any session that is contributing to a tracked +project. The context injected by the context-injection hook (Claude Code) or +extension (Pi) provides the ``project_slug``; ``session_id`` should be the +session id — ``$CLAUDE_SESSION_ID`` on Claude Code, or any stable unique +string for the session otherwise. + +Returns an explicit session handle (``session_slug``, ``project_slug``, +``phase``, ``session_id``, ``started_at``). Hold on to it and pass +``session_slug`` back to ``workflow_session_end`` — the handle is the only +thing that ties the two calls together, since the protocol carries no session +state of its own and consecutive calls may land on different replicas. + +**Re-entrant.** Calling again for a (``project_slug``, ``session_id``) whose +session is still open returns that same handle with ``existed: true`` +instead of minting a second node — so a hook retry, a transport reconnect, +or the replica failover the paragraph above warns about can't silently +duplicate a session. Any newly-supplied ``repo`` and ``tags`` are merged +into the existing session; ``phase`` is left at what the first call set (use +``workflow_project_advance`` to move a project's phase). Once a session has +been ended, the same ``session_id`` starts a fresh session — one +``$CLAUDE_SESSION_ID`` legitimately spans several working stints. + +Two *simultaneous* starts (a client retrying while the first request is +still in flight) can still both insert, since the check and the insert are +not one atomic operation. That is resolved immediately after the fact rather +than left for the migration to find — see ``_dedupe_open_sessions``. Both +racers return the same handle. + +Because the repo accretion below runs on the re-entrant path too, calling +this once per repo remains a valid way to widen a project's repo set — but +``workflow_project_update(add_repos=[...])`` does it directly, without +needing a session at all. + +When a repo is detected and the caller is on a git branch, also upserts a +``CodeBranch`` (repo, branch) and links it ``ForProject`` to this project — +schema.pg § Code Branches. Best-effort: silently skipped with no +repo/branch context, never fails the session start. Both values come from +the caller (see ``branch`` below); a deployed server has no checkout to +read them from. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `project_slug` | str | **required** | The ``wp-`` slug of the project this session belongs to. | +| `session_id` | str | **required** | Unique identifier for this agent session. | +| `phase` | `discovery` \| `spec` \| `implementation` \| `delivery` | **required** | The phase this session is working in. | +| `repo` | str? | `null` | Repo scoping — see instructions. | +| `branch` | str? | `null` | The git branch the caller is on, for the ``CodeBranch`` above. Filled
in client-side by the remote proxy; omitted, the server falls back to
its own working directory, which is right under local stdio and empty
under a deployment. | +| `tags` | list[str]? | `null` | Optional tags. | + +## `workflow_session_end` + +Close the current session with a summary of work accomplished. + +Call this before ending a session to produce a high-quality corpus record. +The ``Stop`` hook will auto-close sessions that did not call this, but +with a placeholder summary. + +For best corpus quality, write a summary that includes: +- What was done this session +- What remains for the next session +- Any blockers or decisions made + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `session_slug` | str | **required** | The ``ws-`` slug returned by ``workflow_session_start``. | +| `summary` | str | **required** | Description of what was accomplished and what remains. | +| `tools_used` | list[str]? | `null` | List of tool names used. e.g. ``["Edit", "Bash", "Read"]``. | +| `files_changed` | list[str]? | `null` | List of file paths modified in this session. | + +## `workflow_session_list` + +List workflow sessions, newest last. + +Mainly for finding sessions that leaked open — one whose agent died, or +whose Stop hook could not reach the graph. An open session is not cosmetic: +``workflow_project_complete`` folds every linked session into the corpus +trace, so one with no ``ended_at`` inflates ``session_count``, contributes +its phase while having recorded nothing, carries no handoff summary, and +cannot extend ``duration`` (computed from ``max(ended_at)``). It is also +what drives the context hook's "N sessions in " staleness nag. + +Use ``witan session sweep`` to close them in bulk. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `project_slug` | str? | `null` | Restrict to one project's sessions. Omit for every project. | +| `open_only` | bool | `False` | Only sessions with no ``ended_at``. Superseded sessions (deduped by
``witan migrate dedupe-sessions``) are always excluded — they are
already skipped by every aggregate read and are not leaks. | +| `include_superseded` | bool | `False` | Keep superseded rows instead of dropping them. For ``witan session
list``, the one caller that wants to see what
``migrate dedupe-sessions`` did rather than the leaked-session view. | + +## `workflow_trace_list` + +List corpus WorkflowTrace records, optionally filtered by repo, tags, or author. + +Traces are otherwise only reachable by slug (``wt-``) via +``get_trace`` — this is the discovery path for browsing or mining across +many completed projects (e.g. as onboarding case studies of how a project +went end to end). + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `repo` | str? | `null` | Canonical repo URI to filter to (membership test against each trace's
repo set). Auto-detected from ``.git/config`` if omitted. Pass an
empty string to list traces across all repos. | +| `tags` | list[str]? | `null` | Only return traces that carry ALL of these tags. | +| `author` | str? | `null` | Only return traces created by this author. | +| `limit` | int | `50` | Max rows to return (applied after filtering). | + +## `workflow_trace_get` + +Retrieve a single corpus WorkflowTrace by slug. + +Slug handling is simple: a slug already starting with ``wt-`` is used as-is; +**any other** slug is prefixed with ``wt-``. So the trace slug +(``wt-``) and the project slug (``wp-``, which +becomes ``wt-wp-``) both resolve to the same trace, and callers +never have to hand-construct the ``wt-`` slug (the fragile step the +``witan-project-tracker`` skill used to instruct). Returns the full trace +node (title/description/outcome, ``session_count``, ``phases``, ``duration``, +and any mined ``lessons_slug``/``patterns_slug``) or ``None`` if no trace +exists — a project only has a trace once ``workflow_project_complete`` has +sealed it. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `slug` | str | **required** | The ``wt-`` trace slug, or the ``wp-`` project slug it was minted from
(anything not already ``wt-``-prefixed gets the prefix added). | + +## `workflow_trace_annotate` + +Append lesson/pattern memory slugs to an existing WorkflowTrace. + +Lets an agent (or ``workflow_trace_mine``) record which Memory nodes a completed +project's trace produced without re-running ``workflow_project_complete`` +(traces are otherwise immutable after creation). Unions with whatever is +already recorded, so it's safe to call repeatedly as more lessons/patterns +are mined over time. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `trace_slug` | str | **required** | The ``wt-`` slug of the trace to annotate. | +| `lessons_slug` | list[str]? | `null` | ``les-`` memory slugs to add to the trace's ``lessons_slug`` field. | +| `patterns_slug` | list[str]? | `null` | ``pat-`` memory slugs to add to the trace's ``patterns_slug`` field. | + +## `workflow_trace_mine` + +Turn a completed WorkflowTrace into reusable Pattern/Lesson Memory nodes. + +Call with no ``patterns``/``lessons`` first — returns the trace itself +(title, description, outcome) plus every session summary from its project, +the raw material to mine for reusable knowledge. Review that, then call +again with the patterns/lessons you propose to persist them: each becomes +a ``Memory`` node, gets an ``Informed`` edge back to the trace's project, +and its slug is appended to the trace's ``lessons_slug``/``patterns_slug`` +fields. + +These mined memories are read by other agents for self-improvement, but +are equally a corpus of worked examples for people onboarding onto this +system — write ``title``/``content`` so a newcomer unfamiliar with the +project can follow the reasoning, not just a terse note for a future agent. + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `trace_slug` | str | **required** | The ``wt-`` slug of the trace to mine. | +| `patterns` | list[object]? | `null` | Proposed pattern memories to create on this call. Each dict needs
``title`` and ``content``; may also include ``repo``, ``language``,
and ``tags``. | +| `lessons` | list[object]? | `null` | Proposed lesson memories to create on this call. Each dict needs
``title`` and ``content``; may also include ``repo``, ``severity``,
and ``tags``. | +| `session_slug` | str? | `null` | The ``ws-`` handle from ``workflow_session_start``, recorded as the
provenance of every memory mined on this call — see ``memory_store``. | diff --git a/justfile b/justfile index c67ec7d4..d3785c35 100644 --- a/justfile +++ b/justfile @@ -242,3 +242,33 @@ bump package part: # bump — `just check-versions` above runs uv and refreshes it. Left behind, # the next `uv lock --check` (and CI) fails on a repo that looks untouched. echo " uv.lock" + +# ── Documentation site (witan-context, published on Read the Docs) ────────── +# +# Three kinds of page live under docs/: GENERATED (docs/reference/, built from +# the live tool objects, cyclopts command tree, and .pg schemas), MIRRORED +# (guides and ADRs copied from the package that owns them), and HANDWRITTEN +# (tutorials, overviews, explanation). `docs-gen` produces the first two; +# `docs-check` is the CI gate that stops them drifting from the code. + +# Regenerate and re-mirror every non-handwritten docs page. +docs-gen: + ./bin/gen_docs.py + +# Fail if any generated or mirrored page is out of date. Runs in CI. +docs-check: + ./bin/gen_docs.py --check + +# Build the static site into site/. +# +# Pinned to the same Zensical as .github/workflows/docs.yml and +# .readthedocs.yaml. Zensical is pre-1.0 and its output changes between patch +# releases, so an unpinned local preview would render a different site from the +# one CI checks and Read the Docs publishes — which is the exact drift the CI +# pin exists to prevent. Renovate bumps all three together. +docs-build: + uvx zensical@0.0.56 build + +# Serve the docs locally with incremental rebuilds. +docs-serve: + uvx zensical@0.0.56 serve diff --git a/mcp/servers/witan-code/README.md b/mcp/servers/witan-code/README.md index e6a0d51c..885a7dbd 100644 --- a/mcp/servers/witan-code/README.md +++ b/mcp/servers/witan-code/README.md @@ -14,7 +14,7 @@ graph. > **Wiring it into your agents locally** (MCP server, the `PostToolUse` reindex > hook, and the indexer CLI, run straight from your checkout): see -> [Local Development Setup](../../../docs/agent-memory.md#local-development-setup). +> [Local Development Setup](../../../docs/internals/agent-memory.md#local-development-setup). ## When to use this vs. grep / the `Explore` agent diff --git a/mcp/servers/witan/README.md b/mcp/servers/witan/README.md index 5ae035ca..ee31a976 100644 --- a/mcp/servers/witan/README.md +++ b/mcp/servers/witan/README.md @@ -81,7 +81,7 @@ export WITAN_AUTHOR="Your Name" > **Local development setup** (both servers, hooks, and skills wired from your > checkout): see -> [Local Development Setup](../../../docs/agent-memory.md#local-development-setup). +> [Local Development Setup](../../../docs/internals/agent-memory.md#local-development-setup). ## Environment Variables @@ -303,7 +303,7 @@ export WITAN_MEMORY_TOKEN= export WITAN_AUTHOR="Alice Smith" ``` -See [`docs/agent-memory.md`](../../../docs/agent-memory.md) for full +See [`docs/internals/agent-memory.md`](../../../docs/internals/agent-memory.md) for full deployment instructions, the graph schema, and the v2 roadmap. ## Project Structure diff --git a/mcp/servers/witan/docs/USER_GUIDE.md b/mcp/servers/witan/docs/USER_GUIDE.md index f6776701..a36a0013 100644 --- a/mcp/servers/witan/docs/USER_GUIDE.md +++ b/mcp/servers/witan/docs/USER_GUIDE.md @@ -198,7 +198,7 @@ Three ways to point witan at a store, in increasing order of shared-ness: To join a deployed service, follow [**Pointing your CLI and agent at the deployed witan**](deployed-witan-onboarding.md), and migrate the history you already have with the -[migration runbook](migration-runbook.md#local--shared-the-cutover). Sequence +[migration runbook](migration-runbook.md#local-shared-the-cutover). Sequence the two together — a store you keep writing to after its export was taken has a tail nobody will merge. @@ -206,7 +206,7 @@ Note that pointing `WITAN_MEMORY_URI` straight at an omnigraph-server is a *different*, lower-level mode: it addresses the data tier directly with a shared bearer token and no per-user identity. That is how a self-hosted or in-cluster maintenance process connects, not how a person does. See -[`docs/agent-memory.md` § Operating Modes](../../../../docs/agent-memory.md#6-operating-modes) +[`docs/internals/agent-memory.md` § Operating Modes](../../../../docs/internals/agent-memory.md#6-operating-modes) for the graph schema and server-deployment mechanics. `config.toml` can also define named `[targets.*]` sections that route diff --git a/mcp/servers/witan/docs/adr/0008-optional-task-phase-tag.md b/mcp/servers/witan/docs/adr/0008-optional-task-phase-tag.md index 77dc0718..b01fe574 100644 --- a/mcp/servers/witan/docs/adr/0008-optional-task-phase-tag.md +++ b/mcp/servers/witan/docs/adr/0008-optional-task-phase-tag.md @@ -9,7 +9,7 @@ - Deciders: witan platform owners - Tracking: task `tk-optional-task-phase-tag-per-phase-task-rollup-de-c95687`, project `wp-witan-hooks-workflow-ux-progression-improvements-852aaf` - Supersedes: — -- Related: eval `docs/design/witan-workflow-hooks-elicitation-evaluation.md` §B8; `docs/adr/0003-atomic-task-claims-cas.md` +- Related: eval `docs/internals/design/witan-workflow-hooks-elicitation-evaluation.md` §B8; `docs/adr/0003-atomic-task-claims-cas.md` ## Context diff --git a/mcp/servers/witan/docs/deployed-witan-onboarding.md b/mcp/servers/witan/docs/deployed-witan-onboarding.md index 1a4711fc..8d1385cb 100644 --- a/mcp/servers/witan/docs/deployed-witan-onboarding.md +++ b/mcp/servers/witan/docs/deployed-witan-onboarding.md @@ -5,7 +5,7 @@ the shared, deployed service — so your agent sessions read and write the same graph as everyone else's. This is one half of the cutover. The other half is -[migrating the history you already have](migration-runbook.md#local--shared-the-cutover); +[migrating the history you already have](migration-runbook.md#local-shared-the-cutover); they should be sequenced together, because a store you keep writing to after its export was taken is a store whose tail nobody will merge. @@ -271,7 +271,7 @@ Reads are unaffected by all of this and stay fast under the same load. `svc-witan-admin` (ADR-0005 path b). **`witan migrate merge` is the exception** — it has a per-actor form and is how you bring your own history across. See the - [migration runbook](migration-runbook.md#local--shared-the-cutover). + [migration runbook](migration-runbook.md#local-shared-the-cutover). - **`witan migrate merge --target …` is refused.** Against a deployment the target is the deployment's own graph, resolved server-side. Name the deployment with `--to ` instead, or unset `remote_url` to merge diff --git a/packages/agent-config-kit/README.md b/packages/agent-config-kit/README.md index ef6f3948..1d49f4db 100644 --- a/packages/agent-config-kit/README.md +++ b/packages/agent-config-kit/README.md @@ -19,7 +19,7 @@ bundle = RegistrationBundle( apply_all(bundle) ``` -See `docs/design/agent-config-kit-spec.md` in this repo for the full design. +See `docs/internals/design/agent-config-kit-spec.md` in this repo for the full design. ## CLI (`agent-kit`) @@ -85,8 +85,8 @@ skills = ["cyclopts-cli-scripts"] Table/field names mirror the Python model field names exactly (`kind`, `command`, `args`, `env`, `event`, `entry_path`, ...) — see -`docs/design/agent-config-kit-cli-spec.md` for the full schema and -rationale, and `docs/design/agent-config-kit-profiles-composition-spec.md` +`docs/internals/design/agent-config-kit-cli-spec.md` for the full schema and +rationale, and `docs/internals/design/agent-config-kit-profiles-composition-spec.md` for profiles, `include`, the global config file, and directory-prefix/org scoping (everything from here through "Precedence & resolution order" below). @@ -312,7 +312,7 @@ Exit codes: `0` success, `2` the manifest failed to load. Bootstraps the global config file (`${XDG_CONFIG_HOME:-~/.config}/agent-config-kit/config.toml`, overridable with `--config` or `AC_KIT_CONFIG`) — see -`docs/design/agent-config-kit-profiles-composition-spec.md` §7 for the full +`docs/internals/design/agent-config-kit-profiles-composition-spec.md` §7 for the full schema this file drives (per-org and per-directory-prefix default manifests for zero-arg `apply`). diff --git a/packages/agent-config-kit/agent_config_kit/cli.py b/packages/agent-config-kit/agent_config_kit/cli.py index 960a4b86..4676f56f 100644 --- a/packages/agent-config-kit/agent_config_kit/cli.py +++ b/packages/agent-config-kit/agent_config_kit/cli.py @@ -125,7 +125,7 @@ def _render_config_toml( file is always a usable *and* self-documenting starting point (spec §7.1).""" lines = [ "# agent-config-kit global config.", - "# See docs/design/agent-config-kit-profiles-composition-spec.md §7 for", + "# See docs/internals/design/agent-config-kit-profiles-composition-spec.md §7 for", "# the full schema. Every key below is optional.", "", ] @@ -367,7 +367,7 @@ def _render_manifest_toml(skills: list[tuple[str, str]]) -> str: "# the target repo, keyed by its frontmatter 'name' (falling back to", "# its parent directory name). Review names/paths, then add", "# [mcp_servers]/[[hooks]]/[profiles] by hand as needed — see", - "# docs/design/agent-config-kit-cli-spec.md for the full schema.", + "# docs/internals/design/agent-config-kit-cli-spec.md for the full schema.", "", "[skills]", ] diff --git a/packages/agent-config-kit/agent_config_kit/diff.py b/packages/agent-config-kit/agent_config_kit/diff.py index f9362906..19916c95 100644 --- a/packages/agent-config-kit/agent_config_kit/diff.py +++ b/packages/agent-config-kit/agent_config_kit/diff.py @@ -2,7 +2,7 @@ platform without touching disk. Reuses ``plan.py``'s target resolution and each platform's own ``mcp_serialize``/``hooks_merge`` projections so there is exactly one place that knows a platform's wire format (see -``docs/design/agent-config-kit-cli-spec.md`` §4.2). +``docs/internals/design/agent-config-kit-cli-spec.md`` §4.2). Deviates from that spec's sketch in one way: ``Drift`` has no single ``path`` field. A platform can write MCP servers, hooks, and skills to three different diff --git a/packages/agent-config-kit/agent_config_kit/manifest.py b/packages/agent-config-kit/agent_config_kit/manifest.py index a3668305..f07bdca8 100644 --- a/packages/agent-config-kit/agent_config_kit/manifest.py +++ b/packages/agent-config-kit/agent_config_kit/manifest.py @@ -4,7 +4,7 @@ Part of the base package (only ``tomllib`` + existing ``models``/``plan``/ ``registry`` — no ``cli``-extra dependency) so a programmatic caller can load a manifest without pulling in ``cyclopts``/``rich``. See -``docs/design/agent-config-kit-cli-spec.md`` §3 for the full format spec. +``docs/internals/design/agent-config-kit-cli-spec.md`` §3 for the full format spec. """ from __future__ import annotations diff --git a/packages/agent-config-kit/agent_config_kit/prune.py b/packages/agent-config-kit/agent_config_kit/prune.py index 4e20037f..e688c90e 100644 --- a/packages/agent-config-kit/agent_config_kit/prune.py +++ b/packages/agent-config-kit/agent_config_kit/prune.py @@ -5,7 +5,7 @@ a manifest. This module tracks what a manifest last wrote in a *state file* and, on a later ``apply --prune``, removes exactly the entries that were written before but are no longer in the manifest — never touching a key the -manifest never owned. See ``docs/design/agent-config-kit-cli-spec.md`` §5 for +manifest never owned. See ``docs/internals/design/agent-config-kit-cli-spec.md`` §5 for the full design. """ diff --git a/packages/witan-core/README.md b/packages/witan-core/README.md index bc107b2d..714332e9 100644 --- a/packages/witan-core/README.md +++ b/packages/witan-core/README.md @@ -17,7 +17,7 @@ in lockstep by a fragile Renovate custom manager). `witan-core` deliberately reverses that convention. The full rationale, scope, and per-extraction contracts live in -[`docs/design/witan-core-extraction-spec.md`](../../docs/design/witan-core-extraction-spec.md). +[`docs/internals/design/witan-core-extraction-spec.md`](../../docs/internals/design/witan-core-extraction-spec.md). ## Invariant diff --git a/packages/witan-core/witan_core/__init__.py b/packages/witan-core/witan_core/__init__.py index ccfc1a52..a84f54af 100644 --- a/packages/witan-core/witan_core/__init__.py +++ b/packages/witan-core/witan_core/__init__.py @@ -2,7 +2,7 @@ A leaf package that ``witan`` (witan-council) and ``witan_code`` both depend on, extracted from the surface they were previously carrying as copy-paste-and-diverge -duplicates. See ``docs/design/witan-core-extraction-spec.md``. +duplicates. See ``docs/internals/design/witan-core-extraction-spec.md``. ``witan_core`` imports neither ``witan`` nor ``witan_code``: it sits below both, preserving the one-directional ``witan`` → ``witan_code`` optional-mount DAG. diff --git a/packages/witan-core/witan_core/omnigraph_http.py b/packages/witan-core/witan_core/omnigraph_http.py index 3847ac58..308c489a 100644 --- a/packages/witan-core/witan_core/omnigraph_http.py +++ b/packages/witan-core/witan_core/omnigraph_http.py @@ -8,7 +8,7 @@ worker; 45.8 vs 10.7 at eight). The bare fork/exec floor is only 6.7-14.7ms, so the subprocess costs 2-3x the raw spawn — the rest is per-invocation CLI work (config resolution, ``.gq`` parse, client construction, a fresh TCP connection -every time). See docs/design/omnigraph-remote-call-overhead-spike.md. +every time). See docs/internals/design/omnigraph-remote-call-overhead-spike.md. This module is that connection. It is deliberately NOT a second ``OmnigraphClient``: omnigraph-server 0.8.1 exposes only ``query``, ``mutate`` diff --git a/zensical.toml b/zensical.toml new file mode 100644 index 00000000..2f745632 --- /dev/null +++ b/zensical.toml @@ -0,0 +1,175 @@ +# Zensical configuration for the witan-context documentation site. +# +# Zensical is the Material for MkDocs team's successor generator; Material +# itself entered maintenance mode in early 2026. The config is MkDocs-shaped +# (same nav structure, same markdown extensions) so a fall back to +# mkdocs-material stays cheap if a plugin gap bites — which is also why this +# site deliberately uses no plugins. +# +# Build: zensical build (output in site/) +# Serve: zensical serve (incremental; rebuilds only changed files) + +[project] +site_name = "witan-context" +site_description = "Shared memory, work coordination, and a code graph for coding agents." +site_author = "MIT Open Learning" + +# Read the Docs does not yet support variable interpolation in Zensical config, +# so this is the literal published URL rather than $READTHEDOCS_CANONICAL_URL. +# It sets the canonical link tag; getting it wrong costs search ranking, not a +# broken build, so it fails quietly — check it after the first deploy. +site_url = "https://witan-context.readthedocs.io/" + +repo_url = "https://github.com/mitodl/agent-kit" +repo_name = "mitodl/agent-kit" +edit_uri = "edit/main/docs/" + +copyright = "Copyright © 2026 MIT Open Learning" + +nav = [ + { "Home" = "index.md" }, + { "Get started" = [ + { "Overview" = "getting-started/index.md" }, + { "Installation" = "getting-started/installation.md" }, + { "Your first memory" = "getting-started/first-memory.md" }, + { "Tasks and projects" = "getting-started/tasks-and-projects.md" }, + { "Indexing a repository" = "getting-started/code-graph.md" }, + ] }, + { "Guides" = [ + { "Overview" = "guides/index.md" }, + { "witan user guide" = "guides/witan-user-guide.md" }, + { "witan-code user guide" = "guides/witan-code-user-guide.md" }, + { "Branch indexing" = "guides/branch-indexing.md" }, + { "Write-path scanning" = "guides/write-path-scanning.md" }, + { "Using a deployed witan" = "guides/deployed-witan.md" }, + { "Migration runbook" = "guides/migration-runbook.md" }, + ] }, + { "Reference" = [ + { "Overview" = "reference/index.md" }, + { "MCP tools" = [ + { "Overview" = "reference/mcp-tools/index.md" }, + { "Memory & recall" = "reference/mcp-tools/memory.md" }, + { "Tasks" = "reference/mcp-tools/tasks.md" }, + { "Workflow" = "reference/mcp-tools/workflow.md" }, + { "Code graph" = "reference/mcp-tools/code.md" }, + ] }, + { "CLI" = "reference/cli.md" }, + { "Environment variables" = "reference/environment.md" }, + { "Graph schema" = "reference/graph-schema.md" }, + { "Bridge schema" = "reference/bridge-schema.md" }, + ] }, + { "Explanation" = [ + { "Overview" = "explanation/index.md" }, + { "Architecture" = "explanation/architecture.md" }, + { "The memory model" = "explanation/memory-model.md" }, + { "Coordinating work" = "explanation/task-coordination.md" }, + { "Code graph" = [ + { "Symbol format" = "explanation/code-graph/symbol-format.md" }, + { "Symbol table" = "explanation/code-graph/symbol-table.md" }, + { "Package map" = "explanation/code-graph/package-map.md" }, + { "Edge precision tiers" = "explanation/code-graph/edge-precision-tiers.md" }, + { "Stage-2 stitching" = "explanation/code-graph/stage2-stitching.md" }, + ] }, + { "Decisions (ADRs)" = [ + { "0001 Write-path content scanning" = "explanation/decisions/0001-write-path-content-scanning.md" }, + { "0002 Cedar authorization bundle" = "explanation/decisions/0002-witan-cedar-authorization-bundle.md" }, + { "0003 Atomic task claims (CAS)" = "explanation/decisions/0003-atomic-task-claims-cas.md" }, + { "0004 Keycloak JWT actor mapping" = "explanation/decisions/0004-keycloak-jwt-per-user-actor-mapping.md" }, + { "0008 Optional task phase tag" = "explanation/decisions/0008-optional-task-phase-tag.md" }, + { "0005 Secure CLI path into a deployed witan" = "explanation/decisions/0005-secure-cli-path-into-deployed-witan.md" }, + { "0006 Code-graph branch ownership" = "explanation/decisions/0006-code-graph-branch-ownership-and-reaping.md" }, + { "0009 Stateless MCP protocol era" = "explanation/decisions/0009-stateless-mcp-protocol-era.md" }, + { "0007 Local-to-shared store migration" = "explanation/decisions/0007-local-to-shared-store-migration-transport.md" }, + ] }, + ] }, + { "Internals" = [ + { "Overview" = "internals/index.md" }, + { "Agent memory implementation guide" = "internals/agent-memory.md" }, + { "Design notes" = [ + { "Graph-structured memory" = "internals/design/graph-structured-memory.md" }, + { "Graph-structured memory (spec)" = "internals/design/graph-structured-memory-spec.md" }, + { "witan-core extraction" = "internals/design/witan-core-extraction-spec.md" }, + { "witan surface refinement" = "internals/design/witan-surface-refinement-spec.md" }, + { "Workflow UX (P1)" = "internals/design/witan-workflow-ux-p1-spec.md" }, + { "Workflow hooks & elicitation" = "internals/design/witan-workflow-hooks-elicitation-evaluation.md" }, + { "Remote call overhead spike" = "internals/design/omnigraph-remote-call-overhead-spike.md" }, + { "agent-config-kit" = "internals/design/agent-config-kit-spec.md" }, + { "agent-config-kit CLI" = "internals/design/agent-config-kit-cli-spec.md" }, + { "agent-config-kit profiles" = "internals/design/agent-config-kit-profiles-composition-spec.md" }, + ] }, + ] }, +] + +[project.theme] +language = "en" +features = [ + "announce.dismiss", + "content.action.edit", + "content.code.annotate", + "content.code.copy", + "content.code.select", + "content.footnote.tooltips", + "content.tabs.link", + "content.tooltips", + "navigation.indexes", + "navigation.instant", + "navigation.instant.prefetch", + "navigation.path", + # The reference pages are long (the CLI page is ~1,500 lines); a followed ToC + # and a top button are what make them navigable rather than a scroll. + "navigation.top", + "navigation.tracking", + "search.highlight", + "search.suggest", + "toc.follow", +] + +[[project.theme.palette]] +media = "(prefers-color-scheme)" +toggle.icon = "lucide/sun-moon" +toggle.name = "Switch to light mode" + +[[project.theme.palette]] +media = "(prefers-color-scheme: light)" +scheme = "default" +primary = "indigo" +accent = "indigo" +toggle.icon = "lucide/sun" +toggle.name = "Switch to dark mode" + +[[project.theme.palette]] +media = "(prefers-color-scheme: dark)" +scheme = "slate" +primary = "indigo" +accent = "indigo" +toggle.icon = "lucide/moon" +toggle.name = "Switch to system preference" + +[[project.extra.social]] +icon = "fontawesome/brands/github" +link = "https://github.com/mitodl/agent-kit" + +[project.markdown_extensions] +abbr = {} +admonition = {} +attr_list = {} +def_list = {} +footnotes = {} +md_in_html = {} +tables = {} +toc.permalink = true +pymdownx.betterem = {} +pymdownx.caret = {} +pymdownx.details = {} +pymdownx.highlight.anchor_linenums = true +pymdownx.highlight.line_spans = "__span" +pymdownx.highlight.pygments_lang_class = true +pymdownx.inlinehilite = {} +pymdownx.keys = {} +pymdownx.mark = {} +pymdownx.superfences.custom_fences = [ + { name = "mermaid", class = "mermaid", format = "pymdownx.superfences.fence_code_format" }, +] +pymdownx.tabbed.alternate_style = true +pymdownx.tasklist.custom_checkbox = true +pymdownx.tilde = {}