diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 13ea9756..347b4be4 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -23,6 +23,7 @@ on: - "packages/witan-core/**" - "docker/**" - "bin/gen_docs.py" + - "bin/gen_llms.py" - "zensical.toml" - ".readthedocs.yaml" - "uv.lock" @@ -37,6 +38,7 @@ on: - "packages/witan-core/**" - "docker/**" - "bin/gen_docs.py" + - "bin/gen_llms.py" - "zensical.toml" - ".readthedocs.yaml" - "uv.lock" @@ -120,3 +122,13 @@ jobs: echo "::error::Zensical did not report a completed build." exit 1 fi + + - name: Generate llms.txt and the markdown mirror + # Same script Read the Docs runs in .readthedocs.yaml's build.jobs, so + # this is what actually catches a mismatch between zensical.toml's nav + # and docs/ before it reaches the published site — a nav entry with no + # matching file, or vice versa, fails here as a plain traceback. + run: | + ./bin/gen_llms.py + test -s site/llms.txt + test -s site/llms-full.txt diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 7b4e1729..b49f2fce 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -9,6 +9,11 @@ # Markdown and never has to resolve tree-sitter, fastmcp, or an omnigraph # binary. `just docs-check` in CI is what guarantees the committed output # matches the code. +# +# bin/gen_llms.py (llms.txt, llms-full.txt, and a raw-markdown mirror for +# agents) runs right after the HTML build, as plain `python3` rather than +# `uv run` — same reasoning as above: nothing in this job may need the uv +# workspace synced. version: 2 @@ -26,6 +31,7 @@ build: build: html: - zensical build + - python3 bin/gen_llms.py post_build: - mkdir -p $READTHEDOCS_OUTPUT/html/ - cp --recursive site/* $READTHEDOCS_OUTPUT/html/ diff --git a/README.md b/README.md index 13b41705..537f281d 100644 --- a/README.md +++ b/README.md @@ -13,8 +13,14 @@ install helpers, and sample configurations. Documentation for the witan packages lives in [`docs/`](./docs/) — tutorials, guides, a generated reference for every MCP tool, CLI command and environment variable, and the architecture notes. Build it locally with `just docs-serve`. -It is set up to publish on Read the Docs as **witan-context** once that project -is registered. +It publishes on Read the Docs as +[**witan-context**](https://witan-context.readthedocs.io/). An agent consuming +the published site rather than a person browsing it can start at +[`/llms.txt`](https://witan-context.readthedocs.io/llms.txt) — a link index +into every page's raw markdown — or fetch +[`/llms-full.txt`](https://witan-context.readthedocs.io/llms-full.txt) for the +whole corpus in one file; both are generated by `bin/gen_llms.py` as part of +the site build, not committed. ## Repository Structure diff --git a/bin/gen_llms.py b/bin/gen_llms.py new file mode 100755 index 00000000..5f7d176f --- /dev/null +++ b/bin/gen_llms.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +"""Generate llms.txt, llms-full.txt, and a raw-markdown mirror for the built site. + +WHY. An agent consuming these docs shouldn't have to parse rendered HTML to get +plain text back out of it. The convention (https://llmstxt.org) is a compact +link index at ``/llms.txt`` plus, ideally, a raw-markdown copy of every page an +agent can fetch directly instead of the HTML. This script produces both, plus +``/llms-full.txt`` — the whole corpus concatenated into one file, for an agent +that would rather fetch once than crawl. + +Nothing here is hand-maintained. The link index, its section grouping, and the +per-page descriptions are all derived from ``docs/`` and the ``nav`` already +declared in ``zensical.toml`` — the same source of truth the rendered site +uses — so there is no second copy of the site structure to keep in sync by +hand. A description is the page's own first paragraph, lightly stripped of +markdown syntax; there is no separate blurb to write or forget to update. + +WHY THIS SCRIPT IS STDLIB-ONLY, UNLIKE ITS SIBLINGS IN ``bin/``. Every other +generator here runs via ``uv run --package witan-core ...``, because Read the +Docs deliberately does NOT sync the uv workspace for this build — see the note +in ``.readthedocs.yaml``: it installs zensical alone, specifically so the docs +build never has to resolve tree-sitter, fastmcp, or an omnigraph binary. This +script runs in that same build job, right after ``zensical build``, so it has +to work with nothing beyond what a bare ``python3`` on RTD's image already +has. Reaching for ``uv run --package witan-core`` here would drag the exact +dependency weight RTD's config was written to avoid back in through this side +door. + +Usage: + zensical build && ./bin/gen_llms.py # after the HTML build, before publish +""" + +from __future__ import annotations + +import re +import shutil +import sys +import tomllib +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +DOCS = REPO_ROOT / "docs" +SITE = REPO_ROOT / "site" +ZENSICAL_TOML = REPO_ROOT / "zensical.toml" + +_HEADING_OR_BLOCK = ("#", "<", "!!!", "```", "|", "-", "*", "{") + + +def _first_paragraph(md_path: Path) -> str: + """The first prose paragraph after the H1, as a single markdown-stripped line.""" + lines = md_path.read_text().splitlines() + seen_h1 = False + body: list[str] = [] + for line in lines: + stripped = re.sub(r"^>+\s*", "", line.strip()) + if not seen_h1: + if stripped.startswith("# "): + seen_h1 = True + continue + if not stripped: + if body: + break + continue + if stripped.startswith(_HEADING_OR_BLOCK): + if body: + break + continue + if not body and line.startswith((" ", "\t")): + # Wrapped continuation of a skipped bullet/blockquote, e.g. a + # `- Related: …` metadata line that wraps onto an unindented-looking + # but source-indented second line. Only applies before real prose + # has started — an indented line inside a found paragraph is left + # alone. + continue + body.append(stripped) + + para = " ".join(body) + para = re.sub(r"`([^`]*)`", r"\1", para) + para = re.sub(r"\[([^\]]+)\]\([^)]+\)", r"\1", para) + para = re.sub(r"\*\*([^*]+)\*\*", r"\1", para) + para = re.sub(r"\s+", " ", para).strip() + if len(para) > 160: + para = para[:160].rsplit(" ", 1)[0].rstrip(",.;:") + "…" + return para + + +def _flatten(nodes: list, trail: list[str]) -> list[tuple[str, str]]: + """(breadcrumb title, docs-relative .md path) for every leaf under `nodes`.""" + leaves: list[tuple[str, str]] = [] + for node in nodes: + ((title, value),) = node.items() + if isinstance(value, str): + crumb = " — ".join([*trail, title]) if trail else title + leaves.append((crumb, value)) + else: + leaves.extend(_flatten(value, [*trail, title])) + return leaves + + +def _sections(nav: list) -> list[tuple[str, list[tuple[str, str]]]]: + """One (section title, leaves) pair per top-level nav entry, in nav order.""" + sections = [] + for node in nav: + ((title, value),) = node.items() + leaves = [(title, value)] if isinstance(value, str) else _flatten(value, []) + sections.append((title, leaves)) + return sections + + +def _mirror_markdown(sections: list[tuple[str, list[tuple[str, str]]]]) -> None: + """Copy every page's source markdown to the same relative path under site/.""" + for _title, leaves in sections: + for _crumb, relpath in leaves: + dest = SITE / relpath + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(DOCS / relpath, dest) + + +def _write_llms_txt( + site_name: str, + site_description: str, + site_url: str, + sections: list[tuple[str, list[tuple[str, str]]]], +) -> None: + base = site_url.rstrip("/") + lines = [f"# {site_name}", "", f"> {site_description}", ""] + for title, leaves in sections: + lines.append(f"## {title}") + lines.append("") + for crumb, relpath in leaves: + desc = _first_paragraph(DOCS / relpath) + lines.append(f"- [{crumb}]({base}/{relpath}): {desc}") + lines.append("") + (SITE / "llms.txt").write_text("\n".join(lines).rstrip() + "\n") + + +def _write_llms_full_txt( + site_name: str, + site_url: str, + sections: list[tuple[str, list[tuple[str, str]]]], +) -> None: + base = site_url.rstrip("/") + parts = [f"# {site_name} — full corpus\n"] + for _title, leaves in sections: + for _crumb, relpath in leaves: + # No injected heading: every page already opens with its own H1, + # so one more here would just stack two titles on top of it. + parts.append(f"\n\n") + parts.append((DOCS / relpath).read_text().rstrip() + "\n\n---\n\n") + (SITE / "llms-full.txt").write_text("".join(parts).rstrip() + "\n") + + +def _check_nav_matches_docs(sections: list[tuple[str, list[tuple[str, str]]]]) -> None: + """Every page under docs/ must be in the nav, and vice versa. + + llms.txt is only as complete as the nav it's built from. A page added to + docs/ and forgotten in zensical.toml's nav is invisible to it (and to the + rendered site, for the same reason); a nav entry with no file behind it + would otherwise surface as a bare FileNotFoundError deep in + ``_mirror_markdown``. Catch both here, together, with a message that says + which file and which fix. + """ + on_disk = { + str(p.relative_to(DOCS)) for p in DOCS.rglob("*.md") if "_data" not in p.parts + } + in_nav = {relpath for _title, leaves in sections for _crumb, relpath in leaves} + + missing_from_nav = sorted(on_disk - in_nav) + missing_from_disk = sorted(in_nav - on_disk) + if not missing_from_nav and not missing_from_disk: + return + + lines = ["docs/ and zensical.toml's nav disagree:"] + if missing_from_nav: + lines.append(" in docs/ but not in nav (add a nav entry, or delete the page):") + lines += [f" {p}" for p in missing_from_nav] + if missing_from_disk: + lines.append(" in nav but no file on disk (fix the path, or drop the entry):") + lines += [f" {p}" for p in missing_from_disk] + sys.exit("\n".join(lines)) + + +def main() -> None: + if not SITE.is_dir(): + sys.exit(f"{SITE} does not exist — run `zensical build` first.") + + config = tomllib.loads(ZENSICAL_TOML.read_text()) + project = config["project"] + site_name, site_description = project["site_name"], project["site_description"] + site_url, nav = project["site_url"], project["nav"] + + sections = _sections(nav) + _check_nav_matches_docs(sections) + _mirror_markdown(sections) + _write_llms_txt(site_name, site_description, site_url, sections) + _write_llms_full_txt(site_name, site_url, sections) + + page_count = sum(len(leaves) for _title, leaves in sections) + print(f"wrote llms.txt, llms-full.txt, and {page_count} mirrored markdown pages") + + +if __name__ == "__main__": + main() diff --git a/docker/omnigraph-server.Dockerfile b/docker/omnigraph-server.Dockerfile index 2e5807ce..64bd8eb3 100644 --- a/docker/omnigraph-server.Dockerfile +++ b/docker/omnigraph-server.Dockerfile @@ -36,8 +36,8 @@ ARG OMNIGRAPH_VERSION=0.10.0 # Kept separate because on a moving tag the two differ — see # witan_core/omnigraph_install.py :: _OMNIGRAPH_RELEASE_TAG. ARG OMNIGRAPH_RELEASE_TAG=edge -ARG OMNIGRAPH_SHA256_X86_64=37b1333d83eeb18a30bff841e4801dd269a90f1b720d8ce9e69fc6c2c6c4add5 -ARG OMNIGRAPH_SHA256_ARM64=8c02e1c0426debd809a355129adf315dd284f0afb87a5e5ee89af6a0188a475a +ARG OMNIGRAPH_SHA256_X86_64=7633416d2192eb3b419f7e047759576587758bda80d0278eae6b11579a5e0943 +ARG OMNIGRAPH_SHA256_ARM64=cec6d1ce1ac3bb16f1114d17bbd219ad003127f3d277d9fadd5bfb58cf2dce7c # ── Fetch + checksum-verify the release, extract both binaries ──────────────── FROM debian:trixie-slim@sha256:3a39a0592364683e6bab97937b72cad5a8fa6dcbbee90edb3bb48c7f8e94f258 AS fetch diff --git a/docker/witan.Dockerfile b/docker/witan.Dockerfile index 53f83711..46b86036 100644 --- a/docker/witan.Dockerfile +++ b/docker/witan.Dockerfile @@ -28,8 +28,8 @@ ARG OMNIGRAPH_VERSION=0.10.0 # Kept separate because on a moving tag the two differ — see # witan_core/omnigraph_install.py :: _OMNIGRAPH_RELEASE_TAG. ARG OMNIGRAPH_RELEASE_TAG=edge -ARG OMNIGRAPH_SHA256_X86_64=37b1333d83eeb18a30bff841e4801dd269a90f1b720d8ce9e69fc6c2c6c4add5 -ARG OMNIGRAPH_SHA256_ARM64=8c02e1c0426debd809a355129adf315dd284f0afb87a5e5ee89af6a0188a475a +ARG OMNIGRAPH_SHA256_X86_64=7633416d2192eb3b419f7e047759576587758bda80d0278eae6b11579a5e0943 +ARG OMNIGRAPH_SHA256_ARM64=cec6d1ce1ac3bb16f1114d17bbd219ad003127f3d277d9fadd5bfb58cf2dce7c # Keep in lockstep with witan-council's version (mcp/servers/witan/pyproject.toml # [project].version / [tool.bumpversion]); it labels the built image. ARG WITAN_VERSION=0.8.0 diff --git a/docs/concepts/graph.md b/docs/concepts/graph.md new file mode 100644 index 00000000..7e7bf14e --- /dev/null +++ b/docs/concepts/graph.md @@ -0,0 +1,115 @@ +# The task and project graph + +Two questions live at this layer, and they're answered by different node +types: **what needs doing** (`Task`), and **what we're trying to achieve** +(`WorkflowProject`). You'll touch tasks constantly and projects occasionally — +most work is a task with no project at all. + +## Tasks: dependency-aware, hierarchical + +A `Task` has a `status` (`open` → `in_progress` → `closed`, with `blocked` +alongside `open` for anything waiting on a dependency) and can relate to other +tasks three ways: + +```mermaid +flowchart TB + EPIC["tk-… (epic)"] + SUB1["tk-… (sub-issue)"] + SUB2["tk-… (sub-issue)"] + OTHER["tk-… (unrelated task)"] + FOUND["tk-… (found mid-work)"] + + EPIC -->|ParentOf| SUB1 + EPIC -->|ParentOf| SUB2 + SUB2 -->|Blocks| OTHER + FOUND -->|DiscoveredFrom| OTHER +``` + +- **`ParentOf`** — hierarchy. An epic decomposes into sub-issues; closing the + epic doesn't require closing its children, but listing an epic's children is + a one-hop query. +- **`Blocks`** — dependency. `task_ready` computes "ready" from this: a task is + ready once everything that `Blocks` it has closed. Closing a blocker + automatically makes its dependents eligible — nobody maintains the ready + list by hand. +- **`DiscoveredFrom`** — provenance. Follow-up work found mid-task links back + to the task that surfaced it, which is the detail people skip and later wish + they'd kept. + +See [Coordinating work](../explanation/task-coordination.md) for what a claim +on a task actually guarantees (less than a lock, more than nothing), and the +[graph schema reference](../reference/graph-schema.md#task) for every field. + +## Projects: an objective across sessions + +A `WorkflowProject` tracks something bigger than one sitting — it moves +through four phases (`discovery` → `spec` → `implementation` → `delivery`) and +accumulates one `WorkflowSession` per agent session that contributes to it: + +```mermaid +flowchart LR + WS1["WorkflowSession
(Monday)"] -->|BelongsTo| WP["WorkflowProject
wp-…"] + WS2["WorkflowSession
(Wednesday,
different agent)"] -->|BelongsTo| WP + WP -->|Produced| WT["WorkflowTrace
(after completion)"] + WS1 -->|SessionProduced| MEM1["Memory"] + WP -->|Informed| MEM1 + WP -->|Informed| MEM2["Memory"] + TK["Task"] -->|TaskBelongsTo| WP +``` + +The mechanism that makes this useful is the **session boundary**. Each session +calls `workflow_session_start` on arrival and `workflow_session_end` with a +summary before it stops. That summary is what a *different* session — a +different agent, days later — reads to pick the thread back up. There is no +hand-off document to go stale, because the hand-off is a graph edge. + +When the project finishes, `workflow_project_complete` rolls every session up +into one immutable `WorkflowTrace` — session count, phases traversed, total +duration, and an outcome narrative — kept for later pattern mining, not for +day-to-day reading. + +## Where a git branch fits in + +Claiming a task or starting a session both quietly write a `CodeBranch` node, +linking your current repo+branch to whatever's in flight: + +```mermaid +flowchart LR + CB["CodeBranch
(this repo, this branch)"] -->|WorksOn| TK["Task"] + CB -->|ForProject| WP["WorkflowProject"] +``` + +No command, no configuration — it rides along on `task_claim` and +`workflow_session_start`, and it's why "which branch carries task X" is a +one-hop query instead of something someone has to remember to note down. + +## A worked thread + +One concrete path through all of it, the way it actually accretes during real +work: + +1. You claim `tk-retry-drops-last-error-4f9c21` → `task_claim` writes a + `CodeBranch` linking your branch to that task. +2. While fixing it you notice the retry loop also swallows a *different* + error class → `task_create(title="…", description="…", + discovered_from=["tk-retry-drops-last-error-4f9c21"])` (both `title` and + `description` are required — `discovered_from` alone isn't a valid call) + files `tk-swallowed-cancel-errors-b81a02`, linked `DiscoveredFrom` back to + the task you were on. +3. You store what you learned → + `memory_store(kind="lesson", title="…", …)` returns `les-…`, then + `task_link(from_slug="tk-retry-drops-last-error-4f9c21", to_slug="les-…", kind="addresses")` + ties the fix to the lesson it produced. +4. If this is one session in a longer piece of work, `workflow_session_start` + already wrote `BelongsTo` to the project, and `memory_store`'s + `session_slug` argument wrote `SessionProduced` from this session to that + lesson — so `workflow_project_memories` can later answer "what did this + project teach us?" without anyone curating a list. + +Every edge above is one MCP call, made once, at the moment the fact became +true. Nothing here is a separate bookkeeping step you have to remember to go +back and do. + +--- + +**Next:** [Three ways in: CLI, agent, skills →](interfaces.md) diff --git a/docs/concepts/index.md b/docs/concepts/index.md new file mode 100644 index 00000000..3f06ee72 --- /dev/null +++ b/docs/concepts/index.md @@ -0,0 +1,32 @@ +# Concepts + +The mental model, before the tutorial steps or the tool reference make full +sense. Read these when you want to know *what a thing is* — not yet *how to do +it* ([Get started](../getting-started/index.md), [Guides](../guides/index.md)) +or *why it was built that way* ([Explanation](../explanation/index.md)). + +
+ +- **[Memory and its four kinds](memory.md)** + + What a memory is, why it has a `kind`, and how the graph tells a pattern + from a lesson from a fact that is no longer true. + +- **[The task and project graph](graph.md)** + + Tasks, workflow projects, sessions, and how a memory or a git branch ends + up linked to any of them. + +- **[Three ways in: CLI, agent, skills](interfaces.md)** + + The same graph, three different callers. Which one you're using at any + moment, and why it matters. + +
+ +## If you want a worked example instead + +[Walkthroughs](../walkthroughs/index.md) shows the same small scenario played +out three ways — typed at a terminal, called directly by an agent, and run +through a packaged skill — so you can see where the three interfaces actually +diverge. diff --git a/docs/concepts/interfaces.md b/docs/concepts/interfaces.md new file mode 100644 index 00000000..e5b4ed08 --- /dev/null +++ b/docs/concepts/interfaces.md @@ -0,0 +1,99 @@ +# Three ways in: CLI, agent, skills + +Every witan operation is ultimately a function call against the graph. Three +different things can make that call, and knowing which one you're using — or +which one a doc page is describing — matters, because they don't all have +access to the same operations and they don't all involve a human at the same +point. + +
+ +- **CLI — you, at a terminal** + + Typing `witan tasks`, `witan memory "…"`, `witan run tk-…`. Read-heavy, + scriptable, no agent required. + +- **Agent — direct MCP tool calls** + + Your coding agent calling `task_create`, `memory_store`, `recall` as part + of its own reasoning, because it decided to, not because you ran a + command. + +- **Skills — a guided, interactive script** + + You invoke `/witan-task` or `/witan-workflow`; a packaged set of + instructions asks you questions and makes the MCP calls on your behalf. + +
+ +## CLI: for a human, not for an agent + +The `witan` binary is what you type. It's good for triage, browsing, and +anything you'd rather see in a terminal than have summarized back to you — +`witan tasks --ready`, `witan memory "flaky retry"`, `witan project status +wp-…`. + +**It is deliberately not a full interface to the graph.** There is no `witan +memory store` command — writing a memory is something an agent does when it +learns something, and the CLI has no comparable moment to hang that on. There +is no `witan code find-definition` either: code-graph queries return rows +meant to be reasoned over, not read on a screen. The CLI covers what a person +sitting at a keyboard actually wants to do directly. + +Where the CLI and the MCP tools both offer an operation, **they are the same +implementation** — `witan tasks` calls the identical function that `task_list` +exposes over MCP, then formats the result. They cannot disagree about what the +graph says, because there's only one code path to disagree with itself. See +[Architecture](../explanation/architecture.md#the-cli-is-not-a-second-implementation) +for why that's structural rather than a coincidence of the current code. + +## Agent: tool calls without a human step in between + +Once witan's MCP server is registered with your agent platform, the agent can +call any of its ~60 tools directly, in the middle of ordinary work, with no +slash command and no CLI involved. This is the mode that makes witan a +*shared* graph rather than a personal notebook: `AGENTS.md`/`CLAUDE.md` +instructions (or the agent's own judgment) tell it to check `recall` before +starting work, or to `memory_store` a lesson after fixing something +non-obvious, and it just does — the same way it decides to read a file or run +a test, without you typing a command for each one. + +This is also where most task and memory *writes* actually originate. A person +files a bug with the CLI or a skill; an agent mid-task discovers a second bug +and calls `task_create(discovered_from=[...])` on its own, because that's what +"don't lose follow-up work" means in practice. + +## Skills: interactive, but scripted + +A skill (`/witan-task`, `/witan-workflow`, `/witan-memory`, +`/witan-project-tracker`) is a packaged set of instructions — not a program, +a `SKILL.md` file the agent reads and follows. Invoking one is still "the +agent calling MCP tools", but with a human back in the loop for the decisions +that shouldn't be made silently: which of several ready tasks to claim, what +this session's phase is, what to write in a hand-off summary. + +Skills exist for the operations that are easy to get wrong by skipping a step +— claiming before working, remembering to end a session, picking the right +project when several are active — by turning "the agent should do the right +sequence of calls" into "the agent follows a script that has the sequence +built in." `/witan-task`, for instance, is also the thing to reach for +whenever you're about to start *any* `tk-` task, not only ones you found +through the skill itself, because claiming before the first edit is the one +step everything else depends on. + +## Picking one + +| You want to… | Reach for | +| --- | --- | +| Skim ready work, check a task's status, browse from a terminal | CLI | +| Let the agent record what it just learned, without asking you | Agent (direct MCP calls) | +| Pick a task to claim, or wire this session to a project, with prompts | A skill | +| Script something (CI, a cron job, a report) | CLI | +| Do the *same* thing a skill does but from an already-running session that skipped the slash command | MCP tools directly — `task_claim`, `task_close`, etc. | + +They're not mutually exclusive within one piece of work. A typical session +might start with `/witan-workflow` (skill) to link the session, then have the +agent call `recall` and `memory_store` on its own several times (agent), while +you check progress with `witan tasks` in a second terminal (CLI). See +[Walkthroughs](../walkthroughs/index.md) for the same scenario worked through +each way end to end. diff --git a/docs/concepts/memory.md b/docs/concepts/memory.md new file mode 100644 index 00000000..94a041bd --- /dev/null +++ b/docs/concepts/memory.md @@ -0,0 +1,93 @@ +# Memory and its four kinds + +A memory is one durable fact worth keeping past the end of the session that +learned it — not a transcript, not a to-do, a thing that will still be true +and still useful later. It is the unit witan is built around: everything else +(tasks, projects, code symbols) exists so memories can be attached to it. + +## Why "kind" instead of one big note pile + +Every memory carries a required `kind`, and it's the field a person or an +agent reaches for when *they* know what moment they're in — filtering +`recall`'s search seed, or calling `memory_list(kind=…)` to browse one kind +directly: + +| Kind | Answers | Reach for it when | +| --- | --- | --- | +| `pattern` | "How do we do X here?" | About to write similar code | +| `project_fact` | "What is true about this repo/service?" | Orienting in an unfamiliar codebase | +| `lesson` | "What went wrong last time?" | Something has broken, or is about to | +| `agent_context` | "What should the next session on *this task* know?" | Picking up someone else's in-flight work | + +**This is a filter you opt into, not something the store enforces on your +behalf.** `recall`'s default (`kind` omitted) searches every kind at once — +`kind` only narrows the query seed when you pass it explicitly. Picking the +right kind at write time is still what makes the *narrowed* search useful +later; it just doesn't gate the *default* one. + +`agent_context` is intended for handoff notes scoped to one piece of +work — link it to the task with `symbol_refs`/`tagged`/`addresses`, or it's +just as findable by everyone as any other memory. Nothing in the store ages +it out automatically either: unlike `supersedes`, there's no expiry +mechanism, so a stale `agent_context` memory stays fully live in `recall` +until someone updates, supersedes, or deletes it once the task it was about +is done. + +## Memories are a graph, not a table + +A pile of typed notes is still just a search index. What makes it a *graph* +is that memories point at each other, with the edge meaning something: + +| Edge | Meaning | What happens on read | +| --- | --- | --- | +| `supersedes` | This replaces that | The old one stops appearing by default | +| `refines` | This sharpens that, without replacing it | Both appear; the newer one ranks higher | +| `applies_to` | This pattern/lesson applies in that project's context | Following it pulls in the context | +| `related_to` | Soft association, no stronger claim than "these two are connected" | `recall` expands across it like `applies_to` | +| `contradicts` | These two disagree | **Both** appear, flagged — nothing is auto-resolved | +| `tagged` | This memory is about that topic | Everything else tagged the same way expands with it | + +The one to internalize first is **superseding is not deleting**. When +something you knew changes, you don't edit the old memory in place — you store +a new one and link it `supersedes` the old. The old memory is hidden from +normal reads but never destroyed, so "why did we think that, before?" always +has an answer. Editing in place would erase the fact that the knowledge ever +changed. + +Two memories that genuinely conflict are never silently resolved either — both +keep surfacing, flagged, because a heuristic guessing which one is "right" is +usually wrong about exactly the cases that matter. + +The full rationale for that design — including when to `memory_update` +instead of superseding — lives in [The memory +model](../explanation/memory-model.md). + +## Topics: how unrelated memories end up connected + +A free-text tag like `"vault"` on two different memories doesn't connect them +— they share a string, not a link. So a tag is promoted to a `Topic` node the +first time it's used, and `tagged` becomes a real, traversable edge. Two +memories tagged `vault` are then one hop apart in the graph, and asking "what +do we know about vault?" is a graph query, not a grep. + +One topic kind is worth calling out: a `contract` topic's name is a bridge +key — an environment variable, an HTTP endpoint, a package. That's the join +between the memory graph and the [code graph](../getting-started/code-graph.md): +`memory_for_contract("DATABASE_URL", kind="env_var")` returns both what's been +*written down* about that env var and what code *actually provides or +consumes it* — the `kind` argument is what turns on the second half; omit it +and you get only the memories, no code-graph lookup. + +## How you actually read this back + +You will almost never call a narrow "get me memories of kind X" query +yourself. The default read is `recall` — seed it with a search query, a task +slug, a code symbol, or a topic, and it does the graph expansion, drops +superseded memories, flags contradictions, and re-ranks the result for you. +See [`recall`](../reference/mcp-tools/memory.md#recall) for the full call +shape, or [Your first memory](../getting-started/first-memory.md) to store and +recall one in about five minutes. + +--- + +**Next:** [The task and project graph →](graph.md) diff --git a/docs/explanation/index.md b/docs/explanation/index.md index b120fd14..588d4079 100644 --- a/docs/explanation/index.md +++ b/docs/explanation/index.md @@ -4,6 +4,10 @@ Why witan is built the way it is. These pages are for understanding rather than doing — read them when a design decision seems arbitrary, or when you are about to work against the grain of one. +Looking for *what a thing is* rather than *why it's built that way*? Start +with [Concepts](../concepts/index.md) instead — these pages assume that +groundwork. +
- **[Architecture](architecture.md)** diff --git a/docs/getting-started/tasks-and-projects.md b/docs/getting-started/tasks-and-projects.md index 814e255b..734e9f44 100644 --- a/docs/getting-started/tasks-and-projects.md +++ b/docs/getting-started/tasks-and-projects.md @@ -130,6 +130,10 @@ pattern mining. `/witan-task` and `/witan-workflow` automate the picking and linking. The CLI shown here is what they call underneath, and what you want for triage. + See [Three ways in](../concepts/interfaces.md) for how the CLI, an agent's + own tool calls, and a skill relate — and + [Walkthroughs](../walkthroughs/index.md) for this exact task worked + through all three. ## Branch tracking, for free diff --git a/docs/index.md b/docs/index.md index fe9bab3f..a40768d3 100644 --- a/docs/index.md +++ b/docs/index.md @@ -52,6 +52,8 @@ tools; one entry in your agent's config gets you the whole surface. | If you want to… | Go to | | --- | --- | | Install it and store your first memory | [Get started](getting-started/index.md) | +| Understand the mental model — memory kinds, the task graph, CLI vs. agent vs. skills | [Concepts](concepts/index.md) | +| See the same scenario worked through the CLI, an agent, and a skill | [Walkthroughs](walkthroughs/index.md) | | Do a specific thing — index a repo, run against a deployed service, migrate a store | [Guides](guides/index.md) | | Look up a tool, flag, env var, or node type | [Reference](reference/index.md) | | Understand *why* it works the way it does | [Explanation](explanation/index.md) | @@ -99,3 +101,10 @@ witan-context covers three published packages, all developed in the Storage is [omnigraph](https://github.com/ModernRelay/omnigraph) — a local file, an `s3://` bucket, or a shared `omnigraph-server`. The same tools work against all three; only [`WITAN_MEMORY_URI`](reference/environment.md) changes. + +!!! note "For agents reading this site" + + Every page here has a raw-markdown twin at the same URL with `.md` + appended — this page is also [`/index.md`](index.md). [`/llms.txt`](llms.txt) + is a link index into all of them; [`/llms-full.txt`](llms-full.txt) is the + whole site concatenated into one file. diff --git a/docs/walkthroughs/agent-driven.md b/docs/walkthroughs/agent-driven.md new file mode 100644 index 00000000..11982751 --- /dev/null +++ b/docs/walkthroughs/agent-driven.md @@ -0,0 +1,90 @@ +# Walkthrough: Agent-driven + +Same task, but now you're inside a running agent session (Claude Code, Pi, +whichever). Nobody types a `/` command for any of this — the agent calls MCP +tools because the work in front of it calls for them, the same way it decides +to open a file or run a test. + +You say: + +> Pick up `tk-retry-logic-drops-the-last-attempt-s-e-4f9c21` and fix it. + +## The agent claims before touching anything + +Because `AGENTS.md`/`CLAUDE.md` instructions (or the `witan-task` skill's own +guidance, which the agent has read even without you invoking it) say a task +gets claimed before the first edit, it calls: + +```python +task_claim(slug="tk-retry-logic-drops-the-last-attempt-s-e-4f9c21", assignee="claude-session-8f21") +# → {"claimed": true, "status": "in_progress"} +``` + +If that had come back `{"claimed": false, "held_by": "someone-else"}`, the +agent's next move is to tell you and stop — not to barrel ahead and clobber +someone else's claim. + +## It checks what's already known + +Before writing a fix, a well-behaved agent checks whether this has come up +before: + +```python +recall(query="retry loop exception handling", task="tk-retry-logic-drops-the-last-attempt-s-e-4f9c21") +``` + +`recall` seeds from both the query and the task, expands a hop across +`applies_to`/`related_to` edges plus topic and provenance siblings (not, say, +a merely `refines`-linked memory — see [the memory +model](../concepts/memory.md#memories-are-a-graph-not-a-table) for which +edges actually widen a read), and comes back empty here — nobody has hit this +before. If it hadn't been empty, the agent would read the existing lesson +before writing code that repeats it. + +## It finds the second bug on its own + +Reading the retry loop to fix the reported issue, the agent notices the +`CancelledError` problem too — and files it without being asked, because +losing a follow-up mid-task is the exact failure `discovered_from` exists to +prevent: + +```python +task_create( + title="Retry loop swallows CancelledError", + description="The same retry loop discards CancelledError on the final attempt, same root cause as the reported bug.", + type="bug", + priority="p2", + discovered_from=["tk-retry-logic-drops-the-last-attempt-s-e-4f9c21"], +) +# → {"slug": "tk-retry-loop-swallows-cancellederror-b81a02"} +``` + +## It writes the fix, then records the lesson + +After the fix and a passing test: + +```python +task_close( + slug="tk-retry-logic-drops-the-last-attempt-s-e-4f9c21", + resolution="Re-raise the final exception instead of swallowing it; test added", +) + +memory_store( + kind="lesson", + title="Retry loop must re-raise the last attempt's exception", + content="The final exception in a retry loop was being discarded, making a permanent failure look like a plain timeout to the caller. Re-raise on the last attempt.", + tags=["retry", "error-handling"], +) +# → {"slug": "les-retry-loop-must-re-raise-…"} +``` + +This is the step [CLI-driven](cli-driven.md#the-thing-the-cli-cant-do) +couldn't do — an agent is *always* the one storing memory, whether it got +there by your instruction, a skill's script, or its own judgment mid-task. +That's the whole distinction this page is drawing: the same `memory_store` +call, made because the agent decided to, not because a script told it to ask +you first. + +--- + +**Next:** [Skills-driven →](skills-driven.md) diff --git a/docs/walkthroughs/cli-driven.md b/docs/walkthroughs/cli-driven.md new file mode 100644 index 00000000..ed9f5878 --- /dev/null +++ b/docs/walkthroughs/cli-driven.md @@ -0,0 +1,83 @@ +# Walkthrough: CLI-driven + +You're at a terminal, in a checkout of `mitodl/agent-kit`. No agent session is +running — you're about to fix this one by hand. + +## Find and claim the task + +```bash +witan tasks --ready +``` + +Prints a table titled "Ready tasks — mitodl/agent-kit" — priority, status, +type, slug, title, and a few more columns — with +`tk-retry-logic-drops-the-last-attempt-s-e-4f9c21` in it. + +```bash +witan task claim tk-retry-logic-drops-the-last-attempt-s-e-4f9c21 +``` + +``` +Claimed tk-retry-logic-drops-the-last-attempt-s-e-4f9c21 (assignee=tmacey) +``` + +This sets `in_progress` with a lease under your author name and refuses if +someone else already holds it (`--force` overrides, but see [what a claim +actually guarantees](../getting-started/tasks-and-projects.md#what-a-claim-actually-guarantees) +before reaching for it). + +## Fix it, and file what you found + +While reading the retry loop you notice it also swallows `CancelledError`, +which is a separate bug from the one you were sent to fix. File it before you +forget, linked to the task that surfaced it: + +```bash +witan task create "Retry loop swallows CancelledError" \ + --type bug --priority p2 \ + --discovered-from tk-retry-logic-drops-the-last-attempt-s-e-4f9c21 +``` + +``` +Created task: tk-retry-loop-swallows-cancellederror-b81a02 +``` + +`--discovered-from` writes the `DiscoveredFrom` edge — see [the task +graph](../concepts/graph.md#tasks-dependency-aware-hierarchical) for what that +buys you later. + +## Close it out + +```bash +witan task close tk-retry-logic-drops-the-last-attempt-s-e-4f9c21 \ + --resolution "Re-raise the final exception instead of swallowing it; test added" +``` + +``` +Closed tk-retry-logic-drops-the-last-attempt-s-e-4f9c21 +``` + +## The thing the CLI can't do + +You just fixed a bug caused by a subtle assumption — the retry loop assumed +every exception on the last attempt was safe to discard. That's exactly the +kind of thing worth recording as a `lesson`, so nobody rediscovers it the hard +way. + +**The CLI can't do this part.** There is no `witan memory store` command — +storing memory is deliberately agent-only, because the intended author of a +memory is the agent that just learned the thing, and there's no comparable +moment for a human typing at a prompt to hang it on (see [Three ways +in](../concepts/interfaces.md#cli-for-a-human-not-for-an-agent)). If you want +this fix on record, you need an agent in the loop for at least that one step — +which is exactly what the other two walkthroughs cover. + +```bash +witan memory "retry" --kind lesson +``` + +still works, from the CLI, once someone (or something) has written it. + +--- + +**Next:** [Agent-driven →](agent-driven.md) diff --git a/docs/walkthroughs/index.md b/docs/walkthroughs/index.md new file mode 100644 index 00000000..01f44b43 --- /dev/null +++ b/docs/walkthroughs/index.md @@ -0,0 +1,52 @@ +# Walkthroughs + +One scenario, worked through three times — once for each of the [three ways +in](../concepts/interfaces.md). Same task, same follow-up discovery, same +lesson worth keeping; only the caller changes. + +## The scenario + +Someone already filed the bug from [Tasks and +projects](../getting-started/tasks-and-projects.md#file-a-task): + +``` +tk-retry-logic-drops-the-last-attempt-s-e-4f9c21 + "Retry logic drops the last attempt's error" + bug · p1 · open +``` + +It's your turn to pick it up. Along the way you'll notice a second, related +bug, and want to leave something behind for the next person who touches this +code. + +
+ +- **[CLI-driven](cli-driven.md)** + + A person, at a terminal, no agent session running. Claim, fix, close — + and the one thing the CLI flatly cannot do for you. + +- **[Agent-driven](agent-driven.md)** + + Inside an agent session, tools called directly as part of the work — no + slash command in sight. + +- **[Skills-driven](skills-driven.md)** + + The same session, but reaching for `/witan-task` and letting it drive the + claim-and-triage decisions. + +
+ +## What actually differs + +| | CLI-driven | Agent-driven | Skills-driven | +| --- | --- | --- | --- | +| Who initiates each step | You, every time | The agent, on its own judgment | You, via a slash command; the agent follows a script | +| Can store a memory? | No — `witan memory` only reads | Yes — `memory_store`, on the agent's own judgment | Yes, same as agent-driven — `/witan-task`/`/witan-workflow` don't call `memory_store` themselves, but the agent still can mid-session | +| Picking which task to claim | You already know the slug | The agent reads it from context, or asks | `/witan-task` shows a picker | +| Best for | Triage, scripting, a terminal you already have open | Work happening inside a normal agent session | The moments easy to get wrong by hand — claiming, session hand-off | + +None of these are exclusive. A real session usually mixes all three — see the +close of [Three ways in](../concepts/interfaces.md#picking-one) for how they +typically layer. diff --git a/docs/walkthroughs/skills-driven.md b/docs/walkthroughs/skills-driven.md new file mode 100644 index 00000000..a7e31c49 --- /dev/null +++ b/docs/walkthroughs/skills-driven.md @@ -0,0 +1,109 @@ +# Walkthrough: Skills-driven + +Same session, same task — but instead of naming the slug yourself, you reach +for the packaged triage flow. + +## Triage with `/witan-task` + +``` +/witan-task +``` + +The skill calls `task_ready()` for the current repo and asks you to pick, via +an interactive question rather than a wall of text: + +``` +Claim task +Which task do you want to work on? + ○ Retry logic drops the last attempt's error + [p1] slug: tk-retry-logic-drops-the-last-attempt-s-e-4f9c21 + ○ (other ready tasks…) + ○ Create a task + ○ None +``` + +You pick the first one. The skill claims it on your behalf: + +```python +task_claim(slug="tk-retry-logic-drops-the-last-attempt-s-e-4f9c21", assignee="") +``` + +and confirms: *"Claimed **Retry logic drops the last attempt's error** +(`tk-retry-logic-drops-the-last-attempt-s-e-4f9c21`). Close it with +`/witan-task close`, or `task_release` it if you step away."* + +Nothing here is different under the hood from [the agent calling `task_claim` +directly](agent-driven.md#the-agent-claims-before-touching-anything) — the +skill is still an agent making an MCP call. What's different is that *which* +task to claim was a question put to you, not a decision the agent made alone. + +## Fix it, then close through the skill + +You (or the agent, mid-fix) notice the same `CancelledError` issue as before. +That part isn't skill-gated — filing a discovered task is ordinary agent +behavior, skill or not: + +```python +task_create( + title="Retry loop swallows CancelledError", + description="The same retry loop discards CancelledError on the final attempt, same root cause as the reported bug.", + type="bug", priority="p2", + discovered_from=["tk-retry-logic-drops-the-last-attempt-s-e-4f9c21"], +) +``` + +Once the fix is in and tested: + +``` +/witan-task close +``` + +The skill asks which task and for a resolution note, then calls: + +```python +task_close( + slug="tk-retry-logic-drops-the-last-attempt-s-e-4f9c21", + resolution="Re-raise the final exception instead of swallowing it; test added", +) +``` + +and offers to run `task_ready()` again so you can see what just unblocked. +`/witan-task` doesn't store memory itself — that's not part of what it does — +but the agent still records the lesson unprompted, same as in +[Agent-driven](agent-driven.md#it-writes-the-fix-then-records-the-lesson): + +```python +memory_store( + kind="lesson", + title="Retry loop must re-raise the last attempt's exception", + content="The final exception in a retry loop was being discarded, making a permanent failure look like a plain timeout to the caller. Re-raise on the last attempt.", + tags=["retry", "error-handling"], +) +``` + +## Where a second skill would come in + +If this task were part of a multi-session effort rather than a one-off fix, +`/witan-workflow` at the start of the session would have asked which +`WorkflowProject` this work belongs to (or offered to create one), then called +`workflow_session_start` — the same re-entrant call [the task +graph](../concepts/graph.md#projects-an-objective-across-sessions) describes — +before any of the above. That's a second, independent skill: `/witan-task` +picks *what* to work on; `/witan-workflow` tracks *the session's place* in a +longer effort. A session can use either, both, or neither. + +## Why bother, if it's the same calls underneath + +Because the calls it's easy to skip by hand are exactly the ones a lease-based +system depends on. Claiming *before* the first edit only works if it actually +happens every time — a skill that asks "which task?" and then claims it as +part of answering removes the chance to start editing first and claim later +"once you remember." The [task manager skill's own +instructions](https://github.com/mitodl/agent-kit/blob/main/mcp/servers/witan/witan/skills/witan-task/SKILL.md) +are blunt about this: two sessions have already written the same fix for the +same task on the same day, each unaware of the other, because neither claimed +it first. + +--- + +**Back to:** [Concepts: three ways in](../concepts/interfaces.md) · [Walkthroughs overview](index.md) diff --git a/justfile b/justfile index d3785c35..768ccc12 100644 --- a/justfile +++ b/justfile @@ -259,7 +259,9 @@ docs-gen: docs-check: ./bin/gen_docs.py --check -# Build the static site into site/. +# Build the static site into site/, plus llms.txt and a markdown mirror for +# agents (bin/gen_llms.py) — see .readthedocs.yaml and docs.yml for why that +# step runs as plain `python3`, not `uv run`. # # Pinned to the same Zensical as .github/workflows/docs.yml and # .readthedocs.yaml. Zensical is pre-1.0 and its output changes between patch @@ -268,6 +270,7 @@ docs-check: # pin exists to prevent. Renovate bumps all three together. docs-build: uvx zensical@0.0.56 build + ./bin/gen_llms.py # Serve the docs locally with incremental rebuilds. docs-serve: diff --git a/packages/witan-core/witan_core/omnigraph_install.py b/packages/witan-core/witan_core/omnigraph_install.py index d927cd91..684b95ee 100644 --- a/packages/witan-core/witan_core/omnigraph_install.py +++ b/packages/witan-core/witan_core/omnigraph_install.py @@ -217,19 +217,51 @@ #: `bin/check_omnigraph_format.py` ("omnigraph 0.10.0 reads storage format 6, #: as declared."), so this is not a rebuild-every-graph event — the v0.10.0 #: notes independently confirm the manifest schema stays at v6. +#: +#: ★ REFRESHED AGAIN 2026-08-26, after `witan-code (code graph)` went red on +#: agent-kit#289 with the by-now-familiar checksum mismatch. `edge` had moved +#: from bb0e3dc8bf to f714e5961147, two commits later: +#: #551 `feat(engine): expose branch-merge table-walk timing` — confined to +#: `crates/omnigraph/src/exec/merge.rs` and `instrumentation.rs`, plus +#: tests/docs. Opt-in developer instrumentation +#: (`MergeWriteProbes::merge_timing_snapshot`); the release notes say +#: outright "Production leaves the task-local probe unset and performs +#: no timing clock reads," and separately reconfirm "Internal manifest +#: schema remains v6." +#: #553 `ci: move Azure and vocabulary audits off PRs` — workflow files +#: only (`.github/workflows/*`, a new gating script). Ships nothing in +#: the binary. +#: Re-ran the same vocabulary check as every prior refresh — a tree-wide `git +#: grep` for every `_RETRYABLE`/`_NEEDS_REPAIR`/`_PRECONDITION_FAILED`/ +#: `_RECOVERY_REQUIRED` substring and the `"storage: "` prefix, at both refs — +#: and every match count came back identical; no rename, no removal. +#: `bin/check_omnigraph_format.py` against the freshly-installed binary again +#: read "omnigraph 0.10.0 reads storage format 6, as declared." +#: +#: Each digest below was independently confirmed twice: against the release's +#: published `.sha256`, and against a tarball downloaded fresh and hashed +#: locally, in the same sitting. Worth naming why that second check matters — +#: the first `omnigraph-linux-arm64.tar.gz` download here truncated +#: mid-transfer (a plain connection reset, `curl` exit 56) and hashed to a +#: THIRD value, distinct from both the published digest and the one below; a +#: retry matched. A truncated download can produce a stable, wrong hash +#: rather than an obvious error — check the transfer actually completed +#: before trusting a locally-computed digest, not just that `curl` printed +#: something. +#: #: Reverting the experiment means restoring the v0.9.0 triple, which was: #: linux-x86_64 507a36f385bea073e7f284fe476befbb4cd788b32bfa85d6f4cd5e943b663197 #: linux-arm64 6742a7fcf2761cb5841a38990c38383d7a884da2c65e3e7cc884afbbf2b2d881 #: macos-arm64 69f78c93e661e8ea2b92deafe6330650a0921a003c2099b75b226482a90dc03e _OMNIGRAPH_ASSET_SHA256: dict[str, str] = { "omnigraph-linux-x86_64.tar.gz": ( - "37b1333d83eeb18a30bff841e4801dd269a90f1b720d8ce9e69fc6c2c6c4add5" + "7633416d2192eb3b419f7e047759576587758bda80d0278eae6b11579a5e0943" ), "omnigraph-linux-arm64.tar.gz": ( - "8c02e1c0426debd809a355129adf315dd284f0afb87a5e5ee89af6a0188a475a" + "cec6d1ce1ac3bb16f1114d17bbd219ad003127f3d277d9fadd5bfb58cf2dce7c" ), "omnigraph-macos-arm64.tar.gz": ( - "a272a7830f4d2ddfd6c4295b9b4626aad9cd2caaa6fe30c6162b82d84b732adb" + "245bea28172dfafc231b6ee39c8b9fd0e6feeb928a96458f501e58b234b65f08" ), } _VERSION_RE = re.compile(r"\d+\.\d+\.\d+") diff --git a/zensical.toml b/zensical.toml index f1ded06f..0778328a 100644 --- a/zensical.toml +++ b/zensical.toml @@ -35,6 +35,18 @@ nav = [ { "Tasks and projects" = "getting-started/tasks-and-projects.md" }, { "Indexing a repository" = "getting-started/code-graph.md" }, ] }, + { "Concepts" = [ + { "Overview" = "concepts/index.md" }, + { "Memory and its four kinds" = "concepts/memory.md" }, + { "The task and project graph" = "concepts/graph.md" }, + { "Three ways in: CLI, agent, skills" = "concepts/interfaces.md" }, + ] }, + { "Walkthroughs" = [ + { "Overview" = "walkthroughs/index.md" }, + { "CLI-driven" = "walkthroughs/cli-driven.md" }, + { "Agent-driven" = "walkthroughs/agent-driven.md" }, + { "Skills-driven" = "walkthroughs/skills-driven.md" }, + ] }, { "Guides" = [ { "Overview" = "guides/index.md" }, { "witan user guide" = "guides/witan-user-guide.md" },