From 84b5ca754b92a2e07bddf648bd366e321bbf713b Mon Sep 17 00:00:00 2001 From: Shashank Shekhar Singh Date: Thu, 30 Jul 2026 18:44:28 +0530 Subject: [PATCH] Add a propose kind: the model authors from what the registry read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit plan_docs could read and summarise but a goal ending "and then create a plan for merging them" was satisfied by the mechanical goal check without the proposal ever existing. The fourth kind closes that: its body hands the deterministic notes to the run's model and records the recommendation as `proposal (model-authored):` — the one body in this registry that speaks to a model, built per-run by build_registry(model) via the CLI's existing factory contract. Honesty at the edges: with no real model behind the run (None, or the planner's own ScriptedChatModel, whose replies a body must not consume) the note says authoring needs --model instead of pretending; a model call that raises becomes a note too, because the deterministic reads already in state are worth keeping when the proposal step cannot run. Co-Authored-By: Claude Fable 5 --- docs/cookbook/07-slack.md | 17 +++-- grapharc/examples/plan_docs.py | 116 ++++++++++++++++++++++++++------- tests/test_plan_docs.py | 52 ++++++++++++++- 3 files changed, 153 insertions(+), 32 deletions(-) diff --git a/docs/cookbook/07-slack.md b/docs/cookbook/07-slack.md index c2380dc..3dde88f 100644 --- a/docs/cookbook/07-slack.md +++ b/docs/cookbook/07-slack.md @@ -121,17 +121,20 @@ The default planning registry is the incident-response demo: its node bodies are stubs, so a goal like "summarise the docs here" gets an honest negative (or, if phrased vaguely enough, a hollow success). The shipped alternative has bodies that really read — `survey` / `read` / `summarise`, read-only, -confined to the bot's working directory: +confined to the bot's working directory — plus one kind, `propose`, whose +body hands what was read to the model and records the recommendation that +comes back, labelled `proposal (model-authored):`: ``` -@grapharc plan "summarise the docs in this workspace" --registry grapharc.examples.plan_docs:build_registry --trace docs.jsonl --run-id docs-1 +@grapharc plan "summarise the docs and propose how to merge them" --registry grapharc.examples.plan_docs:build_registry --model claude-cli/claude-sonnet-5 --trace docs.jsonl --run-id docs-1 ``` -Works with the scripted planner (free) and with `--model`; either way the -notes in the final state carry actual file names, titles and excerpts, -because the reading is operator code, not model output. `--registry` from -Slack accepts exactly these two shipped modules and nothing else — the flag's -general form imports arbitrary code, which stays refused. +The scripted planner (no `--model`) runs the same chain free: the reading +notes are identical — file names, titles, excerpts are operator code, not +model output — and the `propose` note says plainly that authoring needs a +real model rather than pretending. `--registry` from Slack accepts exactly +these two shipped modules and nothing else — the flag's general form imports +arbitrary code, which stays refused. ## The `agent` opt-in diff --git a/grapharc/examples/plan_docs.py b/grapharc/examples/plan_docs.py index 76f533a..3f3c96e 100644 --- a/grapharc/examples/plan_docs.py +++ b/grapharc/examples/plan_docs.py @@ -4,16 +4,21 @@ like "summarise the docs in this workspace" gets either an honest negative or a hollow success, depending on how vague the goal is. This registry closes that gap for one bounded job: **reading documentation under the current -working directory and reporting what is there.** Three kinds: +working directory and reporting what is there.** Four kinds: survey list the documentation files under cwd read read them and record an excerpt of each summarise one note distilling title + first paragraph per file + propose the model authors a recommendation from what was read -Everything is deterministic operator code. The model still does the -*planning* — which kinds, in what order, replanning on refusal — but no node -body contains a model call, so a run with a scripted planner produces real -file content and a run with `--model` spends tokens only on rounds. +The first three are deterministic operator code. The model still does the +*planning* — which kinds, in what order, replanning on refusal — and those +bodies never call it, so a scripted run produces real file content and spends +nothing. `propose` is the one deliberate exception: its body hands the notes +the deterministic kinds collected to the model and records what comes back, +labelled as the model's. With no real model behind the run it says so in its +note instead of pretending — a scripted planner cannot author a proposal, and +the note is honest about which kind of run this was. The confinement matters more than the capability: bodies read, never write, and only under `Path.cwd()` — which, launched from the Slack bot, is the @@ -97,43 +102,107 @@ def _summarise_body(state: DocsState) -> dict: return {"notes": [*state.notes, summary]} -_BODIES = {"survey": _survey_body, "read": _read_body, "summarise": _summarise_body} - +def _propose_body_for(model: Any) -> Any: + """The one body with a model in it, built per-run so it closes over the + run's own backend rather than importing one. + + `model` is whatever drove the planner. A scripted planner's model cannot + author anything (invoking it would consume the planner's own scripted + replies), so anything without a usable `invoke` — or a call that fails — + becomes an honest note instead of an exception: the deterministic notes + already in state are worth keeping even when the proposal step cannot run. + """ + + def body(state: DocsState) -> dict: + evidence = "\n".join(state.notes) + real = model is not None and not isinstance(model, _scripted_type()) + if not real: + note = ( + "propose: needs a real model (--model); no proposal was " + "authored — the notes above are deterministic reads only" + ) + return {"notes": [*state.notes, note]} + try: + reply = model.invoke( + "You are the propose step of a documentation-review plan. " + f"The goal: {state.goal!r}. The evidence collected so far:\n" + f"{evidence}\n\n" + "Write one concise, concrete recommendation that satisfies " + "the goal, grounded only in the evidence above." + ) + content = getattr(reply, "content", reply) + note = f"proposal (model-authored): {str(content).strip()[:2000]}" + except Exception as exc: # noqa: BLE001 — a failed proposal is a note, not a crash + note = f"propose: model call failed ({exc}); deterministic notes stand" + return {"notes": [*state.notes, note]} -def _factory(build: Any) -> Any: - # `build` is the materialiser's NodeBuild: `name` is the instance the - # planner chose ("readme_survey"), `kind` is what the registry licensed. - # Behaviour keys on the kind; the name is the planner's business. - body = _BODIES[build.kind] body.writes = {"notes"} return body -WRITES: dict[str, set[str]] = {kind: {"notes"} for kind in _BODIES} +def _scripted_type() -> type: + from grapharc.testing import ScriptedChatModel + + return ScriptedChatModel + + +_BODIES = {"survey": _survey_body, "read": _read_body, "summarise": _summarise_body} + + +def _factory_for(model: Any): + def factory(build: Any) -> Any: + # `build` is the materialiser's NodeBuild: `name` is the instance the + # planner chose ("readme_survey"), `kind` is what the registry + # licensed. Behaviour keys on the kind; the name is the planner's. + if build.kind == "propose": + return _propose_body_for(model) + body = _BODIES[build.kind] + body.writes = {"notes"} + return body + + return factory + + +WRITES: dict[str, set[str]] = {kind: {"notes"} for kind in (*_BODIES, "propose")} -def build_registry() -> NodeRegistry: - """Three read-only kinds. Absence is refusal; there is no write kind to deny.""" +def build_registry(model: Any = None) -> NodeRegistry: + """Three read-only kinds and one that writes what the model recommends. + + Accepting `model` is the CLI's contract: a registry factory with a + positional parameter is handed the run's backend (`resolve_registry` in + `grapharc.cli.plan`), which is how `propose` speaks with the same model + that planned — and stays honest when that model is scripted. + """ + factory = _factory_for(model) return NodeRegistry( [ NodeSpec( name="survey", description="list the documentation files under the working directory", - factory=_factory, + factory=factory, worst_case=CostEstimate(iterations=1, tokens=300), ), NodeSpec( name="read", description="read each documentation file and record an excerpt", - factory=_factory, + factory=factory, worst_case=CostEstimate(iterations=1, tokens=1500), ), NodeSpec( name="summarise", description="distil what was read into one summary note", - factory=_factory, + factory=factory, worst_case=CostEstimate(iterations=1, tokens=800), ), + NodeSpec( + name="propose", + description=( + "author a recommendation from the collected notes; needs a real model" + ), + factory=factory, + worst_case=CostEstimate(iterations=1, tokens=2500), + ), ] ) @@ -144,15 +213,16 @@ def default_edge_policy() -> EdgePolicy: def scripted_planner_replies() -> list[str]: - """One reply: survey → read → summarise. Read by `grapharc plan` when no - `--model` is given, so the free path exercises the same registry the paid - one does. In an empty directory the chain still yields three notes, so the - loop's goal check is satisfied either way.""" + """One reply: survey → read → summarise → propose. Read by `grapharc plan` + when no `--model` is given, so the free path exercises the same registry + the paid one does — including the `propose` note honestly declining to + author without a real model. In an empty directory the chain still yields + enough notes for the loop's goal check either way.""" import json from grapharc.runtime.graph import END, START - chain = ["survey", "read", "summarise"] + chain = ["survey", "read", "summarise", "propose"] endpoints = [START, *chain, END] return [ json.dumps( diff --git a/tests/test_plan_docs.py b/tests/test_plan_docs.py index 7ca9409..d95046c 100644 --- a/tests/test_plan_docs.py +++ b/tests/test_plan_docs.py @@ -77,11 +77,56 @@ def test_the_cli_scripted_path_uses_this_registrys_replies(docs_dir, capsys): assert "A widget that frobs." in printed +class _FakeReply: + def __init__(self, content: str) -> None: + self.content = content + + +class _FakeModel: + """A real-model stand-in: has `invoke`, is not the scripted type.""" + + def __init__(self) -> None: + self.prompts: list[str] = [] + + def invoke(self, prompt: str) -> _FakeReply: + self.prompts.append(prompt) + return _FakeReply("merge 07 and 08 into one Slack page; keep the walkthrough") + + +def test_propose_hands_the_collected_notes_to_the_model(docs_dir): + model = _FakeModel() + body = plan_docs._propose_body_for(model) + state = plan_docs.DocsState( + goal="plan a merge", notes=["read README.md: Widget — A widget that frobs."] + ) + notes = body(state)["notes"] + assert any("proposal (model-authored): merge 07 and 08" in n for n in notes) + assert "A widget that frobs." in model.prompts[0] + assert "plan a merge" in model.prompts[0] + + +def test_propose_without_a_real_model_declines_honestly(docs_dir): + from grapharc.testing import ScriptedChatModel + + for model in (None, ScriptedChatModel(responses=["{}"])): + notes = plan_docs._propose_body_for(model)(plan_docs.DocsState())["notes"] + assert any("needs a real model" in n for n in notes) + + +def test_a_failing_model_becomes_a_note_not_a_crash(docs_dir): + class Exploding: + def invoke(self, prompt): + raise RuntimeError("backend fell over") + + notes = plan_docs._propose_body_for(Exploding())(plan_docs.DocsState())["notes"] + assert any("model call failed" in n and "backend fell over" in n for n in notes) + + def test_the_governed_loop_reaches_goal_met_with_substantive_notes(docs_dir, tmp_path): # Instance names differ from kinds on purpose: a real planner invents # names ("docs_survey" of kind "survey"), and behaviour must key on the # kind — this is the shape that catches a factory keyed on the name. - chain = ["survey", "read", "summarise"] + chain = ["survey", "read", "summarise", "propose"] names = [f"docs_{kind}" for kind in chain] endpoints = [START, *names, END] reply = json.dumps( @@ -98,7 +143,9 @@ def test_the_governed_loop_reaches_goal_met_with_substantive_notes(docs_dir, tmp ) loop = build_loop( ScriptedChatModel(responses=[reply]), - registry=plan_docs.build_registry(), + # The registry carries a "real" model for `propose` while the planner + # stays scripted — the two roles are deliberately separable. + registry=plan_docs.build_registry(_FakeModel()), state_schema=plan_docs.DocsState, writes=plan_docs.WRITES, edge_policy=plan_docs.default_edge_policy(), @@ -109,3 +156,4 @@ def test_the_governed_loop_reaches_goal_met_with_substantive_notes(docs_dir, tmp assert result.succeeded assert any("summary of 2 file(s)" in note for note in result.state.notes) assert any("Widget" in note for note in result.state.notes) + assert any("proposal (model-authored):" in note for note in result.state.notes)