From 70900f257173cb10756ebf86982198540395f748 Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Sun, 13 Sep 2026 09:35:20 +0000 Subject: [PATCH 01/11] Agents: test that `override(tools=...)` reaches a run and is memoized `Agent.override` rejects `toolsets=`, because the per-run override each entry point installs would silently shadow it. The same entry points also install `tools=[]`, which looks like it should shadow an override's `tools=` the same way, and nothing tested it either way. It does not. Each entry point builds its wrapped toolsets while the caller's override is still in effect, so the overridden tools are picked up and wrapped before its own `tools=[]` takes over: a tool given with `override(tools=...)` reaches a run inside the override, and a retried workflow gets its memoized result rather than calling it again. The test pins that ordering down, so a change that stops honoring it fails here rather than dropping the tools silently. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LRFggUcgVpLqhgb7h1cK6H --- .../reboot/agents/pydantic_ai/agent_tests.py | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tests/reboot/agents/pydantic_ai/agent_tests.py b/tests/reboot/agents/pydantic_ai/agent_tests.py index 1219f4c3c..ba3e2edcc 100644 --- a/tests/reboot/agents/pydantic_ai/agent_tests.py +++ b/tests/reboot/agents/pydantic_ai/agent_tests.py @@ -1230,6 +1230,44 @@ async def test_override_toolsets_raises(self) -> None: self.assertIn("Overriding `toolsets`", str(raised.exception)) self.assertIn("not currently supported", str(raised.exception)) + async def test_override_tools_reach_the_run_and_are_memoized( + self, + ) -> None: + """A tool given with `agent.override(tools=[...])` reaches a run + made inside the override, and is memoized like any other tool: + a retried workflow does not call it again. + + Unlike `toolsets=`, which `override` rejects, this works because + each entry point builds its wrapped toolsets while the caller's + override is still in effect, before installing its own + `tools=[]`; this pins that ordering down. + """ + invocations = 0 + + async def tool(run: RunContext[None], query: str) -> str: + nonlocal invocations + invocations += 1 + return "Unimportant" + + model = ToolCallingModel([[("tool", {"query": "hello"})]]) + agent = Agent(model, name="Agent") + + attempts = 0 + + async def workflow(context: WorkflowContext) -> None: + nonlocal attempts + attempts += 1 + with agent.override(tools=[tool]): + await agent.run(context, "Some prompt") + if attempts == 1: + raise RuntimeError("Trigger retry") + + await self.call(workflow) + + self.assertEqual(attempts, 2) + # Tool SHOULD NOT be re-invoked on the retry! + self.assertEqual(invocations, 1) + async def test_override_model_uses_inner(self) -> None: """Inside `agent.override(model=inner)`, an `agent.run()` dispatches to the override, not to the agent's default From a24b4187aaf53a9f6249dd4b06df8e6476d5c8f7 Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Mon, 14 Sep 2026 08:55:58 +0000 Subject: [PATCH 02/11] Dashboard: analyze every file again when the analysis changes A restarted dashboard carries a file forward, unanalyzed, when its bytes and its dependencies' are what they were when it was last analyzed. The state recorded nothing about which analysis produced what it holds, so a state written by an older analysis went on showing what that analysis found: after upgrading to a Reboot whose analysis records something new, nothing new would appear for a file until the file itself was edited. The state now records `code_analysis_version` beside what the analysis recorded, and the watch writes `CODE_ANALYSIS_VERSION` with every update. A restart that finds another version, including a state written before the field existed, analyzes every file again. What was recorded is kept, only without the digest that would carry it forward, so the changelog still tells what changed from what was always there. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LRFggUcgVpLqhgb7h1cK6H --- rbt/dashboard/v1/dashboard.proto | 8 ++++++ reboot/dashboard/backend/code_watcher.py | 27 ++++++++++++++++++- reboot/dashboard/backend/servicers.py | 1 + tests/reboot/dashboard/code_watcher_tests.py | 28 ++++++++++++++++++++ 4 files changed, 63 insertions(+), 1 deletion(-) diff --git a/rbt/dashboard/v1/dashboard.proto b/rbt/dashboard/v1/dashboard.proto index 28e82374a..9137598ce 100644 --- a/rbt/dashboard/v1/dashboard.proto +++ b/rbt/dashboard/v1/dashboard.proto @@ -49,6 +49,13 @@ message Dashboard { // whoever reads this to make of what they will. repeated Servicer servicers = 6; + // Represents the version of the analysis that recorded `servicers` + // and `code_files`. A dashboard whose analysis records something + // new analyzes every file again rather than carrying forward what + // an earlier one recorded, which would never say it; see + // `CODE_ANALYSIS_VERSION`. + uint32 code_analysis_version = 16; + // Represents every file the walk of the application analyzed, // keyed by its path, relative to the working directory for a file // under it and absolute otherwise, as of the last write. A @@ -157,6 +164,7 @@ message DashboardUpdateApiResponse {} message DashboardUpdateCodeRequest { repeated Servicer servicers = 1; + uint32 code_analysis_version = 6; map code_files = 2; map generated = 3; diff --git a/reboot/dashboard/backend/code_watcher.py b/reboot/dashboard/backend/code_watcher.py index 40d4b1cef..aa9c7b2e7 100644 --- a/reboot/dashboard/backend/code_watcher.py +++ b/reboot/dashboard/backend/code_watcher.py @@ -84,6 +84,13 @@ class extends: a servicer is a class with a base whose type is # reads are one message. Call = Servicer.Method.Call +# The version of what the analysis records, which the dashboard's +# state records beside it. Counted up whenever the analysis starts +# recording something it did not, or records something differently: +# a file that has not changed is otherwise carried forward as an +# earlier analysis recorded it, which never says the new thing. +CODE_ANALYSIS_VERSION = 1 + @dataclass(frozen=True, kw_only=True) class AnalyzedFile: @@ -994,6 +1001,23 @@ def _reconstitute_known( } +def _known_from(state: DashboardState) -> dict[Path, AnalyzedFile]: + """Returns what a restarted watch starts from: the analyzed files + the state records, joined back together by `_reconstitute_known`. + + Recorded by an analysis of another version, which may not record + what this one does, each is kept -- so that what changes is still + told apart from what was there all along -- but without the digest + that says it need not be analyzed again, so every file is. + """ + known = _reconstitute_known(state) + if state.code_analysis_version == CODE_ANALYSIS_VERSION: + return known + return { + filename: replace(file, digest=b'') for filename, file in known.items() + } + + async def _analyze( *, parsed: Mapping[Path, ParsedFile], @@ -1212,7 +1236,7 @@ async def watch( # analyzed again, and an iteration that reproduces exactly what # the state already records writes nothing. state = await Dashboard.ref().always().read(context) - known: Mapping[Path, AnalyzedFile] = _reconstitute_known(state) + known: Mapping[Path, AnalyzedFile] = _known_from(state) generated: Mapping[str, Generated] = state.generated # Whether this process has yet to wait for a save: a restart @@ -1294,6 +1318,7 @@ async def watch( await Dashboard.ref().per_iteration('Update').UpdateCode( context, servicers=servicers, + code_analysis_version=CODE_ANALYSIS_VERSION, code_files=files, generated=dict(generated_now), changes=changes, diff --git a/reboot/dashboard/backend/servicers.py b/reboot/dashboard/backend/servicers.py index d7e07dc64..3c312be46 100644 --- a/reboot/dashboard/backend/servicers.py +++ b/reboot/dashboard/backend/servicers.py @@ -150,6 +150,7 @@ async def UpdateCode( changed, newest last.""" del self.state.servicers[:] self.state.servicers.extend(request.servicers) + self.state.code_analysis_version = request.code_analysis_version self.state.code_files.clear() self.state.code_files.MergeFrom(request.code_files) self.state.generated.clear() diff --git a/tests/reboot/dashboard/code_watcher_tests.py b/tests/reboot/dashboard/code_watcher_tests.py index a5373c606..e34099706 100644 --- a/tests/reboot/dashboard/code_watcher_tests.py +++ b/tests/reboot/dashboard/code_watcher_tests.py @@ -25,10 +25,12 @@ ENVVAR_RBT_GENERATED_DIRECTORY, ) from reboot.dashboard.backend.code_watcher import ( + CODE_ANALYSIS_VERSION, AnalyzedFile, MethodDefinition, _analyze, _generated_definitions, + _known_from, _list_generated, _try_extract_api_digest, _modified_at, @@ -1470,6 +1472,32 @@ async def test_reconstituting_keeps_stored_spellings(self) -> None: ['shop.v1.Shop'], ) + async def test_a_state_from_another_analysis_is_analyzed_again( + self, + ) -> None: + """What an analysis of another version recorded is kept, so what + changes is still told apart from what was there all along, but + every file is analyzed again, since that analysis may not have + recorded what this one does.""" + state = DashboardState() + state.code_files['backend/x.py'].digest = b'digest' + servicer = state.servicers.add() + servicer.state_type = 'shop.v1.Shop' + servicer.filename = 'backend/x.py' + + known = _known_from(state)[Path('backend/x.py')] + self.assertEqual(known.digest, b'') + self.assertEqual( + [servicer.state_type for servicer in known.servicers], + ['shop.v1.Shop'], + ) + + state.code_analysis_version = CODE_ANALYSIS_VERSION + self.assertEqual( + _known_from(state)[Path('backend/x.py')].digest, + b'digest', + ) + def _state_types_and_files( self, files: dict[Path, AnalyzedFile], From e6ae94cb8b86715324dfc965280c711f0173b7b6 Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Mon, 14 Sep 2026 14:03:31 +0000 Subject: [PATCH 03/11] Dashboard: stop recording where a servicer is written A servicer recorded the line and column of its class, which nothing reads. A position changes whenever anything above it does, so a file whose servicer only moved down recorded something different, and a record pointing into another file would go stale the moment that file shifted -- which rules out ever carrying forward the analysis of a function that did not change. What the analysis records is now only what the code says, so moving a servicer down its file records the same thing. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LRFggUcgVpLqhgb7h1cK6H --- rbt/dashboard/v1/dashboard.proto | 6 ------ reboot/dashboard/backend/code_watcher.py | 2 -- tests/reboot/dashboard/code_watcher_tests.py | 18 ++++++++++-------- 3 files changed, 10 insertions(+), 16 deletions(-) diff --git a/rbt/dashboard/v1/dashboard.proto b/rbt/dashboard/v1/dashboard.proto index 9137598ce..43fbb7f68 100644 --- a/rbt/dashboard/v1/dashboard.proto +++ b/rbt/dashboard/v1/dashboard.proto @@ -542,12 +542,6 @@ message Servicer { // open it, e.g. "backend/src/account_servicer.py". string filename = 2; - // Represents the line its class is written on, counting from one. - uint32 line = 4; - - // Represents the column its class starts at, counting from zero. - uint32 character = 5; - // Represents every method it defines, in the order they are // written, which is the order somebody reading the file meets them. repeated Method methods = 3; diff --git a/reboot/dashboard/backend/code_watcher.py b/reboot/dashboard/backend/code_watcher.py index aa9c7b2e7..b690858f0 100644 --- a/reboot/dashboard/backend/code_watcher.py +++ b/reboot/dashboard/backend/code_watcher.py @@ -874,8 +874,6 @@ async def _analyze_class( servicer = Servicer( state_type=definition.state_type, filename=str(filename), - line=class_definition.lineno, - character=class_definition.col_offset, ) for statement in class_definition.body: diff --git a/tests/reboot/dashboard/code_watcher_tests.py b/tests/reboot/dashboard/code_watcher_tests.py index e34099706..74e6af9f0 100644 --- a/tests/reboot/dashboard/code_watcher_tests.py +++ b/tests/reboot/dashboard/code_watcher_tests.py @@ -2039,17 +2039,19 @@ async def test_records_the_methods_a_servicer_defines(self) -> None: [method.name for method in found[0].methods], ['look'] ) - async def test_where_the_servicer_is_written(self) -> None: - """The line and column of the class, for a reader to be taken - to it.""" - servicer_file = self._write('shop_servicer.py', source=SHOP) + async def test_a_servicer_moved_down_records_the_same(self) -> None: + """Where in its file a servicer is written is not recorded, so + moving it records nothing different.""" + self._write('shop_servicer.py', source=SHOP) application = self._write('main.py', source=APPLICATION) - found = extract_and_sort_servicers(await self._analyze(application)) + before = extract_and_sort_servicers(await self._analyze(application)) + + self._write('shop_servicer.py', source='\n\n# Moved.\n\n' + SHOP) + + after = extract_and_sort_servicers(await self._analyze(application)) - lines = servicer_file.read_text().splitlines() - line = lines.index('class ShopServicer(Shop.Servicer):') + 1 - self.assertEqual((found[0].line, found[0].character), (line, 0)) + self.assertEqual(after, before) async def test_a_method_reformatted_digests_the_same(self) -> None: """The digest is over what the method says, so laying it out From 4a1be8d66de1bcf9453b54fd989e4e1e5dc5a81f Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Mon, 14 Sep 2026 21:34:33 +0000 Subject: [PATCH 04/11] Dashboard: record where a method runs an agent The call graph stopped at the agent boundary. A workflow that hands its work to a model showed one unremarkable method, and nothing said it runs an agent at all. A run of an agent is now recognized by its definition rather than by how it is spelled, which is how the analysis already recognizes a Reboot call: a call whose own definition lands on one of `Agent`'s entry points -- `run`, `iter`, `run_stream`, `run_stream_events` -- in the module Reboot writes them in, flattened through the helpers a method calls. Reading Reboot's agents module is recorded as an external dependency, so upgrading the installed `reboot` reanalyzes exactly the files that read it. Which agent a run is made on is not resolved yet, so each run is recorded as what the analysis did not follow rather than guessed at or dropped: a `Servicer.Method.Hazard`, a message whose `oneof` says what went unseen and carries what is written there. Its one case so far is `run_on_unresolved_agent`, with the callee of the run. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LRFggUcgVpLqhgb7h1cK6H --- rbt/dashboard/v1/dashboard.proto | 30 +++ reboot/dashboard/backend/code_watcher.py | 103 +++++++-- tests/reboot/dashboard/code_watcher_tests.py | 207 +++++++++++++++++++ 3 files changed, 327 insertions(+), 13 deletions(-) diff --git a/rbt/dashboard/v1/dashboard.proto b/rbt/dashboard/v1/dashboard.proto index 43fbb7f68..93bcc614e 100644 --- a/rbt/dashboard/v1/dashboard.proto +++ b/rbt/dashboard/v1/dashboard.proto @@ -532,6 +532,36 @@ message Servicer { // possibly nothing of Reboot's. Spelled the way `ast.unparse` // writes the callee, e.g. "self._transfer". repeated string ambiguous = 4; + + // One thing the method's implementation does, itself or through + // the functions it calls, that the analysis did not follow, + // recorded rather than guessed at or dropped, so the dashboard + // can show where what it draws may be incomplete. + message Hazard { + // A run of an agent the analysis did not resolve: the call's own + // definition proves an agent is run, through any of `Agent`'s + // entry points, `run`, `iter`, `run_stream` or + // `run_stream_events`, but which agent is not known. Nothing + // about the run is recorded, what it is given included. + message RunOnUnresolvedAgent { + // Represents what is called, spelled the way `ast.unparse` + // writes it: the receiver and the method, without the call's + // arguments, e.g. `make_agent().run`. + string callee = 1; + } + + // Represents the file the hazard is written in, spelled the way + // the developer would open it. + string filename = 1; + + oneof hazard { + RunOnUnresolvedAgent run_on_unresolved_agent = 2; + } + } + + // Represents everything the method's implementation does that the + // analysis did not follow, in the order it met them. + repeated Hazard hazards = 6; } // The state type it services, spelled as the runtime names one, diff --git a/reboot/dashboard/backend/code_watcher.py b/reboot/dashboard/backend/code_watcher.py index b690858f0..712a0e383 100644 --- a/reboot/dashboard/backend/code_watcher.py +++ b/reboot/dashboard/backend/code_watcher.py @@ -27,6 +27,12 @@ class extends: a servicer is a class with a base whose type is the servicers, so that whoever reads both can tell a state type with nothing generated for it. +The same walk finds where the application runs an agent: a call whose +definition lands on one of Reboot's `Agent` entry points -- `run`, +`iter`, `run_stream`, `run_stream_events`. Which agent is run is not +resolved yet, so each run is recorded as a hazard, so that nothing +the analysis could not see is silently missing. + Where following stops is what makes this the developer's code rather than somebody else's. A module resolves to a file only if a root holds it, so an import of an installed package leads nowhere. @@ -84,6 +90,19 @@ class extends: a servicer is a class with a base whose type is # reads are one message. Call = Servicer.Method.Call +# Where Reboot's `Agent` is written, as the last parts of the path +# pyright answers with, so that the module is recognized wherever +# `reboot` is installed. What tells a run of an agent from any other +# call: the call's own definition is one of the methods below, +# whichever way the agent was come by. +AGENT_MODULE = ('reboot', 'agents', 'pydantic_ai', '_agent.py') + +# The methods of `Agent` a run is made through, which is what a +# run's own definition lands on. `run_sync` and `run_stream_sync` are +# not here: a Reboot `Agent` raises on both, since a workflow is +# always async. +RUN_NAMES = ('run', 'iter', 'run_stream', 'run_stream_events') + # The version of what the analysis records, which the dashboard's # state records beside it. Counted up whenever the analysis starts # recording something it did not, or records something differently: @@ -92,6 +111,14 @@ class extends: a servicer is a class with a base whose type is CODE_ANALYSIS_VERSION = 1 +def _is_agent_module(filename: Path) -> bool: + """Returns whether a file is the module Reboot's `Agent` is + written in, which is what makes a call to something defined + there a run of an agent rather than a call of the developer's + own.""" + return filename.parts[-len(AGENT_MODULE):] == AGENT_MODULE + + @dataclass(frozen=True, kw_only=True) class AnalyzedFile: """What analyzing one of the developer's files found.""" @@ -700,6 +727,16 @@ async def _generated_definition_at( return await analysis.generated_definition_at(location) +@dataclass(frozen=True, kw_only=True) +class Implementation: + """What analyzing one function's body found, which is what a + servicer method records of what it does.""" + + calls: tuple[Call, ...] + hazards: tuple[Servicer.Method.Hazard, ...] + ambiguous: tuple[str, ...] + + async def _analyze_function( function: ast.FunctionDef | ast.AsyncFunctionDef, *, @@ -707,10 +744,10 @@ async def _analyze_function( text: str, analysis: Analysis, visited: frozenset[tuple[Path, int]], -) -> tuple[list[Call], list[str], Analysis]: - """Returns the Reboot calls a function's body makes, itself or - through the functions it calls, and the calls it makes that are - ambiguous. +) -> tuple[Implementation, Analysis]: + """Returns what a function's body does, itself or through the + functions it calls: the Reboot calls it makes, what it does that + is not followed, and the calls it makes that are ambiguous. A Reboot call is one whose own definition pyright places at a method stub of a state type. However the reference was come by, @@ -718,6 +755,12 @@ async def _analyze_function( elsewhere, the called method's definition is the same, so one question decides. + A run of an agent is the same question answered by one of + `Agent`'s entry points, `run`, `iter`, `run_stream` or + `run_stream_events`, in the module Reboot writes them in. Which + agent is run is not resolved, so the run is recorded as a hazard + of the function; see `Servicer.Method.Hazard`. + A call defined by a function the generator did not write, the developer's own or an installed package's, is followed: that function's body is analyzed the same way and its calls are the @@ -736,9 +779,11 @@ async def _analyze_function( one whose definition is no function: a stub's, which has no body to follow, or a class's. A call whose definition is the generator's own machinery, such as the `ref` or `schedule` - inside a chain, or the standard library's, is neither. + inside a chain, Reboot's own machinery around a run, or the + standard library's, is neither. """ calls: list[Call] = [] + hazards: list[Servicer.Method.Hazard] = [] ambiguous: list[str] = [] # The function itself is walked here, and everything defined @@ -796,6 +841,32 @@ async def _analyze_function( ) continue + if _is_agent_module(location.filename): + entry_point, analysis = await analysis.helper_definition_at( + location + ) + entry_point_name = ( + None if entry_point is None else entry_point.syntax.name + ) + + if entry_point_name not in RUN_NAMES: + # Reboot's own machinery around a run, such as a + # construction. Followed no further than the + # generator's machinery is. + continue + + hazards.append( + Servicer.Method.Hazard( + filename=str(filename), + run_on_unresolved_agent=( + Servicer.Method.Hazard.RunOnUnresolvedAgent( + callee=ast.unparse(callee), + ) + ), + ) + ) + continue + helper, analysis = await analysis.helper_definition_at(location) if helper is None: ambiguous.append(ast.unparse(callee)) @@ -805,17 +876,22 @@ async def _analyze_function( if key in visited: continue - helper_calls, helper_ambiguous, analysis = await _analyze_function( + followed, analysis = await _analyze_function( helper.syntax, filename=helper.filename, text=helper.text, analysis=analysis, visited=visited | {key}, ) - calls.extend(helper_calls) - ambiguous.extend(helper_ambiguous) - - return calls, ambiguous, analysis + calls.extend(followed.calls) + hazards.extend(followed.hazards) + ambiguous.extend(followed.ambiguous) + + return Implementation( + calls=tuple(calls), + hazards=tuple(hazards), + ambiguous=tuple(ambiguous), + ), analysis async def _analyze_class( @@ -882,7 +958,7 @@ async def _analyze_class( ast.FunctionDef(name=str(name)) | ast.AsyncFunctionDef(name=str(name)) ): - calls, ambiguous, analysis = await _analyze_function( + implementation, analysis = await _analyze_function( statement, filename=filename, text=analysis.parsed[filename].text, @@ -893,8 +969,9 @@ async def _analyze_class( Servicer.Method( name=name, digest=_digest(statement), - calls=calls, - ambiguous=ambiguous, + calls=implementation.calls, + hazards=implementation.hazards, + ambiguous=implementation.ambiguous, ) ) diff --git a/tests/reboot/dashboard/code_watcher_tests.py b/tests/reboot/dashboard/code_watcher_tests.py index 74e6af9f0..1cb4a262a 100644 --- a/tests/reboot/dashboard/code_watcher_tests.py +++ b/tests/reboot/dashboard/code_watcher_tests.py @@ -259,6 +259,54 @@ async def main(): await Application(servicers=[ShopServicer]).run() ''' +# The shape of Reboot's own `Agent`, as far as the analysis needs: +# the entry points a run is made through, and the construction whose +# arguments say what the agent is. Written where an installed `reboot` would be, since the +# analysis recognizes the module by its path. +AGENTS_MODULE = ''' +class Agent: + + def __init__( + self, + model=None, + *, + name=None, + system_prompt=(), + instructions=None, + description=None, + **kwargs, + ): + pass + + async def run(self, context=None, user_prompt=None, **kwargs): + pass + + def iter(self, context=None, user_prompt=None, **kwargs): + pass + + def run_stream(self, context=None, user_prompt=None, **kwargs): + pass + + async def run_stream_events( + self, + context=None, + user_prompt=None, + **kwargs, + ): + pass +''' + + +def _write_agents_module(directory: Path) -> None: + """Writes an installed `reboot` holding the agents module the + analysis recognizes.""" + package = directory / 'reboot' / 'agents' / 'pydantic_ai' + package.mkdir(parents=True, exist_ok=True) + (directory / 'reboot' / '__init__.py').write_text('') + (directory / 'reboot' / 'agents' / '__init__.py').write_text('') + (package / '_agent.py').write_text(AGENTS_MODULE) + (package / '__init__.py').write_text('from ._agent import Agent\n') + class ImplementationWatcherTest(unittest.IsolatedAsyncioTestCase): @@ -836,6 +884,10 @@ async def asyncSetUp(self) -> None: installed.parent.mkdir(parents=True) installed.write_text(GENERATED.format(state='Ext')) + # Reboot's own agents module, installed the way it is in an + # application that uses agents. + _write_agents_module(self.installed) + # The generator's actual output, checked in as a golden and # rewritten with the templates by `make goldens`, so that # resolving is tested against the real templates and not @@ -1267,6 +1319,161 @@ async def test_an_installed_helper_is_followed_and_recorded( }, ) + async def test_every_way_of_running_an_agent(self) -> None: + """Each of `Agent`'s four entry points is a run, which is a + hazard of the method while which agent is run is not resolved.""" + servicer = self._write( + 'shop_servicer.py', + source=( + 'from reboot.agents.pydantic_ai import Agent\n' + 'from shop.v1.shop_rbt import Shop\n' + '\n' + '\n' + "librarian = Agent('test', name='librarian')\n" + '\n' + '\n' + 'class ShopServicer(Shop.Servicer):\n' + '\n' + ' async def look(self, context, request):\n' + " await librarian.run(context, 'a')\n" + " async with librarian.iter(context, 'b'):\n" + ' pass\n' + " async with librarian.run_stream(context, 'c'):\n" + ' pass\n' + ' async for _ in librarian.run_stream_events(\n' + " context, 'd'\n" + ' ):\n' + ' pass\n' + ), + ) + application = self._write('main.py', source=APPLICATION) + + found = await self._analyze(application) + + [found_servicer] = found[servicer].servicers + [method] = found_servicer.methods + self.assertEqual( + sorted( + hazard.run_on_unresolved_agent.callee + for hazard in method.hazards + ), + [ + 'librarian.iter', + 'librarian.run', + 'librarian.run_stream', + 'librarian.run_stream_events', + ], + ) + + def _hazards(self, hazards) -> list[tuple[str, dict[str, str]]]: + """Returns hazards as the name of each one's case and the fields + it sets, which is what reads well in a failed assertion.""" + return [ + ( + case, + { + field.name: value + for field, value in getattr(hazard, case).ListFields() + }, + ) + for hazard in hazards + if (case := hazard.WhichOneof('hazard')) is not None + ] + + async def test_a_run_whose_agent_cannot_be_resolved(self) -> None: + """A run whose own definition proves an agent is run -- made on + an agent a factory returns, an alias, a parameter, an element of + a collection -- is recorded as a hazard of the method, with + nothing about the run, rather than as a guess at which agent it + is. A run on something pyright cannot type at all is not known + to be a run, and stays an ambiguous call.""" + servicer = self._write( + 'shop_servicer.py', + source=( + 'from reboot.agents.pydantic_ai import Agent\n' + 'from shop.v1.shop_rbt import Shop\n' + '\n' + '\n' + 'def make_agent():\n' + " return Agent('test', name='made')\n" + '\n' + '\n' + "librarian = Agent('test', name='librarian')\n" + 'agents = [librarian]\n' + 'research = None\n' + '\n' + '\n' + 'async def ask(agent: Agent, context):\n' + " await agent.run(context, 'Tidy up')\n" + '\n' + '\n' + 'async def ask_untyped(agent, context):\n' + " await agent.run(context, 'Tidy up')\n" + '\n' + '\n' + 'class ShopServicer(Shop.Servicer):\n' + '\n' + ' async def factory(self, context, request):\n' + ' await make_agent().run(\n' + " context, 'Tidy up', toolsets=[research]\n" + ' )\n' + '\n' + ' async def alias(self, context, request):\n' + ' helper = librarian\n' + " await helper.run(context, 'Tidy up')\n" + '\n' + ' async def parameter(self, context, request):\n' + ' await ask(librarian, context)\n' + '\n' + ' async def element(self, context, request):\n' + " await agents[0].run(context, 'Tidy up')\n" + '\n' + ' async def untyped(self, context, request):\n' + ' await ask_untyped(librarian, context)\n' + ), + ) + application = self._write('main.py', source=APPLICATION) + + found = await self._analyze(application) + + [found_servicer] = found[servicer].servicers + methods = {method.name: method for method in found_servicer.methods} + self.assertEqual( + { + name: self._hazards(method.hazards) + for name, method in methods.items() + }, + { + 'factory': + [ + ( + 'run_on_unresolved_agent', + { + 'callee': 'make_agent().run' + }, + ), + ], + 'alias': + [('run_on_unresolved_agent', { + 'callee': 'helper.run' + })], + 'parameter': + [('run_on_unresolved_agent', { + 'callee': 'agent.run' + })], + 'element': + [('run_on_unresolved_agent', { + 'callee': 'agents[0].run' + })], + 'untyped': [], + }, + ) + self.assertEqual( + [hazard.filename for hazard in methods['factory'].hazards], + [str(servicer)], + ) + self.assertEqual(list(methods['untyped'].ambiguous), ['agent.run']) + async def test_a_base_from_a_function_return_type_is_resolved( self, ) -> None: From 8675f288af83189fc3e2d33b848f75ed22de1543 Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Mon, 14 Sep 2026 21:34:56 +0000 Subject: [PATCH 05/11] Dashboard: find the agents a method runs Every run of an agent was a hazard, since nothing said which agent a run is made on. Which agent is run is now what pyright resolves the receiver to, whether the agent is used in the file constructing it or imported from another. To keep this first version simple, only an `Agent(...)` constructed with a string literal `name=` and bound to a name at the top level of a module is resolved. The `name`, which the runtime requires to be unique, is what tells it from every other agent and what a run records; no position is recorded, so moving code records nothing new. Its construction also says its model, prompt and description; only literals are read. A method records its runs, flattened through the helpers it calls, and the file records each agent it runs, in `Agent`. Records are per file, the way servicers are, because the watch's cache is keyed by file, so an agent run in several files has a record in each. An agent whose `name=` is computed or missing, one a factory builds, one held on `self`, bound inside a function or constructed where it is run, an alias, a parameter, an element of a collection, or a name bound twice at the top level -- which pyright resolves to its first binding whichever one a run holds -- is still not resolved, and its runs are still `run_on_unresolved_agent` hazards. What a resolved run is given on top of its agent is not followed, but recorded as an `Agent.Run.Hazard`: `run_arguments`, for `toolsets=`, `model=` and `instructions=` passed to the run. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LRFggUcgVpLqhgb7h1cK6H --- rbt/dashboard/v1/dashboard.proto | 116 ++++- reboot/dashboard/backend/code_watcher.py | 469 ++++++++++++++++++- reboot/dashboard/backend/servicers.py | 3 + tests/reboot/dashboard/code_watcher_tests.py | 265 ++++++++++- 4 files changed, 804 insertions(+), 49 deletions(-) diff --git a/rbt/dashboard/v1/dashboard.proto b/rbt/dashboard/v1/dashboard.proto index 93bcc614e..19dd1cf51 100644 --- a/rbt/dashboard/v1/dashboard.proto +++ b/rbt/dashboard/v1/dashboard.proto @@ -49,13 +49,17 @@ message Dashboard { // whoever reads this to make of what they will. repeated Servicer servicers = 6; - // Represents the version of the analysis that recorded `servicers` - // and `code_files`. A dashboard whose analysis records something - // new analyzes every file again rather than carrying forward what - // an earlier one recorded, which would never say it; see - // `CODE_ANALYSIS_VERSION`. + // Represents the version of the analysis that recorded + // `servicers`, `agents` and `code_files`. A dashboard whose analysis + // records something new analyzes every file again rather than + // carrying forward what an earlier one recorded, which would never + // say it; see `CODE_ANALYSIS_VERSION`. uint32 code_analysis_version = 16; + // Every agent found in the developer's application, sorted. One + // agent has a record per file whose analysis found it; see `Agent`. + repeated Agent agents = 17; + // Represents every file the walk of the application analyzed, // keyed by its path, relative to the working directory for a file // under it and absolute otherwise, as of the last write. A @@ -126,6 +130,7 @@ message DashboardGetResponse { map apis = 4; map api_digests = 5; repeated Servicer servicers = 6; + repeated Agent agents = 16; map generated = 7; // The worst reason over all the API files, and absent when @@ -165,6 +170,7 @@ message DashboardUpdateApiResponse {} message DashboardUpdateCodeRequest { repeated Servicer servicers = 1; uint32 code_analysis_version = 6; + repeated Agent agents = 7; map code_files = 2; map generated = 3; @@ -533,16 +539,25 @@ message Servicer { // writes the callee, e.g. "self._transfer". repeated string ambiguous = 4; + // Represents every agent the method's implementation runs, + // itself or through the functions it calls. What each agent is, + // is in `Agent`, which each of these names. + repeated Agent.Run runs = 5; + // One thing the method's implementation does, itself or through // the functions it calls, that the analysis did not follow, // recorded rather than guessed at or dropped, so the dashboard // can show where what it draws may be incomplete. message Hazard { - // A run of an agent the analysis did not resolve: the call's own - // definition proves an agent is run, through any of `Agent`'s - // entry points, `run`, `iter`, `run_stream` or - // `run_stream_events`, but which agent is not known. Nothing - // about the run is recorded, what it is given included. + // A run of an agent the analysis could not resolve: the call's + // own definition proves an agent is run, but what it is run on + // is not an `Agent(...)` with a string literal `name=`, bound to + // a name at the top level of a module. E.g. an agent whose + // `name=` is computed or missing, one a factory builds, an + // alias, a parameter, an element of a collection, one held on + // `self`, one constructed inside a function or where it is run, + // or a name bound to more than one thing. Nothing about the run + // is recorded, what it is given included. message RunOnUnresolvedAgent { // Represents what is called, spelled the way `ast.unparse` // writes it: the receiver and the method, without the call's @@ -577,6 +592,87 @@ message Servicer { repeated Method methods = 3; } +// One agent the developer's application uses, as one file's analysis +// found it: a Reboot `Agent` some method runs, constructed with a +// string literal `name=` and bound to a name at the top level of a +// module. +// +// A record per file, because the analysis keys everything by the +// file it read, and an agent is met in more files than the one +// constructing it: a workflow in one file runs an agent constructed +// in another. Every record says the same thing about the agent, since +// each is built from the construction the analysis resolved. Whoever +// reads these joins them on `name`, which the runtime requires to be +// unique. +message Agent { + // One run of an agent, made by a servicer method's implementation + // through any of `Agent`'s entry points: `run`, `iter`, `run_stream` + // or `run_stream_events`. Which one says nothing about what the + // application does, so it is not recorded. + message Run { + // One thing a run is given on top of its agent that the analysis + // did not follow, recorded rather than guessed at or dropped. + message Hazard { + // The run is given arguments that change what its agent is or + // can reach. Each represents the value passed, spelled the way + // `ast.unparse` writes it, and is absent when it is not passed. + message RunArguments { + // `toolsets=`, which give the run tools. + optional string toolsets = 1; + + // `model=`, which the run uses instead of `Agent.model`. + optional string model = 2; + + // `instructions=`, which the run is given on top of those in + // `Agent.system_prompt`. + optional string instructions = 3; + } + + // Represents the file the hazard is written in, spelled the way + // the developer would open it. + string filename = 1; + + oneof hazard { + RunArguments run_arguments = 2; + } + } + + // Represents the agent run, spelled as `Agent.name` spells it. + string agent = 1; + + // Represents everything the run is given on top of its agent that + // the analysis did not follow. + repeated Hazard hazards = 2; + } + + // Represents the agent's `name`, which the runtime requires and + // which is unique in an application, so it is what a run names and + // what records of one agent join on. + string name = 1; + + // The file whose analysis found this record, spelled the way the + // developer would open it, e.g. "backend/src/librarian.py". What + // the records are keyed by, and not necessarily where the agent is + // constructed. + string filename = 2; + + // Represents the model the agent is constructed with, e.g. + // "anthropic:claude-sonnet-4-6". Absent when it is not a literal + // the analysis can read. + optional string model = 3; + + // Represents what the agent tells the model about itself: every + // literal string its `system_prompt` and its `instructions` are + // constructed with, in that order. A prompt computed rather than + // written down contributes nothing. + repeated string system_prompt = 4; + + // Represents the agent's `description`, which is what another + // agent handing work to this one is told it does. Absent when it + // has none the analysis can read. + optional string description = 5; +} + // What analyzing one of the developer's files used: the digest // of its bytes and what each of its imports observed. What a // restarted dashboard reconstitutes its knowledge from, so that only diff --git a/reboot/dashboard/backend/code_watcher.py b/reboot/dashboard/backend/code_watcher.py index 712a0e383..dd028da22 100644 --- a/reboot/dashboard/backend/code_watcher.py +++ b/reboot/dashboard/backend/code_watcher.py @@ -27,11 +27,16 @@ class extends: a servicer is a class with a base whose type is the servicers, so that whoever reads both can tell a state type with nothing generated for it. -The same walk finds where the application runs an agent: a call whose -definition lands on one of Reboot's `Agent` entry points -- `run`, -`iter`, `run_stream`, `run_stream_events`. Which agent is run is not -resolved yet, so each run is recorded as a hazard, so that nothing -the analysis could not see is silently missing. +The same walk finds the agents the application uses. A run of one is +a call whose definition lands on one of Reboot's `Agent` entry +points -- `run`, `iter`, `run_stream`, `run_stream_events` -- and +which agent is run is what the receiver of that call resolves to: an +`Agent(...)` with a string literal `name=`, bound to a name at the +top level of a module, which says the agent's name, its model and +the prompt it is given. Whatever is not followed -- a run of any +other agent, what a run is given on top of its agent -- is recorded +as a hazard instead, so that nothing the analysis could not see is +silently missing. Where following stops is what makes this the developer's code rather than somebody else's. A module resolves to a file only if a root @@ -58,7 +63,7 @@ class extends: a servicer is a class with a base whose type is from functools import partial from google.protobuf.timestamp_pb2 import Timestamp from pathlib import Path -from rbt.dashboard.v1.dashboard_pb2 import Change +from rbt.dashboard.v1.dashboard_pb2 import Agent, Change from rbt.dashboard.v1.dashboard_pb2 import Dashboard as DashboardState from rbt.dashboard.v1.dashboard_pb2 import File, Generated, Servicer from rbt.dashboard.v1.dashboard_rbt import Dashboard @@ -103,6 +108,11 @@ class extends: a servicer is a class with a base whose type is # always async. RUN_NAMES = ('run', 'iter', 'run_stream', 'run_stream_events') +# The keyword arguments a run may be given that change what its agent +# is or can reach, which the analysis does not follow; see +# `Agent.Run.Hazard.RunArguments`. +RUN_ARGUMENTS = ('toolsets', 'model', 'instructions') + # The version of what the analysis records, which the dashboard's # state records beside it. Counted up whenever the analysis starts # recording something it did not, or records something differently: @@ -148,6 +158,11 @@ class AnalyzedFile: # services and the calls each method makes. servicers: tuple[Servicer, ...] + # Every agent this file's analysis found it runs. An agent met in + # more than one file has a record in each, which whoever reads + # them joins; see `Agent`. + agents: tuple[Agent, ...] + def _position_at_last_character( node: ast.Name | ast.Attribute, @@ -512,6 +527,212 @@ def _helper_definitions( ) +def _keyword(call: ast.Call, name: str) -> Optional[ast.expr]: + """Returns what a call passes a keyword argument, and `None` for + one it does not pass.""" + for keyword in call.keywords: + if keyword.arg == name: + return keyword.value + return None + + +def _passes(call: ast.Call, name: str) -> bool: + """Returns whether a call passes a keyword argument something: one + it passes anything but `None`, or an empty list or tuple written + out, which give nothing.""" + match _keyword(call, name): + case None | ast.Constant(value=None): + return False + case ast.List(elts=[]) | ast.Tuple(elts=[]): + return False + return True + + +def _passed(call: ast.Call, keywords: Sequence[str]) -> dict[str, str]: + """Returns the value a call passes each of some keyword arguments + that gives something, by keyword, spelled the way `ast.unparse` + writes it; see `_passes`.""" + return { + keyword: ast.unparse(value) + for keyword in keywords + if (value := _keyword(call, keyword)) is not None and + _passes(call, keyword) + } + + +def _try_constant_string(node: Optional[ast.expr]) -> Optional[str]: + """Returns the string an expression is, and `None` for an + expression that is anything else: a name, an f-string, a call. + Adjacent string literals are one expression, so a prompt written + across a screenful of quoted lines reads as the one string the + developer meant.""" + match node: + case ast.Constant(value=str(value)): + return value + return None + + +def _elements(node: Optional[ast.expr]) -> list[ast.expr]: + """Returns what a list or a tuple written out holds, and nothing + for anything else, such as a name the elements were gathered + under.""" + match node: + case ast.List(elts=elements) | ast.Tuple(elts=elements): + return list(elements) + return [] + + +def _constant_strings(node: Optional[ast.expr]) -> list[str]: + """Returns the strings an expression is: the one it is, or every + one written out in the list or tuple it is.""" + string = _try_constant_string(node) + if string is not None: + return [string] + return [ + string for element in _elements(node) + if (string := _try_constant_string(element)) is not None + ] + + +@dataclass(frozen=True, kw_only=True) +class AgentDefinition: + """A line an agent comes from, e.g. + `librarian = Agent('anthropic:claude-sonnet-4-6', name='librarian')`. + + What a run of an agent leads to: the agent it is made on, with + everything the line says about it. + """ + + # The `name` the agent is constructed with, which tells it from + # every other agent; see `Agent.name`. + name: str + + # What its construction says, each absent when it says nothing a + # literal can be read from. + model: Optional[str] + system_prompt: tuple[str, ...] + description: Optional[str] + + +def _is_agent_construction(call: ast.Call) -> bool: + """Returns whether a call is written as constructing an agent, + `Agent(...)` or `pydantic_ai.Agent(...)`, rather than as a call to + something that builds one, such as a factory, whose arguments say + what the factory is given rather than what the agent is. + + Read from how the call is spelled, since it is asked only of a + line a run already proved is an agent: an `Agent` imported under + another name is not resolved, and says so, rather than being read + as something it is not. + """ + match call.func: + case ast.Name(id='Agent') | ast.Attribute(attr='Agent'): + return True + return False + + +def _agent_definition(value: ast.Call) -> Optional[AgentDefinition]: + """Returns what the call an agent comes from says the agent is, + and `None` when it is not an `Agent(...)` with a string literal + `name=`, which is the one agent the analysis can tell from every + other: an agent a factory builds, or one whose `name=` is computed + or missing, is not resolved.""" + if not _is_agent_construction(value): + return None + construction = value + + name = _try_constant_string(_keyword(construction, 'name')) + if name is None: + return None + + # The model is the first argument, wherever it is passed: + # `Agent('anthropic:claude-sonnet-4-6')` or `Agent(model=...)`. + model = _keyword(construction, 'model') + if model is None and len(construction.args) > 0: + model = construction.args[0] + + return AgentDefinition( + name=name, + model=_try_constant_string(model), + system_prompt=tuple( + _constant_strings(_keyword(construction, 'system_prompt')) + + _constant_strings(_keyword(construction, 'instructions')) + ), + description=_try_constant_string( + _keyword(construction, 'description') + ), + ) + + +def _agent_definitions(parse: Parse) -> Mapping[int, AgentDefinition]: + """Returns what each line at the top level of a module constructs + an agent as, by the line the assignment is written on, which is + the line pyright places the definition of the name it binds at. + + Only an agent bound to a name at the top level of a module is + indexed. One constructed anywhere else -- held on `self`, bound + inside a function, or constructed where it is run -- is not, so a + run of it is not resolved, and says so. + + Every top-level assignment of a call is indexed, because what + makes one of them an agent is the type of what it binds, which + only the run that led here has established. A name bound more than + once at the top level is not indexed at all: pyright places every + use of it at the first binding, whichever one the use holds, so + reading any of them risks recording a run of one agent as a run of + another. + """ + definitions: dict[int, tuple[str, AgentDefinition]] = {} + + # How many times each name is assigned at the top level. + assigned: dict[str, int] = {} + + for statement in parse.syntax.body: + match statement: + case ast.Assign(targets=targets): + for target in targets: + if isinstance(target, ast.Name): + assigned[target.id] = assigned.get(target.id, 0) + 1 + case ast.AnnAssign(target=ast.Name(id=str(name))): + assigned[name] = assigned.get(name, 0) + 1 + + match statement: + case ast.Assign( + targets=[ast.Name(id=str(bound_to))], + value=ast.Call() as value, + ) | ast.AnnAssign( + target=ast.Name(id=str(bound_to)), + value=ast.Call() as value, + ): + definition = _agent_definition(value) + if definition is not None: + definitions[statement.lineno] = (bound_to, definition) + + return MappingProxyType( + { + line: definition + for line, (bound_to, definition) in definitions.items() + if assigned[bound_to] == 1 + } + ) + + +@dataclass(frozen=True, kw_only=True) +class AgentFile: + """What reading one file an agent is constructed in during an + analysis said.""" + + # What each line of the file constructs, empty when the file + # could not be read or would not parse; see `_agent_definitions`. + definitions: Mapping[int, AgentDefinition] + + # The dependency each file whose analysis reads this file + # records in its `external`: present only for a file outside + # every root, which the walk never finds or digests, and always + # carrying a digest of bytes actually read. + external: Optional[Dependency] + + @dataclass(frozen=True, kw_only=True) class Analysis: """One iteration's analysis: what the walk read, for the asking. @@ -544,6 +765,10 @@ class Analysis: # indexed at most once per analysis. helpers: Mapping[Path, HelperFile] + # Every file an agent was found constructed in so far, keyed the + # same way and indexed at most once per analysis. + agents: Mapping[Path, AgentFile] + # The dependencies on files outside every root read since the # map was last emptied, keyed by filename in the spelling # `_standardized_path` returns. Emptied by `_analyze_file` for @@ -693,6 +918,57 @@ async def helper_definition_at( return helper.definitions.get(location.line), analysis + async def agent_definition_at( + self, + location: Location, + ) -> tuple[Optional[AgentDefinition], 'Analysis']: + """Returns the agent a file constructs at a location, and + `None` for a location that constructs none: a name bound to + an agent another name already held, or one a function + returned, whose construction is wherever that function + wrote it. The file is read, parsed and indexed at most once + per analysis.""" + if location.filename.suffix != '.py': + return None, self + + # Standardized because pyright spells its locations + # absolutely, while everything here is keyed by the + # spelling `_standardized_path` returns. + agent, analysis = await self._agent_file( + _standardized_path(location.filename) + ) + return agent.definitions.get(location.line), analysis + + async def _agent_file( + self, + filename: Path, + ) -> tuple[AgentFile, 'Analysis']: + """Returns what a file constructs, reading, parsing and + indexing it the first time it is asked about.""" + analysis = self + + agent = analysis.agents.get(filename) + if agent is None: + parse, external = await analysis._parse(filename) + agent = AgentFile( + definitions=( + _agent_definitions(parse) + if parse is not None else MappingProxyType({}) + ), + external=external, + ) + analysis = replace( + analysis, + agents=MappingProxyType({ + **analysis.agents, + filename: agent, + }), + ) + + analysis = analysis._with_external_dependency(agent.external) + + return agent, analysis + async def _generated_definition_at( filename: Path, @@ -733,6 +1009,7 @@ class Implementation: servicer method records of what it does.""" calls: tuple[Call, ...] + runs: tuple[Agent.Run, ...] hazards: tuple[Servicer.Method.Hazard, ...] ambiguous: tuple[str, ...] @@ -744,10 +1021,12 @@ async def _analyze_function( text: str, analysis: Analysis, visited: frozenset[tuple[Path, int]], + agents: dict[str, Agent], ) -> tuple[Implementation, Analysis]: """Returns what a function's body does, itself or through the - functions it calls: the Reboot calls it makes, what it does that - is not followed, and the calls it makes that are ambiguous. + functions it calls: the Reboot calls it makes, the agents it + runs, what it does that is not followed, and the calls it makes + that are ambiguous. A Reboot call is one whose own definition pyright places at a method stub of a state type. However the reference was come by, @@ -758,8 +1037,13 @@ async def _analyze_function( A run of an agent is the same question answered by one of `Agent`'s entry points, `run`, `iter`, `run_stream` or `run_stream_events`, in the module Reboot writes them in. Which - agent is run is not resolved, so the run is recorded as a hazard - of the function; see `Servicer.Method.Hazard`. + agent is run is what the receiver of the call resolves to, an + `Agent(...)` with a literal `name=` at the top level of a module; + the record of that agent joins `agents`, which is every agent the + file being analyzed has found so far. What a run is given on top of its + agent is not followed, but recorded on the run, and a run whose + agent cannot be resolved is recorded as a hazard of the function; + see `Agent.Run.Hazard` and `Servicer.Method.Hazard`. A call defined by a function the generator did not write, the developer's own or an installed package's, is followed: that @@ -783,6 +1067,7 @@ async def _analyze_function( standard library's, is neither. """ calls: list[Call] = [] + runs: list[Agent.Run] = [] hazards: list[Servicer.Method.Hazard] = [] ambiguous: list[str] = [] @@ -855,14 +1140,43 @@ async def _analyze_function( # generator's machinery is. continue - hazards.append( - Servicer.Method.Hazard( - filename=str(filename), - run_on_unresolved_agent=( - Servicer.Method.Hazard.RunOnUnresolvedAgent( - callee=ast.unparse(callee), - ) - ), + constructed, analysis = await _agent_at( + callee, + filename=filename, + text=text, + analysis=analysis, + ) + if constructed is None: + hazards.append( + Servicer.Method.Hazard( + filename=str(filename), + run_on_unresolved_agent=( + Servicer.Method.Hazard.RunOnUnresolvedAgent( + callee=ast.unparse(callee), + ) + ), + ) + ) + continue + + _agent_record(constructed, agents=agents) + + # What the run is given on top of its agent by its own + # arguments is not followed, but said on the run. + run_hazards: list[Agent.Run.Hazard] = [] + given = _passed(node, RUN_ARGUMENTS) + if len(given) > 0: + run_hazards.append( + Agent.Run.Hazard( + filename=str(filename), + run_arguments=Agent.Run.Hazard.RunArguments(**given), + ) + ) + + runs.append( + Agent.Run( + agent=constructed.name, + hazards=run_hazards, ) ) continue @@ -882,23 +1196,94 @@ async def _analyze_function( text=helper.text, analysis=analysis, visited=visited | {key}, + agents=agents, ) calls.extend(followed.calls) + runs.extend(followed.runs) hazards.extend(followed.hazards) ambiguous.extend(followed.ambiguous) return Implementation( calls=tuple(calls), + runs=tuple(runs), hazards=tuple(hazards), ambiguous=tuple(ambiguous), ), analysis +async def _agent_at( + callee: ast.expr, + *, + filename: Path, + text: str, + analysis: Analysis, +) -> tuple[Optional[AgentDefinition], Analysis]: + """Returns the agent a call is made on, from the expression the + call is made through, e.g. `librarian.run(...)`. + + What the receiver refers to, pyright answers, resolving however + the agent gets there: a name in this file, one imported from + another, or an attribute of something held. Where it lands is an + agent only when it is an `Agent(...)` with a string literal + `name=`, bound at the top level of a module, which says what the + agent is. + """ + match callee: + case ast.Attribute(value=receiver): + pass + case _: + # A call reached any other way is made on nothing that + # can be resolved to an agent. + return None, analysis + + match receiver: + case ast.Name() | ast.Attribute(): + line, character = _position_at_last_character(receiver) + location = await analysis.pyright.definition_at( + filename=filename, + line=line, + character=character, + text=text, + ) + if location is None: + return None, analysis + return await analysis.agent_definition_at(location) + + return None, analysis + + +def _agent_record( + definition: AgentDefinition, + *, + agents: dict[str, Agent], +) -> Agent: + """Returns the record the file being analyzed keeps of an agent, + making it, from what the line it comes from says, the first time + the file finds the agent. + + `agents` is every agent the file has found so far, by name, so + that an agent run in several of the file's methods is one record. + """ + found = agents.get(definition.name) + if found is not None: + return found + + agent = Agent( + name=definition.name, + model=definition.model, + system_prompt=definition.system_prompt, + description=definition.description, + ) + agents[definition.name] = agent + return agent + + async def _analyze_class( class_definition: ast.ClassDef, *, filename: Path, analysis: Analysis, + agents: dict[str, Agent], ) -> tuple[Optional[Servicer], Analysis]: """Returns the servicer a class is, and `None` when it is not one. @@ -964,12 +1349,14 @@ async def _analyze_class( text=analysis.parsed[filename].text, analysis=analysis, visited=frozenset(), + agents=agents, ) servicer.methods.append( Servicer.Method( name=name, digest=_digest(statement), calls=implementation.calls, + runs=implementation.runs, hazards=implementation.hazards, ambiguous=implementation.ambiguous, ) @@ -986,12 +1373,13 @@ async def _analyze_file( ) -> tuple[AnalyzedFile, Analysis]: """Returns one parsed file analyzed: an `AnalyzedFile` built from its `ParsedFile` -- the dependencies the parse recorded -- - with the external files the analysis read and every servicer - the file defines.""" + with the external files the analysis read, every servicer the + file defines, and every agent it runs.""" parsed = analysis.parsed[filename] - # A generated file defines no servicer, the generator writes - # none, so only the developer's own files are searched. + # A generated file defines no servicer and runs no agent, the + # generator writes neither, so only the developer's own files + # are searched. if filename.name.endswith(GENERATED_SUFFIXES): return AnalyzedFile( filename=parsed.filename, @@ -999,6 +1387,7 @@ async def _analyze_file( dependencies=parsed.dependencies, external=(), servicers=(), + agents=(), ), analysis # Emptied so that what gathers in `external` below is what @@ -1007,6 +1396,11 @@ async def _analyze_file( servicers: list[Servicer] = [] + # Every agent this file finds, by name, gathered as the servicers + # are walked, so that an agent run in two of the file's methods is + # one record. + agents: dict[str, Agent] = {} + for node in ast.walk(parsed.syntax): match node: case ast.ClassDef(): @@ -1014,10 +1408,16 @@ async def _analyze_file( node, filename=filename, analysis=analysis, + agents=agents, ) if servicer is not None: servicers.append(servicer) + # Said once, here, because it is this file's analysis that found + # every one of them, wherever each is constructed. + for agent in agents.values(): + agent.filename = str(filename) + # Discarded so that pyright holds only what analyzing one file # asked about, rather than accumulating every file asked about # across an analysis; the next file syncs what it asks about. @@ -1032,6 +1432,7 @@ async def _analyze_file( dependency for _, dependency in sorted(analysis.external.items()) ), servicers=tuple(servicers), + agents=tuple(agents.values()), ), analysis @@ -1052,6 +1453,22 @@ def extract_and_sort_servicers( ) +def extract_and_sort_agents( + files: Mapping[Path, AnalyzedFile], +) -> list[Agent]: + """Returns every agent found, sorted by name and the file whose + analysis found it. + + Records of one name are one agent, which more than one file + running it found, every record saying the same thing about it. + See `Agent`. + """ + return sorted( + (agent for file in files.values() for agent in file.agents), + key=lambda agent: (agent.name, agent.filename), + ) + + def _reconstitute_known( state: DashboardState, ) -> dict[Path, AnalyzedFile]: @@ -1064,6 +1481,10 @@ def _reconstitute_known( for servicer in state.servicers: servicers.setdefault(servicer.filename, []).append(servicer) + agents: dict[str, list[Agent]] = {} + for agent in state.agents: + agents.setdefault(agent.filename, []).append(agent) + return { Path(filename): AnalyzedFile( @@ -1072,6 +1493,7 @@ def _reconstitute_known( dependencies=MappingProxyType(dict(file.dependencies)), external=tuple(file.external), servicers=tuple(servicers.get(filename, [])), + agents=tuple(agents.get(filename, [])), ) for filename, file in state.code_files.items() } @@ -1121,6 +1543,7 @@ async def _analyze( roots=tuple(roots), reads=MappingProxyType({}), helpers=MappingProxyType({}), + agents=MappingProxyType({}), external=MappingProxyType({}), ) @@ -1374,6 +1797,7 @@ async def watch( # on its own. if known_now is not None: servicers = extract_and_sort_servicers(known_now) + agents = extract_and_sort_agents(known_now) # The file messages the write below records, # built before the write so that the state is @@ -1393,6 +1817,7 @@ async def watch( await Dashboard.ref().per_iteration('Update').UpdateCode( context, servicers=servicers, + agents=agents, code_analysis_version=CODE_ANALYSIS_VERSION, code_files=files, generated=dict(generated_now), diff --git a/reboot/dashboard/backend/servicers.py b/reboot/dashboard/backend/servicers.py index 3c312be46..01e2b1237 100644 --- a/reboot/dashboard/backend/servicers.py +++ b/reboot/dashboard/backend/servicers.py @@ -73,6 +73,7 @@ async def Get( apis=self.state.apis, api_digests=self.state.api_digests, servicers=self.state.servicers, + agents=self.state.agents, generated=self.state.generated, needs_generate_reason=needs_generate_reason(self.state), features=self.state.features, @@ -150,6 +151,8 @@ async def UpdateCode( changed, newest last.""" del self.state.servicers[:] self.state.servicers.extend(request.servicers) + del self.state.agents[:] + self.state.agents.extend(request.agents) self.state.code_analysis_version = request.code_analysis_version self.state.code_files.clear() self.state.code_files.MergeFrom(request.code_files) diff --git a/tests/reboot/dashboard/code_watcher_tests.py b/tests/reboot/dashboard/code_watcher_tests.py index 1cb4a262a..95feab947 100644 --- a/tests/reboot/dashboard/code_watcher_tests.py +++ b/tests/reboot/dashboard/code_watcher_tests.py @@ -35,6 +35,7 @@ _try_extract_api_digest, _modified_at, _reconstitute_known, + extract_and_sort_agents, extract_and_sort_servicers, ) from reboot.dashboard.backend.main import application @@ -1319,9 +1320,58 @@ async def test_an_installed_helper_is_followed_and_recorded( }, ) + async def test_a_method_records_the_agent_it_runs(self) -> None: + """A run of an agent is recorded on the method that makes it, + naming the agent; the agent is recorded with what its + construction says it is.""" + servicer = self._write( + 'shop_servicer.py', + source=( + 'from reboot.agents.pydantic_ai import Agent\n' + 'from shop.v1.shop_rbt import Shop\n' + '\n' + '\n' + 'librarian = Agent(\n' + " 'anthropic:claude-sonnet-4-6',\n" + " name='librarian',\n" + " description='Keeps the shelves in order.',\n" + " system_prompt='You are the librarian.',\n" + ')\n' + '\n' + '\n' + 'class ShopServicer(Shop.Servicer):\n' + '\n' + ' async def look(self, context, request):\n' + " await librarian.run(context, 'Tidy up')\n" + ), + ) + application = self._write('main.py', source=APPLICATION) + + found = await self._analyze(application) + + [found_servicer] = found[servicer].servicers + [method] = found_servicer.methods + self.assertEqual( + [run.agent for run in method.runs], + ['librarian'], + ) + self.assertEqual(list(method.ambiguous), []) + + [agent] = found[servicer].agents + self.assertEqual(agent.name, 'librarian') + self.assertEqual(agent.filename, str(servicer)) + self.assertEqual(agent.model, 'anthropic:claude-sonnet-4-6') + self.assertEqual(list(agent.system_prompt), ['You are the librarian.']) + self.assertEqual(agent.description, 'Keeps the shelves in order.') + + # Everything about the run is analyzed, so there is nothing to + # say about it. + self.assertEqual(list(method.hazards), []) + [run] = method.runs + self.assertEqual(list(run.hazards), []) + async def test_every_way_of_running_an_agent(self) -> None: - """Each of `Agent`'s four entry points is a run, which is a - hazard of the method while which agent is run is not resolved.""" + """Each of `Agent`'s four entry points is a run.""" servicer = self._write( 'shop_servicer.py', source=( @@ -1353,16 +1403,51 @@ async def test_every_way_of_running_an_agent(self) -> None: [found_servicer] = found[servicer].servicers [method] = found_servicer.methods self.assertEqual( - sorted( - hazard.run_on_unresolved_agent.callee - for hazard in method.hazards + [run.agent for run in method.runs], + ['librarian'] * 4, + ) + + async def test_an_agent_constructed_in_another_file(self) -> None: + """An agent run in one file and constructed in another is + recorded by the file running it, saying where it is + constructed, since that is where the analysis read what it + is.""" + self._write( + 'agents.py', + source=( + 'from reboot.agents.pydantic_ai import Agent\n' + '\n' + '\n' + "librarian = Agent('test', name='librarian')\n" ), + ) + servicer = self._write( + 'shop_servicer.py', + source=( + 'from agents import librarian\n' + 'from shop.v1.shop_rbt import Shop\n' + '\n' + '\n' + 'class ShopServicer(Shop.Servicer):\n' + '\n' + ' async def look(self, context, request):\n' + " await librarian.run(context, 'Tidy up')\n" + ), + ) + application = self._write('main.py', source=APPLICATION) + + found = await self._analyze(application) + + [running] = found[servicer].agents + self.assertEqual(running.name, 'librarian') + self.assertEqual(running.filename, str(servicer)) + + self.assertEqual( [ - 'librarian.iter', - 'librarian.run', - 'librarian.run_stream', - 'librarian.run_stream_events', + (agent.name, agent.filename) + for agent in extract_and_sort_agents(found) ], + [('librarian', str(servicer))], ) def _hazards(self, hazards) -> list[tuple[str, dict[str, str]]]: @@ -1381,12 +1466,14 @@ def _hazards(self, hazards) -> list[tuple[str, dict[str, str]]]: ] async def test_a_run_whose_agent_cannot_be_resolved(self) -> None: - """A run whose own definition proves an agent is run -- made on - an agent a factory returns, an alias, a parameter, an element of - a collection -- is recorded as a hazard of the method, with - nothing about the run, rather than as a guess at which agent it - is. A run on something pyright cannot type at all is not known - to be a run, and stays an ambiguous call.""" + """A run whose own definition proves an agent is run, but made + on something that leads to no agent constructed at the top level + of a module -- an agent a factory returns, an alias, a + parameter, an element of a collection -- is recorded as a hazard + of the method, with nothing about the run, rather than as a + guess at which agent it is. A run on something pyright cannot + type at all is not known to be a run, and stays an ambiguous + call.""" servicer = self._write( 'shop_servicer.py', source=( @@ -1438,6 +1525,12 @@ async def test_a_run_whose_agent_cannot_be_resolved(self) -> None: [found_servicer] = found[servicer].servicers methods = {method.name: method for method in found_servicer.methods} + self.assertEqual( + { + name: list(method.runs) for name, method in methods.items() + }, + {name: [] for name in methods}, + ) self.assertEqual( { name: self._hazards(method.hazards) @@ -1473,6 +1566,137 @@ async def test_a_run_whose_agent_cannot_be_resolved(self) -> None: [str(servicer)], ) self.assertEqual(list(methods['untyped'].ambiguous), ['agent.run']) + self.assertEqual(found[servicer].agents, ()) + + async def test_what_changes_a_run_is_said(self) -> None: + """A `model` or `instructions` passed to a run changes what the + run is from what its agent says, and is said on the run.""" + servicer = self._write( + 'shop_servicer.py', + source=( + 'from reboot.agents.pydantic_ai import Agent\n' + 'from shop.v1.shop_rbt import Shop\n' + '\n' + '\n' + "librarian = Agent('test', name='librarian')\n" + '\n' + '\n' + 'class ShopServicer(Shop.Servicer):\n' + '\n' + ' async def look(self, context, request):\n' + ' await librarian.run(\n' + " context, 'a', model='other',\n" + " instructions='Be brief.',\n" + ' )\n' + ), + ) + application = self._write('main.py', source=APPLICATION) + + found = await self._analyze(application) + + [found_servicer] = found[servicer].servicers + [method] = found_servicer.methods + self.assertEqual( + [self._hazards(run.hazards) for run in method.runs], + [ + [ + ( + 'run_arguments', + { + 'model': "'other'", + 'instructions': "'Be brief.'", + }, + ), + ], + ], + ) + + async def test_only_a_named_agent_at_the_top_level_is_resolved( + self, + ) -> None: + """Only an `Agent(...)` with a string literal `name=`, bound to + a name at the top level of its module, is resolved. One whose + `name=` is computed or missing, one a factory builds, one held + on `self`, bound inside a function or constructed where it is + run, and a name bound twice, which could hold either agent, are + not: each run of one is a hazard of the method. Moving everything + down records the same agents.""" + source = ( + 'from reboot.agents.pydantic_ai import Agent\n' + 'from shop.v1.shop_rbt import Shop\n' + '\n' + '\n' + 'def make_agent():\n' + " return Agent('test', name='made')\n" + '\n' + '\n' + "NAME = 'computed'\n" + "helper = Agent('test', name='first')\n" + "helper = Agent('test', name='second')\n" + "tutor = Agent('test', name='tutor')\n" + 'computed = Agent(\'test\', name=NAME)\n' + "unnamed = Agent('test')\n" + "built = make_agent()\n" + '\n' + '\n' + 'class ShopServicer(Shop.Servicer):\n' + '\n' + ' def __init__(self):\n' + " self.agent = Agent('test', name='held')\n" + '\n' + ' async def look(self, context, request):\n' + " local = Agent('test', name='local')\n" + " await helper.run(context, 'Help')\n" + " await tutor.run(context, 'Teach')\n" + " await self.agent.run(context, 'Hold')\n" + " await local.run(context, 'Stay')\n" + " await computed.run(context, 'Count')\n" + " await unnamed.run(context, 'Wander')\n" + " await built.run(context, 'Build')\n" + " await Agent('test', name='inline').run(context, 'Go')\n" + ) + servicer = self._write('shop_servicer.py', source=source) + application = self._write('main.py', source=APPLICATION) + + found = await self._analyze(application) + + [found_servicer] = found[servicer].servicers + [method] = [ + method for method in found_servicer.methods + if method.name == 'look' + ] + self.assertEqual( + [run.agent for run in method.runs], + ['tutor'], + ) + self.assertEqual( + sorted( + hazard.run_on_unresolved_agent.callee + for hazard in method.hazards + ), + sorted( + [ + "Agent('test', name='inline').run", + 'built.run', + 'computed.run', + 'helper.run', + 'local.run', + 'self.agent.run', + 'unnamed.run', + ] + ), + ) + before = found[servicer].agents + self.assertEqual( + [agent.name for agent in before], + ['tutor'], + ) + + self._write('shop_servicer.py', source='\n\n# Moved.\n\n' + source) + + found = await self._analyze(application) + + self.assertEqual(found[servicer].agents, before) async def test_a_base_from_a_function_return_type_is_resolved( self, @@ -1655,8 +1879,8 @@ async def test_a_file_under_the_working_directory_stores_relative( async def test_reconstituting_keeps_stored_spellings(self) -> None: """What a previous run recorded comes back keyed by the - stored spelling, with the servicers recorded for each file - joined back on.""" + stored spelling, with the servicers and the agents recorded for + each file joined back on.""" state = DashboardState() file = state.code_files['backend/x.py'] file.digest = b'digest' @@ -1664,6 +1888,9 @@ async def test_reconstituting_keeps_stored_spellings(self) -> None: servicer = state.servicers.add() servicer.state_type = 'shop.v1.Shop' servicer.filename = 'backend/x.py' + agent = state.agents.add() + agent.name = 'librarian' + agent.filename = 'backend/x.py' known = _reconstitute_known(state) @@ -1678,6 +1905,10 @@ async def test_reconstituting_keeps_stored_spellings(self) -> None: [servicer.state_type for servicer in analyzed.servicers], ['shop.v1.Shop'], ) + self.assertEqual( + [agent.name for agent in analyzed.agents], + ['librarian'], + ) async def test_a_state_from_another_analysis_is_analyzed_again( self, From 18f615a7d606f3dbeeb40139453fb53fbe86397f Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Mon, 14 Sep 2026 21:35:01 +0000 Subject: [PATCH 06/11] Dashboard: read an agent adopted with `Agent.wrap` where it is made `Agent.wrap(...)` is not `Agent(...)`, so an adopted agent was not resolved, and every run of it was a hazard, even when the agent it adopts is constructed right there with a literal `name=`. `Agent.wrap(Agent(...))` is now read from the construction it wraps. An agent that wraps anything else, e.g. `Agent.wrap(existing)`, is still not resolved, and its runs still say so. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LRFggUcgVpLqhgb7h1cK6H --- rbt/dashboard/v1/dashboard.proto | 11 +++-- reboot/dashboard/backend/code_watcher.py | 19 ++++--- tests/reboot/dashboard/code_watcher_tests.py | 52 ++++++++++++++++++++ 3 files changed, 71 insertions(+), 11 deletions(-) diff --git a/rbt/dashboard/v1/dashboard.proto b/rbt/dashboard/v1/dashboard.proto index 19dd1cf51..83fa2b0e8 100644 --- a/rbt/dashboard/v1/dashboard.proto +++ b/rbt/dashboard/v1/dashboard.proto @@ -553,11 +553,12 @@ message Servicer { // own definition proves an agent is run, but what it is run on // is not an `Agent(...)` with a string literal `name=`, bound to // a name at the top level of a module. E.g. an agent whose - // `name=` is computed or missing, one a factory builds, an - // alias, a parameter, an element of a collection, one held on - // `self`, one constructed inside a function or where it is run, - // or a name bound to more than one thing. Nothing about the run - // is recorded, what it is given included. + // `name=` is computed or missing, one a factory builds or + // `Agent.wrap` adopts from one, an alias, a parameter, an + // element of a collection, one held on `self`, one constructed + // inside a function or where it is run, or a name bound to more + // than one thing. Nothing about the run is recorded, what it is + // given included. message RunOnUnresolvedAgent { // Represents what is called, spelled the way `ast.unparse` // writes it: the receiver and the method, without the call's diff --git a/reboot/dashboard/backend/code_watcher.py b/reboot/dashboard/backend/code_watcher.py index dd028da22..7e86cdbef 100644 --- a/reboot/dashboard/backend/code_watcher.py +++ b/reboot/dashboard/backend/code_watcher.py @@ -634,12 +634,19 @@ def _is_agent_construction(call: ast.Call) -> bool: def _agent_definition(value: ast.Call) -> Optional[AgentDefinition]: """Returns what the call an agent comes from says the agent is, and `None` when it is not an `Agent(...)` with a string literal - `name=`, which is the one agent the analysis can tell from every - other: an agent a factory builds, or one whose `name=` is computed - or missing, is not resolved.""" - if not _is_agent_construction(value): - return None - construction = value + `name=`, or an `Agent.wrap(...)` of one, which is the one agent the + analysis can tell from every other: an agent a factory builds, or + one whose `name=` is computed or missing, is not resolved.""" + match value: + case ast.Call( + func=ast.Attribute(attr='wrap'), + args=[ast.Call() as wrapped, *_], + ) if _is_agent_construction(wrapped): + construction = wrapped + case _ if _is_agent_construction(value): + construction = value + case _: + return None name = _try_constant_string(_keyword(construction, 'name')) if name is None: diff --git a/tests/reboot/dashboard/code_watcher_tests.py b/tests/reboot/dashboard/code_watcher_tests.py index 95feab947..3cccb1e33 100644 --- a/tests/reboot/dashboard/code_watcher_tests.py +++ b/tests/reboot/dashboard/code_watcher_tests.py @@ -279,6 +279,10 @@ def __init__( ): pass + @classmethod + def wrap(cls, wrapped, **kwargs) -> 'Agent': + return wrapped + async def run(self, context=None, user_prompt=None, **kwargs): pass @@ -1698,6 +1702,54 @@ async def test_only_a_named_agent_at_the_top_level_is_resolved( self.assertEqual(found[servicer].agents, before) + async def test_an_adopted_agent_is_read_where_it_is_made(self) -> None: + """An agent adopted with `Agent.wrap(...)` is what the call + it adopts was constructed with, since that is where its name + and its prompt are written. One that adopts anything else is not + resolved.""" + servicer = self._write( + 'shop_servicer.py', + source=( + 'import pydantic_ai\n' + 'from reboot.agents.pydantic_ai import Agent\n' + 'from shop.v1.shop_rbt import Shop\n' + '\n' + '\n' + 'librarian = Agent.wrap(\n' + ' Agent(\n' + " 'test',\n" + " name='librarian',\n" + " instructions='Tidy the shelves.',\n" + ' ),\n' + ')\n' + 'other = Agent.wrap(pydantic_ai)\n' + '\n' + '\n' + 'class ShopServicer(Shop.Servicer):\n' + '\n' + ' async def look(self, context, request):\n' + " await librarian.run(context, 'Tidy up')\n" + " await other.run(context, 'Carry on')\n" + ), + ) + application = self._write('main.py', source=APPLICATION) + + found = await self._analyze(application) + + [agent] = found[servicer].agents + self.assertEqual(agent.name, 'librarian') + self.assertEqual(list(agent.system_prompt), ['Tidy the shelves.']) + [found_servicer] = found[servicer].servicers + [method] = found_servicer.methods + self.assertEqual([run.agent for run in method.runs], ['librarian']) + self.assertEqual( + [ + hazard.run_on_unresolved_agent.callee + for hazard in method.hazards + ], + ['other.run'], + ) + async def test_a_base_from_a_function_return_type_is_resolved( self, ) -> None: From 2257a8fd044a2bbc78b39a4d0091a19934b5c840 Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Mon, 14 Sep 2026 21:35:07 +0000 Subject: [PATCH 07/11] Dashboard: find the tools an agent has, and what they do What an agent can do to the application is what its tools do, and nothing recorded them: every tool call back into a state type was invisible, even though those calls are Reboot calls like any other and run under the same workflow. A tool is now found by its definition too: a function whose decorator resolves to the `tool` or `tool_plain` a Reboot `Agent` registers one with, the one way of giving an agent a tool that can always be followed. It is recorded on the agent with the name and description the model is told, and its body is analyzed exactly as a servicer method's is: the Reboot calls it makes, the runs it makes, which is where one agent handing work to another comes from, and what it does that is not followed, in `Agent.Tool.method_hazards`. Two agents handing work to each other terminate. The file registering a tool records the agent too, wherever the agent is constructed. Every other way of giving an agent tools can be written in more shapes than can be followed reliably, so none is, and each is recorded as a hazard instead: - `Agent.Hazard.constructed_with`: `tools=`, `toolsets=` and `prepare_tools=` passed to its construction. - `Agent.Hazard.tool_registered_by_call`: a tool registered by calling `tool` rather than decorating with it. - `Agent.Tool.Hazard.prepared`: `prepare=`, which can hide the tool when the agent runs. - `UnattributedHazard.tool_on_unresolved_agent`: a tool registered on an agent that cannot be resolved, which has no agent to be recorded on, so the file records it. `builtin_tools=` is left out: those tools run in the model provider and cannot reach the application's state. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LRFggUcgVpLqhgb7h1cK6H --- rbt/dashboard/v1/dashboard.proto | 160 +++++++- reboot/dashboard/backend/code_watcher.py | 392 +++++++++++++++++-- reboot/dashboard/backend/servicers.py | 3 + tests/reboot/dashboard/code_watcher_tests.py | 316 ++++++++++++++- 4 files changed, 817 insertions(+), 54 deletions(-) diff --git a/rbt/dashboard/v1/dashboard.proto b/rbt/dashboard/v1/dashboard.proto index 83fa2b0e8..cfd4e3fa9 100644 --- a/rbt/dashboard/v1/dashboard.proto +++ b/rbt/dashboard/v1/dashboard.proto @@ -50,16 +50,21 @@ message Dashboard { repeated Servicer servicers = 6; // Represents the version of the analysis that recorded - // `servicers`, `agents` and `code_files`. A dashboard whose analysis - // records something new analyzes every file again rather than - // carrying forward what an earlier one recorded, which would never - // say it; see `CODE_ANALYSIS_VERSION`. + // `servicers`, `agents`, `unattributed_hazards` and `code_files`. + // A dashboard whose analysis records something new analyzes every + // file again rather than carrying forward what an earlier one + // recorded, which would never say it; see `CODE_ANALYSIS_VERSION`. uint32 code_analysis_version = 16; // Every agent found in the developer's application, sorted. One // agent has a record per file whose analysis found it; see `Agent`. repeated Agent agents = 17; + // Every hazard the analysis of the developer's application recorded + // that belongs to no agent, run, tool or method, sorted; see + // `UnattributedHazard`. + repeated UnattributedHazard unattributed_hazards = 18; + // Represents every file the walk of the application analyzed, // keyed by its path, relative to the working directory for a file // under it and absolute otherwise, as of the last write. A @@ -131,6 +136,7 @@ message DashboardGetResponse { map api_digests = 5; repeated Servicer servicers = 6; repeated Agent agents = 16; + repeated UnattributedHazard unattributed_hazards = 17; map generated = 7; // The worst reason over all the API files, and absent when @@ -171,6 +177,7 @@ message DashboardUpdateCodeRequest { repeated Servicer servicers = 1; uint32 code_analysis_version = 6; repeated Agent agents = 7; + repeated UnattributedHazard unattributed_hazards = 8; map code_files = 2; map generated = 3; @@ -594,22 +601,63 @@ message Servicer { } // One agent the developer's application uses, as one file's analysis -// found it: a Reboot `Agent` some method runs, constructed with a -// string literal `name=` and bound to a name at the top level of a -// module. +// found it: a Reboot `Agent` constructed with a string literal `name=` +// and bound to a name at the top level of a module, which some method +// runs, or some function is decorated as a tool of. // // A record per file, because the analysis keys everything by the // file it read, and an agent is met in more files than the one -// constructing it: a workflow in one file runs an agent constructed -// in another. Every record says the same thing about the agent, since -// each is built from the construction the analysis resolved. Whoever -// reads these joins them on `name`, which the runtime requires to be -// unique. +// constructing it: a workflow runs it in one file, a tool is +// decorated on it in another. Every record says the same thing about +// the agent itself, since each is built from the construction the +// analysis resolved; what differs is the tools and the hazards each +// file contributed. Whoever reads these joins them on `name`, which +// the runtime requires to be unique. message Agent { + // One thing about how an agent is constructed or given tools that + // the analysis did not follow, recorded rather than guessed at or + // dropped, so the dashboard can show where what it draws may be + // incomplete. + message Hazard { + // The agent is constructed with arguments that give it tools, or + // hide them, which are not followed. Each represents the value + // passed, spelled the way `ast.unparse` writes it, and is absent + // when it is not passed. + message ConstructedWith { + // `tools=`. + optional string tools = 1; + + // `toolsets=`. + optional string toolsets = 2; + + // `prepare_tools=`, which can hide any of its tools when it + // runs. + optional string prepare_tools = 3; + } + + // A tool is registered on the agent by calling `tool` or + // `tool_plain` rather than decorating with it, e.g. + // `librarian.tool(lookup)`, which is not followed. + message ToolRegisteredByCall { + // Represents the call, its arguments included, spelled the way + // `ast.unparse` writes it. + string call = 1; + } + + // Represents the file the hazard is written in, spelled the way + // the developer would open it. + string filename = 1; + + oneof hazard { + ConstructedWith constructed_with = 2; + ToolRegisteredByCall tool_registered_by_call = 3; + } + } + // One run of an agent, made by a servicer method's implementation - // through any of `Agent`'s entry points: `run`, `iter`, `run_stream` - // or `run_stream_events`. Which one says nothing about what the - // application does, so it is not recorded. + // or by another agent's tool, through any of `Agent`'s entry points: + // `run`, `iter`, `run_stream` or `run_stream_events`. Which one says + // nothing about what the application does, so it is not recorded. message Run { // One thing a run is given on top of its agent that the analysis // did not follow, recorded rather than guessed at or dropped. @@ -646,6 +694,53 @@ message Agent { repeated Hazard hazards = 2; } + // One tool an agent may call, and what analyzing its + // implementation found. + message Tool { + // One thing about how a tool is registered that the analysis did + // not follow, recorded rather than guessed at or dropped. + message Hazard { + // The tool is registered with `prepare=`, which can hide it + // when the agent runs. + message Prepared { + // Represents the value passed, spelled the way `ast.unparse` + // writes it. + string prepare = 1; + } + + // Represents the file the hazard is written in, spelled the way + // the developer would open it. + string filename = 1; + + oneof hazard { + Prepared prepared = 2; + } + } + + // Represents the tool's name as the model is told it: the + // `name` given where it was registered, and the function's own + // name otherwise. + string name = 1; + + // Represents what the model is told the tool does: the + // `description` given where it was registered, and the + // function's docstring otherwise. Absent when it has neither. + optional string description = 2; + + // Represents every Reboot call, agent run, and ambiguous call the + // tool's implementation makes, and everything it does that the + // analysis did not follow, found exactly the way a servicer + // method's are. + repeated Servicer.Method.Call calls = 3; + repeated Run runs = 4; + repeated string ambiguous = 5; + repeated Servicer.Method.Hazard method_hazards = 7; + + // Represents everything about how the tool is registered that the + // analysis did not follow. + repeated Hazard hazards = 6; + } + // Represents the agent's `name`, which the runtime requires and // which is unique in an application, so it is what a run names and // what records of one agent join on. @@ -672,6 +767,41 @@ message Agent { // agent handing work to this one is told it does. Absent when it // has none the analysis can read. optional string description = 5; + + // Represents every tool this file's analysis found the agent has, + // in the order it met them: each function decorated as one with + // `@agent.tool` or `@agent.tool_plain`, the one way of giving an + // agent a tool that can always be followed. + repeated Tool tools = 6; + + // Represents everything about how the agent is constructed and + // given tools that the analysis did not follow. What a single run + // is given is on the run; see `Run.hazards`. + repeated Hazard hazards = 7; +} + +// One thing the analysis did not follow that belongs to no agent, run, +// tool or method, recorded rather than dropped, for the file it is +// written in. +message UnattributedHazard { + // A tool is registered, decorated or by a call, on an agent that + // cannot be resolved, e.g. `@agents[0].tool`, so there is no agent + // to record it on. + message ToolOnUnresolvedAgent { + // Represents the registration, spelled the way `ast.unparse` + // writes it: the decorator without its arguments, e.g. + // `agents[0].tool`, or the whole call registering by calling, + // e.g. `agents[0].tool_plain(count)`. + string registration = 1; + } + + // Represents the file the hazard is written in, spelled the way the + // developer would open it. + string filename = 1; + + oneof hazard { + ToolOnUnresolvedAgent tool_on_unresolved_agent = 2; + } } // What analyzing one of the developer's files used: the digest diff --git a/reboot/dashboard/backend/code_watcher.py b/reboot/dashboard/backend/code_watcher.py index 7e86cdbef..8e6de50c0 100644 --- a/reboot/dashboard/backend/code_watcher.py +++ b/reboot/dashboard/backend/code_watcher.py @@ -33,10 +33,15 @@ class extends: a servicer is a class with a base whose type is which agent is run is what the receiver of that call resolves to: an `Agent(...)` with a string literal `name=`, bound to a name at the top level of a module, which says the agent's name, its model and -the prompt it is given. Whatever is not followed -- a run of any -other agent, what a run is given on top of its agent -- is recorded -as a hazard instead, so that nothing the analysis could not see is -silently missing. +the prompt it is given. An agent's tools are found by their +definitions rather than their spellings too: a function whose +decorator resolves to the `tool` a Reboot `Agent` registers one +with. Each tool's body is analyzed exactly as a servicer method's +is, so what the model can reach through an agent is recorded beside +what the application calls itself. Whatever is not followed -- a run +of any other agent, tools given any other way, what a run is given +on top of its agent -- is recorded as a hazard instead, so that +nothing the analysis could not see is silently missing. Where following stops is what makes this the developer's code rather than somebody else's. A module resolves to a file only if a root @@ -65,7 +70,12 @@ class extends: a servicer is a class with a base whose type is from pathlib import Path from rbt.dashboard.v1.dashboard_pb2 import Agent, Change from rbt.dashboard.v1.dashboard_pb2 import Dashboard as DashboardState -from rbt.dashboard.v1.dashboard_pb2 import File, Generated, Servicer +from rbt.dashboard.v1.dashboard_pb2 import ( + File, + Generated, + Servicer, + UnattributedHazard, +) from rbt.dashboard.v1.dashboard_rbt import Dashboard from reboot.aio.contexts import WorkflowContext from reboot.aio.cooperatively import cooperatively @@ -87,7 +97,7 @@ class extends: a servicer is a class with a base whose type is _walk, ) from types import MappingProxyType -from typing import Mapping, Optional, Sequence +from typing import Mapping, MutableSequence, Optional, Sequence # One Reboot call a method's implementation makes: which state type, # which method, and how the call is reached. Aliased from where it @@ -108,10 +118,17 @@ class extends: a servicer is a class with a base whose type is # always async. RUN_NAMES = ('run', 'iter', 'run_stream', 'run_stream_events') +# The methods a tool is registered on an agent with, decorating with +# `@agent.tool` or calling `agent.tool(...)`, which is what the +# registration's own definition lands on. +TOOL_REGISTRATION_NAMES = ('tool', 'tool_plain') + # The keyword arguments a run may be given that change what its agent -# is or can reach, which the analysis does not follow; see -# `Agent.Run.Hazard.RunArguments`. +# is or can reach, and those an agent may be constructed with that +# give it tools or hide them, which the analysis does not follow; see +# `Agent.Run.Hazard.RunArguments` and `Agent.Hazard.ConstructedWith`. RUN_ARGUMENTS = ('toolsets', 'model', 'instructions') +CONSTRUCTION_ARGUMENTS = ('tools', 'toolsets', 'prepare_tools') # The version of what the analysis records, which the dashboard's # state records beside it. Counted up whenever the analysis starts @@ -158,11 +175,16 @@ class AnalyzedFile: # services and the calls each method makes. servicers: tuple[Servicer, ...] - # Every agent this file's analysis found it runs. An agent met in + # Every agent this file's analysis found: one it runs, and one a + # function it holds is decorated as a tool of. An agent met in # more than one file has a record in each, which whoever reads # them joins; see `Agent`. agents: tuple[Agent, ...] + # Every hazard this file's analysis recorded that belongs to no + # agent, run, tool or method; see `UnattributedHazard`. + unattributed_hazards: tuple[UnattributedHazard, ...] + def _position_at_last_character( node: ast.Name | ast.Attribute, @@ -560,6 +582,16 @@ def _passed(call: ast.Call, keywords: Sequence[str]) -> dict[str, str]: } +def _add_hazard( + hazards: MutableSequence[Agent.Hazard], + hazard: Agent.Hazard, +) -> None: + """Adds a hazard once: the same thing written in two places a + file's analysis reaches is still one thing to show.""" + if hazard not in hazards: + hazards.append(hazard) + + def _try_constant_string(node: Optional[ast.expr]) -> Optional[str]: """Returns the string an expression is, and `None` for an expression that is anything else: a name, an f-string, a call. @@ -599,8 +631,9 @@ class AgentDefinition: """A line an agent comes from, e.g. `librarian = Agent('anthropic:claude-sonnet-4-6', name='librarian')`. - What a run of an agent leads to: the agent it is made on, with - everything the line says about it. + What a run of an agent, and a tool registered on one, lead to: + the agent both are made on, with everything the line says about + it, and what it does not. """ # The `name` the agent is constructed with, which tells it from @@ -613,6 +646,10 @@ class AgentDefinition: system_prompt: tuple[str, ...] description: Optional[str] + # Everything the line says about the agent that is not followed; + # see `Agent.hazards`. + hazards: tuple[Agent.Hazard, ...] + def _is_agent_construction(call: ast.Call) -> bool: """Returns whether a call is written as constructing an agent, @@ -621,7 +658,8 @@ def _is_agent_construction(call: ast.Call) -> bool: what the factory is given rather than what the agent is. Read from how the call is spelled, since it is asked only of a - line a run already proved is an agent: an `Agent` imported under + line a run or a registration already proved is an agent: an + `Agent` imported under another name is not resolved, and says so, rather than being read as something it is not. """ @@ -631,7 +669,11 @@ def _is_agent_construction(call: ast.Call) -> bool: return False -def _agent_definition(value: ast.Call) -> Optional[AgentDefinition]: +def _agent_definition( + value: ast.Call, + *, + filename: Path, +) -> Optional[AgentDefinition]: """Returns what the call an agent comes from says the agent is, and `None` when it is not an `Agent(...)` with a string literal `name=`, or an `Agent.wrap(...)` of one, which is the one agent the @@ -652,6 +694,16 @@ def _agent_definition(value: ast.Call) -> Optional[AgentDefinition]: if name is None: return None + hazards: list[Agent.Hazard] = [] + given = _passed(construction, CONSTRUCTION_ARGUMENTS) + if len(given) > 0: + hazards.append( + Agent.Hazard( + filename=str(filename), + constructed_with=Agent.Hazard.ConstructedWith(**given), + ) + ) + # The model is the first argument, wherever it is passed: # `Agent('anthropic:claude-sonnet-4-6')` or `Agent(model=...)`. model = _keyword(construction, 'model') @@ -668,10 +720,14 @@ def _agent_definition(value: ast.Call) -> Optional[AgentDefinition]: description=_try_constant_string( _keyword(construction, 'description') ), + hazards=tuple(hazards), ) -def _agent_definitions(parse: Parse) -> Mapping[int, AgentDefinition]: +def _agent_definitions( + filename: Path, + parse: Parse, +) -> Mapping[int, AgentDefinition]: """Returns what each line at the top level of a module constructs an agent as, by the line the assignment is written on, which is the line pyright places the definition of the name it binds at. @@ -683,7 +739,7 @@ def _agent_definitions(parse: Parse) -> Mapping[int, AgentDefinition]: Every top-level assignment of a call is indexed, because what makes one of them an agent is the type of what it binds, which - only the run that led here has established. A name bound more than + only the run or the registration that led here has established. A name bound more than once at the top level is not indexed at all: pyright places every use of it at the first binding, whichever one the use holds, so reading any of them risks recording a run of one agent as a run of @@ -711,7 +767,7 @@ def _agent_definitions(parse: Parse) -> Mapping[int, AgentDefinition]: target=ast.Name(id=str(bound_to)), value=ast.Call() as value, ): - definition = _agent_definition(value) + definition = _agent_definition(value, filename=filename) if definition is not None: definitions[statement.lineno] = (bound_to, definition) @@ -959,7 +1015,7 @@ async def _agent_file( parse, external = await analysis._parse(filename) agent = AgentFile( definitions=( - _agent_definitions(parse) + _agent_definitions(filename, parse) if parse is not None else MappingProxyType({}) ), external=external, @@ -1013,7 +1069,8 @@ async def _generated_definition_at( @dataclass(frozen=True, kw_only=True) class Implementation: """What analyzing one function's body found, which is what a - servicer method records of what it does.""" + servicer method and an agent's tool each record of what they + do.""" calls: tuple[Call, ...] runs: tuple[Agent.Run, ...] @@ -1142,8 +1199,10 @@ async def _analyze_function( ) if entry_point_name not in RUN_NAMES: - # Reboot's own machinery around a run, such as a - # construction. Followed no further than the + # Reboot's own machinery around a run: the `tool` a + # tool is registered with, which the walk over the + # file's registrations finds wherever it is written, + # or a construction. Followed no further than the # generator's machinery is. continue @@ -1226,7 +1285,8 @@ async def _agent_at( analysis: Analysis, ) -> tuple[Optional[AgentDefinition], Analysis]: """Returns the agent a call is made on, from the expression the - call is made through, e.g. `librarian.run(...)`. + call is made through: `librarian.run(...)`, or + `@librarian.tool`. What the receiver refers to, pyright answers, resolving however the agent gets there: a name in this file, one imported from @@ -1269,7 +1329,8 @@ def _agent_record( the file finds the agent. `agents` is every agent the file has found so far, by name, so - that an agent run in several of the file's methods is one record. + that an agent run in several of the file's methods, and registered + a tool on besides, is one record. """ found = agents.get(definition.name) if found is not None: @@ -1280,11 +1341,246 @@ def _agent_record( model=definition.model, system_prompt=definition.system_prompt, description=definition.description, + hazards=definition.hazards, ) agents[definition.name] = agent return agent +async def _registration_at( + registration: ast.Attribute, + *, + filename: Path, + text: str, + analysis: Analysis, +) -> tuple[bool, Optional[AgentDefinition], Analysis]: + """Returns whether an expression spelled `.tool` or + `.tool_plain` is the `tool` or `tool_plain` a Reboot `Agent` + registers a tool with, by its own definition, and the agent it is + made on when it is, `None` when that agent cannot be resolved.""" + line, character = _position_at_last_character(registration) + location = await analysis.pyright.definition_at( + filename=filename, + line=line, + character=character, + text=text, + ) + if location is None or not _is_agent_module(location.filename): + return False, None, analysis + + registered, analysis = await analysis.helper_definition_at(location) + if ( + registered is None or + registered.syntax.name not in TOOL_REGISTRATION_NAMES + ): + return False, None, analysis + + definition, analysis = await _agent_at( + registration, + filename=filename, + text=text, + analysis=analysis, + ) + return True, definition, analysis + + +async def _analyze_decorated_tool( + function: ast.FunctionDef | ast.AsyncFunctionDef, + decorator: ast.expr, + *, + filename: Path, + text: str, + analysis: Analysis, + agents: dict[str, Agent], + unattributed: list[UnattributedHazard], +) -> Analysis: + """Adds a function to the agent a decorator registers it as a tool + of, analyzed the way a servicer method is, when the decorator is + one; and returns the analysis. + + Both forms are read, the bare `@librarian.tool` and the + parametrized `@librarian.tool(retries=2)`, whose `name` and + `description`, if it gives them, are what the model is told, and + whose `prepare`, which can hide the tool when the agent runs, is + said on the tool. A tool decorated on an agent that cannot be + resolved has no agent to be added to, and is said in + `unattributed`. + """ + match decorator: + case ast.Call( + func=ast.Attribute(attr=str(attribute)) as registration, + ) if attribute in TOOL_REGISTRATION_NAMES: + given: Optional[ast.Call] = decorator + case ast.Attribute( + attr=str(attribute), + ) as registration if attribute in TOOL_REGISTRATION_NAMES: + given = None + case _: + return analysis + + proven, definition, analysis = await _registration_at( + registration, + filename=filename, + text=text, + analysis=analysis, + ) + if not proven: + return analysis + + if definition is None: + unattributed.append( + UnattributedHazard( + filename=str(filename), + tool_on_unresolved_agent=( + UnattributedHazard.ToolOnUnresolvedAgent( + registration=ast.unparse(registration), + ) + ), + ) + ) + return analysis + + agent = _agent_record(definition, agents=agents) + + name = ( + None + if given is None else _try_constant_string(_keyword(given, 'name')) + ) or function.name + if any(tool.name == name for tool in agent.tools): + return analysis + + hazards: list[Agent.Tool.Hazard] = [] + if given is not None and _passes(given, 'prepare'): + prepare = _keyword(given, 'prepare') + assert prepare is not None + hazards.append( + Agent.Tool.Hazard( + filename=str(filename), + prepared=Agent.Tool.Hazard.Prepared( + prepare=ast.unparse(prepare) + ), + ) + ) + + implementation, analysis = await _analyze_function( + function, + filename=filename, + text=text, + analysis=analysis, + visited=frozenset(), + agents=agents, + ) + + agent.tools.append( + Agent.Tool( + name=name, + description=( + ( + None if given is None else + _try_constant_string(_keyword(given, 'description')) + ) or ast.get_docstring(function) or None + ), + calls=implementation.calls, + runs=implementation.runs, + ambiguous=implementation.ambiguous, + method_hazards=implementation.hazards, + hazards=hazards, + ) + ) + + return analysis + + +async def _analyze_tool_registrations( + syntax: ast.Module, + *, + filename: Path, + text: str, + analysis: Analysis, + agents: dict[str, Agent], + unattributed: list[UnattributedHazard], +) -> Analysis: + """Adds to each agent the tools a file registers on it, and says + what of that is not followed; and returns the analysis. + + A registration is an expression whose own definition is the `tool` + or `tool_plain` a Reboot `Agent` registers a tool with, wherever + it is written and whichever file the agent is constructed in. A + function decorated with one is followed; see + `_analyze_decorated_tool`. A tool registered by calling one + instead, `librarian.tool(lookup)`, is not, and is said on the + agent, or in `unattributed` when the agent cannot be resolved. + + Asked about only where a registration is spelled the way one has + to be, `.tool` or `.tool_plain`, so that the + decorators and calls every file is full of cost nothing. + """ + # A decorator written as a call, `@librarian.tool(retries=2)`, is + # a call too, and is read as a decorator, not as a call. + decorators = { + id(decorator) for node in ast.walk(syntax) + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + for decorator in node.decorator_list + } + + for node in ast.walk(syntax): + match node: + case ast.FunctionDef() | ast.AsyncFunctionDef(): + for decorator in node.decorator_list: + analysis = await _analyze_decorated_tool( + node, + decorator, + filename=filename, + text=text, + analysis=analysis, + agents=agents, + unattributed=unattributed, + ) + + case ast.Call( + func=ast.Attribute(attr=str(attribute)) as registration, + ) if ( + attribute in TOOL_REGISTRATION_NAMES and + id(node) not in decorators + ): + proven, definition, analysis = await _registration_at( + registration, + filename=filename, + text=text, + analysis=analysis, + ) + if not proven: + continue + + if definition is None: + unattributed.append( + UnattributedHazard( + filename=str(filename), + tool_on_unresolved_agent=( + UnattributedHazard.ToolOnUnresolvedAgent( + registration=ast.unparse(node), + ) + ), + ) + ) + continue + + agent = _agent_record(definition, agents=agents) + _add_hazard( + agent.hazards, + Agent.Hazard( + filename=str(filename), + tool_registered_by_call=( + Agent.Hazard.ToolRegisteredByCall( + call=ast.unparse(node), + ) + ), + ), + ) + + return analysis + + async def _analyze_class( class_definition: ast.ClassDef, *, @@ -1381,7 +1677,7 @@ async def _analyze_file( """Returns one parsed file analyzed: an `AnalyzedFile` built from its `ParsedFile` -- the dependencies the parse recorded -- with the external files the analysis read, every servicer the - file defines, and every agent it runs.""" + file defines, and every agent it runs or decorates a tool on.""" parsed = analysis.parsed[filename] # A generated file defines no servicer and runs no agent, the @@ -1395,6 +1691,7 @@ async def _analyze_file( external=(), servicers=(), agents=(), + unattributed_hazards=(), ), analysis # Emptied so that what gathers in `external` below is what @@ -1404,10 +1701,15 @@ async def _analyze_file( servicers: list[Servicer] = [] # Every agent this file finds, by name, gathered as the servicers - # are walked, so that an agent run in two of the file's methods is - # one record. + # and then the registrations are walked, so that an agent run in + # two of the file's methods and registered a tool on besides is one + # record. agents: dict[str, Agent] = {} + # Every hazard this file's analysis records that belongs to no + # agent, run, tool or method. + unattributed: list[UnattributedHazard] = [] + for node in ast.walk(parsed.syntax): match node: case ast.ClassDef(): @@ -1420,6 +1722,15 @@ async def _analyze_file( if servicer is not None: servicers.append(servicer) + analysis = await _analyze_tool_registrations( + parsed.syntax, + filename=filename, + text=parsed.text, + analysis=analysis, + agents=agents, + unattributed=unattributed, + ) + # Said once, here, because it is this file's analysis that found # every one of them, wherever each is constructed. for agent in agents.values(): @@ -1440,6 +1751,7 @@ async def _analyze_file( ), servicers=tuple(servicers), agents=tuple(agents.values()), + unattributed_hazards=tuple(unattributed), ), analysis @@ -1467,8 +1779,10 @@ def extract_and_sort_agents( analysis found it. Records of one name are one agent, which more than one file - running it found, every record saying the same thing about it. - See `Agent`. + found: the file constructing it, the file running it, the file + registering a tool on it. Every record says the same thing about + the agent itself; what differs is the tools and the hazards each + file contributed, which whoever reads them joins. See `Agent`. """ return sorted( (agent for file in files.values() for agent in file.agents), @@ -1476,6 +1790,21 @@ def extract_and_sort_agents( ) +def extract_and_sort_unattributed_hazards( + files: Mapping[Path, AnalyzedFile], +) -> list[UnattributedHazard]: + """Returns every hazard found that belongs to no agent, run, tool + or method, sorted by the file it is written in, and within a file + in the order the file's analysis met them.""" + return sorted( + ( + hazard for file in files.values() + for hazard in file.unattributed_hazards + ), + key=lambda hazard: hazard.filename, + ) + + def _reconstitute_known( state: DashboardState, ) -> dict[Path, AnalyzedFile]: @@ -1492,6 +1821,10 @@ def _reconstitute_known( for agent in state.agents: agents.setdefault(agent.filename, []).append(agent) + unattributed: dict[str, list[UnattributedHazard]] = {} + for hazard in state.unattributed_hazards: + unattributed.setdefault(hazard.filename, []).append(hazard) + return { Path(filename): AnalyzedFile( @@ -1501,6 +1834,7 @@ def _reconstitute_known( external=tuple(file.external), servicers=tuple(servicers.get(filename, [])), agents=tuple(agents.get(filename, [])), + unattributed_hazards=tuple(unattributed.get(filename, [])), ) for filename, file in state.code_files.items() } @@ -1805,6 +2139,9 @@ async def watch( if known_now is not None: servicers = extract_and_sort_servicers(known_now) agents = extract_and_sort_agents(known_now) + unattributed_hazards = ( + extract_and_sort_unattributed_hazards(known_now) + ) # The file messages the write below records, # built before the write so that the state is @@ -1825,6 +2162,7 @@ async def watch( context, servicers=servicers, agents=agents, + unattributed_hazards=unattributed_hazards, code_analysis_version=CODE_ANALYSIS_VERSION, code_files=files, generated=dict(generated_now), diff --git a/reboot/dashboard/backend/servicers.py b/reboot/dashboard/backend/servicers.py index 01e2b1237..889df0743 100644 --- a/reboot/dashboard/backend/servicers.py +++ b/reboot/dashboard/backend/servicers.py @@ -74,6 +74,7 @@ async def Get( api_digests=self.state.api_digests, servicers=self.state.servicers, agents=self.state.agents, + unattributed_hazards=self.state.unattributed_hazards, generated=self.state.generated, needs_generate_reason=needs_generate_reason(self.state), features=self.state.features, @@ -153,6 +154,8 @@ async def UpdateCode( self.state.servicers.extend(request.servicers) del self.state.agents[:] self.state.agents.extend(request.agents) + del self.state.unattributed_hazards[:] + self.state.unattributed_hazards.extend(request.unattributed_hazards) self.state.code_analysis_version = request.code_analysis_version self.state.code_files.clear() self.state.code_files.MergeFrom(request.code_files) diff --git a/tests/reboot/dashboard/code_watcher_tests.py b/tests/reboot/dashboard/code_watcher_tests.py index 3cccb1e33..3828b2236 100644 --- a/tests/reboot/dashboard/code_watcher_tests.py +++ b/tests/reboot/dashboard/code_watcher_tests.py @@ -261,8 +261,9 @@ async def main(): ''' # The shape of Reboot's own `Agent`, as far as the analysis needs: -# the entry points a run is made through, and the construction whose -# arguments say what the agent is. Written where an installed `reboot` would be, since the +# the entry points a run is made through, the decorators a tool is +# registered with, and the construction whose arguments say what the +# agent is. Written where an installed `reboot` would be, since the # analysis recognizes the module by its path. AGENTS_MODULE = ''' class Agent: @@ -275,6 +276,8 @@ def __init__( system_prompt=(), instructions=None, description=None, + tools=(), + toolsets=None, **kwargs, ): pass @@ -283,6 +286,12 @@ def __init__( def wrap(cls, wrapped, **kwargs) -> 'Agent': return wrapped + def tool(self, function=None, /, **kwargs): + return function + + def tool_plain(self, function=None, /, **kwargs): + return function + async def run(self, context=None, user_prompt=None, **kwargs): pass @@ -301,16 +310,27 @@ async def run_stream_events( pass ''' +# What an agent's tools may be wrapped in where they are given, which +# the analysis reads without asking what either is. +PYDANTIC_AI_MODULE = ''' +class FunctionToolset: + + def __init__(self, tools=(), **kwargs): + pass +''' + def _write_agents_module(directory: Path) -> None: """Writes an installed `reboot` holding the agents module the - analysis recognizes.""" + analysis recognizes, and the `pydantic_ai` an agent's tools are + given through.""" package = directory / 'reboot' / 'agents' / 'pydantic_ai' package.mkdir(parents=True, exist_ok=True) (directory / 'reboot' / '__init__.py').write_text('') (directory / 'reboot' / 'agents' / '__init__.py').write_text('') (package / '_agent.py').write_text(AGENTS_MODULE) (package / '__init__.py').write_text('from ._agent import Agent\n') + (directory / 'pydantic_ai.py').write_text(PYDANTIC_AI_MODULE) class ImplementationWatcherTest(unittest.IsolatedAsyncioTestCase): @@ -1327,11 +1347,14 @@ async def test_an_installed_helper_is_followed_and_recorded( async def test_a_method_records_the_agent_it_runs(self) -> None: """A run of an agent is recorded on the method that makes it, naming the agent; the agent is recorded with what its - construction says it is.""" + construction says it is, and with the tools decorated on it, + each analyzed for the Reboot calls it makes the way a + servicer method is.""" servicer = self._write( 'shop_servicer.py', source=( 'from reboot.agents.pydantic_ai import Agent\n' + 'from shop.v1.depot_rbt import Depot\n' 'from shop.v1.shop_rbt import Shop\n' '\n' '\n' @@ -1343,6 +1366,12 @@ async def test_a_method_records_the_agent_it_runs(self) -> None: ')\n' '\n' '\n' + '@librarian.tool\n' + 'async def look_up(context, run_context, item):\n' + ' """Reads the depot."""\n' + " return await Depot.ref('d').look(context)\n" + '\n' + '\n' 'class ShopServicer(Shop.Servicer):\n' '\n' ' async def look(self, context, request):\n' @@ -1374,6 +1403,14 @@ async def test_a_method_records_the_agent_it_runs(self) -> None: [run] = method.runs self.assertEqual(list(run.hazards), []) + [tool] = agent.tools + self.assertEqual(tool.name, 'look_up') + self.assertEqual(tool.description, 'Reads the depot.') + self.assertEqual( + [(call.state_type, call.method, call.how) for call in tool.calls], + [('shop.v1.Depot', 'look', Servicer.Method.Call.How.CALL)], + ) + async def test_every_way_of_running_an_agent(self) -> None: """Each of `Agent`'s four entry points is a run.""" servicer = self._write( @@ -1411,18 +1448,100 @@ async def test_every_way_of_running_an_agent(self) -> None: ['librarian'] * 4, ) + async def test_tools_and_toolsets_given_are_said_not_followed( + self, + ) -> None: + """A `tools` or a `toolsets` given where an agent is constructed, + or a `toolsets` given where it is run, can be written in more + shapes than can be followed reliably, so neither is followed, + even when it names a function that could be: the agent, or the + run, says it may have tools the analysis has not seen.""" + servicer = self._write( + 'shop_servicer.py', + source=( + 'from pydantic_ai import FunctionToolset\n' + 'from reboot.agents.pydantic_ai import Agent\n' + 'from shop.v1.depot_rbt import Depot\n' + 'from shop.v1.shop_rbt import Shop\n' + '\n' + '\n' + 'async def restock(context):\n' + " await Depot.ref('d').look(context)\n" + '\n' + '\n' + 'librarian = Agent(\n' + " name='librarian',\n" + ' tools=[restock],\n' + ' toolsets=[FunctionToolset(tools=[restock])],\n' + ')\n' + '\n' + '\n' + 'class ShopServicer(Shop.Servicer):\n' + '\n' + ' async def look(self, context, request):\n' + ' await librarian.run(\n' + ' context,\n' + " 'Tidy up',\n" + ' toolsets=[FunctionToolset(tools=[restock])],\n' + ' )\n' + ), + ) + application = self._write('main.py', source=APPLICATION) + + found = await self._analyze(application) + + [agent] = found[servicer].agents + self.assertEqual(list(agent.tools), []) + self.assertEqual( + self._hazards(agent.hazards), + [ + ( + 'constructed_with', + { + 'tools': '[restock]', + 'toolsets': '[FunctionToolset(tools=[restock])]', + }, + ), + ], + ) + self.assertEqual( + [hazard.filename for hazard in agent.hazards], + [str(servicer)], + ) + + [found_servicer] = found[servicer].servicers + [method] = found_servicer.methods + [run] = method.runs + self.assertEqual( + self._hazards(run.hazards), + [ + ( + 'run_arguments', + { + 'toolsets': '[FunctionToolset(tools=[restock])]' + }, + ), + ], + ) + async def test_an_agent_constructed_in_another_file(self) -> None: - """An agent run in one file and constructed in another is - recorded by the file running it, saying where it is - constructed, since that is where the analysis read what it - is.""" - self._write( + """An agent is the same agent wherever it is met: the file + constructing it records it with the tools decorated there, + and the file running it records it too, so that whoever joins + them on the name has both.""" + agents = self._write( 'agents.py', source=( 'from reboot.agents.pydantic_ai import Agent\n' + 'from shop.v1.depot_rbt import Depot\n' '\n' '\n' "librarian = Agent('test', name='librarian')\n" + '\n' + '\n' + '@librarian.tool_plain(name=\'lookup\')\n' + 'async def look_up(context):\n' + " await Depot.ref('d').look(context)\n" ), ) servicer = self._write( @@ -1442,18 +1561,72 @@ async def test_an_agent_constructed_in_another_file(self) -> None: found = await self._analyze(application) + [constructing] = found[agents].agents + self.assertEqual(constructing.filename, str(agents)) + self.assertEqual( + [tool.name for tool in constructing.tools], + ['lookup'], + ) + [running] = found[servicer].agents self.assertEqual(running.name, 'librarian') self.assertEqual(running.filename, str(servicer)) + self.assertEqual(list(running.tools), []) self.assertEqual( [ (agent.name, agent.filename) for agent in extract_and_sort_agents(found) ], - [('librarian', str(servicer))], + [('librarian', str(agents)), ('librarian', str(servicer))], ) + async def test_an_agent_whose_tool_runs_another_agent(self) -> None: + """A run made by a tool is the tool's, so an agent handing + work to another is an edge like any other; two agents handing + work to each other are followed once.""" + servicer = self._write( + 'shop_servicer.py', + source=( + 'from reboot.agents.pydantic_ai import Agent\n' + 'from shop.v1.shop_rbt import Shop\n' + '\n' + '\n' + "librarian = Agent('test', name='librarian')\n" + "scribe = Agent('test', name='scribe')\n" + '\n' + '\n' + '@librarian.tool\n' + 'async def write_it_down(context, run_context):\n' + " await scribe.run(context, 'Write it down')\n" + '\n' + '\n' + '@scribe.tool\n' + 'async def look_it_up(context, run_context):\n' + " await librarian.run(context, 'Look it up')\n" + '\n' + '\n' + 'class ShopServicer(Shop.Servicer):\n' + '\n' + ' async def look(self, context, request):\n' + " await librarian.run(context, 'Tidy up')\n" + ), + ) + application = self._write('main.py', source=APPLICATION) + + found = await self._analyze(application) + + agents = {agent.name: agent for agent in found[servicer].agents} + self.assertEqual(sorted(agents), ['librarian', 'scribe']) + + [writes] = agents['librarian'].tools + self.assertEqual(writes.name, 'write_it_down') + self.assertEqual([run.agent for run in writes.runs], ['scribe']) + + [looks] = agents['scribe'].tools + self.assertEqual(looks.name, 'look_it_up') + self.assertEqual([run.agent for run in looks.runs], ['librarian']) + def _hazards(self, hazards) -> list[tuple[str, dict[str, str]]]: """Returns hazards as the name of each one's case and the fields it sets, which is what reads well in a failed assertion.""" @@ -1615,6 +1788,115 @@ async def test_what_changes_a_run_is_said(self) -> None: ], ) + async def test_tools_registered_in_ways_not_followed_say_so( + self, + ) -> None: + """A `prepare_tools` passed where the agent is constructed, and a + `prepare` passed where a tool is registered, can hide tools when + the agent runs, and are said on the agent and the tool. A tool + registered by calling `tool` rather than decorating is not + followed, and is said on the agent. A tool registered, either + way, on an agent that cannot be resolved has no agent to be said + on, and is said for the file.""" + servicer = self._write( + 'shop_servicer.py', + source=( + 'from reboot.agents.pydantic_ai import Agent\n' + 'from shop.v1.shop_rbt import Shop\n' + '\n' + '\n' + 'def only_reads(context, tools):\n' + ' return tools\n' + '\n' + '\n' + 'def when_admin(context, tool):\n' + ' return tool\n' + '\n' + '\n' + 'async def count(context):\n' + ' pass\n' + '\n' + '\n' + 'librarian = Agent(\n' + " 'test',\n" + " name='librarian',\n" + ' prepare_tools=only_reads,\n' + ')\n' + 'agents = [librarian]\n' + '\n' + '\n' + '@librarian.tool(prepare=when_admin)\n' + 'async def delete_page(context, run_context):\n' + " await Shop.ref('s').look(context)\n" + '\n' + '\n' + '@agents[0].tool\n' + 'async def orphan(context, run_context):\n' + ' pass\n' + '\n' + '\n' + 'librarian.tool(count)\n' + 'agents[0].tool_plain(count)\n' + '\n' + '\n' + 'class ShopServicer(Shop.Servicer):\n' + '\n' + ' async def look(self, context, request):\n' + " await librarian.run(context, 'Tidy up')\n" + ), + ) + application = self._write('main.py', source=APPLICATION) + + found = await self._analyze(application) + + [agent] = found[servicer].agents + self.assertEqual( + self._hazards(agent.hazards), + [ + ('constructed_with', { + 'prepare_tools': 'only_reads' + }), + ( + 'tool_registered_by_call', + { + 'call': 'librarian.tool(count)' + }, + ), + ], + ) + [tool] = agent.tools + self.assertEqual(tool.name, 'delete_page') + self.assertEqual( + self._hazards(tool.hazards), + [('prepared', { + 'prepare': 'when_admin' + })], + ) + self.assertEqual( + self._hazards(found[servicer].unattributed_hazards), + [ + ( + 'tool_on_unresolved_agent', + { + 'registration': 'agents[0].tool' + }, + ), + ( + 'tool_on_unresolved_agent', + { + 'registration': 'agents[0].tool_plain(count)' + }, + ), + ], + ) + self.assertEqual( + { + hazard.filename + for hazard in found[servicer].unattributed_hazards + }, + {str(servicer)}, + ) + async def test_only_a_named_agent_at_the_top_level_is_resolved( self, ) -> None: @@ -1931,8 +2213,8 @@ async def test_a_file_under_the_working_directory_stores_relative( async def test_reconstituting_keeps_stored_spellings(self) -> None: """What a previous run recorded comes back keyed by the - stored spelling, with the servicers and the agents recorded for - each file joined back on.""" + stored spelling, with the servicers, the agents and the + hazards recorded for each file joined back on.""" state = DashboardState() file = state.code_files['backend/x.py'] file.digest = b'digest' @@ -1943,6 +2225,9 @@ async def test_reconstituting_keeps_stored_spellings(self) -> None: agent = state.agents.add() agent.name = 'librarian' agent.filename = 'backend/x.py' + hazard = state.unattributed_hazards.add() + hazard.filename = 'backend/x.py' + hazard.tool_on_unresolved_agent.registration = 'agents[0].tool' known = _reconstitute_known(state) @@ -1961,6 +2246,13 @@ async def test_reconstituting_keeps_stored_spellings(self) -> None: [agent.name for agent in analyzed.agents], ['librarian'], ) + self.assertEqual( + [ + hazard.tool_on_unresolved_agent.registration + for hazard in analyzed.unattributed_hazards + ], + ['agents[0].tool'], + ) async def test_a_state_from_another_analysis_is_analyzed_again( self, From 0506a22333642bb32b8c2dc6d0af295b1e4ef8d6 Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Mon, 14 Sep 2026 21:35:13 +0000 Subject: [PATCH 08/11] Dashboard: say what an `override` changes about an agent `agent.override(...)` changes the agent for whatever runs inside it: the tools it can reach, its model, its instructions, and a name that does not change the one its runs are memoized under. None of that was recorded, so a card drawn from the construction alone would read as the whole of what those runs do. What an `override` changes is not followed, but recorded as an `Agent.Hazard.Overridden`, one per `override`, with each argument it passes: on the agent, and, as `Agent.Run.Hazard.overridden`, on each run written inside the override's `with`. Which runs are inside is decided by where they are written, so a run in a function called from inside the `with` is not said to be, though its agent still is. An `override` of an agent that cannot be resolved is a `Servicer.Method.Hazard.override_on_unresolved_agent`. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LRFggUcgVpLqhgb7h1cK6H --- rbt/dashboard/v1/dashboard.proto | 48 ++++++++- reboot/dashboard/backend/code_watcher.py | 95 +++++++++++++++-- tests/reboot/dashboard/code_watcher_tests.py | 101 ++++++++++++++++++- 3 files changed, 230 insertions(+), 14 deletions(-) diff --git a/rbt/dashboard/v1/dashboard.proto b/rbt/dashboard/v1/dashboard.proto index cfd4e3fa9..cf382180b 100644 --- a/rbt/dashboard/v1/dashboard.proto +++ b/rbt/dashboard/v1/dashboard.proto @@ -573,12 +573,24 @@ message Servicer { string callee = 1; } + // An `override` of an agent the analysis could not resolve, the + // way it could not resolve a run's, e.g. + // `agents[0].override(model=...)`, which changes whatever runs + // inside it. + message OverrideOnUnresolvedAgent { + // Represents what is called, spelled the way `ast.unparse` + // writes it: the receiver and the method, without the call's + // arguments, e.g. `agents[0].override`. + string callee = 1; + } + // Represents the file the hazard is written in, spelled the way // the developer would open it. string filename = 1; oneof hazard { RunOnUnresolvedAgent run_on_unresolved_agent = 2; + OverrideOnUnresolvedAgent override_on_unresolved_agent = 3; } } @@ -644,6 +656,28 @@ message Agent { string call = 1; } + // The agent is overridden, with `agent.override(...)`, for + // whatever runs inside it, with arguments that change what it is + // or can reach, which are not followed. Each represents the value + // passed, spelled the way `ast.unparse` writes it, and is absent + // when it is not passed. + message Overridden { + // `tools=`, which reach the runs inside it. + optional string tools = 1; + + // `name=`, which does not change the name the runs inside it + // are memoized under. + optional string name = 2; + + // `model=`, which the runs inside it use instead of + // `Agent.model`. + optional string model = 3; + + // `instructions=`, which the runs inside it are given instead of + // those in `Agent.system_prompt`. + optional string instructions = 4; + } + // Represents the file the hazard is written in, spelled the way // the developer would open it. string filename = 1; @@ -651,6 +685,7 @@ message Agent { oneof hazard { ConstructedWith constructed_with = 2; ToolRegisteredByCall tool_registered_by_call = 3; + Overridden overridden = 4; } } @@ -683,14 +718,19 @@ message Agent { oneof hazard { RunArguments run_arguments = 2; + + // The run is written inside the `with` of an `override` of its + // agent, which changes what it is or can reach. + Agent.Hazard.Overridden overridden = 3; } } // Represents the agent run, spelled as `Agent.name` spells it. string agent = 1; - // Represents everything the run is given on top of its agent that - // the analysis did not follow. + // Represents everything the run is given on top of its agent, by + // its own arguments or by an `override` it is written inside, + // that the analysis did not follow. repeated Hazard hazards = 2; } @@ -774,8 +814,8 @@ message Agent { // agent a tool that can always be followed. repeated Tool tools = 6; - // Represents everything about how the agent is constructed and - // given tools that the analysis did not follow. What a single run + // Represents everything about how the agent is constructed, given + // tools and overridden that the analysis did not follow. What a single run // is given is on the run; see `Run.hazards`. repeated Hazard hazards = 7; } diff --git a/reboot/dashboard/backend/code_watcher.py b/reboot/dashboard/backend/code_watcher.py index 8e6de50c0..e97b70277 100644 --- a/reboot/dashboard/backend/code_watcher.py +++ b/reboot/dashboard/backend/code_watcher.py @@ -39,9 +39,10 @@ class extends: a servicer is a class with a base whose type is with. Each tool's body is analyzed exactly as a servicer method's is, so what the model can reach through an agent is recorded beside what the application calls itself. Whatever is not followed -- a run -of any other agent, tools given any other way, what a run is given -on top of its agent -- is recorded as a hazard instead, so that -nothing the analysis could not see is silently missing. +of any other agent, tools given any other way, what a run or an +`override` is given on top of its agent -- is recorded as a hazard +instead, so that nothing the analysis could not see is silently +missing. Where following stops is what makes this the developer's code rather than somebody else's. A module resolves to a file only if a root @@ -130,6 +131,11 @@ class extends: a servicer is a class with a base whose type is RUN_ARGUMENTS = ('toolsets', 'model', 'instructions') CONSTRUCTION_ARGUMENTS = ('tools', 'toolsets', 'prepare_tools') +# The keyword arguments an `override` may be given that change what +# the runs inside it are or can reach, which the analysis does not +# follow; see `Agent.Hazard.Overridden`. +OVERRIDE_ARGUMENTS = ('tools', 'name', 'model', 'instructions') + # The version of what the analysis records, which the dashboard's # state records beside it. Counted up whenever the analysis starts # recording something it did not, or records something differently: @@ -1105,7 +1111,8 @@ async def _analyze_function( `Agent(...)` with a literal `name=` at the top level of a module; the record of that agent joins `agents`, which is every agent the file being analyzed has found so far. What a run is given on top of its - agent is not followed, but recorded on the run, and a run whose + agent, by its own arguments or by an `override` it is written + inside, is not followed, but recorded on the run, and a run whose agent cannot be resolved is recorded as a hazard of the function; see `Agent.Run.Hazard` and `Servicer.Method.Hazard`. @@ -1143,6 +1150,28 @@ async def _analyze_function( assert function.end_lineno is not None span = range(function.lineno, function.end_lineno + 1) + # The lines of each `with` a call is entered in, by the call, so + # that a run written inside an `override` can say what the + # override changes about it. Lexical, so a run in a function + # called from inside the `with` is not said to be inside it; the + # agent still says it is overridden. + override_bodies: dict[int, range] = {} + for node in ast.walk(function): + match node: + case ast.With() | ast.AsyncWith(): + assert node.end_lineno is not None + for item in node.items: + override_bodies[id(item.context_expr)] = range( + node.body[0].lineno, + node.end_lineno + 1, + ) + + # Each `override` found so far, with the lines of its `with`, the + # agent it is made on and what it changes. `ast.walk` reaches the + # call a `with` enters before any call in its body, so every run + # inside finds its override here. + overrides: list[tuple[range, str, Agent.Hazard.Overridden]] = [] + for node in ast.walk(function): match node: case ast.Call(func=(ast.Attribute() | ast.Name()) as callee): @@ -1198,6 +1227,51 @@ async def _analyze_function( None if entry_point is None else entry_point.syntax.name ) + if entry_point_name == 'override': + # An `override` changes the agent for whatever runs + # inside it. What it changes is not followed, but said: + # on the agent, and on each run written inside its + # `with`. Its `toolsets=` needs no saying, since a + # Reboot `Agent` rejects it. + changed = _passed(node, OVERRIDE_ARGUMENTS) + if len(changed) == 0: + continue + + constructed, analysis = await _agent_at( + callee, + filename=filename, + text=text, + analysis=analysis, + ) + if constructed is None: + hazards.append( + Servicer.Method.Hazard( + filename=str(filename), + override_on_unresolved_agent=( + Servicer.Method.Hazard. + OverrideOnUnresolvedAgent( + callee=ast.unparse(callee), + ) + ), + ) + ) + continue + + overridden = Agent.Hazard.Overridden(**changed) + agent = _agent_record(constructed, agents=agents) + _add_hazard( + agent.hazards, + Agent.Hazard( + filename=str(filename), + overridden=overridden, + ), + ) + + body = override_bodies.get(id(node)) + if body is not None: + overrides.append((body, constructed.name, overridden)) + continue + if entry_point_name not in RUN_NAMES: # Reboot's own machinery around a run: the `tool` a # tool is registered with, which the walk over the @@ -1227,8 +1301,9 @@ async def _analyze_function( _agent_record(constructed, agents=agents) - # What the run is given on top of its agent by its own - # arguments is not followed, but said on the run. + # What the run is given on top of its agent, by its own + # arguments or by an `override` it is written inside, is + # not followed, but said on the run. run_hazards: list[Agent.Run.Hazard] = [] given = _passed(node, RUN_ARGUMENTS) if len(given) > 0: @@ -1238,6 +1313,14 @@ async def _analyze_function( run_arguments=Agent.Run.Hazard.RunArguments(**given), ) ) + for body, overridden_name, overridden in overrides: + if node.lineno in body and overridden_name == constructed.name: + run_hazards.append( + Agent.Run.Hazard( + filename=str(filename), + overridden=overridden, + ) + ) runs.append( Agent.Run( diff --git a/tests/reboot/dashboard/code_watcher_tests.py b/tests/reboot/dashboard/code_watcher_tests.py index 3828b2236..dc86c6ea8 100644 --- a/tests/reboot/dashboard/code_watcher_tests.py +++ b/tests/reboot/dashboard/code_watcher_tests.py @@ -292,6 +292,9 @@ def tool(self, function=None, /, **kwargs): def tool_plain(self, function=None, /, **kwargs): return function + def override(self, **kwargs): + pass + async def run(self, context=None, user_prompt=None, **kwargs): pass @@ -1524,6 +1527,51 @@ async def test_tools_and_toolsets_given_are_said_not_followed( ], ) + async def test_override_tools_are_said_not_followed(self) -> None: + """`tools=` passed to `override`, which reach the runs inside + it, are said on the agent and on each run inside its `with`, but + not on one after it, rather than followed.""" + servicer = self._write( + 'shop_servicer.py', + source=( + 'from reboot.agents.pydantic_ai import Agent\n' + 'from shop.v1.shop_rbt import Shop\n' + '\n' + '\n' + 'async def lookup(context):\n' + " await Shop.ref('s').look(context)\n" + '\n' + '\n' + "librarian = Agent('test', name='librarian')\n" + '\n' + '\n' + 'class ShopServicer(Shop.Servicer):\n' + '\n' + ' async def look(self, context, request):\n' + ' with librarian.override(tools=[lookup]):\n' + " await librarian.run(context, 'Tidy up')\n" + " await librarian.run(context, 'Carry on')\n" + ), + ) + application = self._write('main.py', source=APPLICATION) + + found = await self._analyze(application) + + overridden = ('overridden', {'tools': '[lookup]'}) + [agent] = found[servicer].agents + self.assertEqual(list(agent.tools), []) + self.assertEqual(self._hazards(agent.hazards), [overridden]) + + [found_servicer] = found[servicer].servicers + [method] = found_servicer.methods + self.assertEqual( + sorted( + (self._hazards(run.hazards) for run in method.runs), + key=len, + ), + [[], [overridden]], + ) + async def test_an_agent_constructed_in_another_file(self) -> None: """An agent is the same agent wherever it is met: the file constructing it records it with the tools decorated there, @@ -1694,6 +1742,10 @@ async def test_a_run_whose_agent_cannot_be_resolved(self) -> None: '\n' ' async def untyped(self, context, request):\n' ' await ask_untyped(librarian, context)\n' + '\n' + ' async def overridden(self, context, request):\n' + " with agents[0].override(model='other'):\n" + ' pass\n' ), ) application = self._write('main.py', source=APPLICATION) @@ -1736,6 +1788,15 @@ async def test_a_run_whose_agent_cannot_be_resolved(self) -> None: 'callee': 'agents[0].run' })], 'untyped': [], + 'overridden': + [ + ( + 'override_on_unresolved_agent', + { + 'callee': 'agents[0].override' + }, + ), + ], }, ) self.assertEqual( @@ -1746,8 +1807,11 @@ async def test_a_run_whose_agent_cannot_be_resolved(self) -> None: self.assertEqual(found[servicer].agents, ()) async def test_what_changes_a_run_is_said(self) -> None: - """A `model` or `instructions` passed to a run changes what the - run is from what its agent says, and is said on the run.""" + """A `model` or `instructions` passed to a run, and a `name`, + `model` or `instructions` passed to an `override`, change what + a run is from what its agent says, and are said: a run's own + arguments on the run, and an `override` on the agent and on + each run written inside its `with`, but not on one after it.""" servicer = self._write( 'shop_servicer.py', source=( @@ -1765,6 +1829,14 @@ async def test_what_changes_a_run_is_said(self) -> None: " context, 'a', model='other',\n" " instructions='Be brief.',\n" ' )\n' + '\n' + ' async def overridden(self, context, request):\n' + ' with librarian.override(\n' + " name='archivist', model='other',\n" + " instructions='In French.',\n" + ' ):\n' + " await librarian.run(context, 'b')\n" + " await librarian.run(context, 'c')\n" ), ) application = self._write('main.py', source=APPLICATION) @@ -1772,9 +1844,17 @@ async def test_what_changes_a_run_is_said(self) -> None: found = await self._analyze(application) [found_servicer] = found[servicer].servicers - [method] = found_servicer.methods + methods = {method.name: method for method in found_servicer.methods} + overridden = ( + 'overridden', + { + 'name': "'archivist'", + 'model': "'other'", + 'instructions': "'In French.'", + }, + ) self.assertEqual( - [self._hazards(run.hazards) for run in method.runs], + [self._hazards(run.hazards) for run in methods['look'].runs], [ [ ( @@ -1787,6 +1867,19 @@ async def test_what_changes_a_run_is_said(self) -> None: ], ], ) + self.assertEqual( + sorted( + ( + self._hazards(run.hazards) + for run in methods['overridden'].runs + ), + key=len, + ), + [[], [overridden]], + ) + + [agent] = found[servicer].agents + self.assertEqual(self._hazards(agent.hazards), [overridden]) async def test_tools_registered_in_ways_not_followed_say_so( self, From 1d62d8737305c4fdae29f24c005797d4ca7667ac Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Mon, 14 Sep 2026 21:35:19 +0000 Subject: [PATCH 09/11] Dashboard: draw the agents an application runs The analysis records every agent an application runs, the tools each has, and the calls those tools make, but the page drew none of it: a workflow handing its work to a model still showed one unremarkable method. The page grows a card per agent, marked with a robot, in the workflow's colour, since an agent only runs inside a workflow: its name, its model, the first lines of its prompt, and a row per tool, whose dot is a ring, since a tool runs in the workflow but is called by the agent. A run lands on the card's head as a labelled arrow, folding into one counted arrow when the calling package is collapsed, and each tool's own calls leave its row, dashed the way a workflow's calls are. The card's head opens the agent in the types pane, where the prompt is shown whole beside each tool's description and what it calls. The page joins an agent's records on its name, which the runtime requires to be unique, and does not show hazards yet. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LRFggUcgVpLqhgb7h1cK6H --- reboot/dashboard/web/dashboard.css | 130 ++- reboot/dashboard/web/src/callgraph.ts | 209 +++-- reboot/dashboard/web/src/graph.tsx | 1041 +++++++++++++++++-------- reboot/dashboard/web/src/main.tsx | 249 ++++-- 4 files changed, 1206 insertions(+), 423 deletions(-) diff --git a/reboot/dashboard/web/dashboard.css b/reboot/dashboard/web/dashboard.css index 8e1b2ddb0..5cae7bce8 100644 --- a/reboot/dashboard/web/dashboard.css +++ b/reboot/dashboard/web/dashboard.css @@ -877,6 +877,50 @@ header h1 { gap: 6px; } +/* The agent's prompt on the pane, whole and in the shape it was + written in: it is a piece of prose the developer wrote, and its + paragraphs and its lists are what it means. */ +.agent-prompt { + border: 1px solid hsl(var(--border-soft)); + border-radius: var(--radius); + background: hsl(var(--surface-sunken)); + padding: 12px 14px; + font-size: 12px; + line-height: 1.55; + color: hsl(var(--foreground)); + white-space: pre-wrap; + overflow-wrap: anywhere; +} + +/* One tool, set the way a method is. */ +.agent-tool { + border: 1px solid hsl(var(--border-soft)); + border-radius: var(--radius); + background: hsl(var(--surface-sunken)); + padding: 10px 14px; +} + +.agent-tool-head { + display: flex; + align-items: baseline; + gap: 8px; +} + +.agent-tool-name { + font-family: ui-monospace, Menlo, monospace; + font-size: 12px; + font-weight: 650; +} + +/* What the tool reaches: each a link to the method, or the agent, in + the pane. */ +.agent-tool-calls { + display: flex; + flex-wrap: wrap; + gap: 6px; + margin-top: 8px; +} + /* Deliberately not a scroll container. Nothing inside a card paints out to its edges, so there is no corner for `overflow: hidden` to clip, and it would clip two things that must escape: a pill's @@ -1327,8 +1371,12 @@ header h1 { } /* An unknown method's dot is a ring: nothing declares it, only a - call names it, so there is nothing to paint it in with. */ -.graph-kind-unknown .graph-method-dot { + call names it, so there is nothing to paint it in with. An agent's + tool is a ring too, for the same reason -- no API declares it -- + in the workflow colour, since it runs in the workflow that runs + the agent, called by the agent rather than by the application. */ +.graph-kind-unknown .graph-method-dot, +.graph-tool .graph-method-dot { background: transparent; border: 1.5px solid hsl(var(--kind)); } @@ -1348,6 +1396,84 @@ header h1 { text-decoration: underline; } +/* An agent: a card like a state type's, with a row per tool, its + head tinted in the workflow colour, since an agent only ever runs + inside a workflow. Never in a package's box, so it carries its own + tint rather than sitting in one. */ +.graph-agent { + width: 100%; + background: hsl(var(--card)); + border: 1px solid hsl(36 85% 42% / 0.4); + border-radius: var(--radius); + box-shadow: 0 1px 3px hsl(240 10% 40% / 0.1); + overflow: hidden; +} + +/* 40px tall, which `graph.tsx` lays out by: the name on one line and + the model under it, rather than the two sharing a line, where a + long model name would eat the name the reader came for. */ +.graph-agent-head { + position: relative; + display: flex; + flex-direction: column; + justify-content: center; + height: 40px; + padding: 0 12px; + background: hsl(36 85% 42% / 0.08); + border-bottom: 1px solid hsl(36 85% 42% / 0.25); + color: hsl(var(--foreground)); +} + +.graph-agent-title { + display: flex; + align-items: center; + gap: 6px; + font-size: 12.5px; + font-weight: 650; + line-height: 1.3; +} + +.graph-agent-emoji { + flex: none; + font-size: 13px; +} + +.graph-agent-name { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.graph-agent-model { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + font-family: ui-monospace, Menlo, monospace; + font-size: 9.5px; + font-weight: 500; + line-height: 1.3; + color: hsl(var(--muted-foreground)); +} + +/* The first of the prompt, which is what the agent is: three lines + of it, clamped to the 48px `graph.tsx` lays out by, which is + exactly three at this line height plus the padding above them. The + whole of it, in the shape it was written in, is in the types pane, + a click on the head away; here its lines run on, so that three of + them carry as much of it as they can. */ +.graph-agent-prompt { + height: 48px; + padding: 5px 12px 1px; + overflow: hidden; + display: -webkit-box; + -webkit-box-orient: vertical; + -webkit-line-clamp: 3; + line-clamp: 3; + font-size: 10px; + line-height: 14px; + color: hsl(var(--muted-foreground)); +} + /* The stubs of the arrows at the chosen row's flanks: the right one lights what the method calls, the left one who calls it. */ .graph-cone { diff --git a/reboot/dashboard/web/src/callgraph.ts b/reboot/dashboard/web/src/callgraph.ts index d6973ab45..6803be696 100644 --- a/reboot/dashboard/web/src/callgraph.ts +++ b/reboot/dashboard/web/src/callgraph.ts @@ -1,9 +1,13 @@ // The call graph's data: the API's state types and methods, joined // with the Reboot calls the analysis of the developer's application -// found in each method's implementation. +// found in each method's implementation, and the agents it found the +// application runs, joined with the tools each agent has. import type { + Agent, + Agent_Run, Servicer, Servicer_Method, + Servicer_Method_Call, Servicer_Method_Call_How, } from "../../../../rbt/dashboard/v1/dashboard_pb"; import type { APIs, Kind } from "./link_properties_to_data_types"; @@ -27,6 +31,15 @@ export interface GraphCall { count: number; } +// One agent a method's or a tool's implementation runs, counted the +// same way. +export interface GraphRun { + // The agent's name, which is what a run names and what the records + // of one agent are joined on. + agentName: string; + count: number; +} + export interface GraphMethod { name: string; // Only the API's declaration says the kind, so a method known only @@ -34,6 +47,7 @@ export interface GraphMethod { kind?: Kind; factory: boolean; calls: GraphCall[]; + runs: GraphRun[]; } export interface GraphStateType { @@ -51,6 +65,30 @@ export interface GraphPackage { stateTypes: GraphStateType[]; } +// One tool an agent may call: a row of its card, and what the model +// reaches the application through. +export interface GraphTool { + name: string; + // What the model is told it does: the description it was + // registered with, or the function's docstring. + description?: string; + calls: GraphCall[]; + runs: GraphRun[]; +} + +export interface GraphAgent { + // `agent:librarian`: kept apart from the state types' ids, which + // are qualified names, so that one id names one thing. + id: string; + name: string; + model?: string; + // Every literal string its prompt is made of, in the order the + // agent is constructed with them. + systemPrompt: string[]; + description?: string; + tools: GraphTool[]; +} + // Packages in the order their first state type comes. export const groupStateTypesByPackage = ( stateTypes: GraphStateType[] @@ -72,37 +110,132 @@ export const groupStateTypesByPackage = ( export const methodId = (stateTypeName: string, methodName: string): string => `${stateTypeName}.${methodName}`; +// A key unique to one agent, in the same space as the state types', +// which cannot hold a colon. +export const agentId = (agentName: string): string => `agent:${agentName}`; + +// A key unique to one of an agent's tools, the way a method's is +// unique to one method: `agent:librarian.get_page`. +export const toolId = (agentId: string, toolName: string): string => + `${agentId}.${toolName}`; + +// Whether an id names an agent or one of its tools, which only an +// agent's id begins the way it does. +export const isAgentRowId = (id: string): boolean => id.startsWith("agent:"); + // Folds the calls the analysis lists into one per distinct call, // counted. -const countCalls = ( - analyzedMethod: Servicer_Method | undefined -): GraphCall[] => { - const calls = new Map(); - for (const call of analyzedMethod?.calls ?? []) { +const countCalls = (calls: Servicer_Method_Call[] | undefined): GraphCall[] => { + const counted = new Map(); + for (const call of calls ?? []) { const key = `${call.stateType}|${call.method}|${call.how}`; - const counted = calls.get(key); - if (counted === undefined) { - calls.set(key, { + const already = counted.get(key); + if (already === undefined) { + counted.set(key, { stateTypeName: call.stateType, methodName: call.method, how: call.how, count: 1, }); } else { - counted.count += 1; + already.count += 1; + } + } + return [...counted.values()]; +}; + +// The same, for the agents an implementation runs. +const countRuns = (runs: Agent_Run[] | undefined): GraphRun[] => { + const counted = new Map(); + for (const run of runs ?? []) { + const already = counted.get(run.agent); + if (already === undefined) { + counted.set(run.agent, { agentName: run.agent, count: 1 }); + } else { + already.count += 1; + } + } + return [...counted.values()]; +}; + +// Joins the records of each agent the analysis found: one per file +// that met the agent, every one saying the same thing about the +// agent itself, and each carrying the tools its own file +// contributed. The tools gather, one per name; the agents come in +// the order the records do, which is by name. +export const joinAgents = (agents: Agent[]): GraphAgent[] => { + const joined = new Map(); + for (const agent of agents) { + let joinedAgent = joined.get(agent.name); + if (joinedAgent === undefined) { + joinedAgent = { + id: agentId(agent.name), + name: agent.name, + model: agent.model, + systemPrompt: agent.systemPrompt, + description: agent.description, + tools: [], + }; + joined.set(agent.name, joinedAgent); + } + for (const tool of agent.tools) { + // By name alone, since the name is what the model calls and + // what the agent's row is: a tool the agent is given in two + // places is one tool. + if (joinedAgent.tools.some((known) => known.name === tool.name)) { + continue; + } + joinedAgent.tools.push({ + name: tool.name, + description: tool.description, + calls: countCalls(tool.calls), + runs: countRuns(tool.runs), + }); + } + } + return [...joined.values()]; +}; + +// Adds, as a target with no kind and no calls of its own, every +// state type and method some call names that the API does not +// declare. +const addCalled = ( + stateTypes: Map, + calls: GraphCall[] +): void => { + for (const call of calls) { + let calledStateType = stateTypes.get(call.stateTypeName); + if (calledStateType === undefined) { + calledStateType = { + id: call.stateTypeName, + name: shortNameOfTypeName(call.stateTypeName), + methods: [], + }; + stateTypes.set(call.stateTypeName, calledStateType); + } + if ( + !calledStateType.methods.some((known) => known.name === call.methodName) + ) { + calledStateType.methods.push({ + name: call.methodName, + factory: false, + calls: [], + runs: [], + }); } } - return [...calls.values()]; }; // Joins the state types the API files declare with the calls the // analysis found in each declared method. Servicer methods the API // does not declare, such as helpers, are dropped. Anything a call // names that the API does not declare is added as a target, with no -// kind and no calls. +// kind and no calls, whether a method calls it or one of the +// `agents`' tools does. export const joinStateTypes = ( apis: APIs, - servicers: Servicer[] + servicers: Servicer[], + agents: GraphAgent[] ): GraphStateType[] => { // A state type can have more than one servicer in `servicers`, sorted // by file; where they define the same method, the first wins. @@ -125,44 +258,32 @@ export const joinStateTypes = ( { id: name, name: stateType.name, - methods: stateType.methods.map((method) => ({ - name: method.name, - kind: kindOfMethod(method), - factory: method.factory, - calls: countCalls( - analyzedMethodsById.get(methodId(name, method.name)) - ), - })), + methods: stateType.methods.map((method) => { + const analyzed = analyzedMethodsById.get( + methodId(name, method.name) + ); + return { + name: method.name, + kind: kindOfMethod(method), + factory: method.factory, + calls: countCalls(analyzed?.calls), + runs: countRuns(analyzed?.runs), + }; + }), }, ]; }) ) ); - for (const stateType of graphStateTypes.values()) { + for (const stateType of [...graphStateTypes.values()]) { for (const method of stateType.methods) { - for (const call of method.calls) { - let calledStateType = graphStateTypes.get(call.stateTypeName); - if (calledStateType === undefined) { - calledStateType = { - id: call.stateTypeName, - name: shortNameOfTypeName(call.stateTypeName), - methods: [], - }; - graphStateTypes.set(call.stateTypeName, calledStateType); - } - if ( - !calledStateType.methods.some( - (known) => known.name === call.methodName - ) - ) { - calledStateType.methods.push({ - name: call.methodName, - factory: false, - calls: [], - }); - } - } + addCalled(graphStateTypes, method.calls); + } + } + for (const agent of agents) { + for (const tool of agent.tools) { + addCalled(graphStateTypes, tool.calls); } } diff --git a/reboot/dashboard/web/src/graph.tsx b/reboot/dashboard/web/src/graph.tsx index f5dbe82c3..78f2e6f5a 100644 --- a/reboot/dashboard/web/src/graph.tsx +++ b/reboot/dashboard/web/src/graph.tsx @@ -6,6 +6,11 @@ // package; a collapsed box hides its cards, and the calls leaving it // fold into one counted arrow per box they reach. // +// A card per agent too, with a row per tool, which runs land on and +// the calls the tools make leave from: an agent stands beside the +// packages rather than in one, since it belongs to no package of the +// API, and is never collapsed. +// // React Flow draws; ELK places. React Flow deliberately has no layout // of its own. import { Servicer_Method_Call_How as How } from "../../../../rbt/dashboard/v1/dashboard_pb"; @@ -31,14 +36,21 @@ import type { Viewport } from "@xyflow/react"; import { useLocation, useNavigationType } from "react-router"; import ELK from "elkjs/lib/elk.bundled.js"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; -import type { FC, MouseEvent } from "react"; +import type { FC, MouseEvent, ReactNode } from "react"; import type { + GraphAgent, GraphCall, - GraphMethod, GraphPackage, + GraphRun, GraphStateType, } from "./callgraph"; -import { groupStateTypesByPackage, methodId } from "./callgraph"; +import { + agentId, + groupStateTypesByPackage, + isAgentRowId, + methodId, + toolId, +} from "./callgraph"; import type { Kind } from "./link_properties_to_data_types"; import { labelOfKind, @@ -122,16 +134,34 @@ const COLLAPSED_PACKAGE_HEIGHT = 78; const EXPANDED_PACKAGE_HEAD_HEIGHT = 38; const EXPANDED_PACKAGE_PAD = 20; +// An agent's card is wider, because what it is is a prompt, and its +// head is taller: the emoji and the name, then the model, then the +// first lines of the prompt, which `.graph-agent-prompt` clamps to +// the height below. An agent with no prompt to show has a head of +// `AGENT_HEAD_HEIGHT` alone. +const AGENT_CARD_WIDTH = 260; +const AGENT_HEAD_HEIGHT = 40; +const AGENT_PROMPT_HEIGHT = 48; + const heightOfStateType = (stateType: GraphStateType): number => HEAD_HEIGHT + stateType.methods.length * ROW_HEIGHT + CARD_SLACK; +// What an agent's tool rows start below, which is its head with the +// prompt it shows, if it shows one. +const headHeightOfAgent = (agent: GraphAgent): number => + AGENT_HEAD_HEIGHT + (agent.systemPrompt.length > 0 ? AGENT_PROMPT_HEIGHT : 0); + +const heightOfAgent = (agent: GraphAgent): number => + headHeightOfAgent(agent) + agent.tools.length * ROW_HEIGHT + CARD_SLACK; + // A package's node id, kept apart from state type ids, which are // fully qualified names and could equal a package's. const packageNodeId = (name: string): string => `pkg:${name}`; -// `bank.v1.Account` for `bank.v1.Account.deposit`. -const stateTypeNameOfMethodId = (id: string): string => - id.slice(0, id.lastIndexOf(".")); +// The card a row belongs to: `bank.v1.Account` for +// `bank.v1.Account.deposit`, and `agent:librarian` for +// `agent:librarian.get_page`. +const cardOfRowId = (id: string): string => id.slice(0, id.lastIndexOf(".")); // --------------------------------------------------------------- // Layout. @@ -149,6 +179,19 @@ interface ExpandedPackageData extends Record { onCollapse?: (name: string) => void; } +interface AgentData extends Record { + agent: GraphAgent; + // The chosen row's id, when one is chosen, which may be a tool of + // this agent or anything else the graph draws. + selectedRow?: string | null; + onSelectRow?: (id: string, cones: Cones) => void; + onOpenAgent?: (id: string) => void; + cones?: Cones; + onToggleCone?: (cone: keyof Cones) => void; + withCallers?: Set; + withCalls?: Set; +} + // Which cones of the chosen method the graph lights: what it calls // (downstream), who calls it (upstream), or both. interface Cones { @@ -180,14 +223,14 @@ const CONES_OF_THIRD: Record = { interface StateTypeData extends Record { stateType: GraphStateType; - // The chosen method's id, when one is chosen. - selectedMethod?: string | null; - onSelectMethod?: (id: string, cones: Cones) => void; + // The chosen row's id, when one is chosen. + selectedRow?: string | null; + onSelectRow?: (id: string, cones: Cones) => void; onOpenStateType?: (id: string) => void; cones?: Cones; onToggleCone?: (cone: keyof Cones) => void; - // The methods some drawn call lands on, and the methods that make - // one; a button with nothing to light is never shown. + // The rows some drawn arrow lands on, and the rows that make one; + // a button with nothing to light is never shown. withCallers?: Set; withCalls?: Set; } @@ -195,7 +238,8 @@ interface StateTypeData extends Record { type GraphNode = | Node | Node - | Node; + | Node + | Node; interface Point { x: number; @@ -213,12 +257,13 @@ const ELK_LAYERED_OPTIONS = { // Where everything goes: callers to the left of what they call, the // way an edge leaves a row on its right and enters one on its left. -// Each expanded box's cards are laid out alone, then the boxes are -// laid out at the size their cards came to, so an open box never -// lands on a neighbour. A card or box calling itself has no say in -// where it goes. +// Each expanded box's cards are laid out alone, then the boxes and +// the agents' cards are laid out at the size their contents came to, +// so an open box never lands on a neighbour. A card or box calling +// itself has no say in where it goes. const layoutPackages = async ( packages: GraphPackage[], + agents: GraphAgent[], collapsed: ReadonlySet ): Promise => { const cardLayoutsByPackage = new Map< @@ -286,19 +331,54 @@ const layoutPackages = async ( }); } - const callPairsBetweenPackages = new Set(); + // Between the boxes and the agents' cards, which are laid out + // together: a package's calls reach the packages they name and the + // agents its methods run, and an agent's tools reach the packages + // they call and the agents they run. + const pairsBetweenBoxes = new Set(); + const agentIds = new Set(agents.map((agent) => agent.id)); + const reaches = (source: string, target: string): void => { + if (source !== target) { + pairsBetweenBoxes.add(`${source}\u0000${target}`); + } + }; for (const pkg of packages) { + const source = packageNodeId(pkg.name); for (const stateType of pkg.stateTypes) { for (const method of stateType.methods) { for (const call of method.calls) { - const target = packageOfStateTypeName(call.stateTypeName); - if (isDrawn(call) && target !== pkg.name) { - callPairsBetweenPackages.add(`${pkg.name}>${target}`); + if (isDrawn(call)) { + reaches( + source, + packageNodeId(packageOfStateTypeName(call.stateTypeName)) + ); + } + } + for (const run of method.runs) { + if (agentIds.has(agentId(run.agentName))) { + reaches(source, agentId(run.agentName)); } } } } } + for (const agent of agents) { + for (const tool of agent.tools) { + for (const call of tool.calls) { + if (isDrawn(call)) { + reaches( + agent.id, + packageNodeId(packageOfStateTypeName(call.stateTypeName)) + ); + } + } + for (const run of tool.runs) { + if (agentIds.has(agentId(run.agentName))) { + reaches(agent.id, agentId(run.agentName)); + } + } + } + } const elkPackageLayout = await elk.layout({ id: "root", @@ -307,37 +387,49 @@ const layoutPackages = async ( "elk.spacing.nodeNode": "60", "elk.layered.spacing.nodeNodeBetweenLayers": "140", }, - children: packages.map((pkg) => { - const cardLayout = cardLayoutsByPackage.get(pkg.name); - return { - id: packageNodeId(pkg.name), - width: cardLayout?.width ?? COLLAPSED_PACKAGE_WIDTH, - height: cardLayout?.height ?? COLLAPSED_PACKAGE_HEIGHT, - }; - }), - edges: [...callPairsBetweenPackages].map((pair) => { - const [source, target] = pair.split(">"); - return { - id: pair, - sources: [packageNodeId(source)], - targets: [packageNodeId(target)], - }; + children: [ + ...packages.map((pkg) => { + const cardLayout = cardLayoutsByPackage.get(pkg.name); + return { + id: packageNodeId(pkg.name), + width: cardLayout?.width ?? COLLAPSED_PACKAGE_WIDTH, + height: cardLayout?.height ?? COLLAPSED_PACKAGE_HEIGHT, + }; + }), + ...agents.map((agent) => ({ + id: agent.id, + width: AGENT_CARD_WIDTH, + height: heightOfAgent(agent), + })), + ], + edges: [...pairsBetweenBoxes].map((pair) => { + const [source, target] = pair.split("\u0000"); + return { id: pair, sources: [source], targets: [target] }; }), }); - const packagePositions = new Map( - (elkPackageLayout.children ?? []).map((elkPackage) => [ - elkPackage.id, - { x: elkPackage.x ?? 0, y: elkPackage.y ?? 0 }, + const boxPositions = new Map( + (elkPackageLayout.children ?? []).map((elkBox) => [ + elkBox.id, + { x: elkBox.x ?? 0, y: elkBox.y ?? 0 }, ]) ); // A parent precedes its children: React Flow resolves a elkCard's // position, relative to its parent, in array order. const nodes: GraphNode[] = []; + for (const agent of agents) { + nodes.push({ + id: agent.id, + type: "agent", + position: boxPositions.get(agent.id) ?? { x: 0, y: 0 }, + width: AGENT_CARD_WIDTH, + data: { agent }, + }); + } for (const pkg of packages) { const boxId = packageNodeId(pkg.name); - const position = packagePositions.get(boxId) ?? { x: 0, y: 0 }; + const position = boxPositions.get(boxId) ?? { x: 0, y: 0 }; const cardLayout = cardLayoutsByPackage.get(pkg.name); if (cardLayout === undefined) { nodes.push({ @@ -390,93 +482,152 @@ const layoutPackages = async ( // Edges. interface CallEdgeData extends Record { - // Absent on a folded edge, which carries calls reached every way. + // Absent on a folded edge, which carries calls reached every way, + // and on an edge that is a run rather than a call. how?: How; - // The calling method's kind, which is the edge's colour. Absent - // for a method the API does not declare, and on a folded edge. + // Set on an edge that is a run of an agent rather than a call, + // which always says so. + run?: boolean; + // The calling method's kind, which is the edge's colour. An agent's + // tools run in the workflow that runs the agent, so an edge leaving + // one is a workflow's. Absent for a method the API does not + // declare, and on a folded edge. kind?: Kind; count: number; - // Every calling method whose calls this edge carries: one for an - // edge from a method row, each contributor for a folded edge. - // What choosing a method keeps, transitively. - sourceMethodIds: string[]; - // Every called method the same way, which is what says whether + // Every row whose arrows this edge carries: one for an edge from a + // method's or a tool's row, each contributor for a folded edge. + // What choosing a row keeps, transitively. + sourceIds: string[]; + // Every row it lands on the same way, which is what says whether // the edge lands inside the upstream cone. - targetMethodIds: string[]; - // Set while another method is chosen. The label fades off this + targetIds: string[]; + // Set while another row is chosen. The label fades off this // rather than off the edge's class: `EdgeLabelRenderer` draws // labels in a layer of their own, out of the class's reach. faded?: boolean; } -// Every method the chosen one calls, transitively, itself included: -// the downstream closure over the drawn calls. Collapse-blind, so -// the path continues through a collapsed box. -const reachableMethodIds = ( - from: string, - packages: GraphPackage[] -): Set => { - const callsByMethodId = new Map( - packages.flatMap((pkg) => - pkg.stateTypes.flatMap((stateType) => - stateType.methods.map( - (method) => - [methodId(stateType.id, method.name), method.calls] as const - ) - ) - ) - ); - const reached = new Set([from]); - const frontier = [from]; - while (frontier.length > 0) { - for (const call of callsByMethodId.get(frontier.pop()!) ?? []) { - if (!isDrawn(call)) { - continue; - } - const callee = methodId(call.stateTypeName, call.methodName); - if (!reached.has(callee)) { - reached.add(callee); - frontier.push(callee); - } +// Where a run lands: an agent's card is one thing to run, however +// many tools it has, so every run enters it at its head. +const AGENT_TARGET_HANDLE = "t:agent"; + +// One arrow, drawn or folded into the one already drawn between the +// same two places, which is what a collapsed box's arrows become. +const addEdge = ( + edgesById: Map>, + edge: { + id: string; + source: string; + sourceHandle?: string; + target: string; + targetHandle?: string; + kind?: Kind; + how?: How; + run?: boolean; + count: number; + sourceId: string; + targetId: string; + } +): void => { + const folded = edgesById.get(edge.id); + if (folded !== undefined) { + const data = folded.data!; + data.count += edge.count; + if (!data.sourceIds.includes(edge.sourceId)) { + data.sourceIds.push(edge.sourceId); + } + if (!data.targetIds.includes(edge.targetId)) { + data.targetIds.push(edge.targetId); } + return; } - return reached; + edgesById.set(edge.id, { + id: edge.id, + source: edge.source, + sourceHandle: edge.sourceHandle, + target: edge.target, + targetHandle: edge.targetHandle, + type: "call", + data: { + how: edge.how, + run: edge.run, + kind: edge.kind, + count: edge.count, + sourceIds: [edge.sourceId], + targetIds: [edge.targetId], + }, + markerEnd: { + type: MarkerType.ArrowClosed, + color: colorOfKind(edge.kind), + width: 16, + height: 16, + }, + }); }; -// Every method that calls the chosen one, transitively, itself -// included: the upstream closure over the same drawn calls, -// collapse-blind the same way. -const reachingMethodIds = ( - to: string, - packages: GraphPackage[] -): Set => { - const callersByMethodId = new Map(); +// Who calls whom, over everything the graph draws, by row: a method +// leads to the methods its calls name and to the agents it runs; an +// agent leads to each of its tools, which is what running it +// reaches; and a tool leads on the way a method does. Collapse-blind, +// so a path continues through a collapsed box. +interface Reachability { + callees: Map; + callers: Map; +} + +const reachability = ( + packages: GraphPackage[], + agents: GraphAgent[] +): Reachability => { + const callees = new Map(); + const callers = new Map(); + const leads = (from: string, to: string): void => { + callees.set(from, [...(callees.get(from) ?? []), to]); + callers.set(to, [...(callers.get(to) ?? []), from]); + }; + const leadsFrom = (id: string, calls: GraphCall[], runs: GraphRun[]) => { + for (const call of calls) { + if (isDrawn(call)) { + leads(id, methodId(call.stateTypeName, call.methodName)); + } + } + for (const run of runs) { + leads(id, agentId(run.agentName)); + } + }; + for (const pkg of packages) { for (const stateType of pkg.stateTypes) { for (const method of stateType.methods) { - const caller = methodId(stateType.id, method.name); - for (const call of method.calls) { - if (!isDrawn(call)) { - continue; - } - const callee = methodId(call.stateTypeName, call.methodName); - const callers = callersByMethodId.get(callee); - if (callers === undefined) { - callersByMethodId.set(callee, [caller]); - } else { - callers.push(caller); - } - } + leadsFrom( + methodId(stateType.id, method.name), + method.calls, + method.runs + ); } } } - const reached = new Set([to]); - const frontier = [to]; + for (const agent of agents) { + for (const tool of agent.tools) { + const id = toolId(agent.id, tool.name); + leads(agent.id, id); + leadsFrom(id, tool.calls, tool.runs); + } + } + + return { callees, callers }; +}; + +// Everything the chosen row leads to, transitively, itself included, +// and everything that leads to it, over the same graph. +const closure = (from: string, edges: Map): Set => { + const reached = new Set([from]); + const frontier = [from]; while (frontier.length > 0) { - for (const caller of callersByMethodId.get(frontier.pop()!) ?? []) { - if (!reached.has(caller)) { - reached.add(caller); - frontier.push(caller); + for (const next of edges.get(frontier.pop()!) ?? []) { + if (!reached.has(next)) { + reached.add(next); + frontier.push(next); } } } @@ -486,16 +637,25 @@ const reachingMethodIds = ( // The edges as the boxes show them. A call whose box is expanded // leaves from its own method row; otherwise it leaves from the box, // and every call the box hides folds into one counted edge per node -// they reach. -const edgesOfPackages = ( +// they reach. An agent's card is never in a box and never collapsed, +// so its tools' arrows always leave their own rows, and a run always +// lands on the agent's head. +const edgesOf = ( packages: GraphPackage[], + agents: GraphAgent[], collapsed: ReadonlySet ): Edge[] => { const edgesById = new Map>(); + const agentsByName = new Map(agents.map((agent) => [agent.name, agent])); + for (const pkg of packages) { const sourceExpanded = !collapsed.has(pkg.name); for (const stateType of pkg.stateTypes) { for (const method of stateType.methods) { + const source = sourceExpanded ? stateType.id : packageNodeId(pkg.name); + const sourceHandle = sourceExpanded ? `s:${method.name}` : undefined; + const caller = methodId(stateType.id, method.name); + for (const call of method.calls) { if (!isDrawn(call)) { continue; @@ -508,59 +668,109 @@ const edgesOfPackages = ( continue; } - const source = sourceExpanded - ? stateType.id - : packageNodeId(pkg.name); - const sourceHandle = sourceExpanded ? `s:${method.name}` : undefined; const target = targetExpanded ? call.stateTypeName : packageNodeId(targetPackage); const targetHandle = targetExpanded ? `t:${call.methodName}` : undefined; - const id = sourceExpanded - ? `${source}|${sourceHandle}>${target}|${targetHandle}:${call.how}` - : `${source}>${target}|${targetHandle}`; - - const caller = methodId(stateType.id, method.name); - const callee = methodId(call.stateTypeName, call.methodName); - const edgeFoldedInto = edgesById.get(id); - if (edgeFoldedInto !== undefined) { - edgeFoldedInto.data!.count += call.count; - if (!edgeFoldedInto.data!.sourceMethodIds.includes(caller)) { - edgeFoldedInto.data!.sourceMethodIds.push(caller); - } - if (!edgeFoldedInto.data!.targetMethodIds.includes(callee)) { - edgeFoldedInto.data!.targetMethodIds.push(callee); - } - continue; - } - const kind = sourceExpanded ? method.kind : undefined; - edgesById.set(id, { - id, + + addEdge(edgesById, { + id: sourceExpanded + ? `${source}|${sourceHandle}>${target}|${targetHandle}:${call.how}` + : `${source}>${target}|${targetHandle}`, source, sourceHandle, target, targetHandle, - type: "call", - data: { - how: sourceExpanded ? call.how : undefined, - kind, - count: call.count, - sourceMethodIds: [caller], - targetMethodIds: [callee], - }, - markerEnd: { - type: MarkerType.ArrowClosed, - color: colorOfKind(kind), - width: 16, - height: 16, - }, + kind: sourceExpanded ? method.kind : undefined, + how: sourceExpanded ? call.how : undefined, + count: call.count, + sourceId: caller, + targetId: methodId(call.stateTypeName, call.methodName), }); } + + for (const run of method.runs) { + const agent = agentsByName.get(run.agentName); + // A run naming an agent the analysis never recorded has + // nowhere to land. + if (agent === undefined) { + continue; + } + addEdge(edgesById, { + id: sourceExpanded + ? `${source}|${sourceHandle}>${agent.id}` + : `${source}>${agent.id}`, + source, + sourceHandle, + target: agent.id, + targetHandle: AGENT_TARGET_HANDLE, + kind: sourceExpanded ? method.kind : undefined, + run: true, + count: run.count, + sourceId: caller, + targetId: agent.id, + }); + } + } + } + } + + for (const agent of agents) { + for (const tool of agent.tools) { + const source = agent.id; + const sourceHandle = `s:${tool.name}`; + const caller = toolId(agent.id, tool.name); + + for (const call of tool.calls) { + if (!isDrawn(call)) { + continue; + } + const targetPackage = packageOfStateTypeName(call.stateTypeName); + const targetExpanded = !collapsed.has(targetPackage); + const target = targetExpanded + ? call.stateTypeName + : packageNodeId(targetPackage); + const targetHandle = targetExpanded + ? `t:${call.methodName}` + : undefined; + + addEdge(edgesById, { + id: `${source}|${sourceHandle}>${target}|${targetHandle}:${call.how}`, + source, + sourceHandle, + target, + targetHandle, + kind: "workflow", + how: call.how, + count: call.count, + sourceId: caller, + targetId: methodId(call.stateTypeName, call.methodName), + }); + } + + for (const run of tool.runs) { + const target = agentsByName.get(run.agentName); + if (target === undefined) { + continue; + } + addEdge(edgesById, { + id: `${source}|${sourceHandle}>${target.id}`, + source, + sourceHandle, + target: target.id, + targetHandle: AGENT_TARGET_HANDLE, + kind: "workflow", + run: true, + count: run.count, + sourceId: caller, + targetId: target.id, + }); } } } + return [...edgesById.values()]; }; @@ -602,58 +812,70 @@ const ExpandedPackageNode: FC< ); -// an edge landing on its left or leaving on its right. The handles -// are invisible: the edge just needs somewhere to land. Hovering the -// row shows, beside the card, the cones a click there lights: the -// left third of the row, the arrow in, who calls the method; the -// right third, the arrow out, what it calls; the middle, both. A -// click chooses the method with those cones, which also opens it in -// the types pane, and a click asking for what is already lit lets it -// go. -const MethodRow: FC<{ +// One row of a card: a method of a state type, or a tool of an +// agent, with an edge landing on its left or leaving on its right. +interface CardRow { + // The row's id, in the one space every arrow's ends are named in. id: string; - method: GraphMethod; + name: string; + kind?: Kind; + // Set on an agent's tool, whose dot is a ring: it runs in the + // workflow that runs the agent, but no API declares it and it is + // the agent, not the application, that calls it. + tool?: boolean; + // What the row says of itself when the pointer rests on it. + title: string; + // A word the row wears at its right, e.g. `factory`. + badge?: string; +} + +// The handles are invisible: the edge just needs somewhere to land. +// Hovering the row shows, beside the card, the cones a click there +// lights: the left third of the row, the arrow in, who calls it; the +// right third, the arrow out, what it calls; the middle, both. A +// click chooses the row with those cones, which also opens it in the +// types pane, and a click asking for what is already lit lets it go. +const Row: FC<{ + row: CardRow; selected: boolean; onHover: (third: RowThird | null) => void; onSelect: (id: string, cones: Cones) => void; -}> = ({ id, method, selected, onHover, onSelect }) => ( +}> = ({ row, selected, onHover, onSelect }) => (
onHover(thirdOfPointer(event))} onMouseLeave={() => onHover(null)} onClick={(event) => { event.stopPropagation(); - onSelect(id, CONES_OF_THIRD[thirdOfPointer(event)]); + onSelect(row.id, CONES_OF_THIRD[thirdOfPointer(event)]); }} - title={ - method.kind === undefined - ? "unknown" - : `${labelOfKind(method.kind)}${method.factory ? ", factory" : ""}` - } + title={row.title} >
); // A button beside the card, level with a row, for one of the row's -// cones: lit in the method's kind colour while that cone is shown, -// and unlit again under the pointer, since a click then puts it out. +// cones: lit in the row's kind colour while that cone is shown, and unlit +// again under the pointer, since a click then puts it out. const ConeButton: FC<{ cone: keyof Cones; top: number; @@ -716,21 +938,39 @@ const ConeButton: FC<{ // the card, past the row's edge. const HOVER_LINGER_MS = 250; -const StateTypeNode: FC>> = ({ - data, +// What a state type and an agent are both drawn as: a head, a row +// each, and the cone buttons flanking the chosen row. `headHeight` +// is what the head comes out at, which is what places the buttons, +// since the card clips its contents and they are siblings of it. +const RowsCard: FC<{ + className: string; + head: ReactNode; + headHeight: number; + rows: CardRow[]; + selectedRow?: string | null; + onSelectRow?: (id: string, cones: Cones) => void; + cones?: Cones; + onToggleCone?: (cone: keyof Cones) => void; + // The rows some drawn arrow lands on, and the rows that make one; + // a button with nothing to light is never shown. + withCallers?: Set; + withCalls?: Set; +}> = ({ + className, + head, + headHeight, + rows, + selectedRow, + onSelectRow, + cones, + onToggleCone, + withCallers, + withCalls, }) => { - // The chosen row's place in the card, for the cone buttons that - // flank it. The card clips its contents, so the buttons are - // siblings of it, placed by the layout's own row arithmetic. const selectedIndex = - data.selectedMethod == null - ? -1 - : data.stateType.methods.findIndex( - (method) => - methodId(data.stateType.id, method.name) === data.selectedMethod - ); + selectedRow == null ? -1 : rows.findIndex((row) => row.id === selectedRow); const topOfRow = (index: number): number => - 1 + HEAD_HEIGHT + index * ROW_HEIGHT + ROW_HEIGHT / 2; + 1 + headHeight + index * ROW_HEIGHT + ROW_HEIGHT / 2; // The row the pointer is over and which third of it, with the // hide put off a moment when the pointer leaves, so it can reach @@ -755,13 +995,13 @@ const StateTypeNode: FC>> = ({ }, [keepShown]); useEffect(() => keepShown, [keepShown]); - // The cones a method has anything to light in. + // The cones a row has anything to light in. const availableCones = (id: string): Cones => ({ - upstream: data.withCallers?.has(id) ?? false, - downstream: data.withCalls?.has(id) ?? false, + upstream: withCallers?.has(id) ?? false, + downstream: withCalls?.has(id) ?? false, }); - // Chooses a method with the cones asked for, of those it has; a + // Chooses a row with the cones asked for, of those it has; a // click asking only for a cone it lacks lights what it has. const select = (id: string, asked: Cones): void => { const available = availableCones(id); @@ -769,7 +1009,7 @@ const StateTypeNode: FC>> = ({ upstream: asked.upstream && available.upstream, downstream: asked.downstream && available.downstream, }; - data.onSelectMethod?.( + onSelectRow?.( id, wanted.upstream || wanted.downstream ? wanted : available ); @@ -780,25 +1020,23 @@ const StateTypeNode: FC>> = ({ // asks for a cone it has not lit shows that cone's button too, // which lights it. Any other hovered row shows the buttons for the // cones its third asks for, of those it has, and clicking one - // chooses the method with that cone alone. + // chooses the row with that cone alone. const asked = hovered === null ? undefined : CONES_OF_THIRD[hovered.third]; const buttonsOf = ( index: number - ): { id: string; method: GraphMethod; show: Cones } | undefined => { + ): { row: CardRow; show: Cones } | undefined => { if (index === -1) { return undefined; } - const method = data.stateType.methods[index]; - const id = methodId(data.stateType.id, method.name); - const available = availableCones(id); + const row = rows[index]; + const available = availableCones(row.id); const lit = index === selectedIndex - ? data.cones ?? DEFAULT_CONES + ? cones ?? DEFAULT_CONES : { upstream: false, downstream: false }; const hoveredHere = hovered?.index === index && asked !== undefined; return { - id, - method, + row, show: { upstream: available.upstream && @@ -817,38 +1055,24 @@ const StateTypeNode: FC>> = ({ return ( <> -
- {/* The name is the way to the state type in the types pane. */} -
{ - event.stopPropagation(); - data.onOpenStateType?.(data.stateType.id); - }} - > - {data.stateType.name} -
- {data.stateType.methods.map((method, index) => { - const id = methodId(data.stateType.id, method.name); - return ( - { - if (third === null) { - hideSoon(); - } else { - keepShown(); - setHovered({ index, third }); - } - }} - onSelect={select} - key={method.name} - /> - ); - })} +
+ {head} + {rows.map((row, index) => ( + { + if (third === null) { + hideSoon(); + } else { + keepShown(); + setHovered({ index, third }); + } + }} + onSelect={select} + key={row.id} + /> + ))}
{selectedButtons !== undefined && (["upstream", "downstream"] as const).map( @@ -857,18 +1081,18 @@ const StateTypeNode: FC>> = ({ data.onToggleCone?.(cone)} + onClick={() => onToggleCone?.(cone)} onMouseEnter={keepShown} onMouseLeave={hideSoon} key={cone} @@ -884,14 +1108,14 @@ const StateTypeNode: FC>> = ({ cone={cone} top={topOfRow(hovered.index)} lit={false} - color={colorOfKind(hoveredButtons.method.kind)} + color={colorOfKind(hoveredButtons.row.kind)} title={ cone === "upstream" - ? "Show who calls this method" - : "Show what this method calls" + ? "Show what calls this" + : "Show what this calls" } onClick={() => - select(hoveredButtons.id, { + select(hoveredButtons.row.id, { upstream: cone === "upstream", downstream: cone === "downstream", }) @@ -906,6 +1130,101 @@ const StateTypeNode: FC>> = ({ ); }; +const StateTypeNode: FC>> = ({ + data, +}) => ( + { + event.stopPropagation(); + data.onOpenStateType?.(data.stateType.id); + }} + > + {data.stateType.name} +
+ } + rows={data.stateType.methods.map((method) => ({ + id: methodId(data.stateType.id, method.name), + name: method.name, + kind: method.kind, + badge: method.factory ? "factory" : undefined, + title: + method.kind === undefined + ? "unknown" + : `${labelOfKind(method.kind)}${method.factory ? ", factory" : ""}`, + }))} + selectedRow={data.selectedRow} + onSelectRow={data.onSelectRow} + cones={data.cones} + onToggleCone={data.onToggleCone} + withCallers={data.withCallers} + withCalls={data.withCalls} + /> +); + +// An agent: the robot, its name and its model, the first of its +// prompt, and a row per tool. Runs land on its head, since running +// it is running the whole of it, and each tool's own calls leave its +// row. +const AgentNode: FC>> = ({ data }) => ( + +
{ + event.stopPropagation(); + data.onOpenAgent?.(data.agent.id); + }} + > + +
+ + {data.agent.name} +
+ {data.agent.model !== undefined && ( + {data.agent.model} + )} +
+ {data.agent.systemPrompt.length > 0 && ( +
+ {data.agent.systemPrompt.join("\n\n")} +
+ )} + + } + rows={data.agent.tools.map((tool) => ({ + id: toolId(data.agent.id, tool.name), + name: tool.name, + kind: "workflow" as const, + tool: true, + title: tool.description ?? "an agent's tool", + }))} + selectedRow={data.selectedRow} + onSelectRow={data.onSelectRow} + cones={data.cones} + onToggleCone={data.onToggleCone} + withCallers={data.withCallers} + withCalls={data.withCalls} + /> +); + const CallEdge: FC>> = ({ id, source, @@ -952,7 +1271,13 @@ const CallEdge: FC>> = ({ const kind = data?.kind; const how = data?.how; const count = data?.count ?? 1; - const howWord = how === undefined ? undefined : HOW_LABEL[how]; + // A run always says so; a plain call says nothing, since it is the + // ordinary case, and labelling every edge "calls" would be noise. + const howWord = data?.run + ? "runs" + : how === undefined + ? undefined + : HOW_LABEL[how]; const label = count > 1 ? `${howWord ?? "calls"} ×${count}` : howWord; const dashPattern = (how === undefined ? undefined : HOW_DASH[how]) ?? @@ -1029,6 +1354,14 @@ const Legend: FC = () => ( unknown +
+
@@ -1044,6 +1377,12 @@ const Legend: FC = () => ( a workflow's calls
+
+
+ + an agent and its tools +
+
factory @@ -1059,6 +1398,7 @@ const nodeTypes = { package: PackageNode, expanded: ExpandedPackageNode, stateType: StateTypeNode, + agent: AgentNode, }; const edgeTypes = { call: CallEdge }; @@ -1080,12 +1420,22 @@ const graphViews = new Map< const GraphCanvas: FC<{ packages: GraphPackage[]; - // The chosen method's id, which is what the URL names: choosing - // one is a navigation, so back steps to the one chosen before. - selectedMethodId: string | null; - onSelectMethod: (id: string | null, replace?: boolean) => void; + agents: GraphAgent[]; + // The chosen row's id, a method's or a tool's, which is what the + // URL names: choosing one is a navigation, so back steps to the + // one chosen before. + selectedRowId: string | null; + onSelectRow: (id: string | null, replace?: boolean) => void; onOpenStateType: (id: string) => void; -}> = ({ packages, selectedMethodId, onSelectMethod, onOpenStateType }) => { + onOpenAgent: (id: string) => void; +}> = ({ + packages, + agents, + selectedRowId, + onSelectRow, + onOpenStateType, + onOpenAgent, +}) => { const location = useLocation(); const saved = useNavigationType() === "POP" ? graphViews.get(location.key) : undefined; @@ -1134,7 +1484,7 @@ const GraphCanvas: FC<{ useEffect(() => { const thisLayoutRun = ++layoutRun.current; - layoutPackages(packages, collapsed).then((nodes) => { + layoutPackages(packages, agents, collapsed).then((nodes) => { if (layoutRun.current !== thisLayoutRun) { return; } @@ -1185,11 +1535,18 @@ const GraphCanvas: FC<{ }); } }); - }, [packages, collapsed, fitView]); + }, [packages, agents, collapsed, fitView]); const edges = useMemo( - () => edgesOfPackages(packages, collapsed), - [packages, collapsed] + () => edgesOf(packages, agents, collapsed), + [packages, agents, collapsed] + ); + + // Who leads to whom over everything drawn, which the cones and the + // cone buttons are both read off. + const reach = useMemo( + () => reachability(packages, agents), + [packages, agents] ); const togglePackage = useCallback( @@ -1206,17 +1563,18 @@ const GraphCanvas: FC<{ }); // Closing the chosen method's own box lets it go, since its // row is gone; replaced rather than pushed, since the reader - // clicked the box, not the choice. + // clicked the box, not the choice. An agent's row is in no + // box, so it stays. if ( - selectedMethodId !== null && - packageOfStateTypeName(stateTypeNameOfMethodId(selectedMethodId)) === - name && + selectedRowId !== null && + !isAgentRowId(selectedRowId) && + packageOfStateTypeName(cardOfRowId(selectedRowId)) === name && !collapsed.has(name) ) { - onSelectMethod(null, true); + onSelectRow(null, true); } }, - [collapsed, selectedMethodId, onSelectMethod] + [collapsed, selectedRowId, onSelectRow] ); const setAllCollapsed = useCallback( @@ -1226,11 +1584,15 @@ const GraphCanvas: FC<{ setCollapsed( new Set(allCollapsed ? packages.map((pkg) => pkg.name) : []) ); - if (allCollapsed && selectedMethodId !== null) { - onSelectMethod(null, true); + if ( + allCollapsed && + selectedRowId !== null && + !isAgentRowId(selectedRowId) + ) { + onSelectRow(null, true); } }, - [packages, selectedMethodId, onSelectMethod] + [packages, selectedRowId, onSelectRow] ); // Which cones of the chosen method the graph lights: what the @@ -1247,58 +1609,32 @@ const GraphCanvas: FC<{ useEffect(() => { setCones(askedCones.current ?? DEFAULT_CONES); askedCones.current = null; - }, [selectedMethodId]); + }, [selectedRowId]); - const toggleMethodSelection = useCallback( + const toggleRowSelection = useCallback( (id: string, asked: Cones) => { - if (selectedMethodId !== id) { + if (selectedRowId !== id) { askedCones.current = asked; - onSelectMethod(id); + onSelectRow(id); } else if (sameCones(cones, asked)) { - onSelectMethod(null); + onSelectRow(null); } else { setCones(asked); } }, - [selectedMethodId, cones, onSelectMethod] + [selectedRowId, cones, onSelectRow] ); const toggleCone = useCallback((cone: keyof Cones): void => { setCones((current) => ({ ...current, [cone]: !current[cone] })); }, []); - // The methods some drawn call lands on, self-calls included, and - // the methods that make one: what a cone button needs to have - // anything to light. - const withCallers = useMemo(() => { - const ids = new Set(); - for (const pkg of packages) { - for (const stateType of pkg.stateTypes) { - for (const method of stateType.methods) { - for (const call of method.calls) { - if (isDrawn(call)) { - ids.add(methodId(call.stateTypeName, call.methodName)); - } - } - } - } - } - return ids; - }, [packages]); - - const withCalls = useMemo(() => { - const ids = new Set(); - for (const pkg of packages) { - for (const stateType of pkg.stateTypes) { - for (const method of stateType.methods) { - if (method.calls.some(isDrawn)) { - ids.add(methodId(stateType.id, method.name)); - } - } - } - } - return ids; - }, [packages]); + // The rows some drawn arrow lands on, self-calls included, and the + // rows that make one: what a cone button needs to have anything to + // light. + const withCallers = useMemo(() => new Set(reach.callers.keys()), [reach]); + + const withCalls = useMemo(() => new Set(reach.callees.keys()), [reach]); // With a method chosen, its lit cones: downstream, the methods it // calls transitively and the arrows carrying those calls; @@ -1308,12 +1644,10 @@ const GraphCanvas: FC<{ // in a cone when any method folded into it is. An expanded box // never fades: it is the room its cards are in. const unfaded = useMemo(() => { - if (selectedMethodId === null) { + if (selectedRowId === null) { return null; } - const nodeIds = new Set([ - stateTypeNameOfMethodId(selectedMethodId), - ]); + const nodeIds = new Set([cardOfRowId(selectedRowId)]); const edgeIds = new Set(); const light = (edge: Edge): void => { edgeIds.add(edge.id); @@ -1321,26 +1655,26 @@ const GraphCanvas: FC<{ nodeIds.add(edge.target); }; if (cones.downstream) { - const reached = reachableMethodIds(selectedMethodId, packages); + const reached = closure(selectedRowId, reach.callees); for (const edge of edges) { - if (edge.data!.sourceMethodIds.some((id) => reached.has(id))) { + if (edge.data!.sourceIds.some((id) => reached.has(id))) { light(edge); } } } if (cones.upstream) { - const reaching = reachingMethodIds(selectedMethodId, packages); + const reaching = closure(selectedRowId, reach.callers); for (const edge of edges) { if ( - edge.data!.sourceMethodIds.some((id) => reaching.has(id)) && - edge.data!.targetMethodIds.some((id) => reaching.has(id)) + edge.data!.sourceIds.some((id) => reaching.has(id)) && + edge.data!.targetIds.some((id) => reaching.has(id)) ) { light(edge); } } } return { nodeIds, edgeIds }; - }, [selectedMethodId, cones, packages, edges]); + }, [selectedRowId, cones, reach, edges]); const shownNodes = useMemo( () => @@ -1363,8 +1697,8 @@ const GraphCanvas: FC<{ className, data: { ...node.data, - selectedMethod: selectedMethodId, - onSelectMethod: toggleMethodSelection, + selectedRow: selectedRowId, + onSelectRow: toggleRowSelection, onOpenStateType, cones, onToggleCone: toggleCone, @@ -1372,6 +1706,21 @@ const GraphCanvas: FC<{ withCalls, }, }; + case "agent": + return { + ...node, + className, + data: { + ...node.data, + selectedRow: selectedRowId, + onSelectRow: toggleRowSelection, + onOpenAgent, + cones, + onToggleCone: toggleCone, + withCallers, + withCalls, + }, + }; default: return { ...node, className }; } @@ -1379,10 +1728,11 @@ const GraphCanvas: FC<{ [ nodes, unfaded, - selectedMethodId, - toggleMethodSelection, + selectedRowId, + toggleRowSelection, togglePackage, onOpenStateType, + onOpenAgent, cones, toggleCone, withCallers, @@ -1415,8 +1765,8 @@ const GraphCanvas: FC<{ } }} onPaneClick={() => { - if (selectedMethodId !== null) { - onSelectMethod(null); + if (selectedRowId !== null) { + onSelectRow(null); } }} // ELK places the nodes, so they don't move one by one. Left @@ -1457,25 +1807,52 @@ const GraphCanvas: FC<{ ); }; -// How many calls the graph draws: counted from the data rather than -// the edges, which collapse when their box is collapsed. -export const drawnCallCount = (stateTypes: GraphStateType[]): number => - stateTypes.reduce( - (count, stateType) => - count + - stateType.methods.reduce( - (count, method) => count + method.calls.filter(isDrawn).length, - 0 - ), - 0 +// How many arrows the graph draws: every call and every run, +// whoever makes it, counted from the data rather than the edges, +// which fold when their box is collapsed. +export const drawnCallCount = ( + stateTypes: GraphStateType[], + agents: GraphAgent[] +): number => { + const drawn = (calls: GraphCall[], runs: GraphRun[]): number => + calls.filter(isDrawn).length + runs.length; + return ( + stateTypes.reduce( + (count, stateType) => + count + + stateType.methods.reduce( + (count, method) => count + drawn(method.calls, method.runs), + 0 + ), + 0 + ) + + agents.reduce( + (count, agent) => + count + + agent.tools.reduce( + (count, tool) => count + drawn(tool.calls, tool.runs), + 0 + ), + 0 + ) ); +}; export const GraphPage: FC<{ stateTypes: GraphStateType[]; - selectedMethodId: string | null; - onSelectMethod: (id: string | null, replace?: boolean) => void; + agents: GraphAgent[]; + selectedRowId: string | null; + onSelectRow: (id: string | null, replace?: boolean) => void; onOpenStateType: (id: string) => void; -}> = ({ stateTypes, selectedMethodId, onSelectMethod, onOpenStateType }) => { + onOpenAgent: (id: string) => void; +}> = ({ + stateTypes, + agents, + selectedRowId, + onSelectRow, + onOpenStateType, + onOpenAgent, +}) => { const packages = useMemo( () => groupStateTypesByPackage(stateTypes), [stateTypes] @@ -1486,9 +1863,11 @@ export const GraphPage: FC<{
diff --git a/reboot/dashboard/web/src/main.tsx b/reboot/dashboard/web/src/main.tsx index ff3a9385e..80f209011 100644 --- a/reboot/dashboard/web/src/main.tsx +++ b/reboot/dashboard/web/src/main.tsx @@ -107,7 +107,14 @@ import { import { DashboardGetResponse_NeedsGenerateReason as NeedsGenerateReason } from "../../../../rbt/dashboard/v1/dashboard_pb"; import type * as feature_pb from "../../../../rbt/v1alpha1/bdd/feature_pb"; import type * as grammar_pb from "../../../../rbt/v1alpha1/bdd/grammar_pb"; -import { joinStateTypes, type GraphStateType } from "./callgraph"; +import { + agentId, + isAgentRowId, + joinAgents, + joinStateTypes, + type GraphAgent, + type GraphStateType, +} from "./callgraph"; import { exercisedMethods, graphStateTypeNamed, @@ -138,6 +145,11 @@ const DEFINITIONS: Record = { "Brings a state into existence: it is called with a new id " + "rather than on a state that already exists.", mcp: "Callable by AI agents as a tool, over the Model Context " + "Protocol.", + agent: + "A model the application hands work to, with tools it may call " + + "back with. Reboot runs one inside a workflow, so every model " + + "call and every tool call is durable: a restart replays what " + + "already happened rather than asking again.", "state type": "A durable data type. Each instance, named by an id, has properties " + "that Reboot persists for you. Methods are the way to read and " + @@ -323,40 +335,51 @@ interface PaneRows { const PaneRowsContext = createContext(undefined); // What the `type` search parameter names: one state type or one of -// its methods, `bank.v1.Account` or `bank.v1.Account.deposit`, or one -// data type, `bank.v1.bank.CustomerAccount`. The pane exists only -// while the parameter names something. +// its methods, `bank.v1.Account` or `bank.v1.Account.deposit`, one +// data type, `bank.v1.bank.CustomerAccount`, or one agent or one of +// its tools, `agent:librarian` or `agent:librarian.get_page`. The +// pane exists only while the parameter names something. type PaneTarget = - | { stateTypeId: string; method?: string; dataTypeId?: undefined } - | { dataTypeId: string; stateTypeId?: undefined; method?: undefined }; + | { kind: "stateType"; id: string; method?: string } + | { kind: "dataType"; id: string } + | { kind: "agent"; id: string; tool?: string }; -// The id of the type a target shows, whichever kind it is. -const typeIdOfTarget = (target: PaneTarget): string => - target.dataTypeId ?? target.stateTypeId; +// The id of the thing a target shows, whichever kind it is, which is +// what the URL carries. +const typeIdOfTarget = (target: PaneTarget): string => target.id; const paneTargetOf = ( raw: string | null, isStateTypeId: (id: string) => boolean, - isDataTypeId: (id: string) => boolean + isDataTypeId: (id: string) => boolean, + isAgentId: (id: string) => boolean ): PaneTarget | undefined => { if (raw === null) { return undefined; } if (isDataTypeId(raw)) { - return { dataTypeId: raw }; + return { kind: "dataType", id: raw }; } const separator = raw.lastIndexOf("."); - if ( - !isStateTypeId(raw) && - separator !== -1 && - isStateTypeId(raw.slice(0, separator)) - ) { + const before = separator === -1 ? undefined : raw.slice(0, separator); + if (isAgentRowId(raw)) { + if (!isAgentId(raw) && before !== undefined && isAgentId(before)) { + return { + kind: "agent", + id: before, + tool: raw.slice(separator + 1), + }; + } + return { kind: "agent", id: raw }; + } + if (!isStateTypeId(raw) && before !== undefined && isStateTypeId(before)) { return { - stateTypeId: raw.slice(0, separator), + kind: "stateType", + id: before, method: raw.slice(separator + 1), }; } - return { stateTypeId: raw }; + return { kind: "stateType", id: raw }; }; // The search string a link to a type produces. The path is left @@ -930,12 +953,102 @@ const DataType: FC<{ ); }; -// The types pane: one type, state or data, slid open by a link to it -// from the graph or a page, every method expanded; the X closes it. -// A link naming a method flashes the method. +// One agent, on the pane the way a state type is: what it is told to +// be, and what it may call back with. The prompt is what the agent +// is, so it is shown whole, in the shape it was written in. +const AgentPane: FC<{ agent: GraphAgent }> = ({ agent }) => ( +
+
+ +
+
+
+

+ + {agent.name} +

+ + + {countWithNoun(agent.tools.length, "tool")} + +
+
+ {agent.model !== undefined &&
{agent.model}
} + {agent.description !== undefined && ( + + )} + +
system prompt
+ {agent.systemPrompt.length === 0 ? ( +
+ No prompt written down. One computed rather than written is not + something the dashboard can read. +
+ ) : ( +
{agent.systemPrompt.join("\n\n")}
+ )} + +
tools
+ {agent.tools.length === 0 ? ( +
+ No tools. The agent answers out of the prompt alone, or out of tools the + dashboard cannot read, such as an MCP server's. +
+ ) : ( +
+ {agent.tools.map((tool) => ( +
+
+ {tool.name} +
+ {tool.description !== undefined && ( + + )} + {tool.calls.length + tool.runs.length > 0 && ( +
+ {tool.calls.map((call) => ( + + + {shortNameOfTypeName(call.stateTypeName)}. + {call.methodName} + + + ))} + {tool.runs.map((run) => ( + + 🤖 {run.agentName} + + ))} +
+ )} +
+ ))} +
+ )} +
+); + +// The types pane: one type, state or data, or one agent, slid open +// by a link to it from the graph or a page, every method expanded; +// the X closes it. A link naming a method flashes the method. const TypesPane: FC<{ apis: APIs; linkedDataTypes: LinkedDataType[]; + agents: GraphAgent[]; target: PaneTarget; // The property a followed link named, if any. propertyName?: string; @@ -950,6 +1063,7 @@ const TypesPane: FC<{ }> = ({ apis, linkedDataTypes, + agents, target, propertyName, flashKey, @@ -959,7 +1073,7 @@ const TypesPane: FC<{ }) => { const typeId = typeIdOfTarget(target); const found = - target.stateTypeId === undefined + target.kind !== "stateType" ? undefined : sortedAPIs(apis) .flatMap((api) => @@ -967,14 +1081,18 @@ const TypesPane: FC<{ ) .find( ({ api, stateType }) => - qualifiedName({ api, stateType }) === target.stateTypeId + qualifiedName({ api, stateType }) === target.id ); const foundDataType = - target.dataTypeId === undefined + target.kind !== "dataType" ? undefined : linkedDataTypes.find( - (linkedDataType) => linkedDataType.id === target.dataTypeId + (linkedDataType) => linkedDataType.id === target.id ); + const foundAgent = + target.kind !== "agent" + ? undefined + : agents.find((agent) => agent.id === target.id); const flashProperty = propertyName === undefined || flashKey === undefined ? undefined @@ -982,7 +1100,9 @@ const TypesPane: FC<{ return (
- {typeId} + + {foundAgent === undefined ? typeId : `🤖 ${foundAgent.name}`} +