diff --git a/rbt/dashboard/v1/dashboard.proto b/rbt/dashboard/v1/dashboard.proto index 28e82374a..f3b83eef9 100644 --- a/rbt/dashboard/v1/dashboard.proto +++ b/rbt/dashboard/v1/dashboard.proto @@ -49,6 +49,22 @@ message Dashboard { // whoever reads this to make of what they will. repeated Servicer servicers = 6; + // Represents the version of the analysis that recorded + // `servicers`, `agents`, `unattributed_hazards` and `code_files`. + // A dashboard whose analysis records something new analyzes every + // file again rather than carrying forward what an earlier one + // recorded, which would never say it; see `CODE_ANALYSIS_VERSION`. + uint32 code_analysis_version = 16; + + // Every agent found in the developer's application, sorted. One + // agent has a record per file whose analysis found it; see `Agent`. + repeated Agent agents = 17; + + // Every hazard the analysis of the developer's application recorded + // that belongs to no agent, run, tool or method, sorted; see + // `UnattributedHazard`. + repeated UnattributedHazard unattributed_hazards = 18; + // Represents every file the walk of the application analyzed, // keyed by its path, relative to the working directory for a file // under it and absolute otherwise, as of the last write. A @@ -119,6 +135,8 @@ message DashboardGetResponse { map apis = 4; map api_digests = 5; repeated Servicer servicers = 6; + repeated Agent agents = 16; + repeated UnattributedHazard unattributed_hazards = 17; map generated = 7; // The worst reason over all the API files, and absent when @@ -157,6 +175,9 @@ message DashboardUpdateApiResponse {} message DashboardUpdateCodeRequest { repeated Servicer servicers = 1; + uint32 code_analysis_version = 6; + repeated Agent agents = 7; + repeated UnattributedHazard unattributed_hazards = 8; map code_files = 2; map generated = 3; @@ -518,12 +539,71 @@ message Servicer { // makes. repeated Call calls = 3; - // Represents every call the analysis could not resolve to a - // Reboot definition: possibly a helper that itself makes - // Reboot calls, to be followed by a future analysis, and - // possibly nothing of Reboot's. Spelled the way `ast.unparse` - // writes the callee, e.g. "self._transfer". - repeated string ambiguous = 4; + // Represents every agent the method's implementation runs, + // itself or through the functions it calls. What each agent is, + // is in `Agent`, which each of these names. + repeated Agent.Run runs = 4; + + // One thing the method's implementation does, itself or through + // the functions it calls, that the analysis did not follow, + // recorded rather than guessed at or dropped, so the dashboard + // can show where what it draws may be incomplete. + message Hazard { + // A call the analysis could not resolve to a Reboot definition: + // one with no definition pyright can say, or one whose + // definition is no function, such as a stub's or a class's. + // Possibly a helper that itself makes Reboot calls, to be + // followed by a future analysis, and possibly nothing of + // Reboot's. + message AmbiguousCall { + // Represents what is called, spelled the way `ast.unparse` + // writes it, without the call's arguments, e.g. + // "self._transfer". + string callee = 1; + } + + // A run of an agent the analysis could not resolve: the call's + // own definition proves an agent is run, but what it is run on + // is not an `Agent(...)` with a string literal `name=`, bound to + // a name at the top level of a module. E.g. an agent whose + // `name=` is computed or missing, one a factory builds or + // `Agent.wrap` adopts from one, an alias, a parameter, an + // element of a collection, one held on `self`, one constructed + // inside a function or where it is run, or a name bound to more + // than one thing. Nothing about the run is recorded, what it is + // given included. + message RunOnUnresolvedAgent { + // Represents what is called, spelled the way `ast.unparse` + // writes it: the receiver and the method, without the call's + // arguments, e.g. `make_agent().run`. + string callee = 1; + } + + // An `override` of an agent the analysis could not resolve, the + // way it could not resolve a run's, e.g. + // `agents[0].override(model=...)`, which changes whatever runs + // inside it. + message OverrideOnUnresolvedAgent { + // Represents what is called, spelled the way `ast.unparse` + // writes it: the receiver and the method, without the call's + // arguments, e.g. `agents[0].override`. + string callee = 1; + } + + // Represents the file the hazard is written in, spelled the way + // the developer would open it. + string filename = 1; + + oneof hazard { + RunOnUnresolvedAgent run_on_unresolved_agent = 2; + OverrideOnUnresolvedAgent override_on_unresolved_agent = 3; + AmbiguousCall ambiguous_call = 4; + } + } + + // Represents everything the method's implementation does that the + // analysis did not follow, in the order it met them. + repeated Hazard hazards = 5; } // The state type it services, spelled as the runtime names one, @@ -534,17 +614,241 @@ message Servicer { // open it, e.g. "backend/src/account_servicer.py". string filename = 2; - // Represents the line its class is written on, counting from one. - uint32 line = 4; - - // Represents the column its class starts at, counting from zero. - uint32 character = 5; - // Represents every method it defines, in the order they are // written, which is the order somebody reading the file meets them. repeated Method methods = 3; } +// One agent the developer's application uses, as one file's analysis +// found it: a Reboot `Agent` constructed with a string literal `name=` +// and bound to a name at the top level of a module, which some method +// runs, or some function is decorated as a tool of. +// +// A record per file, because the analysis keys everything by the +// file it read, and an agent is met in more files than the one +// constructing it: a workflow runs it in one file, a tool is +// decorated on it in another. Every record says the same thing about +// the agent itself, since each is built from the construction the +// analysis resolved; what differs is the tools and the hazards each +// file contributed. Whoever reads these joins them on `name`, which +// the runtime requires to be unique. +message Agent { + // One thing about how an agent is constructed or given tools that + // the analysis did not follow, recorded rather than guessed at or + // dropped, so the dashboard can show where what it draws may be + // incomplete. + message Hazard { + // The agent is constructed with arguments that give it tools, or + // hide them, which are not followed. Each represents the value + // passed, spelled the way `ast.unparse` writes it, and is absent + // when it is not passed. + message ConstructedWith { + // `tools=`. + optional string tools = 1; + + // `toolsets=`. + optional string toolsets = 2; + + // `prepare_tools=`, which can hide any of its tools when it + // runs. + optional string prepare_tools = 3; + } + + // A tool is registered on the agent by calling `tool` or + // `tool_plain` rather than decorating with it, e.g. + // `librarian.tool(lookup)`, which is not followed. + message ToolRegisteredByCall { + // Represents the call, its arguments included, spelled the way + // `ast.unparse` writes it. + string call = 1; + } + + // The agent is overridden, with `agent.override(...)`, for + // whatever runs inside it, with arguments that change what it is + // or can reach, which are not followed. Each represents the value + // passed, spelled the way `ast.unparse` writes it, and is absent + // when it is not passed. + message Overridden { + // `tools=`, which reach the runs inside it. + optional string tools = 1; + + // `name=`, which does not change the name the runs inside it + // are memoized under. + optional string name = 2; + + // `model=`, which the runs inside it use instead of + // `Agent.model`. + optional string model = 3; + + // `instructions=`, which the runs inside it are given instead of + // those in `Agent.system_prompt`. + optional string instructions = 4; + } + + // Represents the file the hazard is written in, spelled the way + // the developer would open it. + string filename = 1; + + oneof hazard { + ConstructedWith constructed_with = 2; + ToolRegisteredByCall tool_registered_by_call = 3; + Overridden overridden = 4; + } + } + + // One run of an agent, made by a servicer method's implementation + // or by another agent's tool, through any of `Agent`'s entry points: + // `run`, `iter`, `run_stream` or `run_stream_events`. Which one says + // nothing about what the application does, so it is not recorded. + message Run { + // One thing a run is given on top of its agent that the analysis + // did not follow, recorded rather than guessed at or dropped. + message Hazard { + // The run is given arguments that change what its agent is or + // can reach. Each represents the value passed, spelled the way + // `ast.unparse` writes it, and is absent when it is not passed. + message RunArguments { + // `toolsets=`, which give the run tools. + optional string toolsets = 1; + + // `model=`, which the run uses instead of `Agent.model`. + optional string model = 2; + + // `instructions=`, which the run is given on top of those in + // `Agent.system_prompt`. + optional string instructions = 3; + } + + // Represents the file the hazard is written in, spelled the way + // the developer would open it. + string filename = 1; + + oneof hazard { + RunArguments run_arguments = 2; + + // The run is written inside the `with` of an `override` of its + // agent, which changes what it is or can reach. + Agent.Hazard.Overridden overridden = 3; + } + } + + // Represents the agent run, spelled as `Agent.name` spells it. + string agent = 1; + + // Represents everything the run is given on top of its agent, by + // its own arguments or by an `override` it is written inside, + // that the analysis did not follow. + repeated Hazard hazards = 2; + } + + // One tool an agent may call, and what analyzing its + // implementation found. + message Tool { + // One thing about how a tool is registered that the analysis did + // not follow, recorded rather than guessed at or dropped. + message Hazard { + // The tool is registered with `prepare=`, which can hide it + // when the agent runs. + message Prepared { + // Represents the value passed, spelled the way `ast.unparse` + // writes it. + string prepare = 1; + } + + // Represents the file the hazard is written in, spelled the way + // the developer would open it. + string filename = 1; + + oneof hazard { + Prepared prepared = 2; + } + } + + // Represents the tool's name as the model is told it: the + // `name` given where it was registered, and the function's own + // name otherwise. + string name = 1; + + // Represents what the model is told the tool does: the + // `description` given where it was registered, and the + // function's docstring otherwise. Absent when it has neither. + optional string description = 2; + + // Represents every Reboot call and agent run the tool's + // implementation makes, and everything it does that the analysis + // did not follow, found exactly the way a servicer method's are. + repeated Servicer.Method.Call calls = 3; + repeated Run runs = 4; + repeated Servicer.Method.Hazard method_hazards = 5; + + // Represents everything about how the tool is registered that the + // analysis did not follow. + repeated Hazard hazards = 6; + } + + // Represents the agent's `name`, which the runtime requires and + // which is unique in an application, so it is what a run names and + // what records of one agent join on. + string name = 1; + + // The file whose analysis found this record, spelled the way the + // developer would open it, e.g. "backend/src/librarian.py". What + // the records are keyed by, and not necessarily where the agent is + // constructed. + string filename = 2; + + // Represents the model the agent is constructed with, e.g. + // "anthropic:claude-sonnet-4-6". Absent when it is not a literal + // the analysis can read. + optional string model = 3; + + // Represents what the agent tells the model about itself: every + // literal string its `system_prompt` and its `instructions` are + // constructed with, in that order. A prompt computed rather than + // written down contributes nothing. + repeated string system_prompt = 4; + + // Represents the agent's `description`, which is what another + // agent handing work to this one is told it does. Absent when it + // has none the analysis can read. + optional string description = 5; + + // Represents every tool this file's analysis found the agent has, + // in the order it met them: each function decorated as one with + // `@agent.tool` or `@agent.tool_plain`, the one way of giving an + // agent a tool that can always be followed. + repeated Tool tools = 6; + + // Represents everything about how the agent is constructed, given + // tools and overridden that the analysis did not follow. What a single run + // is given is on the run; see `Run.hazards`. + repeated Hazard hazards = 7; +} + +// One thing the analysis did not follow that belongs to no agent, run, +// tool or method, recorded rather than dropped, for the file it is +// written in. +message UnattributedHazard { + // A tool is registered, decorated or by a call, on an agent that + // cannot be resolved, e.g. `@agents[0].tool`, so there is no agent + // to record it on. + message ToolOnUnresolvedAgent { + // Represents the registration, spelled the way `ast.unparse` + // writes it: the decorator without its arguments, e.g. + // `agents[0].tool`, or the whole call registering by calling, + // e.g. `agents[0].tool_plain(count)`. + string registration = 1; + } + + // Represents the file the hazard is written in, spelled the way the + // developer would open it. + string filename = 1; + + oneof hazard { + ToolOnUnresolvedAgent tool_on_unresolved_agent = 2; + } +} + // What analyzing one of the developer's files used: the digest // of its bytes and what each of its imports observed. What a // restarted dashboard reconstitutes its knowledge from, so that only diff --git a/reboot/dashboard/backend/code_watcher.py b/reboot/dashboard/backend/code_watcher.py index 40d4b1cef..f752571ea 100644 --- a/reboot/dashboard/backend/code_watcher.py +++ b/reboot/dashboard/backend/code_watcher.py @@ -27,6 +27,23 @@ class extends: a servicer is a class with a base whose type is the servicers, so that whoever reads both can tell a state type with nothing generated for it. +The same walk finds the agents the application uses. A run of one is +a call whose definition lands on one of Reboot's `Agent` entry +points -- `run`, `iter`, `run_stream`, `run_stream_events` -- and +which agent is run is what the receiver of that call resolves to: an +`Agent(...)` with a string literal `name=`, bound to a name at the +top level of a module, which says the agent's name, its model and +the prompt it is given. An agent's tools are found by their +definitions rather than their spellings too: a function whose +decorator resolves to the `tool` a Reboot `Agent` registers one +with. Each tool's body is analyzed exactly as a servicer method's +is, so what the model can reach through an agent is recorded beside +what the application calls itself. Whatever is not followed -- a run +of any other agent, tools given any other way, what a run or an +`override` is given on top of its agent -- is recorded as a hazard +instead, so that nothing the analysis could not see is silently +missing. + Where following stops is what makes this the developer's code rather than somebody else's. A module resolves to a file only if a root holds it, so an import of an installed package leads nowhere. @@ -52,9 +69,14 @@ class extends: a servicer is a class with a base whose type is from functools import partial from google.protobuf.timestamp_pb2 import Timestamp from pathlib import Path -from rbt.dashboard.v1.dashboard_pb2 import Change +from rbt.dashboard.v1.dashboard_pb2 import Agent, Change from rbt.dashboard.v1.dashboard_pb2 import Dashboard as DashboardState -from rbt.dashboard.v1.dashboard_pb2 import File, Generated, Servicer +from rbt.dashboard.v1.dashboard_pb2 import ( + File, + Generated, + Servicer, + UnattributedHazard, +) from rbt.dashboard.v1.dashboard_rbt import Dashboard from reboot.aio.contexts import WorkflowContext from reboot.aio.cooperatively import cooperatively @@ -76,13 +98,52 @@ class extends: a servicer is a class with a base whose type is _walk, ) from types import MappingProxyType -from typing import Mapping, Optional, Sequence - -# One Reboot call a method's implementation makes: which state type, -# which method, and how the call is reached. Aliased from where it -# is defined so that what an analysis records and what a reader -# reads are one message. -Call = Servicer.Method.Call +from typing import Mapping, MutableSequence, Optional, Sequence + +# Where Reboot's `Agent` is written, as the last parts of the path +# pyright answers with, so that the module is recognized wherever +# `reboot` is installed. What tells a run of an agent from any other +# call: the call's own definition is one of the methods below, +# whichever way the agent was come by. +AGENT_MODULE = ('reboot', 'agents', 'pydantic_ai', '_agent.py') + +# The methods of `Agent` a run is made through, which is what a +# run's own definition lands on. `run_sync` and `run_stream_sync` are +# not here: a Reboot `Agent` raises on both, since a workflow is +# always async. +RUN_NAMES = ('run', 'iter', 'run_stream', 'run_stream_events') + +# The methods a tool is registered on an agent with, decorating with +# `@agent.tool` or calling `agent.tool(...)`, which is what the +# registration's own definition lands on. +TOOL_REGISTRATION_NAMES = ('tool', 'tool_plain') + +# The keyword arguments a run may be given that change what its agent +# is or can reach, and those an agent may be constructed with that +# give it tools or hide them, which the analysis does not follow; see +# `Agent.Run.Hazard.RunArguments` and `Agent.Hazard.ConstructedWith`. +RUN_ARGUMENTS = ('toolsets', 'model', 'instructions') +CONSTRUCTION_ARGUMENTS = ('tools', 'toolsets', 'prepare_tools') + +# The keyword arguments an `override` may be given that change what +# the runs inside it are or can reach, which the analysis does not +# follow; see `Agent.Hazard.Overridden`. +OVERRIDE_ARGUMENTS = ('tools', 'name', 'model', 'instructions') + +# The version of what the analysis records, which the dashboard's +# state records beside it. Counted up whenever the analysis starts +# recording something it did not, or records something differently: +# a file that has not changed is otherwise carried forward as an +# earlier analysis recorded it, which never says the new thing. +CODE_ANALYSIS_VERSION = 1 + + +def _is_agent_module(filename: Path) -> bool: + """Returns whether a file is the module Reboot's `Agent` is + written in, which is what makes a call to something defined + there a run of an agent rather than a call of the developer's + own.""" + return filename.parts[-len(AGENT_MODULE):] == AGENT_MODULE @dataclass(frozen=True, kw_only=True) @@ -114,6 +175,16 @@ class AnalyzedFile: # services and the calls each method makes. servicers: tuple[Servicer, ...] + # Every agent this file's analysis found: one it runs, and one a + # function it holds is decorated as a tool of. An agent met in + # more than one file has a record in each, which whoever reads + # them joins; see `Agent`. + agents: tuple[Agent, ...] + + # Every hazard this file's analysis recorded that belongs to no + # agent, run, tool or method; see `UnattributedHazard`. + unattributed_hazards: tuple[UnattributedHazard, ...] + def _position_at_last_character( node: ast.Name | ast.Attribute, @@ -222,7 +293,7 @@ class MethodDefinition: # is written: e.g. `CONSTRUCT` for one on the state type's own # class, and `SCHEDULE` for one inside its # `WeakReference._Schedule`. - how: 'Call.How.ValueType' + how: 'Servicer.Method.Call.How.ValueType' GeneratedDefinition = ( @@ -236,14 +307,14 @@ class MethodDefinition: # `.forall(ids)`; for `.idempotently(...)`, a plain call or a # construction made idempotent. HOWS_BY_CLASS_NAME = { - '_ConstructIdempotently': Call.How.CONSTRUCT, - '_Forall': Call.How.FORALL, - '_Idempotently': Call.How.CALL, - '_Schedule': Call.How.SCHEDULE, - '_SelfIdempotently': Call.How.CALL, - '_SelfSchedule': Call.How.SCHEDULE, - '_Spawn': Call.How.SPAWN, - '_Until': Call.How.UNTIL, + '_ConstructIdempotently': Servicer.Method.Call.How.CONSTRUCT, + '_Forall': Servicer.Method.Call.How.FORALL, + '_Idempotently': Servicer.Method.Call.How.CALL, + '_Schedule': Servicer.Method.Call.How.SCHEDULE, + '_SelfIdempotently': Servicer.Method.Call.How.CALL, + '_SelfSchedule': Servicer.Method.Call.How.SCHEDULE, + '_Spawn': Servicer.Method.Call.How.SPAWN, + '_Until': Servicer.Method.Call.How.UNTIL, } @@ -263,7 +334,7 @@ def _method_definitions_in( body: Sequence[ast.stmt], *, state_type: str, - how: Call.How.ValueType, + how: Servicer.Method.Call.How.ValueType, ) -> dict[int, MethodDefinition]: """Returns the method stubs a class body defines, by line: each def taking `__context__` second, and each alias the generator @@ -367,7 +438,7 @@ def _generated_definitions( _method_definitions_in( statement.body, state_type=state_type, - how=Call.How.CONSTRUCT, + how=Servicer.Method.Call.How.CONSTRUCT, ) ) for inner in statement.body: @@ -387,7 +458,7 @@ def _generated_definitions( _method_definitions_in( inner.body, state_type=state_type, - how=Call.How.CALL, + how=Servicer.Method.Call.How.CALL, ) ) for node in inner.body: @@ -478,6 +549,253 @@ def _helper_definitions( ) +def _keyword(call: ast.Call, name: str) -> Optional[ast.expr]: + """Returns what a call passes a keyword argument, and `None` for + one it does not pass.""" + for keyword in call.keywords: + if keyword.arg == name: + return keyword.value + return None + + +def _passes(call: ast.Call, name: str) -> bool: + """Returns whether a call passes a keyword argument something: one + it passes anything but `None`, or an empty list or tuple written + out, which give nothing.""" + match _keyword(call, name): + case None | ast.Constant(value=None): + return False + case ast.List(elts=[]) | ast.Tuple(elts=[]): + return False + return True + + +def _passed(call: ast.Call, keywords: Sequence[str]) -> dict[str, str]: + """Returns the value a call passes each of some keyword arguments + that gives something, by keyword, spelled the way `ast.unparse` + writes it; see `_passes`.""" + return { + keyword: ast.unparse(value) + for keyword in keywords + if (value := _keyword(call, keyword)) is not None and + _passes(call, keyword) + } + + +def _add_hazard( + hazards: MutableSequence[Agent.Hazard], + hazard: Agent.Hazard, +) -> None: + """Adds a hazard once: the same thing written in two places a + file's analysis reaches is still one thing to show.""" + if hazard not in hazards: + hazards.append(hazard) + + +def _try_constant_string(node: Optional[ast.expr]) -> Optional[str]: + """Returns the string an expression is, and `None` for an + expression that is anything else: a name, an f-string, a call. + Adjacent string literals are one expression, so a prompt written + across a screenful of quoted lines reads as the one string the + developer meant.""" + match node: + case ast.Constant(value=str(value)): + return value + return None + + +def _elements(node: Optional[ast.expr]) -> list[ast.expr]: + """Returns what a list or a tuple written out holds, and nothing + for anything else, such as a name the elements were gathered + under.""" + match node: + case ast.List(elts=elements) | ast.Tuple(elts=elements): + return list(elements) + return [] + + +def _constant_strings(node: Optional[ast.expr]) -> list[str]: + """Returns the strings an expression is: the one it is, or every + one written out in the list or tuple it is.""" + string = _try_constant_string(node) + if string is not None: + return [string] + return [ + string for element in _elements(node) + if (string := _try_constant_string(element)) is not None + ] + + +@dataclass(frozen=True, kw_only=True) +class AgentDefinition: + """A line an agent comes from, e.g. + `librarian = Agent('anthropic:claude-sonnet-4-6', name='librarian')`. + + What a run of an agent, and a tool registered on one, lead to: + the agent both are made on, with everything the line says about + it, and what it does not. + """ + + # The `name` the agent is constructed with, which tells it from + # every other agent; see `Agent.name`. + name: str + + # What its construction says, each absent when it says nothing a + # literal can be read from. + model: Optional[str] + system_prompt: tuple[str, ...] + description: Optional[str] + + # Everything the line says about the agent that is not followed; + # see `Agent.hazards`. + hazards: tuple[Agent.Hazard, ...] + + +def _is_agent_construction(call: ast.Call) -> bool: + """Returns whether a call is written as constructing an agent, + `Agent(...)` or `pydantic_ai.Agent(...)`, rather than as a call to + something that builds one, such as a factory, whose arguments say + what the factory is given rather than what the agent is. + + Read from how the call is spelled, since it is asked only of a + line a run or a registration already proved is an agent: an + `Agent` imported under + another name is not resolved, and says so, rather than being read + as something it is not. + """ + match call.func: + case ast.Name(id='Agent') | ast.Attribute(attr='Agent'): + return True + return False + + +def _agent_definition( + value: ast.Call, + *, + filename: Path, +) -> Optional[AgentDefinition]: + """Returns what the call an agent comes from says the agent is, + and `None` when it is not an `Agent(...)` with a string literal + `name=`, or an `Agent.wrap(...)` of one, which is the one agent the + analysis can tell from every other: an agent a factory builds, or + one whose `name=` is computed or missing, is not resolved.""" + match value: + case ast.Call( + func=ast.Attribute(attr='wrap'), + args=[ast.Call() as wrapped, *_], + ) if _is_agent_construction(wrapped): + construction = wrapped + case _ if _is_agent_construction(value): + construction = value + case _: + return None + + name = _try_constant_string(_keyword(construction, 'name')) + if name is None: + return None + + hazards: list[Agent.Hazard] = [] + given = _passed(construction, CONSTRUCTION_ARGUMENTS) + if len(given) > 0: + hazards.append( + Agent.Hazard( + filename=str(filename), + constructed_with=Agent.Hazard.ConstructedWith(**given), + ) + ) + + # The model is the first argument, wherever it is passed: + # `Agent('anthropic:claude-sonnet-4-6')` or `Agent(model=...)`. + model = _keyword(construction, 'model') + if model is None and len(construction.args) > 0: + model = construction.args[0] + + return AgentDefinition( + name=name, + model=_try_constant_string(model), + system_prompt=tuple( + _constant_strings(_keyword(construction, 'system_prompt')) + + _constant_strings(_keyword(construction, 'instructions')) + ), + description=_try_constant_string( + _keyword(construction, 'description') + ), + hazards=tuple(hazards), + ) + + +def _agent_definitions( + filename: Path, + parse: Parse, +) -> Mapping[int, AgentDefinition]: + """Returns what each line at the top level of a module constructs + an agent as, by the line the assignment is written on, which is + the line pyright places the definition of the name it binds at. + + Only an agent bound to a name at the top level of a module is + indexed. One constructed anywhere else -- held on `self`, bound + inside a function, or constructed where it is run -- is not, so a + run of it is not resolved, and says so. + + Every top-level assignment of a call is indexed, because what + makes one of them an agent is the type of what it binds, which + only the run or the registration that led here has established. A name bound more than + once at the top level is not indexed at all: pyright places every + use of it at the first binding, whichever one the use holds, so + reading any of them risks recording a run of one agent as a run of + another. + """ + definitions: dict[int, tuple[str, AgentDefinition]] = {} + + # How many times each name is assigned at the top level. + assigned: dict[str, int] = {} + + for statement in parse.syntax.body: + match statement: + case ast.Assign(targets=targets): + for target in targets: + if isinstance(target, ast.Name): + assigned[target.id] = assigned.get(target.id, 0) + 1 + case ast.AnnAssign(target=ast.Name(id=str(name))): + assigned[name] = assigned.get(name, 0) + 1 + + match statement: + case ast.Assign( + targets=[ast.Name(id=str(bound_to))], + value=ast.Call() as value, + ) | ast.AnnAssign( + target=ast.Name(id=str(bound_to)), + value=ast.Call() as value, + ): + definition = _agent_definition(value, filename=filename) + if definition is not None: + definitions[statement.lineno] = (bound_to, definition) + + return MappingProxyType( + { + line: definition + for line, (bound_to, definition) in definitions.items() + if assigned[bound_to] == 1 + } + ) + + +@dataclass(frozen=True, kw_only=True) +class AgentFile: + """What reading one file an agent is constructed in during an + analysis said.""" + + # What each line of the file constructs, empty when the file + # could not be read or would not parse; see `_agent_definitions`. + definitions: Mapping[int, AgentDefinition] + + # The dependency each file whose analysis reads this file + # records in its `external`: present only for a file outside + # every root, which the walk never finds or digests, and always + # carrying a digest of bytes actually read. + external: Optional[Dependency] + + @dataclass(frozen=True, kw_only=True) class Analysis: """One iteration's analysis: what the walk read, for the asking. @@ -510,6 +828,10 @@ class Analysis: # indexed at most once per analysis. helpers: Mapping[Path, HelperFile] + # Every file an agent was found constructed in so far, keyed the + # same way and indexed at most once per analysis. + agents: Mapping[Path, AgentFile] + # The dependencies on files outside every root read since the # map was last emptied, keyed by filename in the spelling # `_standardized_path` returns. Emptied by `_analyze_file` for @@ -659,6 +981,57 @@ async def helper_definition_at( return helper.definitions.get(location.line), analysis + async def agent_definition_at( + self, + location: Location, + ) -> tuple[Optional[AgentDefinition], 'Analysis']: + """Returns the agent a file constructs at a location, and + `None` for a location that constructs none: a name bound to + an agent another name already held, or one a function + returned, whose construction is wherever that function + wrote it. The file is read, parsed and indexed at most once + per analysis.""" + if location.filename.suffix != '.py': + return None, self + + # Standardized because pyright spells its locations + # absolutely, while everything here is keyed by the + # spelling `_standardized_path` returns. + agent, analysis = await self._agent_file( + _standardized_path(location.filename) + ) + return agent.definitions.get(location.line), analysis + + async def _agent_file( + self, + filename: Path, + ) -> tuple[AgentFile, 'Analysis']: + """Returns what a file constructs, reading, parsing and + indexing it the first time it is asked about.""" + analysis = self + + agent = analysis.agents.get(filename) + if agent is None: + parse, external = await analysis._parse(filename) + agent = AgentFile( + definitions=( + _agent_definitions(filename, parse) + if parse is not None else MappingProxyType({}) + ), + external=external, + ) + analysis = replace( + analysis, + agents=MappingProxyType({ + **analysis.agents, + filename: agent, + }), + ) + + analysis = analysis._with_external_dependency(agent.external) + + return agent, analysis + async def _generated_definition_at( filename: Path, @@ -693,6 +1066,30 @@ async def _generated_definition_at( return await analysis.generated_definition_at(location) +@dataclass(frozen=True, kw_only=True) +class Implementation: + """What analyzing one function's body found, which is what a + servicer method and an agent's tool each record of what they + do.""" + + calls: tuple[Servicer.Method.Call, ...] + runs: tuple[Agent.Run, ...] + hazards: tuple[Servicer.Method.Hazard, ...] + + +def _ambiguous_call( + callee: ast.expr, *, filename: Path +) -> Servicer.Method.Hazard: + """Returns the hazard a call the analysis could not resolve is; see + `Servicer.Method.Hazard.AmbiguousCall`.""" + return Servicer.Method.Hazard( + filename=str(filename), + ambiguous_call=Servicer.Method.Hazard.AmbiguousCall( + callee=ast.unparse(callee) + ), + ) + + async def _analyze_function( function: ast.FunctionDef | ast.AsyncFunctionDef, *, @@ -700,10 +1097,12 @@ async def _analyze_function( text: str, analysis: Analysis, visited: frozenset[tuple[Path, int]], -) -> tuple[list[Call], list[str], Analysis]: - """Returns the Reboot calls a function's body makes, itself or - through the functions it calls, and the calls it makes that are - ambiguous. + agents: dict[str, Agent], +) -> tuple[Implementation, Analysis]: + """Returns what a function's body does, itself or through the + functions it calls: the Reboot calls it makes, the agents it + runs, and what it does that is not followed, the calls it makes + that are ambiguous included. A Reboot call is one whose own definition pyright places at a method stub of a state type. However the reference was come by, @@ -711,6 +1110,18 @@ async def _analyze_function( elsewhere, the called method's definition is the same, so one question decides. + A run of an agent is the same question answered by one of + `Agent`'s entry points, `run`, `iter`, `run_stream` or + `run_stream_events`, in the module Reboot writes them in. Which + agent is run is what the receiver of the call resolves to, an + `Agent(...)` with a literal `name=` at the top level of a module; + the record of that agent joins `agents`, which is every agent the + file being analyzed has found so far. What a run is given on top of its + agent, by its own arguments or by an `override` it is written + inside, is not followed, but recorded on the run, and a run whose + agent cannot be resolved is recorded as a hazard of the function; + see `Agent.Run.Hazard` and `Servicer.Method.Hazard`. + A call defined by a function the generator did not write, the developer's own or an installed package's, is followed: that function's body is analyzed the same way and its calls are the @@ -725,14 +1136,16 @@ async def _analyze_function( function already walked on the way here, by file and line, so that functions calling each other are followed once. - An ambiguous call is one with no definition pyright can say, or - one whose definition is no function: a stub's, which has no body - to follow, or a class's. A call whose definition is the + An ambiguous call, a hazard like any other, is one with no + definition pyright can say, or one whose definition is no + function: a stub's, which has no body to follow, or a class's. A call whose definition is the generator's own machinery, such as the `ref` or `schedule` - inside a chain, or the standard library's, is neither. + inside a chain, Reboot's own machinery around a run, or the + standard library's, is neither. """ - calls: list[Call] = [] - ambiguous: list[str] = [] + calls: list[Servicer.Method.Call] = [] + runs: list[Agent.Run] = [] + hazards: list[Servicer.Method.Hazard] = [] # The function itself is walked here, and everything defined # in it with it: an `async def inner` written in the body, or a @@ -742,6 +1155,28 @@ async def _analyze_function( assert function.end_lineno is not None span = range(function.lineno, function.end_lineno + 1) + # The lines of each `with` a call is entered in, by the call, so + # that a run written inside an `override` can say what the + # override changes about it. Lexical, so a run in a function + # called from inside the `with` is not said to be inside it; the + # agent still says it is overridden. + override_bodies: dict[int, range] = {} + for node in ast.walk(function): + match node: + case ast.With() | ast.AsyncWith(): + assert node.end_lineno is not None + for item in node.items: + override_bodies[id(item.context_expr)] = range( + node.body[0].lineno, + node.end_lineno + 1, + ) + + # Each `override` found so far, with the lines of its `with`, the + # agent it is made on and what it changes. `ast.walk` reaches the + # call a `with` enters before any call in its body, so every run + # inside finds its override here. + overrides: list[tuple[range, str, Agent.Hazard.Overridden]] = [] + for node in ast.walk(function): match node: case ast.Call(func=(ast.Attribute() | ast.Name()) as callee): @@ -758,7 +1193,7 @@ async def _analyze_function( text=text, ) if location is None: - ambiguous.append(ast.unparse(callee)) + hazards.append(_ambiguous_call(callee, filename=filename)) continue # Nothing in the standard library takes a context, so a call @@ -785,30 +1220,452 @@ async def _analyze_function( how=how, ): calls.append( - Call(state_type=state_type, method=name, how=how) + Servicer.Method.Call( + state_type=state_type, method=name, how=how + ) + ) + continue + + if _is_agent_module(location.filename): + entry_point, analysis = await analysis.helper_definition_at( + location + ) + entry_point_name = ( + None if entry_point is None else entry_point.syntax.name + ) + + if entry_point_name == 'override': + # An `override` changes the agent for whatever runs + # inside it. What it changes is not followed, but said: + # on the agent, and on each run written inside its + # `with`. Its `toolsets=` needs no saying, since a + # Reboot `Agent` rejects it. + changed = _passed(node, OVERRIDE_ARGUMENTS) + if len(changed) == 0: + continue + + constructed, analysis = await _agent_at( + callee, + filename=filename, + text=text, + analysis=analysis, + ) + if constructed is None: + hazards.append( + Servicer.Method.Hazard( + filename=str(filename), + override_on_unresolved_agent=( + Servicer.Method.Hazard. + OverrideOnUnresolvedAgent( + callee=ast.unparse(callee), + ) + ), + ) + ) + continue + + overridden = Agent.Hazard.Overridden(**changed) + agent = _agent_record(constructed, agents=agents) + _add_hazard( + agent.hazards, + Agent.Hazard( + filename=str(filename), + overridden=overridden, + ), + ) + + body = override_bodies.get(id(node)) + if body is not None: + overrides.append((body, constructed.name, overridden)) + continue + + if entry_point_name not in RUN_NAMES: + # Reboot's own machinery around a run: the `tool` a + # tool is registered with, which the walk over the + # file's registrations finds wherever it is written, + # or a construction. Followed no further than the + # generator's machinery is. + continue + + constructed, analysis = await _agent_at( + callee, + filename=filename, + text=text, + analysis=analysis, + ) + if constructed is None: + hazards.append( + Servicer.Method.Hazard( + filename=str(filename), + run_on_unresolved_agent=( + Servicer.Method.Hazard.RunOnUnresolvedAgent( + callee=ast.unparse(callee), + ) + ), ) + ) + continue + + _agent_record(constructed, agents=agents) + + # What the run is given on top of its agent, by its own + # arguments or by an `override` it is written inside, is + # not followed, but said on the run. + run_hazards: list[Agent.Run.Hazard] = [] + given = _passed(node, RUN_ARGUMENTS) + if len(given) > 0: + run_hazards.append( + Agent.Run.Hazard( + filename=str(filename), + run_arguments=Agent.Run.Hazard.RunArguments(**given), + ) + ) + for body, overridden_name, overridden in overrides: + if node.lineno in body and overridden_name == constructed.name: + run_hazards.append( + Agent.Run.Hazard( + filename=str(filename), + overridden=overridden, + ) + ) + + runs.append( + Agent.Run( + agent=constructed.name, + hazards=run_hazards, + ) + ) continue helper, analysis = await analysis.helper_definition_at(location) if helper is None: - ambiguous.append(ast.unparse(callee)) + hazards.append(_ambiguous_call(callee, filename=filename)) continue key = (helper.filename, helper.syntax.lineno) if key in visited: continue - helper_calls, helper_ambiguous, analysis = await _analyze_function( + followed, analysis = await _analyze_function( helper.syntax, filename=helper.filename, text=helper.text, analysis=analysis, visited=visited | {key}, + agents=agents, + ) + calls.extend(followed.calls) + runs.extend(followed.runs) + hazards.extend(followed.hazards) + + return Implementation( + calls=tuple(calls), + runs=tuple(runs), + hazards=tuple(hazards), + ), analysis + + +async def _agent_at( + callee: ast.expr, + *, + filename: Path, + text: str, + analysis: Analysis, +) -> tuple[Optional[AgentDefinition], Analysis]: + """Returns the agent a call is made on, from the expression the + call is made through: `librarian.run(...)`, or + `@librarian.tool`. + + What the receiver refers to, pyright answers, resolving however + the agent gets there: a name in this file, one imported from + another, or an attribute of something held. Where it lands is an + agent only when it is an `Agent(...)` with a string literal + `name=`, bound at the top level of a module, which says what the + agent is. + """ + match callee: + case ast.Attribute(value=receiver): + pass + case _: + # A call reached any other way is made on nothing that + # can be resolved to an agent. + return None, analysis + + match receiver: + case ast.Name() | ast.Attribute(): + line, character = _position_at_last_character(receiver) + location = await analysis.pyright.definition_at( + filename=filename, + line=line, + character=character, + text=text, + ) + if location is None: + return None, analysis + return await analysis.agent_definition_at(location) + + return None, analysis + + +def _agent_record( + definition: AgentDefinition, + *, + agents: dict[str, Agent], +) -> Agent: + """Returns the record the file being analyzed keeps of an agent, + making it, from what the line it comes from says, the first time + the file finds the agent. + + `agents` is every agent the file has found so far, by name, so + that an agent run in several of the file's methods, and registered + a tool on besides, is one record. + """ + found = agents.get(definition.name) + if found is not None: + return found + + agent = Agent( + name=definition.name, + model=definition.model, + system_prompt=definition.system_prompt, + description=definition.description, + hazards=definition.hazards, + ) + agents[definition.name] = agent + return agent + + +async def _registration_at( + registration: ast.Attribute, + *, + filename: Path, + text: str, + analysis: Analysis, +) -> tuple[bool, Optional[AgentDefinition], Analysis]: + """Returns whether an expression spelled `.tool` or + `.tool_plain` is the `tool` or `tool_plain` a Reboot `Agent` + registers a tool with, by its own definition, and the agent it is + made on when it is, `None` when that agent cannot be resolved.""" + line, character = _position_at_last_character(registration) + location = await analysis.pyright.definition_at( + filename=filename, + line=line, + character=character, + text=text, + ) + if location is None or not _is_agent_module(location.filename): + return False, None, analysis + + registered, analysis = await analysis.helper_definition_at(location) + if ( + registered is None or + registered.syntax.name not in TOOL_REGISTRATION_NAMES + ): + return False, None, analysis + + definition, analysis = await _agent_at( + registration, + filename=filename, + text=text, + analysis=analysis, + ) + return True, definition, analysis + + +async def _analyze_decorated_tool( + function: ast.FunctionDef | ast.AsyncFunctionDef, + decorator: ast.expr, + *, + filename: Path, + text: str, + analysis: Analysis, + agents: dict[str, Agent], + unattributed: list[UnattributedHazard], +) -> Analysis: + """Adds a function to the agent a decorator registers it as a tool + of, analyzed the way a servicer method is, when the decorator is + one; and returns the analysis. + + Both forms are read, the bare `@librarian.tool` and the + parametrized `@librarian.tool(retries=2)`, whose `name` and + `description`, if it gives them, are what the model is told, and + whose `prepare`, which can hide the tool when the agent runs, is + said on the tool. A tool decorated on an agent that cannot be + resolved has no agent to be added to, and is said in + `unattributed`. + """ + match decorator: + case ast.Call( + func=ast.Attribute(attr=str(attribute)) as registration, + ) if attribute in TOOL_REGISTRATION_NAMES: + given: Optional[ast.Call] = decorator + case ast.Attribute( + attr=str(attribute), + ) as registration if attribute in TOOL_REGISTRATION_NAMES: + given = None + case _: + return analysis + + proven, definition, analysis = await _registration_at( + registration, + filename=filename, + text=text, + analysis=analysis, + ) + if not proven: + return analysis + + if definition is None: + unattributed.append( + UnattributedHazard( + filename=str(filename), + tool_on_unresolved_agent=( + UnattributedHazard.ToolOnUnresolvedAgent( + registration=ast.unparse(registration), + ) + ), + ) + ) + return analysis + + agent = _agent_record(definition, agents=agents) + + name = ( + None + if given is None else _try_constant_string(_keyword(given, 'name')) + ) or function.name + if any(tool.name == name for tool in agent.tools): + return analysis + + hazards: list[Agent.Tool.Hazard] = [] + if given is not None and _passes(given, 'prepare'): + prepare = _keyword(given, 'prepare') + assert prepare is not None + hazards.append( + Agent.Tool.Hazard( + filename=str(filename), + prepared=Agent.Tool.Hazard.Prepared( + prepare=ast.unparse(prepare) + ), + ) ) - calls.extend(helper_calls) - ambiguous.extend(helper_ambiguous) - return calls, ambiguous, analysis + implementation, analysis = await _analyze_function( + function, + filename=filename, + text=text, + analysis=analysis, + visited=frozenset(), + agents=agents, + ) + + agent.tools.append( + Agent.Tool( + name=name, + description=( + ( + None if given is None else + _try_constant_string(_keyword(given, 'description')) + ) or ast.get_docstring(function) or None + ), + calls=implementation.calls, + runs=implementation.runs, + method_hazards=implementation.hazards, + hazards=hazards, + ) + ) + + return analysis + + +async def _analyze_tool_registrations( + syntax: ast.Module, + *, + filename: Path, + text: str, + analysis: Analysis, + agents: dict[str, Agent], + unattributed: list[UnattributedHazard], +) -> Analysis: + """Adds to each agent the tools a file registers on it, and says + what of that is not followed; and returns the analysis. + + A registration is an expression whose own definition is the `tool` + or `tool_plain` a Reboot `Agent` registers a tool with, wherever + it is written and whichever file the agent is constructed in. A + function decorated with one is followed; see + `_analyze_decorated_tool`. A tool registered by calling one + instead, `librarian.tool(lookup)`, is not, and is said on the + agent, or in `unattributed` when the agent cannot be resolved. + + Asked about only where a registration is spelled the way one has + to be, `.tool` or `.tool_plain`, so that the + decorators and calls every file is full of cost nothing. + """ + # A decorator written as a call, `@librarian.tool(retries=2)`, is + # a call too, and is read as a decorator, not as a call. + decorators = { + id(decorator) for node in ast.walk(syntax) + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + for decorator in node.decorator_list + } + + for node in ast.walk(syntax): + match node: + case ast.FunctionDef() | ast.AsyncFunctionDef(): + for decorator in node.decorator_list: + analysis = await _analyze_decorated_tool( + node, + decorator, + filename=filename, + text=text, + analysis=analysis, + agents=agents, + unattributed=unattributed, + ) + + case ast.Call( + func=ast.Attribute(attr=str(attribute)) as registration, + ) if ( + attribute in TOOL_REGISTRATION_NAMES and + id(node) not in decorators + ): + proven, definition, analysis = await _registration_at( + registration, + filename=filename, + text=text, + analysis=analysis, + ) + if not proven: + continue + + if definition is None: + unattributed.append( + UnattributedHazard( + filename=str(filename), + tool_on_unresolved_agent=( + UnattributedHazard.ToolOnUnresolvedAgent( + registration=ast.unparse(node), + ) + ), + ) + ) + continue + + agent = _agent_record(definition, agents=agents) + _add_hazard( + agent.hazards, + Agent.Hazard( + filename=str(filename), + tool_registered_by_call=( + Agent.Hazard.ToolRegisteredByCall( + call=ast.unparse(node), + ) + ), + ), + ) + + return analysis async def _analyze_class( @@ -816,6 +1673,7 @@ async def _analyze_class( *, filename: Path, analysis: Analysis, + agents: dict[str, Agent], ) -> tuple[Optional[Servicer], Analysis]: """Returns the servicer a class is, and `None` when it is not one. @@ -867,8 +1725,6 @@ async def _analyze_class( servicer = Servicer( state_type=definition.state_type, filename=str(filename), - line=class_definition.lineno, - character=class_definition.col_offset, ) for statement in class_definition.body: @@ -877,19 +1733,21 @@ async def _analyze_class( ast.FunctionDef(name=str(name)) | ast.AsyncFunctionDef(name=str(name)) ): - calls, ambiguous, analysis = await _analyze_function( + implementation, analysis = await _analyze_function( statement, filename=filename, text=analysis.parsed[filename].text, analysis=analysis, visited=frozenset(), + agents=agents, ) servicer.methods.append( Servicer.Method( name=name, digest=_digest(statement), - calls=calls, - ambiguous=ambiguous, + calls=implementation.calls, + runs=implementation.runs, + hazards=implementation.hazards, ) ) @@ -904,12 +1762,13 @@ async def _analyze_file( ) -> tuple[AnalyzedFile, Analysis]: """Returns one parsed file analyzed: an `AnalyzedFile` built from its `ParsedFile` -- the dependencies the parse recorded -- - with the external files the analysis read and every servicer - the file defines.""" + with the external files the analysis read, every servicer the + file defines, and every agent it runs or decorates a tool on.""" parsed = analysis.parsed[filename] - # A generated file defines no servicer, the generator writes - # none, so only the developer's own files are searched. + # A generated file defines no servicer and runs no agent, the + # generator writes neither, so only the developer's own files + # are searched. if filename.name.endswith(GENERATED_SUFFIXES): return AnalyzedFile( filename=parsed.filename, @@ -917,6 +1776,8 @@ async def _analyze_file( dependencies=parsed.dependencies, external=(), servicers=(), + agents=(), + unattributed_hazards=(), ), analysis # Emptied so that what gathers in `external` below is what @@ -925,6 +1786,16 @@ async def _analyze_file( servicers: list[Servicer] = [] + # Every agent this file finds, by name, gathered as the servicers + # and then the registrations are walked, so that an agent run in + # two of the file's methods and registered a tool on besides is one + # record. + agents: dict[str, Agent] = {} + + # Every hazard this file's analysis records that belongs to no + # agent, run, tool or method. + unattributed: list[UnattributedHazard] = [] + for node in ast.walk(parsed.syntax): match node: case ast.ClassDef(): @@ -932,10 +1803,25 @@ async def _analyze_file( node, filename=filename, analysis=analysis, + agents=agents, ) if servicer is not None: servicers.append(servicer) + analysis = await _analyze_tool_registrations( + parsed.syntax, + filename=filename, + text=parsed.text, + analysis=analysis, + agents=agents, + unattributed=unattributed, + ) + + # Said once, here, because it is this file's analysis that found + # every one of them, wherever each is constructed. + for agent in agents.values(): + agent.filename = str(filename) + # Discarded so that pyright holds only what analyzing one file # asked about, rather than accumulating every file asked about # across an analysis; the next file syncs what it asks about. @@ -950,6 +1836,8 @@ async def _analyze_file( dependency for _, dependency in sorted(analysis.external.items()) ), servicers=tuple(servicers), + agents=tuple(agents.values()), + unattributed_hazards=tuple(unattributed), ), analysis @@ -970,6 +1858,39 @@ def extract_and_sort_servicers( ) +def extract_and_sort_agents( + files: Mapping[Path, AnalyzedFile], +) -> list[Agent]: + """Returns every agent found, sorted by name and the file whose + analysis found it. + + Records of one name are one agent, which more than one file + found: the file constructing it, the file running it, the file + registering a tool on it. Every record says the same thing about + the agent itself; what differs is the tools and the hazards each + file contributed, which whoever reads them joins. See `Agent`. + """ + return sorted( + (agent for file in files.values() for agent in file.agents), + key=lambda agent: (agent.name, agent.filename), + ) + + +def extract_and_sort_unattributed_hazards( + files: Mapping[Path, AnalyzedFile], +) -> list[UnattributedHazard]: + """Returns every hazard found that belongs to no agent, run, tool + or method, sorted by the file it is written in, and within a file + in the order the file's analysis met them.""" + return sorted( + ( + hazard for file in files.values() + for hazard in file.unattributed_hazards + ), + key=lambda hazard: hazard.filename, + ) + + def _reconstitute_known( state: DashboardState, ) -> dict[Path, AnalyzedFile]: @@ -982,6 +1903,14 @@ def _reconstitute_known( for servicer in state.servicers: servicers.setdefault(servicer.filename, []).append(servicer) + agents: dict[str, list[Agent]] = {} + for agent in state.agents: + agents.setdefault(agent.filename, []).append(agent) + + unattributed: dict[str, list[UnattributedHazard]] = {} + for hazard in state.unattributed_hazards: + unattributed.setdefault(hazard.filename, []).append(hazard) + return { Path(filename): AnalyzedFile( @@ -990,10 +1919,29 @@ def _reconstitute_known( dependencies=MappingProxyType(dict(file.dependencies)), external=tuple(file.external), servicers=tuple(servicers.get(filename, [])), + agents=tuple(agents.get(filename, [])), + unattributed_hazards=tuple(unattributed.get(filename, [])), ) for filename, file in state.code_files.items() } +def _known_from(state: DashboardState) -> dict[Path, AnalyzedFile]: + """Returns what a restarted watch starts from: the analyzed files + the state records, joined back together by `_reconstitute_known`. + + Recorded by an analysis of another version, which may not record + what this one does, each is kept -- so that what changes is still + told apart from what was there all along -- but without the digest + that says it need not be analyzed again, so every file is. + """ + known = _reconstitute_known(state) + if state.code_analysis_version == CODE_ANALYSIS_VERSION: + return known + return { + filename: replace(file, digest=b'') for filename, file in known.items() + } + + async def _analyze( *, parsed: Mapping[Path, ParsedFile], @@ -1022,6 +1970,7 @@ async def _analyze( roots=tuple(roots), reads=MappingProxyType({}), helpers=MappingProxyType({}), + agents=MappingProxyType({}), external=MappingProxyType({}), ) @@ -1212,7 +2161,7 @@ async def watch( # analyzed again, and an iteration that reproduces exactly what # the state already records writes nothing. state = await Dashboard.ref().always().read(context) - known: Mapping[Path, AnalyzedFile] = _reconstitute_known(state) + known: Mapping[Path, AnalyzedFile] = _known_from(state) generated: Mapping[str, Generated] = state.generated # Whether this process has yet to wait for a save: a restart @@ -1275,6 +2224,10 @@ async def watch( # on its own. if known_now is not None: servicers = extract_and_sort_servicers(known_now) + agents = extract_and_sort_agents(known_now) + unattributed_hazards = ( + extract_and_sort_unattributed_hazards(known_now) + ) # The file messages the write below records, # built before the write so that the state is @@ -1294,6 +2247,9 @@ async def watch( await Dashboard.ref().per_iteration('Update').UpdateCode( context, servicers=servicers, + agents=agents, + unattributed_hazards=unattributed_hazards, + code_analysis_version=CODE_ANALYSIS_VERSION, code_files=files, generated=dict(generated_now), changes=changes, diff --git a/reboot/dashboard/backend/servicers.py b/reboot/dashboard/backend/servicers.py index d7e07dc64..889df0743 100644 --- a/reboot/dashboard/backend/servicers.py +++ b/reboot/dashboard/backend/servicers.py @@ -73,6 +73,8 @@ async def Get( apis=self.state.apis, api_digests=self.state.api_digests, servicers=self.state.servicers, + agents=self.state.agents, + unattributed_hazards=self.state.unattributed_hazards, generated=self.state.generated, needs_generate_reason=needs_generate_reason(self.state), features=self.state.features, @@ -150,6 +152,11 @@ async def UpdateCode( changed, newest last.""" del self.state.servicers[:] self.state.servicers.extend(request.servicers) + del self.state.agents[:] + self.state.agents.extend(request.agents) + del self.state.unattributed_hazards[:] + self.state.unattributed_hazards.extend(request.unattributed_hazards) + self.state.code_analysis_version = request.code_analysis_version self.state.code_files.clear() self.state.code_files.MergeFrom(request.code_files) self.state.generated.clear() diff --git a/reboot/dashboard/web/dashboard.css b/reboot/dashboard/web/dashboard.css index 8e1b2ddb0..5cae7bce8 100644 --- a/reboot/dashboard/web/dashboard.css +++ b/reboot/dashboard/web/dashboard.css @@ -877,6 +877,50 @@ header h1 { gap: 6px; } +/* The agent's prompt on the pane, whole and in the shape it was + written in: it is a piece of prose the developer wrote, and its + paragraphs and its lists are what it means. */ +.agent-prompt { + border: 1px solid hsl(var(--border-soft)); + border-radius: var(--radius); + background: hsl(var(--surface-sunken)); + padding: 12px 14px; + font-size: 12px; + line-height: 1.55; + color: hsl(var(--foreground)); + white-space: pre-wrap; + overflow-wrap: anywhere; +} + +/* One tool, set the way a method is. */ +.agent-tool { + border: 1px solid hsl(var(--border-soft)); + border-radius: var(--radius); + background: hsl(var(--surface-sunken)); + padding: 10px 14px; +} + +.agent-tool-head { + display: flex; + align-items: baseline; + gap: 8px; +} + +.agent-tool-name { + font-family: ui-monospace, Menlo, monospace; + font-size: 12px; + font-weight: 650; +} + +/* What the tool reaches: each a link to the method, or the agent, in + the pane. */ +.agent-tool-calls { + display: flex; + flex-wrap: wrap; + gap: 6px; + margin-top: 8px; +} + /* Deliberately not a scroll container. Nothing inside a card paints out to its edges, so there is no corner for `overflow: hidden` to clip, and it would clip two things that must escape: a pill's @@ -1327,8 +1371,12 @@ header h1 { } /* An unknown method's dot is a ring: nothing declares it, only a - call names it, so there is nothing to paint it in with. */ -.graph-kind-unknown .graph-method-dot { + call names it, so there is nothing to paint it in with. An agent's + tool is a ring too, for the same reason -- no API declares it -- + in the workflow colour, since it runs in the workflow that runs + the agent, called by the agent rather than by the application. */ +.graph-kind-unknown .graph-method-dot, +.graph-tool .graph-method-dot { background: transparent; border: 1.5px solid hsl(var(--kind)); } @@ -1348,6 +1396,84 @@ header h1 { text-decoration: underline; } +/* An agent: a card like a state type's, with a row per tool, its + head tinted in the workflow colour, since an agent only ever runs + inside a workflow. Never in a package's box, so it carries its own + tint rather than sitting in one. */ +.graph-agent { + width: 100%; + background: hsl(var(--card)); + border: 1px solid hsl(36 85% 42% / 0.4); + border-radius: var(--radius); + box-shadow: 0 1px 3px hsl(240 10% 40% / 0.1); + overflow: hidden; +} + +/* 40px tall, which `graph.tsx` lays out by: the name on one line and + the model under it, rather than the two sharing a line, where a + long model name would eat the name the reader came for. */ +.graph-agent-head { + position: relative; + display: flex; + flex-direction: column; + justify-content: center; + height: 40px; + padding: 0 12px; + background: hsl(36 85% 42% / 0.08); + border-bottom: 1px solid hsl(36 85% 42% / 0.25); + color: hsl(var(--foreground)); +} + +.graph-agent-title { + display: flex; + align-items: center; + gap: 6px; + font-size: 12.5px; + font-weight: 650; + line-height: 1.3; +} + +.graph-agent-emoji { + flex: none; + font-size: 13px; +} + +.graph-agent-name { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.graph-agent-model { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + font-family: ui-monospace, Menlo, monospace; + font-size: 9.5px; + font-weight: 500; + line-height: 1.3; + color: hsl(var(--muted-foreground)); +} + +/* The first of the prompt, which is what the agent is: three lines + of it, clamped to the 48px `graph.tsx` lays out by, which is + exactly three at this line height plus the padding above them. The + whole of it, in the shape it was written in, is in the types pane, + a click on the head away; here its lines run on, so that three of + them carry as much of it as they can. */ +.graph-agent-prompt { + height: 48px; + padding: 5px 12px 1px; + overflow: hidden; + display: -webkit-box; + -webkit-box-orient: vertical; + -webkit-line-clamp: 3; + line-clamp: 3; + font-size: 10px; + line-height: 14px; + color: hsl(var(--muted-foreground)); +} + /* The stubs of the arrows at the chosen row's flanks: the right one lights what the method calls, the left one who calls it. */ .graph-cone { diff --git a/reboot/dashboard/web/src/callgraph.ts b/reboot/dashboard/web/src/callgraph.ts index d6973ab45..6803be696 100644 --- a/reboot/dashboard/web/src/callgraph.ts +++ b/reboot/dashboard/web/src/callgraph.ts @@ -1,9 +1,13 @@ // The call graph's data: the API's state types and methods, joined // with the Reboot calls the analysis of the developer's application -// found in each method's implementation. +// found in each method's implementation, and the agents it found the +// application runs, joined with the tools each agent has. import type { + Agent, + Agent_Run, Servicer, Servicer_Method, + Servicer_Method_Call, Servicer_Method_Call_How, } from "../../../../rbt/dashboard/v1/dashboard_pb"; import type { APIs, Kind } from "./link_properties_to_data_types"; @@ -27,6 +31,15 @@ export interface GraphCall { count: number; } +// One agent a method's or a tool's implementation runs, counted the +// same way. +export interface GraphRun { + // The agent's name, which is what a run names and what the records + // of one agent are joined on. + agentName: string; + count: number; +} + export interface GraphMethod { name: string; // Only the API's declaration says the kind, so a method known only @@ -34,6 +47,7 @@ export interface GraphMethod { kind?: Kind; factory: boolean; calls: GraphCall[]; + runs: GraphRun[]; } export interface GraphStateType { @@ -51,6 +65,30 @@ export interface GraphPackage { stateTypes: GraphStateType[]; } +// One tool an agent may call: a row of its card, and what the model +// reaches the application through. +export interface GraphTool { + name: string; + // What the model is told it does: the description it was + // registered with, or the function's docstring. + description?: string; + calls: GraphCall[]; + runs: GraphRun[]; +} + +export interface GraphAgent { + // `agent:librarian`: kept apart from the state types' ids, which + // are qualified names, so that one id names one thing. + id: string; + name: string; + model?: string; + // Every literal string its prompt is made of, in the order the + // agent is constructed with them. + systemPrompt: string[]; + description?: string; + tools: GraphTool[]; +} + // Packages in the order their first state type comes. export const groupStateTypesByPackage = ( stateTypes: GraphStateType[] @@ -72,37 +110,132 @@ export const groupStateTypesByPackage = ( export const methodId = (stateTypeName: string, methodName: string): string => `${stateTypeName}.${methodName}`; +// A key unique to one agent, in the same space as the state types', +// which cannot hold a colon. +export const agentId = (agentName: string): string => `agent:${agentName}`; + +// A key unique to one of an agent's tools, the way a method's is +// unique to one method: `agent:librarian.get_page`. +export const toolId = (agentId: string, toolName: string): string => + `${agentId}.${toolName}`; + +// Whether an id names an agent or one of its tools, which only an +// agent's id begins the way it does. +export const isAgentRowId = (id: string): boolean => id.startsWith("agent:"); + // Folds the calls the analysis lists into one per distinct call, // counted. -const countCalls = ( - analyzedMethod: Servicer_Method | undefined -): GraphCall[] => { - const calls = new Map(); - for (const call of analyzedMethod?.calls ?? []) { +const countCalls = (calls: Servicer_Method_Call[] | undefined): GraphCall[] => { + const counted = new Map(); + for (const call of calls ?? []) { const key = `${call.stateType}|${call.method}|${call.how}`; - const counted = calls.get(key); - if (counted === undefined) { - calls.set(key, { + const already = counted.get(key); + if (already === undefined) { + counted.set(key, { stateTypeName: call.stateType, methodName: call.method, how: call.how, count: 1, }); } else { - counted.count += 1; + already.count += 1; + } + } + return [...counted.values()]; +}; + +// The same, for the agents an implementation runs. +const countRuns = (runs: Agent_Run[] | undefined): GraphRun[] => { + const counted = new Map(); + for (const run of runs ?? []) { + const already = counted.get(run.agent); + if (already === undefined) { + counted.set(run.agent, { agentName: run.agent, count: 1 }); + } else { + already.count += 1; + } + } + return [...counted.values()]; +}; + +// Joins the records of each agent the analysis found: one per file +// that met the agent, every one saying the same thing about the +// agent itself, and each carrying the tools its own file +// contributed. The tools gather, one per name; the agents come in +// the order the records do, which is by name. +export const joinAgents = (agents: Agent[]): GraphAgent[] => { + const joined = new Map(); + for (const agent of agents) { + let joinedAgent = joined.get(agent.name); + if (joinedAgent === undefined) { + joinedAgent = { + id: agentId(agent.name), + name: agent.name, + model: agent.model, + systemPrompt: agent.systemPrompt, + description: agent.description, + tools: [], + }; + joined.set(agent.name, joinedAgent); + } + for (const tool of agent.tools) { + // By name alone, since the name is what the model calls and + // what the agent's row is: a tool the agent is given in two + // places is one tool. + if (joinedAgent.tools.some((known) => known.name === tool.name)) { + continue; + } + joinedAgent.tools.push({ + name: tool.name, + description: tool.description, + calls: countCalls(tool.calls), + runs: countRuns(tool.runs), + }); + } + } + return [...joined.values()]; +}; + +// Adds, as a target with no kind and no calls of its own, every +// state type and method some call names that the API does not +// declare. +const addCalled = ( + stateTypes: Map, + calls: GraphCall[] +): void => { + for (const call of calls) { + let calledStateType = stateTypes.get(call.stateTypeName); + if (calledStateType === undefined) { + calledStateType = { + id: call.stateTypeName, + name: shortNameOfTypeName(call.stateTypeName), + methods: [], + }; + stateTypes.set(call.stateTypeName, calledStateType); + } + if ( + !calledStateType.methods.some((known) => known.name === call.methodName) + ) { + calledStateType.methods.push({ + name: call.methodName, + factory: false, + calls: [], + runs: [], + }); } } - return [...calls.values()]; }; // Joins the state types the API files declare with the calls the // analysis found in each declared method. Servicer methods the API // does not declare, such as helpers, are dropped. Anything a call // names that the API does not declare is added as a target, with no -// kind and no calls. +// kind and no calls, whether a method calls it or one of the +// `agents`' tools does. export const joinStateTypes = ( apis: APIs, - servicers: Servicer[] + servicers: Servicer[], + agents: GraphAgent[] ): GraphStateType[] => { // A state type can have more than one servicer in `servicers`, sorted // by file; where they define the same method, the first wins. @@ -125,44 +258,32 @@ export const joinStateTypes = ( { id: name, name: stateType.name, - methods: stateType.methods.map((method) => ({ - name: method.name, - kind: kindOfMethod(method), - factory: method.factory, - calls: countCalls( - analyzedMethodsById.get(methodId(name, method.name)) - ), - })), + methods: stateType.methods.map((method) => { + const analyzed = analyzedMethodsById.get( + methodId(name, method.name) + ); + return { + name: method.name, + kind: kindOfMethod(method), + factory: method.factory, + calls: countCalls(analyzed?.calls), + runs: countRuns(analyzed?.runs), + }; + }), }, ]; }) ) ); - for (const stateType of graphStateTypes.values()) { + for (const stateType of [...graphStateTypes.values()]) { for (const method of stateType.methods) { - for (const call of method.calls) { - let calledStateType = graphStateTypes.get(call.stateTypeName); - if (calledStateType === undefined) { - calledStateType = { - id: call.stateTypeName, - name: shortNameOfTypeName(call.stateTypeName), - methods: [], - }; - graphStateTypes.set(call.stateTypeName, calledStateType); - } - if ( - !calledStateType.methods.some( - (known) => known.name === call.methodName - ) - ) { - calledStateType.methods.push({ - name: call.methodName, - factory: false, - calls: [], - }); - } - } + addCalled(graphStateTypes, method.calls); + } + } + for (const agent of agents) { + for (const tool of agent.tools) { + addCalled(graphStateTypes, tool.calls); } } diff --git a/reboot/dashboard/web/src/graph.tsx b/reboot/dashboard/web/src/graph.tsx index f5dbe82c3..78f2e6f5a 100644 --- a/reboot/dashboard/web/src/graph.tsx +++ b/reboot/dashboard/web/src/graph.tsx @@ -6,6 +6,11 @@ // package; a collapsed box hides its cards, and the calls leaving it // fold into one counted arrow per box they reach. // +// A card per agent too, with a row per tool, which runs land on and +// the calls the tools make leave from: an agent stands beside the +// packages rather than in one, since it belongs to no package of the +// API, and is never collapsed. +// // React Flow draws; ELK places. React Flow deliberately has no layout // of its own. import { Servicer_Method_Call_How as How } from "../../../../rbt/dashboard/v1/dashboard_pb"; @@ -31,14 +36,21 @@ import type { Viewport } from "@xyflow/react"; import { useLocation, useNavigationType } from "react-router"; import ELK from "elkjs/lib/elk.bundled.js"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; -import type { FC, MouseEvent } from "react"; +import type { FC, MouseEvent, ReactNode } from "react"; import type { + GraphAgent, GraphCall, - GraphMethod, GraphPackage, + GraphRun, GraphStateType, } from "./callgraph"; -import { groupStateTypesByPackage, methodId } from "./callgraph"; +import { + agentId, + groupStateTypesByPackage, + isAgentRowId, + methodId, + toolId, +} from "./callgraph"; import type { Kind } from "./link_properties_to_data_types"; import { labelOfKind, @@ -122,16 +134,34 @@ const COLLAPSED_PACKAGE_HEIGHT = 78; const EXPANDED_PACKAGE_HEAD_HEIGHT = 38; const EXPANDED_PACKAGE_PAD = 20; +// An agent's card is wider, because what it is is a prompt, and its +// head is taller: the emoji and the name, then the model, then the +// first lines of the prompt, which `.graph-agent-prompt` clamps to +// the height below. An agent with no prompt to show has a head of +// `AGENT_HEAD_HEIGHT` alone. +const AGENT_CARD_WIDTH = 260; +const AGENT_HEAD_HEIGHT = 40; +const AGENT_PROMPT_HEIGHT = 48; + const heightOfStateType = (stateType: GraphStateType): number => HEAD_HEIGHT + stateType.methods.length * ROW_HEIGHT + CARD_SLACK; +// What an agent's tool rows start below, which is its head with the +// prompt it shows, if it shows one. +const headHeightOfAgent = (agent: GraphAgent): number => + AGENT_HEAD_HEIGHT + (agent.systemPrompt.length > 0 ? AGENT_PROMPT_HEIGHT : 0); + +const heightOfAgent = (agent: GraphAgent): number => + headHeightOfAgent(agent) + agent.tools.length * ROW_HEIGHT + CARD_SLACK; + // A package's node id, kept apart from state type ids, which are // fully qualified names and could equal a package's. const packageNodeId = (name: string): string => `pkg:${name}`; -// `bank.v1.Account` for `bank.v1.Account.deposit`. -const stateTypeNameOfMethodId = (id: string): string => - id.slice(0, id.lastIndexOf(".")); +// The card a row belongs to: `bank.v1.Account` for +// `bank.v1.Account.deposit`, and `agent:librarian` for +// `agent:librarian.get_page`. +const cardOfRowId = (id: string): string => id.slice(0, id.lastIndexOf(".")); // --------------------------------------------------------------- // Layout. @@ -149,6 +179,19 @@ interface ExpandedPackageData extends Record { onCollapse?: (name: string) => void; } +interface AgentData extends Record { + agent: GraphAgent; + // The chosen row's id, when one is chosen, which may be a tool of + // this agent or anything else the graph draws. + selectedRow?: string | null; + onSelectRow?: (id: string, cones: Cones) => void; + onOpenAgent?: (id: string) => void; + cones?: Cones; + onToggleCone?: (cone: keyof Cones) => void; + withCallers?: Set; + withCalls?: Set; +} + // Which cones of the chosen method the graph lights: what it calls // (downstream), who calls it (upstream), or both. interface Cones { @@ -180,14 +223,14 @@ const CONES_OF_THIRD: Record = { interface StateTypeData extends Record { stateType: GraphStateType; - // The chosen method's id, when one is chosen. - selectedMethod?: string | null; - onSelectMethod?: (id: string, cones: Cones) => void; + // The chosen row's id, when one is chosen. + selectedRow?: string | null; + onSelectRow?: (id: string, cones: Cones) => void; onOpenStateType?: (id: string) => void; cones?: Cones; onToggleCone?: (cone: keyof Cones) => void; - // The methods some drawn call lands on, and the methods that make - // one; a button with nothing to light is never shown. + // The rows some drawn arrow lands on, and the rows that make one; + // a button with nothing to light is never shown. withCallers?: Set; withCalls?: Set; } @@ -195,7 +238,8 @@ interface StateTypeData extends Record { type GraphNode = | Node | Node - | Node; + | Node + | Node; interface Point { x: number; @@ -213,12 +257,13 @@ const ELK_LAYERED_OPTIONS = { // Where everything goes: callers to the left of what they call, the // way an edge leaves a row on its right and enters one on its left. -// Each expanded box's cards are laid out alone, then the boxes are -// laid out at the size their cards came to, so an open box never -// lands on a neighbour. A card or box calling itself has no say in -// where it goes. +// Each expanded box's cards are laid out alone, then the boxes and +// the agents' cards are laid out at the size their contents came to, +// so an open box never lands on a neighbour. A card or box calling +// itself has no say in where it goes. const layoutPackages = async ( packages: GraphPackage[], + agents: GraphAgent[], collapsed: ReadonlySet ): Promise => { const cardLayoutsByPackage = new Map< @@ -286,19 +331,54 @@ const layoutPackages = async ( }); } - const callPairsBetweenPackages = new Set(); + // Between the boxes and the agents' cards, which are laid out + // together: a package's calls reach the packages they name and the + // agents its methods run, and an agent's tools reach the packages + // they call and the agents they run. + const pairsBetweenBoxes = new Set(); + const agentIds = new Set(agents.map((agent) => agent.id)); + const reaches = (source: string, target: string): void => { + if (source !== target) { + pairsBetweenBoxes.add(`${source}\u0000${target}`); + } + }; for (const pkg of packages) { + const source = packageNodeId(pkg.name); for (const stateType of pkg.stateTypes) { for (const method of stateType.methods) { for (const call of method.calls) { - const target = packageOfStateTypeName(call.stateTypeName); - if (isDrawn(call) && target !== pkg.name) { - callPairsBetweenPackages.add(`${pkg.name}>${target}`); + if (isDrawn(call)) { + reaches( + source, + packageNodeId(packageOfStateTypeName(call.stateTypeName)) + ); + } + } + for (const run of method.runs) { + if (agentIds.has(agentId(run.agentName))) { + reaches(source, agentId(run.agentName)); } } } } } + for (const agent of agents) { + for (const tool of agent.tools) { + for (const call of tool.calls) { + if (isDrawn(call)) { + reaches( + agent.id, + packageNodeId(packageOfStateTypeName(call.stateTypeName)) + ); + } + } + for (const run of tool.runs) { + if (agentIds.has(agentId(run.agentName))) { + reaches(agent.id, agentId(run.agentName)); + } + } + } + } const elkPackageLayout = await elk.layout({ id: "root", @@ -307,37 +387,49 @@ const layoutPackages = async ( "elk.spacing.nodeNode": "60", "elk.layered.spacing.nodeNodeBetweenLayers": "140", }, - children: packages.map((pkg) => { - const cardLayout = cardLayoutsByPackage.get(pkg.name); - return { - id: packageNodeId(pkg.name), - width: cardLayout?.width ?? COLLAPSED_PACKAGE_WIDTH, - height: cardLayout?.height ?? COLLAPSED_PACKAGE_HEIGHT, - }; - }), - edges: [...callPairsBetweenPackages].map((pair) => { - const [source, target] = pair.split(">"); - return { - id: pair, - sources: [packageNodeId(source)], - targets: [packageNodeId(target)], - }; + children: [ + ...packages.map((pkg) => { + const cardLayout = cardLayoutsByPackage.get(pkg.name); + return { + id: packageNodeId(pkg.name), + width: cardLayout?.width ?? COLLAPSED_PACKAGE_WIDTH, + height: cardLayout?.height ?? COLLAPSED_PACKAGE_HEIGHT, + }; + }), + ...agents.map((agent) => ({ + id: agent.id, + width: AGENT_CARD_WIDTH, + height: heightOfAgent(agent), + })), + ], + edges: [...pairsBetweenBoxes].map((pair) => { + const [source, target] = pair.split("\u0000"); + return { id: pair, sources: [source], targets: [target] }; }), }); - const packagePositions = new Map( - (elkPackageLayout.children ?? []).map((elkPackage) => [ - elkPackage.id, - { x: elkPackage.x ?? 0, y: elkPackage.y ?? 0 }, + const boxPositions = new Map( + (elkPackageLayout.children ?? []).map((elkBox) => [ + elkBox.id, + { x: elkBox.x ?? 0, y: elkBox.y ?? 0 }, ]) ); // A parent precedes its children: React Flow resolves a elkCard's // position, relative to its parent, in array order. const nodes: GraphNode[] = []; + for (const agent of agents) { + nodes.push({ + id: agent.id, + type: "agent", + position: boxPositions.get(agent.id) ?? { x: 0, y: 0 }, + width: AGENT_CARD_WIDTH, + data: { agent }, + }); + } for (const pkg of packages) { const boxId = packageNodeId(pkg.name); - const position = packagePositions.get(boxId) ?? { x: 0, y: 0 }; + const position = boxPositions.get(boxId) ?? { x: 0, y: 0 }; const cardLayout = cardLayoutsByPackage.get(pkg.name); if (cardLayout === undefined) { nodes.push({ @@ -390,93 +482,152 @@ const layoutPackages = async ( // Edges. interface CallEdgeData extends Record { - // Absent on a folded edge, which carries calls reached every way. + // Absent on a folded edge, which carries calls reached every way, + // and on an edge that is a run rather than a call. how?: How; - // The calling method's kind, which is the edge's colour. Absent - // for a method the API does not declare, and on a folded edge. + // Set on an edge that is a run of an agent rather than a call, + // which always says so. + run?: boolean; + // The calling method's kind, which is the edge's colour. An agent's + // tools run in the workflow that runs the agent, so an edge leaving + // one is a workflow's. Absent for a method the API does not + // declare, and on a folded edge. kind?: Kind; count: number; - // Every calling method whose calls this edge carries: one for an - // edge from a method row, each contributor for a folded edge. - // What choosing a method keeps, transitively. - sourceMethodIds: string[]; - // Every called method the same way, which is what says whether + // Every row whose arrows this edge carries: one for an edge from a + // method's or a tool's row, each contributor for a folded edge. + // What choosing a row keeps, transitively. + sourceIds: string[]; + // Every row it lands on the same way, which is what says whether // the edge lands inside the upstream cone. - targetMethodIds: string[]; - // Set while another method is chosen. The label fades off this + targetIds: string[]; + // Set while another row is chosen. The label fades off this // rather than off the edge's class: `EdgeLabelRenderer` draws // labels in a layer of their own, out of the class's reach. faded?: boolean; } -// Every method the chosen one calls, transitively, itself included: -// the downstream closure over the drawn calls. Collapse-blind, so -// the path continues through a collapsed box. -const reachableMethodIds = ( - from: string, - packages: GraphPackage[] -): Set => { - const callsByMethodId = new Map( - packages.flatMap((pkg) => - pkg.stateTypes.flatMap((stateType) => - stateType.methods.map( - (method) => - [methodId(stateType.id, method.name), method.calls] as const - ) - ) - ) - ); - const reached = new Set([from]); - const frontier = [from]; - while (frontier.length > 0) { - for (const call of callsByMethodId.get(frontier.pop()!) ?? []) { - if (!isDrawn(call)) { - continue; - } - const callee = methodId(call.stateTypeName, call.methodName); - if (!reached.has(callee)) { - reached.add(callee); - frontier.push(callee); - } +// Where a run lands: an agent's card is one thing to run, however +// many tools it has, so every run enters it at its head. +const AGENT_TARGET_HANDLE = "t:agent"; + +// One arrow, drawn or folded into the one already drawn between the +// same two places, which is what a collapsed box's arrows become. +const addEdge = ( + edgesById: Map>, + edge: { + id: string; + source: string; + sourceHandle?: string; + target: string; + targetHandle?: string; + kind?: Kind; + how?: How; + run?: boolean; + count: number; + sourceId: string; + targetId: string; + } +): void => { + const folded = edgesById.get(edge.id); + if (folded !== undefined) { + const data = folded.data!; + data.count += edge.count; + if (!data.sourceIds.includes(edge.sourceId)) { + data.sourceIds.push(edge.sourceId); + } + if (!data.targetIds.includes(edge.targetId)) { + data.targetIds.push(edge.targetId); } + return; } - return reached; + edgesById.set(edge.id, { + id: edge.id, + source: edge.source, + sourceHandle: edge.sourceHandle, + target: edge.target, + targetHandle: edge.targetHandle, + type: "call", + data: { + how: edge.how, + run: edge.run, + kind: edge.kind, + count: edge.count, + sourceIds: [edge.sourceId], + targetIds: [edge.targetId], + }, + markerEnd: { + type: MarkerType.ArrowClosed, + color: colorOfKind(edge.kind), + width: 16, + height: 16, + }, + }); }; -// Every method that calls the chosen one, transitively, itself -// included: the upstream closure over the same drawn calls, -// collapse-blind the same way. -const reachingMethodIds = ( - to: string, - packages: GraphPackage[] -): Set => { - const callersByMethodId = new Map(); +// Who calls whom, over everything the graph draws, by row: a method +// leads to the methods its calls name and to the agents it runs; an +// agent leads to each of its tools, which is what running it +// reaches; and a tool leads on the way a method does. Collapse-blind, +// so a path continues through a collapsed box. +interface Reachability { + callees: Map; + callers: Map; +} + +const reachability = ( + packages: GraphPackage[], + agents: GraphAgent[] +): Reachability => { + const callees = new Map(); + const callers = new Map(); + const leads = (from: string, to: string): void => { + callees.set(from, [...(callees.get(from) ?? []), to]); + callers.set(to, [...(callers.get(to) ?? []), from]); + }; + const leadsFrom = (id: string, calls: GraphCall[], runs: GraphRun[]) => { + for (const call of calls) { + if (isDrawn(call)) { + leads(id, methodId(call.stateTypeName, call.methodName)); + } + } + for (const run of runs) { + leads(id, agentId(run.agentName)); + } + }; + for (const pkg of packages) { for (const stateType of pkg.stateTypes) { for (const method of stateType.methods) { - const caller = methodId(stateType.id, method.name); - for (const call of method.calls) { - if (!isDrawn(call)) { - continue; - } - const callee = methodId(call.stateTypeName, call.methodName); - const callers = callersByMethodId.get(callee); - if (callers === undefined) { - callersByMethodId.set(callee, [caller]); - } else { - callers.push(caller); - } - } + leadsFrom( + methodId(stateType.id, method.name), + method.calls, + method.runs + ); } } } - const reached = new Set([to]); - const frontier = [to]; + for (const agent of agents) { + for (const tool of agent.tools) { + const id = toolId(agent.id, tool.name); + leads(agent.id, id); + leadsFrom(id, tool.calls, tool.runs); + } + } + + return { callees, callers }; +}; + +// Everything the chosen row leads to, transitively, itself included, +// and everything that leads to it, over the same graph. +const closure = (from: string, edges: Map): Set => { + const reached = new Set([from]); + const frontier = [from]; while (frontier.length > 0) { - for (const caller of callersByMethodId.get(frontier.pop()!) ?? []) { - if (!reached.has(caller)) { - reached.add(caller); - frontier.push(caller); + for (const next of edges.get(frontier.pop()!) ?? []) { + if (!reached.has(next)) { + reached.add(next); + frontier.push(next); } } } @@ -486,16 +637,25 @@ const reachingMethodIds = ( // The edges as the boxes show them. A call whose box is expanded // leaves from its own method row; otherwise it leaves from the box, // and every call the box hides folds into one counted edge per node -// they reach. -const edgesOfPackages = ( +// they reach. An agent's card is never in a box and never collapsed, +// so its tools' arrows always leave their own rows, and a run always +// lands on the agent's head. +const edgesOf = ( packages: GraphPackage[], + agents: GraphAgent[], collapsed: ReadonlySet ): Edge[] => { const edgesById = new Map>(); + const agentsByName = new Map(agents.map((agent) => [agent.name, agent])); + for (const pkg of packages) { const sourceExpanded = !collapsed.has(pkg.name); for (const stateType of pkg.stateTypes) { for (const method of stateType.methods) { + const source = sourceExpanded ? stateType.id : packageNodeId(pkg.name); + const sourceHandle = sourceExpanded ? `s:${method.name}` : undefined; + const caller = methodId(stateType.id, method.name); + for (const call of method.calls) { if (!isDrawn(call)) { continue; @@ -508,59 +668,109 @@ const edgesOfPackages = ( continue; } - const source = sourceExpanded - ? stateType.id - : packageNodeId(pkg.name); - const sourceHandle = sourceExpanded ? `s:${method.name}` : undefined; const target = targetExpanded ? call.stateTypeName : packageNodeId(targetPackage); const targetHandle = targetExpanded ? `t:${call.methodName}` : undefined; - const id = sourceExpanded - ? `${source}|${sourceHandle}>${target}|${targetHandle}:${call.how}` - : `${source}>${target}|${targetHandle}`; - - const caller = methodId(stateType.id, method.name); - const callee = methodId(call.stateTypeName, call.methodName); - const edgeFoldedInto = edgesById.get(id); - if (edgeFoldedInto !== undefined) { - edgeFoldedInto.data!.count += call.count; - if (!edgeFoldedInto.data!.sourceMethodIds.includes(caller)) { - edgeFoldedInto.data!.sourceMethodIds.push(caller); - } - if (!edgeFoldedInto.data!.targetMethodIds.includes(callee)) { - edgeFoldedInto.data!.targetMethodIds.push(callee); - } - continue; - } - const kind = sourceExpanded ? method.kind : undefined; - edgesById.set(id, { - id, + + addEdge(edgesById, { + id: sourceExpanded + ? `${source}|${sourceHandle}>${target}|${targetHandle}:${call.how}` + : `${source}>${target}|${targetHandle}`, source, sourceHandle, target, targetHandle, - type: "call", - data: { - how: sourceExpanded ? call.how : undefined, - kind, - count: call.count, - sourceMethodIds: [caller], - targetMethodIds: [callee], - }, - markerEnd: { - type: MarkerType.ArrowClosed, - color: colorOfKind(kind), - width: 16, - height: 16, - }, + kind: sourceExpanded ? method.kind : undefined, + how: sourceExpanded ? call.how : undefined, + count: call.count, + sourceId: caller, + targetId: methodId(call.stateTypeName, call.methodName), }); } + + for (const run of method.runs) { + const agent = agentsByName.get(run.agentName); + // A run naming an agent the analysis never recorded has + // nowhere to land. + if (agent === undefined) { + continue; + } + addEdge(edgesById, { + id: sourceExpanded + ? `${source}|${sourceHandle}>${agent.id}` + : `${source}>${agent.id}`, + source, + sourceHandle, + target: agent.id, + targetHandle: AGENT_TARGET_HANDLE, + kind: sourceExpanded ? method.kind : undefined, + run: true, + count: run.count, + sourceId: caller, + targetId: agent.id, + }); + } + } + } + } + + for (const agent of agents) { + for (const tool of agent.tools) { + const source = agent.id; + const sourceHandle = `s:${tool.name}`; + const caller = toolId(agent.id, tool.name); + + for (const call of tool.calls) { + if (!isDrawn(call)) { + continue; + } + const targetPackage = packageOfStateTypeName(call.stateTypeName); + const targetExpanded = !collapsed.has(targetPackage); + const target = targetExpanded + ? call.stateTypeName + : packageNodeId(targetPackage); + const targetHandle = targetExpanded + ? `t:${call.methodName}` + : undefined; + + addEdge(edgesById, { + id: `${source}|${sourceHandle}>${target}|${targetHandle}:${call.how}`, + source, + sourceHandle, + target, + targetHandle, + kind: "workflow", + how: call.how, + count: call.count, + sourceId: caller, + targetId: methodId(call.stateTypeName, call.methodName), + }); + } + + for (const run of tool.runs) { + const target = agentsByName.get(run.agentName); + if (target === undefined) { + continue; + } + addEdge(edgesById, { + id: `${source}|${sourceHandle}>${target.id}`, + source, + sourceHandle, + target: target.id, + targetHandle: AGENT_TARGET_HANDLE, + kind: "workflow", + run: true, + count: run.count, + sourceId: caller, + targetId: target.id, + }); } } } + return [...edgesById.values()]; }; @@ -602,58 +812,70 @@ const ExpandedPackageNode: FC< ); -// an edge landing on its left or leaving on its right. The handles -// are invisible: the edge just needs somewhere to land. Hovering the -// row shows, beside the card, the cones a click there lights: the -// left third of the row, the arrow in, who calls the method; the -// right third, the arrow out, what it calls; the middle, both. A -// click chooses the method with those cones, which also opens it in -// the types pane, and a click asking for what is already lit lets it -// go. -const MethodRow: FC<{ +// One row of a card: a method of a state type, or a tool of an +// agent, with an edge landing on its left or leaving on its right. +interface CardRow { + // The row's id, in the one space every arrow's ends are named in. id: string; - method: GraphMethod; + name: string; + kind?: Kind; + // Set on an agent's tool, whose dot is a ring: it runs in the + // workflow that runs the agent, but no API declares it and it is + // the agent, not the application, that calls it. + tool?: boolean; + // What the row says of itself when the pointer rests on it. + title: string; + // A word the row wears at its right, e.g. `factory`. + badge?: string; +} + +// The handles are invisible: the edge just needs somewhere to land. +// Hovering the row shows, beside the card, the cones a click there +// lights: the left third of the row, the arrow in, who calls it; the +// right third, the arrow out, what it calls; the middle, both. A +// click chooses the row with those cones, which also opens it in the +// types pane, and a click asking for what is already lit lets it go. +const Row: FC<{ + row: CardRow; selected: boolean; onHover: (third: RowThird | null) => void; onSelect: (id: string, cones: Cones) => void; -}> = ({ id, method, selected, onHover, onSelect }) => ( +}> = ({ row, selected, onHover, onSelect }) => (
onHover(thirdOfPointer(event))} onMouseLeave={() => onHover(null)} onClick={(event) => { event.stopPropagation(); - onSelect(id, CONES_OF_THIRD[thirdOfPointer(event)]); + onSelect(row.id, CONES_OF_THIRD[thirdOfPointer(event)]); }} - title={ - method.kind === undefined - ? "unknown" - : `${labelOfKind(method.kind)}${method.factory ? ", factory" : ""}` - } + title={row.title} >
); // A button beside the card, level with a row, for one of the row's -// cones: lit in the method's kind colour while that cone is shown, -// and unlit again under the pointer, since a click then puts it out. +// cones: lit in the row's kind colour while that cone is shown, and unlit +// again under the pointer, since a click then puts it out. const ConeButton: FC<{ cone: keyof Cones; top: number; @@ -716,21 +938,39 @@ const ConeButton: FC<{ // the card, past the row's edge. const HOVER_LINGER_MS = 250; -const StateTypeNode: FC>> = ({ - data, +// What a state type and an agent are both drawn as: a head, a row +// each, and the cone buttons flanking the chosen row. `headHeight` +// is what the head comes out at, which is what places the buttons, +// since the card clips its contents and they are siblings of it. +const RowsCard: FC<{ + className: string; + head: ReactNode; + headHeight: number; + rows: CardRow[]; + selectedRow?: string | null; + onSelectRow?: (id: string, cones: Cones) => void; + cones?: Cones; + onToggleCone?: (cone: keyof Cones) => void; + // The rows some drawn arrow lands on, and the rows that make one; + // a button with nothing to light is never shown. + withCallers?: Set; + withCalls?: Set; +}> = ({ + className, + head, + headHeight, + rows, + selectedRow, + onSelectRow, + cones, + onToggleCone, + withCallers, + withCalls, }) => { - // The chosen row's place in the card, for the cone buttons that - // flank it. The card clips its contents, so the buttons are - // siblings of it, placed by the layout's own row arithmetic. const selectedIndex = - data.selectedMethod == null - ? -1 - : data.stateType.methods.findIndex( - (method) => - methodId(data.stateType.id, method.name) === data.selectedMethod - ); + selectedRow == null ? -1 : rows.findIndex((row) => row.id === selectedRow); const topOfRow = (index: number): number => - 1 + HEAD_HEIGHT + index * ROW_HEIGHT + ROW_HEIGHT / 2; + 1 + headHeight + index * ROW_HEIGHT + ROW_HEIGHT / 2; // The row the pointer is over and which third of it, with the // hide put off a moment when the pointer leaves, so it can reach @@ -755,13 +995,13 @@ const StateTypeNode: FC>> = ({ }, [keepShown]); useEffect(() => keepShown, [keepShown]); - // The cones a method has anything to light in. + // The cones a row has anything to light in. const availableCones = (id: string): Cones => ({ - upstream: data.withCallers?.has(id) ?? false, - downstream: data.withCalls?.has(id) ?? false, + upstream: withCallers?.has(id) ?? false, + downstream: withCalls?.has(id) ?? false, }); - // Chooses a method with the cones asked for, of those it has; a + // Chooses a row with the cones asked for, of those it has; a // click asking only for a cone it lacks lights what it has. const select = (id: string, asked: Cones): void => { const available = availableCones(id); @@ -769,7 +1009,7 @@ const StateTypeNode: FC>> = ({ upstream: asked.upstream && available.upstream, downstream: asked.downstream && available.downstream, }; - data.onSelectMethod?.( + onSelectRow?.( id, wanted.upstream || wanted.downstream ? wanted : available ); @@ -780,25 +1020,23 @@ const StateTypeNode: FC>> = ({ // asks for a cone it has not lit shows that cone's button too, // which lights it. Any other hovered row shows the buttons for the // cones its third asks for, of those it has, and clicking one - // chooses the method with that cone alone. + // chooses the row with that cone alone. const asked = hovered === null ? undefined : CONES_OF_THIRD[hovered.third]; const buttonsOf = ( index: number - ): { id: string; method: GraphMethod; show: Cones } | undefined => { + ): { row: CardRow; show: Cones } | undefined => { if (index === -1) { return undefined; } - const method = data.stateType.methods[index]; - const id = methodId(data.stateType.id, method.name); - const available = availableCones(id); + const row = rows[index]; + const available = availableCones(row.id); const lit = index === selectedIndex - ? data.cones ?? DEFAULT_CONES + ? cones ?? DEFAULT_CONES : { upstream: false, downstream: false }; const hoveredHere = hovered?.index === index && asked !== undefined; return { - id, - method, + row, show: { upstream: available.upstream && @@ -817,38 +1055,24 @@ const StateTypeNode: FC>> = ({ return ( <> -
- {/* The name is the way to the state type in the types pane. */} -
{ - event.stopPropagation(); - data.onOpenStateType?.(data.stateType.id); - }} - > - {data.stateType.name} -
- {data.stateType.methods.map((method, index) => { - const id = methodId(data.stateType.id, method.name); - return ( - { - if (third === null) { - hideSoon(); - } else { - keepShown(); - setHovered({ index, third }); - } - }} - onSelect={select} - key={method.name} - /> - ); - })} +
+ {head} + {rows.map((row, index) => ( + { + if (third === null) { + hideSoon(); + } else { + keepShown(); + setHovered({ index, third }); + } + }} + onSelect={select} + key={row.id} + /> + ))}
{selectedButtons !== undefined && (["upstream", "downstream"] as const).map( @@ -857,18 +1081,18 @@ const StateTypeNode: FC>> = ({ data.onToggleCone?.(cone)} + onClick={() => onToggleCone?.(cone)} onMouseEnter={keepShown} onMouseLeave={hideSoon} key={cone} @@ -884,14 +1108,14 @@ const StateTypeNode: FC>> = ({ cone={cone} top={topOfRow(hovered.index)} lit={false} - color={colorOfKind(hoveredButtons.method.kind)} + color={colorOfKind(hoveredButtons.row.kind)} title={ cone === "upstream" - ? "Show who calls this method" - : "Show what this method calls" + ? "Show what calls this" + : "Show what this calls" } onClick={() => - select(hoveredButtons.id, { + select(hoveredButtons.row.id, { upstream: cone === "upstream", downstream: cone === "downstream", }) @@ -906,6 +1130,101 @@ const StateTypeNode: FC>> = ({ ); }; +const StateTypeNode: FC>> = ({ + data, +}) => ( + { + event.stopPropagation(); + data.onOpenStateType?.(data.stateType.id); + }} + > + {data.stateType.name} +
+ } + rows={data.stateType.methods.map((method) => ({ + id: methodId(data.stateType.id, method.name), + name: method.name, + kind: method.kind, + badge: method.factory ? "factory" : undefined, + title: + method.kind === undefined + ? "unknown" + : `${labelOfKind(method.kind)}${method.factory ? ", factory" : ""}`, + }))} + selectedRow={data.selectedRow} + onSelectRow={data.onSelectRow} + cones={data.cones} + onToggleCone={data.onToggleCone} + withCallers={data.withCallers} + withCalls={data.withCalls} + /> +); + +// An agent: the robot, its name and its model, the first of its +// prompt, and a row per tool. Runs land on its head, since running +// it is running the whole of it, and each tool's own calls leave its +// row. +const AgentNode: FC>> = ({ data }) => ( + +
{ + event.stopPropagation(); + data.onOpenAgent?.(data.agent.id); + }} + > + +
+ + {data.agent.name} +
+ {data.agent.model !== undefined && ( + {data.agent.model} + )} +
+ {data.agent.systemPrompt.length > 0 && ( +
+ {data.agent.systemPrompt.join("\n\n")} +
+ )} + + } + rows={data.agent.tools.map((tool) => ({ + id: toolId(data.agent.id, tool.name), + name: tool.name, + kind: "workflow" as const, + tool: true, + title: tool.description ?? "an agent's tool", + }))} + selectedRow={data.selectedRow} + onSelectRow={data.onSelectRow} + cones={data.cones} + onToggleCone={data.onToggleCone} + withCallers={data.withCallers} + withCalls={data.withCalls} + /> +); + const CallEdge: FC>> = ({ id, source, @@ -952,7 +1271,13 @@ const CallEdge: FC>> = ({ const kind = data?.kind; const how = data?.how; const count = data?.count ?? 1; - const howWord = how === undefined ? undefined : HOW_LABEL[how]; + // A run always says so; a plain call says nothing, since it is the + // ordinary case, and labelling every edge "calls" would be noise. + const howWord = data?.run + ? "runs" + : how === undefined + ? undefined + : HOW_LABEL[how]; const label = count > 1 ? `${howWord ?? "calls"} ×${count}` : howWord; const dashPattern = (how === undefined ? undefined : HOW_DASH[how]) ?? @@ -1029,6 +1354,14 @@ const Legend: FC = () => ( unknown +
+
@@ -1044,6 +1377,12 @@ const Legend: FC = () => ( a workflow's calls
+
+
+ + an agent and its tools +
+
factory @@ -1059,6 +1398,7 @@ const nodeTypes = { package: PackageNode, expanded: ExpandedPackageNode, stateType: StateTypeNode, + agent: AgentNode, }; const edgeTypes = { call: CallEdge }; @@ -1080,12 +1420,22 @@ const graphViews = new Map< const GraphCanvas: FC<{ packages: GraphPackage[]; - // The chosen method's id, which is what the URL names: choosing - // one is a navigation, so back steps to the one chosen before. - selectedMethodId: string | null; - onSelectMethod: (id: string | null, replace?: boolean) => void; + agents: GraphAgent[]; + // The chosen row's id, a method's or a tool's, which is what the + // URL names: choosing one is a navigation, so back steps to the + // one chosen before. + selectedRowId: string | null; + onSelectRow: (id: string | null, replace?: boolean) => void; onOpenStateType: (id: string) => void; -}> = ({ packages, selectedMethodId, onSelectMethod, onOpenStateType }) => { + onOpenAgent: (id: string) => void; +}> = ({ + packages, + agents, + selectedRowId, + onSelectRow, + onOpenStateType, + onOpenAgent, +}) => { const location = useLocation(); const saved = useNavigationType() === "POP" ? graphViews.get(location.key) : undefined; @@ -1134,7 +1484,7 @@ const GraphCanvas: FC<{ useEffect(() => { const thisLayoutRun = ++layoutRun.current; - layoutPackages(packages, collapsed).then((nodes) => { + layoutPackages(packages, agents, collapsed).then((nodes) => { if (layoutRun.current !== thisLayoutRun) { return; } @@ -1185,11 +1535,18 @@ const GraphCanvas: FC<{ }); } }); - }, [packages, collapsed, fitView]); + }, [packages, agents, collapsed, fitView]); const edges = useMemo( - () => edgesOfPackages(packages, collapsed), - [packages, collapsed] + () => edgesOf(packages, agents, collapsed), + [packages, agents, collapsed] + ); + + // Who leads to whom over everything drawn, which the cones and the + // cone buttons are both read off. + const reach = useMemo( + () => reachability(packages, agents), + [packages, agents] ); const togglePackage = useCallback( @@ -1206,17 +1563,18 @@ const GraphCanvas: FC<{ }); // Closing the chosen method's own box lets it go, since its // row is gone; replaced rather than pushed, since the reader - // clicked the box, not the choice. + // clicked the box, not the choice. An agent's row is in no + // box, so it stays. if ( - selectedMethodId !== null && - packageOfStateTypeName(stateTypeNameOfMethodId(selectedMethodId)) === - name && + selectedRowId !== null && + !isAgentRowId(selectedRowId) && + packageOfStateTypeName(cardOfRowId(selectedRowId)) === name && !collapsed.has(name) ) { - onSelectMethod(null, true); + onSelectRow(null, true); } }, - [collapsed, selectedMethodId, onSelectMethod] + [collapsed, selectedRowId, onSelectRow] ); const setAllCollapsed = useCallback( @@ -1226,11 +1584,15 @@ const GraphCanvas: FC<{ setCollapsed( new Set(allCollapsed ? packages.map((pkg) => pkg.name) : []) ); - if (allCollapsed && selectedMethodId !== null) { - onSelectMethod(null, true); + if ( + allCollapsed && + selectedRowId !== null && + !isAgentRowId(selectedRowId) + ) { + onSelectRow(null, true); } }, - [packages, selectedMethodId, onSelectMethod] + [packages, selectedRowId, onSelectRow] ); // Which cones of the chosen method the graph lights: what the @@ -1247,58 +1609,32 @@ const GraphCanvas: FC<{ useEffect(() => { setCones(askedCones.current ?? DEFAULT_CONES); askedCones.current = null; - }, [selectedMethodId]); + }, [selectedRowId]); - const toggleMethodSelection = useCallback( + const toggleRowSelection = useCallback( (id: string, asked: Cones) => { - if (selectedMethodId !== id) { + if (selectedRowId !== id) { askedCones.current = asked; - onSelectMethod(id); + onSelectRow(id); } else if (sameCones(cones, asked)) { - onSelectMethod(null); + onSelectRow(null); } else { setCones(asked); } }, - [selectedMethodId, cones, onSelectMethod] + [selectedRowId, cones, onSelectRow] ); const toggleCone = useCallback((cone: keyof Cones): void => { setCones((current) => ({ ...current, [cone]: !current[cone] })); }, []); - // The methods some drawn call lands on, self-calls included, and - // the methods that make one: what a cone button needs to have - // anything to light. - const withCallers = useMemo(() => { - const ids = new Set(); - for (const pkg of packages) { - for (const stateType of pkg.stateTypes) { - for (const method of stateType.methods) { - for (const call of method.calls) { - if (isDrawn(call)) { - ids.add(methodId(call.stateTypeName, call.methodName)); - } - } - } - } - } - return ids; - }, [packages]); - - const withCalls = useMemo(() => { - const ids = new Set(); - for (const pkg of packages) { - for (const stateType of pkg.stateTypes) { - for (const method of stateType.methods) { - if (method.calls.some(isDrawn)) { - ids.add(methodId(stateType.id, method.name)); - } - } - } - } - return ids; - }, [packages]); + // The rows some drawn arrow lands on, self-calls included, and the + // rows that make one: what a cone button needs to have anything to + // light. + const withCallers = useMemo(() => new Set(reach.callers.keys()), [reach]); + + const withCalls = useMemo(() => new Set(reach.callees.keys()), [reach]); // With a method chosen, its lit cones: downstream, the methods it // calls transitively and the arrows carrying those calls; @@ -1308,12 +1644,10 @@ const GraphCanvas: FC<{ // in a cone when any method folded into it is. An expanded box // never fades: it is the room its cards are in. const unfaded = useMemo(() => { - if (selectedMethodId === null) { + if (selectedRowId === null) { return null; } - const nodeIds = new Set([ - stateTypeNameOfMethodId(selectedMethodId), - ]); + const nodeIds = new Set([cardOfRowId(selectedRowId)]); const edgeIds = new Set(); const light = (edge: Edge): void => { edgeIds.add(edge.id); @@ -1321,26 +1655,26 @@ const GraphCanvas: FC<{ nodeIds.add(edge.target); }; if (cones.downstream) { - const reached = reachableMethodIds(selectedMethodId, packages); + const reached = closure(selectedRowId, reach.callees); for (const edge of edges) { - if (edge.data!.sourceMethodIds.some((id) => reached.has(id))) { + if (edge.data!.sourceIds.some((id) => reached.has(id))) { light(edge); } } } if (cones.upstream) { - const reaching = reachingMethodIds(selectedMethodId, packages); + const reaching = closure(selectedRowId, reach.callers); for (const edge of edges) { if ( - edge.data!.sourceMethodIds.some((id) => reaching.has(id)) && - edge.data!.targetMethodIds.some((id) => reaching.has(id)) + edge.data!.sourceIds.some((id) => reaching.has(id)) && + edge.data!.targetIds.some((id) => reaching.has(id)) ) { light(edge); } } } return { nodeIds, edgeIds }; - }, [selectedMethodId, cones, packages, edges]); + }, [selectedRowId, cones, reach, edges]); const shownNodes = useMemo( () => @@ -1363,8 +1697,8 @@ const GraphCanvas: FC<{ className, data: { ...node.data, - selectedMethod: selectedMethodId, - onSelectMethod: toggleMethodSelection, + selectedRow: selectedRowId, + onSelectRow: toggleRowSelection, onOpenStateType, cones, onToggleCone: toggleCone, @@ -1372,6 +1706,21 @@ const GraphCanvas: FC<{ withCalls, }, }; + case "agent": + return { + ...node, + className, + data: { + ...node.data, + selectedRow: selectedRowId, + onSelectRow: toggleRowSelection, + onOpenAgent, + cones, + onToggleCone: toggleCone, + withCallers, + withCalls, + }, + }; default: return { ...node, className }; } @@ -1379,10 +1728,11 @@ const GraphCanvas: FC<{ [ nodes, unfaded, - selectedMethodId, - toggleMethodSelection, + selectedRowId, + toggleRowSelection, togglePackage, onOpenStateType, + onOpenAgent, cones, toggleCone, withCallers, @@ -1415,8 +1765,8 @@ const GraphCanvas: FC<{ } }} onPaneClick={() => { - if (selectedMethodId !== null) { - onSelectMethod(null); + if (selectedRowId !== null) { + onSelectRow(null); } }} // ELK places the nodes, so they don't move one by one. Left @@ -1457,25 +1807,52 @@ const GraphCanvas: FC<{ ); }; -// How many calls the graph draws: counted from the data rather than -// the edges, which collapse when their box is collapsed. -export const drawnCallCount = (stateTypes: GraphStateType[]): number => - stateTypes.reduce( - (count, stateType) => - count + - stateType.methods.reduce( - (count, method) => count + method.calls.filter(isDrawn).length, - 0 - ), - 0 +// How many arrows the graph draws: every call and every run, +// whoever makes it, counted from the data rather than the edges, +// which fold when their box is collapsed. +export const drawnCallCount = ( + stateTypes: GraphStateType[], + agents: GraphAgent[] +): number => { + const drawn = (calls: GraphCall[], runs: GraphRun[]): number => + calls.filter(isDrawn).length + runs.length; + return ( + stateTypes.reduce( + (count, stateType) => + count + + stateType.methods.reduce( + (count, method) => count + drawn(method.calls, method.runs), + 0 + ), + 0 + ) + + agents.reduce( + (count, agent) => + count + + agent.tools.reduce( + (count, tool) => count + drawn(tool.calls, tool.runs), + 0 + ), + 0 + ) ); +}; export const GraphPage: FC<{ stateTypes: GraphStateType[]; - selectedMethodId: string | null; - onSelectMethod: (id: string | null, replace?: boolean) => void; + agents: GraphAgent[]; + selectedRowId: string | null; + onSelectRow: (id: string | null, replace?: boolean) => void; onOpenStateType: (id: string) => void; -}> = ({ stateTypes, selectedMethodId, onSelectMethod, onOpenStateType }) => { + onOpenAgent: (id: string) => void; +}> = ({ + stateTypes, + agents, + selectedRowId, + onSelectRow, + onOpenStateType, + onOpenAgent, +}) => { const packages = useMemo( () => groupStateTypesByPackage(stateTypes), [stateTypes] @@ -1486,9 +1863,11 @@ export const GraphPage: FC<{
diff --git a/reboot/dashboard/web/src/main.tsx b/reboot/dashboard/web/src/main.tsx index ff3a9385e..80f209011 100644 --- a/reboot/dashboard/web/src/main.tsx +++ b/reboot/dashboard/web/src/main.tsx @@ -107,7 +107,14 @@ import { import { DashboardGetResponse_NeedsGenerateReason as NeedsGenerateReason } from "../../../../rbt/dashboard/v1/dashboard_pb"; import type * as feature_pb from "../../../../rbt/v1alpha1/bdd/feature_pb"; import type * as grammar_pb from "../../../../rbt/v1alpha1/bdd/grammar_pb"; -import { joinStateTypes, type GraphStateType } from "./callgraph"; +import { + agentId, + isAgentRowId, + joinAgents, + joinStateTypes, + type GraphAgent, + type GraphStateType, +} from "./callgraph"; import { exercisedMethods, graphStateTypeNamed, @@ -138,6 +145,11 @@ const DEFINITIONS: Record = { "Brings a state into existence: it is called with a new id " + "rather than on a state that already exists.", mcp: "Callable by AI agents as a tool, over the Model Context " + "Protocol.", + agent: + "A model the application hands work to, with tools it may call " + + "back with. Reboot runs one inside a workflow, so every model " + + "call and every tool call is durable: a restart replays what " + + "already happened rather than asking again.", "state type": "A durable data type. Each instance, named by an id, has properties " + "that Reboot persists for you. Methods are the way to read and " + @@ -323,40 +335,51 @@ interface PaneRows { const PaneRowsContext = createContext(undefined); // What the `type` search parameter names: one state type or one of -// its methods, `bank.v1.Account` or `bank.v1.Account.deposit`, or one -// data type, `bank.v1.bank.CustomerAccount`. The pane exists only -// while the parameter names something. +// its methods, `bank.v1.Account` or `bank.v1.Account.deposit`, one +// data type, `bank.v1.bank.CustomerAccount`, or one agent or one of +// its tools, `agent:librarian` or `agent:librarian.get_page`. The +// pane exists only while the parameter names something. type PaneTarget = - | { stateTypeId: string; method?: string; dataTypeId?: undefined } - | { dataTypeId: string; stateTypeId?: undefined; method?: undefined }; + | { kind: "stateType"; id: string; method?: string } + | { kind: "dataType"; id: string } + | { kind: "agent"; id: string; tool?: string }; -// The id of the type a target shows, whichever kind it is. -const typeIdOfTarget = (target: PaneTarget): string => - target.dataTypeId ?? target.stateTypeId; +// The id of the thing a target shows, whichever kind it is, which is +// what the URL carries. +const typeIdOfTarget = (target: PaneTarget): string => target.id; const paneTargetOf = ( raw: string | null, isStateTypeId: (id: string) => boolean, - isDataTypeId: (id: string) => boolean + isDataTypeId: (id: string) => boolean, + isAgentId: (id: string) => boolean ): PaneTarget | undefined => { if (raw === null) { return undefined; } if (isDataTypeId(raw)) { - return { dataTypeId: raw }; + return { kind: "dataType", id: raw }; } const separator = raw.lastIndexOf("."); - if ( - !isStateTypeId(raw) && - separator !== -1 && - isStateTypeId(raw.slice(0, separator)) - ) { + const before = separator === -1 ? undefined : raw.slice(0, separator); + if (isAgentRowId(raw)) { + if (!isAgentId(raw) && before !== undefined && isAgentId(before)) { + return { + kind: "agent", + id: before, + tool: raw.slice(separator + 1), + }; + } + return { kind: "agent", id: raw }; + } + if (!isStateTypeId(raw) && before !== undefined && isStateTypeId(before)) { return { - stateTypeId: raw.slice(0, separator), + kind: "stateType", + id: before, method: raw.slice(separator + 1), }; } - return { stateTypeId: raw }; + return { kind: "stateType", id: raw }; }; // The search string a link to a type produces. The path is left @@ -930,12 +953,102 @@ const DataType: FC<{ ); }; -// The types pane: one type, state or data, slid open by a link to it -// from the graph or a page, every method expanded; the X closes it. -// A link naming a method flashes the method. +// One agent, on the pane the way a state type is: what it is told to +// be, and what it may call back with. The prompt is what the agent +// is, so it is shown whole, in the shape it was written in. +const AgentPane: FC<{ agent: GraphAgent }> = ({ agent }) => ( +
+
+ +
+
+
+

+ + {agent.name} +

+ + + {countWithNoun(agent.tools.length, "tool")} + +
+
+ {agent.model !== undefined &&
{agent.model}
} + {agent.description !== undefined && ( + + )} + +
system prompt
+ {agent.systemPrompt.length === 0 ? ( +
+ No prompt written down. One computed rather than written is not + something the dashboard can read. +
+ ) : ( +
{agent.systemPrompt.join("\n\n")}
+ )} + +
tools
+ {agent.tools.length === 0 ? ( +
+ No tools. The agent answers out of the prompt alone, or out of tools the + dashboard cannot read, such as an MCP server's. +
+ ) : ( +
+ {agent.tools.map((tool) => ( +
+
+ {tool.name} +
+ {tool.description !== undefined && ( + + )} + {tool.calls.length + tool.runs.length > 0 && ( +
+ {tool.calls.map((call) => ( + + + {shortNameOfTypeName(call.stateTypeName)}. + {call.methodName} + + + ))} + {tool.runs.map((run) => ( + + 🤖 {run.agentName} + + ))} +
+ )} +
+ ))} +
+ )} +
+); + +// The types pane: one type, state or data, or one agent, slid open +// by a link to it from the graph or a page, every method expanded; +// the X closes it. A link naming a method flashes the method. const TypesPane: FC<{ apis: APIs; linkedDataTypes: LinkedDataType[]; + agents: GraphAgent[]; target: PaneTarget; // The property a followed link named, if any. propertyName?: string; @@ -950,6 +1063,7 @@ const TypesPane: FC<{ }> = ({ apis, linkedDataTypes, + agents, target, propertyName, flashKey, @@ -959,7 +1073,7 @@ const TypesPane: FC<{ }) => { const typeId = typeIdOfTarget(target); const found = - target.stateTypeId === undefined + target.kind !== "stateType" ? undefined : sortedAPIs(apis) .flatMap((api) => @@ -967,14 +1081,18 @@ const TypesPane: FC<{ ) .find( ({ api, stateType }) => - qualifiedName({ api, stateType }) === target.stateTypeId + qualifiedName({ api, stateType }) === target.id ); const foundDataType = - target.dataTypeId === undefined + target.kind !== "dataType" ? undefined : linkedDataTypes.find( - (linkedDataType) => linkedDataType.id === target.dataTypeId + (linkedDataType) => linkedDataType.id === target.id ); + const foundAgent = + target.kind !== "agent" + ? undefined + : agents.find((agent) => agent.id === target.id); const flashProperty = propertyName === undefined || flashKey === undefined ? undefined @@ -982,7 +1100,9 @@ const TypesPane: FC<{ return (
- {typeId} + + {foundAgent === undefined ? typeId : `🤖 ${foundAgent.name}`} +