From edd008f13e7761a0dbf2f51df52a3e50e3ae8c3f Mon Sep 17 00:00:00 2001 From: Josh Carver Date: Tue, 30 Jun 2026 06:11:44 -0700 Subject: [PATCH 1/7] refactor(aether-cli): Extract pub build settings methods to allow evals to use preset agents --- crates/aether-cli/src/init/build_settings.rs | 156 +++++++++++++------ crates/aether-cli/src/init/mod.rs | 5 +- 2 files changed, 113 insertions(+), 48 deletions(-) diff --git a/crates/aether-cli/src/init/build_settings.rs b/crates/aether-cli/src/init/build_settings.rs index 18e268a59..ec4b3f702 100644 --- a/crates/aether-cli/src/init/build_settings.rs +++ b/crates/aether-cli/src/init/build_settings.rs @@ -3,7 +3,7 @@ use super::harness::HarnessIntegration; use super::recommendations::{ProviderRecommendations, recommended_for_provider}; use aether_core::agent_spec::{ToolFilter, ToolMatcher}; use aether_project::{AetherSettings, AgentConfig, McpSourceSpec, PromptSource}; -use llm::catalog::Provider; +use llm::{ReasoningEffort, catalog::Provider}; use mcp_utils::client::{InMemoryServerConfig, InMemoryType, McpServerConfig}; const SYSTEM_PATH: &str = "SYSTEM.md"; @@ -38,6 +38,96 @@ pub fn supported_providers() -> impl Iterator { Provider::ALL.iter().copied().filter(|p| recommended_for_provider(*p).is_some()) } +pub fn build_batteries_included_settings( + display: &str, + model: String, + reasoning_effort: Option, + scope: InitScope, + harnesses: &[HarnessIntegration], +) -> AetherSettings { + let plan = build_batteries_included_plan_agent(display, model.clone(), reasoning_effort, harnesses); + let build = build_batteries_included_build_agent(display, model.clone(), reasoning_effort, harnesses); + let explore = build_batteries_included_explore_agent(model, reasoning_effort, scope, harnesses); + + AetherSettings { + prompts: batteries_prompts(scope, harnesses), + agents: vec![plan, build, explore], + ..AetherSettings::default() + } +} + +pub fn build_batteries_included_plan_agent( + display: &str, + model: String, + reasoning_effort: Option, + harnesses: &[HarnessIntegration], +) -> AgentConfig { + let coding = coding_args(harnesses); + let skills = skills_args(harnesses); + AgentConfig { + name: "Plan".to_string(), + description: format!("{display} planner (read-only except plan files)"), + model, + reasoning_effort, + user_invocable: true, + mcps: vec![mcps(vec![ + ("plan", vec![]), + ("coding", coding), + ("skills", skills), + ("subagents", vec![]), + ("tasks", vec![]), + ("survey", vec![]), + ])], + tools: read_only_coding_tools(), + ..AgentConfig::default() + } +} + +pub fn build_batteries_included_build_agent( + display: &str, + model: String, + reasoning_effort: Option, + harnesses: &[HarnessIntegration], +) -> AgentConfig { + let coding = coding_args(harnesses); + let skills = skills_args(harnesses); + AgentConfig { + name: "Build".to_string(), + description: format!("{display} implementor"), + model, + reasoning_effort, + user_invocable: true, + mcps: vec![mcps(vec![ + ("coding", coding), + ("skills", skills), + ("subagents", vec![]), + ("tasks", vec![]), + ("survey", vec![]), + ])], + ..AgentConfig::default() + } +} + +pub fn build_batteries_included_explore_agent( + model: String, + reasoning_effort: Option, + scope: InitScope, + harnesses: &[HarnessIntegration], +) -> AgentConfig { + let coding = coding_args(harnesses); + AgentConfig { + name: "Explore".to_string(), + description: "Explores codebases to find relevant files, patterns, and integration points".to_string(), + model, + reasoning_effort, + agent_invocable: true, + prompts: vec![PromptSource::file(scope.asset_path(EXPLORER_AGENTS_PATH))], + mcps: vec![mcps(vec![("coding", coding)])], + tools: read_only_coding_tools(), + ..AgentConfig::default() + } +} + pub(crate) fn build_preset( preset: Preset, provider: Provider, @@ -80,54 +170,26 @@ fn batteries_included_preset( harnesses: &[HarnessIntegration], ) -> ResolvedPreset { let display = provider.display_name(); - let coding = coding_args(harnesses); - let skills = skills_args(harnesses); - - let plan = AgentConfig { - name: "Plan".to_string(), - description: format!("{display} planner (read-only except plan files)"), - model: recs.plan.model.to_string(), - reasoning_effort: recs.plan.reasoning_effort, - user_invocable: true, - mcps: vec![mcps(vec![ - ("plan", vec![]), - ("coding", coding.clone()), - ("skills", skills.clone()), - ("subagents", vec![]), - ("tasks", vec![]), - ("survey", vec![]), - ])], - tools: read_only_coding_tools(), - ..AgentConfig::default() - }; + let plan = build_batteries_included_plan_agent( + display, + recs.plan.model.to_string(), + recs.plan.reasoning_effort, + harnesses, + ); - let build = AgentConfig { - name: "Build".to_string(), - description: format!("{display} implementor"), - model: recs.build.model.to_string(), - reasoning_effort: recs.build.reasoning_effort, - user_invocable: true, - mcps: vec![mcps(vec![ - ("coding", coding.clone()), - ("skills", skills), - ("subagents", vec![]), - ("tasks", vec![]), - ("survey", vec![]), - ])], - ..AgentConfig::default() - }; + let build = build_batteries_included_build_agent( + display, + recs.build.model.to_string(), + recs.build.reasoning_effort, + harnesses, + ); - let explore = AgentConfig { - name: "Explore".to_string(), - description: "Explores codebases to find relevant files, patterns, and integration points".to_string(), - model: recs.explore.model.to_string(), - reasoning_effort: recs.explore.reasoning_effort, - agent_invocable: true, - prompts: vec![PromptSource::file(scope.asset_path(EXPLORER_AGENTS_PATH))], - mcps: vec![mcps(vec![("coding", coding)])], - tools: read_only_coding_tools(), - ..AgentConfig::default() - }; + let explore = build_batteries_included_explore_agent( + recs.explore.model.to_string(), + recs.explore.reasoning_effort, + scope, + harnesses, + ); ResolvedPreset { files: &[ diff --git a/crates/aether-cli/src/init/mod.rs b/crates/aether-cli/src/init/mod.rs index 31e535e3e..fdc1e0375 100644 --- a/crates/aether-cli/src/init/mod.rs +++ b/crates/aether-cli/src/init/mod.rs @@ -3,7 +3,10 @@ mod harness; mod recommendations; mod tui_runner; use aether_project::user_settings_path; -pub use build_settings::Preset; +pub use build_settings::{ + Preset, build_batteries_included_build_agent, build_batteries_included_explore_agent, + build_batteries_included_plan_agent, build_batteries_included_settings, +}; pub use harness::HarnessIntegration; use llm::catalog::Provider; use recommendations::recommended_for_provider; From 72e9b5f44d3ab8f5a2961fd249f43b719fc83dc9 Mon Sep 17 00:00:00 2001 From: Josh Carver Date: Tue, 30 Jun 2026 06:12:47 -0700 Subject: [PATCH 2/7] feat(aether-cli): Headless mode now expands slash commands if present in prompt --- crates/aether-cli/src/acp/error.rs | 12 --- crates/aether-cli/src/acp/slash_commands.rs | 102 ++++---------------- crates/aether-cli/src/headless/run.rs | 66 ++++++++++++- crates/aether-cli/src/lib.rs | 1 + crates/aether-cli/src/slash_commands.rs | 96 ++++++++++++++++++ 5 files changed, 179 insertions(+), 98 deletions(-) create mode 100644 crates/aether-cli/src/slash_commands.rs diff --git a/crates/aether-cli/src/acp/error.rs b/crates/aether-cli/src/acp/error.rs index 24109653d..665d7c4f2 100644 --- a/crates/aether-cli/src/acp/error.rs +++ b/crates/aether-cli/src/acp/error.rs @@ -17,15 +17,3 @@ pub enum SessionError { #[error("active agent runtime is not running")] ActiveRuntimeNotRunning, } - -#[derive(Error, Debug)] -pub(super) enum SlashCommandError { - #[error("command channel error: {0}")] - CommandChannel(String), - #[error("MCP operation failed: {0}")] - McpOperation(String), - #[error("slash command '/{0}' not found")] - NotFound(String), - #[error("prompt result contains no text content")] - NoTextContent, -} diff --git a/crates/aether-cli/src/acp/slash_commands.rs b/crates/aether-cli/src/acp/slash_commands.rs index 5799bfd97..feb3fc984 100644 --- a/crates/aether-cli/src/acp/slash_commands.rs +++ b/crates/aether-cli/src/acp/slash_commands.rs @@ -2,11 +2,14 @@ use acp_utils::server::AcpServerError; use agent_client_protocol::schema::{self as acp, SessionId}; use agent_client_protocol::{Client, ConnectionTo}; use llm::ContentBlock; -use std::collections::HashSet; use tracing::{error, info}; use super::agent_runtime::AgentRuntime; -use super::error::{SessionError, SlashCommandError}; +use crate::acp::error::SessionError; +pub(crate) use crate::slash_commands::dedupe_commands_by_name; +use crate::slash_commands::{ + SlashCommandError, find_prompt_name, parse_slash_command, parse_slash_command_arguments, prompt_result_text, +}; pub(crate) async fn expand_slash_command_in_content( runtime: &AgentRuntime, @@ -15,31 +18,24 @@ pub(crate) async fn expand_slash_command_in_content( if let Some(ContentBlock::Text { text }) = content.first() && text.starts_with('/') { - let expanded = expand_slash_command_if_needed(runtime, text.clone()).await; + let expanded = expand_slash_command_text(runtime, text.clone()).await; content[0] = ContentBlock::text(expanded); } content } -async fn expand_slash_command_if_needed(runtime: &AgentRuntime, text: String) -> String { - let Some(slash_command_text) = text.strip_prefix('/') else { +async fn expand_slash_command_text(runtime: &AgentRuntime, text: String) -> String { + let Some(slash_command) = parse_slash_command(&text) else { return text; }; - let (command_name, args_text) = if let Some(space_idx) = slash_command_text.find(char::is_whitespace) { - let (cmd, args) = slash_command_text.split_at(space_idx); - (cmd, args.trim()) - } else { - (slash_command_text, "") - }; - - match expand_slash_command(runtime, command_name, args_text).await { + match expand_slash_command(runtime, slash_command.command_name, slash_command.args_text).await { Ok(expanded) => { - info!("Expanded slash command '{}' -> {} chars", command_name, expanded.len()); + info!("Expanded slash command -> {} chars", expanded.len()); expanded } Err(e) => { - error!("Failed to expand slash command '{}': {}", command_name, e); + error!("Failed to expand slash command: {}", e); text } } @@ -51,57 +47,20 @@ async fn expand_slash_command( args_text: &str, ) -> Result { let arguments = parse_slash_command_arguments(args_text); + let prompts = runtime.list_prompts().await.map_err(slash_command_error)?; + let prompt_name = find_prompt_name(&prompts, command_name)?; + let prompt_result = runtime.get_prompt(prompt_name, arguments).await.map_err(slash_command_error)?; - let prompts = runtime.list_prompts().await.map_err(|error| match error { - SessionError::CommandChannel(message) => SlashCommandError::CommandChannel(message), - other => SlashCommandError::McpOperation(other.to_string()), - })?; - - let matching_prompt = prompts - .iter() - .find(|p| p.name.split("__").last().unwrap_or("") == command_name) - .ok_or_else(|| SlashCommandError::NotFound(command_name.to_string()))?; - - let namespaced_name = matching_prompt.name.clone(); + prompt_result_text(&prompt_result) +} - let prompt_result = runtime.get_prompt(namespaced_name.clone(), arguments).await.map_err(|error| match error { +fn slash_command_error(error: super::error::SessionError) -> SlashCommandError { + match error { SessionError::CommandChannel(message) => SlashCommandError::CommandChannel(message), other => SlashCommandError::McpOperation(other.to_string()), - })?; - - prompt_result - .messages - .first() - .and_then(|message| match &message.content { - rmcp::model::PromptMessageContent::Text { text } => Some(text.clone()), - _ => None, - }) - .ok_or(SlashCommandError::NoTextContent) -} - -/// Parse slash command arguments into a map with both positional and special variables. -/// -/// Creates an argument map with: -/// - "ARGUMENTS": The full argument string -/// - "1", "2", "3", etc.: Individual positional arguments (1-based) -fn parse_slash_command_arguments(args_text: &str) -> Option> { - if args_text.is_empty() { - None - } else { - let mut arg_map = serde_json::Map::new(); - arg_map.insert("ARGUMENTS".to_string(), serde_json::Value::String(args_text.to_string())); - for (i, arg) in args_text.split_whitespace().enumerate() { - arg_map.insert((i + 1).to_string(), serde_json::Value::String(arg.to_string())); - } - Some(arg_map) } } -pub(crate) fn dedupe_commands_by_name(commands: Vec) -> Vec { - let mut seen_names = HashSet::new(); - commands.into_iter().filter(|command| seen_names.insert(command.name.clone())).collect() -} - pub(crate) fn send_available_commands( connection: &ConnectionTo, acp_session_id: SessionId, @@ -117,28 +76,3 @@ pub(crate) fn send_available_commands( error!("Failed to send available commands update: {:?}", e); } } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_argument_parsing() { - let arg_map = parse_slash_command_arguments("do a thing that has spaces").expect("Expected Some"); - let expected = serde_json::Map::from_iter([ - ("ARGUMENTS".to_string(), serde_json::Value::String("do a thing that has spaces".to_string())), - ("1".to_string(), serde_json::Value::String("do".to_string())), - ("2".to_string(), serde_json::Value::String("a".to_string())), - ("3".to_string(), serde_json::Value::String("thing".to_string())), - ("4".to_string(), serde_json::Value::String("that".to_string())), - ("5".to_string(), serde_json::Value::String("has".to_string())), - ("6".to_string(), serde_json::Value::String("spaces".to_string())), - ]); - assert_eq!(arg_map, expected); - } - - #[test] - fn test_empty_arguments_returns_none() { - assert!(parse_slash_command_arguments("").is_none()); - } -} diff --git a/crates/aether-cli/src/headless/run.rs b/crates/aether-cli/src/headless/run.rs index 0d82fb0ff..df33531f3 100644 --- a/crates/aether-cli/src/headless/run.rs +++ b/crates/aether-cli/src/headless/run.rs @@ -1,13 +1,19 @@ use aether_core::core::Prompt; use aether_core::events::{AgentMessage, Command}; +use aether_core::mcp::run_mcp_task::McpCommand; +use rmcp::model::{GetPromptResult, Prompt as McpPrompt}; use std::io; use std::process::ExitCode; -use tokio::sync::mpsc; +use tokio::sync::{mpsc, oneshot}; +use tracing::error; use super::error::CliError; use super::{CliEventKind, RunConfig}; use crate::output::OutputFormat; use crate::runtime::RuntimeBuilder; +use crate::slash_commands::{ + SlashCommandError, find_prompt_name, parse_slash_command, parse_slash_command_arguments, prompt_result_text, +}; pub async fn run(config: RunConfig) -> Result { setup_tracing(config.verbose); @@ -23,15 +29,71 @@ pub async fn run(config: RunConfig) -> Result { .build_ready(vec![]) .await?; + let prompt = expand_prompt(&agent.mcp_tx, config.prompt).await; + agent .agent_tx - .send(Command::text(&config.prompt)) + .send(Command::text(&prompt)) .await .map_err(|e| CliError::AgentError(format!("Failed to send prompt: {e}")))?; Ok(stream_output(agent.agent_rx, config.output, &config.events).await) } +async fn expand_prompt(mcp_tx: &mpsc::Sender, prompt: String) -> String { + let Some(slash_command) = parse_slash_command(&prompt) else { + return prompt; + }; + + match expand_slash_command(mcp_tx, slash_command.command_name, slash_command.args_text).await { + Ok(expanded) => expanded, + Err(error) => { + error!("Failed to expand slash command: {error}"); + prompt + } + } +} + +async fn expand_slash_command( + mcp_tx: &mpsc::Sender, + command_name: &str, + args_text: &str, +) -> Result { + let arguments = parse_slash_command_arguments(args_text); + let prompts = list_prompts(mcp_tx).await?; + let prompt_name = find_prompt_name(&prompts, command_name)?; + let prompt_result = get_prompt(mcp_tx, prompt_name, arguments).await?; + prompt_result_text(&prompt_result) +} + +async fn list_prompts(mcp_tx: &mpsc::Sender) -> Result, SlashCommandError> { + let (tx, rx) = oneshot::channel(); + mcp_tx + .send(McpCommand::ListPrompts { tx }) + .await + .map_err(|e| SlashCommandError::CommandChannel(format!("failed to send ListPrompts command: {e}")))?; + + rx.await + .map_err(|e| SlashCommandError::CommandChannel(format!("failed to receive prompts: {e}")))? + .map_err(SlashCommandError::McpOperation) +} + +async fn get_prompt( + mcp_tx: &mpsc::Sender, + name: String, + arguments: Option>, +) -> Result { + let (tx, rx) = oneshot::channel(); + mcp_tx + .send(McpCommand::GetPrompt { name, arguments, tx }) + .await + .map_err(|e| SlashCommandError::CommandChannel(format!("failed to send GetPrompt command: {e}")))?; + + rx.await + .map_err(|e| SlashCommandError::CommandChannel(format!("failed to receive prompt: {e}")))? + .map_err(SlashCommandError::McpOperation) +} + async fn stream_output( mut rx: mpsc::Receiver, format: OutputFormat, diff --git a/crates/aether-cli/src/lib.rs b/crates/aether-cli/src/lib.rs index 93f1a37e6..cf449bce6 100644 --- a/crates/aether-cli/src/lib.rs +++ b/crates/aether-cli/src/lib.rs @@ -15,6 +15,7 @@ pub mod sandbox; pub mod settings; pub mod settings_args; pub mod show_prompt; +pub(crate) mod slash_commands; pub mod workspace; pub use acp::map_mcp_prompt_to_available_command; diff --git a/crates/aether-cli/src/slash_commands.rs b/crates/aether-cli/src/slash_commands.rs new file mode 100644 index 000000000..e33fa8e88 --- /dev/null +++ b/crates/aether-cli/src/slash_commands.rs @@ -0,0 +1,96 @@ +use agent_client_protocol::schema::AvailableCommand; +use rmcp::model::{GetPromptResult, Prompt as McpPrompt, PromptMessageContent}; +use std::collections::HashSet; +use thiserror::Error; + +pub(crate) struct SlashCommand<'a> { + pub(crate) command_name: &'a str, + pub(crate) args_text: &'a str, +} + +#[derive(Error, Debug)] +pub(crate) enum SlashCommandError { + #[error("command channel error: {0}")] + CommandChannel(String), + #[error("MCP operation failed: {0}")] + McpOperation(String), + #[error("slash command '/{0}' not found")] + NotFound(String), + #[error("prompt result contains no text content")] + NoTextContent, +} + +pub(crate) fn parse_slash_command(text: &str) -> Option> { + let slash_command_text = text.strip_prefix('/')?; + + let (command_name, args_text) = if let Some(space_idx) = slash_command_text.find(char::is_whitespace) { + let (cmd, args) = slash_command_text.split_at(space_idx); + (cmd, args.trim()) + } else { + (slash_command_text, "") + }; + + Some(SlashCommand { command_name, args_text }) +} + +pub(crate) fn dedupe_commands_by_name(commands: Vec) -> Vec { + let mut seen_names = HashSet::new(); + commands.into_iter().filter(|command| seen_names.insert(command.name.clone())).collect() +} + +pub(crate) fn find_prompt_name(prompts: &[McpPrompt], command_name: &str) -> Result { + prompts + .iter() + .find(|p| p.name.split("__").last().unwrap_or("") == command_name) + .map(|prompt| prompt.name.clone()) + .ok_or_else(|| SlashCommandError::NotFound(command_name.to_string())) +} + +pub(crate) fn prompt_result_text(prompt_result: &GetPromptResult) -> Result { + prompt_result + .messages + .first() + .and_then(|message| match &message.content { + PromptMessageContent::Text { text } => Some(text.clone()), + _ => None, + }) + .ok_or(SlashCommandError::NoTextContent) +} + +pub(crate) fn parse_slash_command_arguments(args_text: &str) -> Option> { + if args_text.is_empty() { + None + } else { + let mut arg_map = serde_json::Map::new(); + arg_map.insert("ARGUMENTS".to_string(), serde_json::Value::String(args_text.to_string())); + for (i, arg) in args_text.split_whitespace().enumerate() { + arg_map.insert((i + 1).to_string(), serde_json::Value::String(arg.to_string())); + } + Some(arg_map) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_argument_parsing() { + let arg_map = parse_slash_command_arguments("do a thing that has spaces").expect("Expected Some"); + let expected = serde_json::Map::from_iter([ + ("ARGUMENTS".to_string(), serde_json::Value::String("do a thing that has spaces".to_string())), + ("1".to_string(), serde_json::Value::String("do".to_string())), + ("2".to_string(), serde_json::Value::String("a".to_string())), + ("3".to_string(), serde_json::Value::String("thing".to_string())), + ("4".to_string(), serde_json::Value::String("that".to_string())), + ("5".to_string(), serde_json::Value::String("has".to_string())), + ("6".to_string(), serde_json::Value::String("spaces".to_string())), + ]); + assert_eq!(arg_map, expected); + } + + #[test] + fn test_empty_arguments_returns_none() { + assert!(parse_slash_command_arguments("").is_none()); + } +} From 416e73c56c68cddfd0329404fe98d7e3a9c6584f Mon Sep 17 00:00:00 2001 From: Josh Carver Date: Tue, 30 Jun 2026 06:13:50 -0700 Subject: [PATCH 3/7] chore(aether-cli): Add serde derives to a few structs --- crates/aether-cli/src/headless/mod.rs | 8 +++++--- crates/aether-cli/src/output.rs | 5 ++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/crates/aether-cli/src/headless/mod.rs b/crates/aether-cli/src/headless/mod.rs index 69ca62b62..9a56fbfca 100644 --- a/crates/aether-cli/src/headless/mod.rs +++ b/crates/aether-cli/src/headless/mod.rs @@ -6,6 +6,8 @@ use aether_project::AetherSettings; use error::CliError; use llm::{ProviderConnectionOverride, ProviderConnectionOverrides}; use mcp_utils::client::McpConfig; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use std::io::{IsTerminal, Read as _, stdin}; use std::path::{Path, PathBuf}; @@ -20,7 +22,7 @@ use crate::settings_args::SettingsSourceArgs; use aether_auth::OAuthCredentialStorage; use std::sync::Arc; -#[derive(Clone, Copy, PartialEq, Eq, Debug, clap::ValueEnum, serde::Deserialize, schemars::JsonSchema)] +#[derive(Clone, Copy, PartialEq, Eq, Debug, clap::ValueEnum, Deserialize, Serialize, JsonSchema)] #[clap(rename_all = "snake_case")] #[serde(rename_all = "snake_case")] pub enum CliEventKind { @@ -54,7 +56,7 @@ pub struct RunConfig { pub oauth_credential_store: Arc, } -#[derive(Clone, Debug, Default, serde::Deserialize, schemars::JsonSchema)] +#[derive(Clone, Debug, Default, Deserialize, Serialize, JsonSchema)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct HeadlessOptions { #[serde(default, skip_serializing_if = "Option::is_none")] @@ -89,7 +91,7 @@ pub async fn run_headless(args: HeadlessArgs) -> Result { #[derive(clap::Args)] pub struct HeadlessArgs { - #[arg(long = "options-json", value_name = "JSON")] + #[arg(long = "options-json", value_name = "JSON", hide = true)] pub options_json: Option, /// Prompt to send (reads stdin if omitted and stdin is not a TTY) diff --git a/crates/aether-cli/src/output.rs b/crates/aether-cli/src/output.rs index 17c77b4e3..cda1643e5 100644 --- a/crates/aether-cli/src/output.rs +++ b/crates/aether-cli/src/output.rs @@ -1,4 +1,7 @@ -#[derive(Clone, Copy, PartialEq, Eq, Debug, clap::ValueEnum, serde::Deserialize, schemars::JsonSchema)] +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Copy, PartialEq, Eq, Debug, clap::ValueEnum, Deserialize, Serialize, JsonSchema)] #[serde(rename_all = "lowercase")] pub enum OutputFormat { Text, From b34cca4357eb12c9b59a887e92289e637ed49100 Mon Sep 17 00:00:00 2001 From: Josh Carver Date: Tue, 30 Jun 2026 06:14:56 -0700 Subject: [PATCH 4/7] feat(internal-evals): Add EvalAgent builder to make it easy to author evals --- Cargo.lock | 4 + crates/internal-evals/Cargo.toml | 12 +- crates/internal-evals/src/lib.rs | 139 ++++++++++++++++++ crates/internal-evals/tests/batched_edits.rs | 13 +- .../internal-evals/tests/coding_edit_file.rs | 22 ++- crates/internal-evals/tests/coding_find.rs | 18 +-- 6 files changed, 176 insertions(+), 32 deletions(-) create mode 100644 crates/internal-evals/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 1a417a926..ddb7e1469 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3411,9 +3411,13 @@ dependencies = [ name = "internal-evals" version = "0.1.0" dependencies = [ + "aether-agent-cli", "aether-agent-core", "aether-evals", + "aether-llm", "aether-project", + "async-stream", + "futures", "serde_json", "tempfile", "testcontainers", diff --git a/crates/internal-evals/Cargo.toml b/crates/internal-evals/Cargo.toml index 404840748..ce9f19a52 100644 --- a/crates/internal-evals/Cargo.toml +++ b/crates/internal-evals/Cargo.toml @@ -10,12 +10,18 @@ dist = false [lints] workspace = true -[dev-dependencies] -aether-core = { package = "aether-agent-core", path = "../aether-core" } +[dependencies] +aether-cli = { package = "aether-agent-cli", path = "../aether-cli" } aether-evals = { path = "../aether-evals" } aether-project = { path = "../aether-project" } +async-stream = { workspace = true } +futures = { workspace = true } +llm = { package = "aether-llm", path = "../llm" } serde_json = { workspace = true } +thiserror = { workspace = true } + +[dev-dependencies] +aether-core = { package = "aether-agent-core", path = "../aether-core" } tempfile = { workspace = true } testcontainers = { workspace = true } -thiserror = { workspace = true } tokio = { workspace = true } diff --git a/crates/internal-evals/src/lib.rs b/crates/internal-evals/src/lib.rs new file mode 100644 index 000000000..e4db24fa8 --- /dev/null +++ b/crates/internal-evals/src/lib.rs @@ -0,0 +1,139 @@ +use aether_cli::{ + headless::HeadlessOptions, + init::{HarnessIntegration, InitScope, build_batteries_included_settings}, + output::OutputFormat, +}; +use aether_evals::{ + Agent, AgentRunResult, CONTAINER_AETHER_HOME, Container, ContainerError, DockerAgent, Image, Task, Workspace, + default_eval_env_vars, +}; +use aether_project::{AetherSettings, SettingsError}; +use async_stream::try_stream; +use futures::{Stream, StreamExt}; +use llm::ReasoningEffort; +use std::{env, path::Path}; +use thiserror::Error; + +pub struct EvalAgent { + settings: Option, + agent: Option, + output: OutputFormat, +} + +#[derive(Debug, Error)] +pub enum EvalHarnessError { + #[error("workspace setup failed: {0}")] + Workspace(#[from] aether_evals::WorkspaceError), + + #[error("transcript collection failed: {0}")] + Transcript(#[from] aether_evals::TranscriptError), + + #[error("container setup failed: {0}")] + Container(#[from] ContainerError), + + #[error("file IO failed: {0}")] + Io(#[from] std::io::Error), + + #[error("AETHER_EVAL_MODEL must be set for this eval")] + MissingEvalModel, + + #[error("invalid AETHER_EVAL_REASONING_EFFORT value '{value}': {error}")] + InvalidEvalReasoningEffort { value: String, error: String }, + + #[error("settings setup failed: {0}")] + Settings(#[from] SettingsError), + + #[error("failed to serialize generated headless options: {0}")] + HeadlessOptionsSerialize(#[from] serde_json::Error), +} + +impl EvalAgent { + pub fn new() -> Self { + Self { settings: None, agent: Some("Build".to_string()), output: OutputFormat::Json } + } + + pub fn settings(mut self, settings: AetherSettings) -> Self { + self.settings = Some(settings); + self + } + + pub fn agent(mut self, agent: impl Into) -> Self { + self.agent = Some(agent.into()); + self + } + + pub async fn run( + self, + workspace: &Workspace, + task: Task, + ) -> Result<(Container, impl Stream + Send), EvalHarnessError> { + let container = Container::builder(Image::new("aether-sandbox", "latest")) + .with_env_vars(default_eval_env_vars()) + .with_ephemeral_mount(CONTAINER_AETHER_HOME) + .start(workspace) + .await?; + + let settings = match self.settings { + Some(settings) => settings, + None => batteries_included_settings()?, + }; + + let options = HeadlessOptions { + prompt: Some(task.prompt().to_string()), + settings: Some(settings), + agent: self.agent, + output: Some(self.output), + ..HeadlessOptions::default() + }; + let options_json = serde_json::to_string(&options)?; + + let agent = DockerAgent::new(container.clone(), vec!["/usr/local/bin/aether-eval-agent".to_string()]) + .with_env_var("AETHER_EVAL_OPTIONS_JSON", options_json); + + let stream = try_stream! { + let messages = agent.run(task); + futures::pin_mut!(messages); + while let Some(message) = messages.next().await { + yield message?; + } + }; + + Ok((container, stream)) + } +} + +impl Default for EvalAgent { + fn default() -> Self { + Self::new() + } +} + +pub fn eval_model() -> Result { + env::var("AETHER_EVAL_MODEL") + .ok() + .map(|model| model.trim().to_string()) + .filter(|model| !model.is_empty()) + .ok_or(EvalHarnessError::MissingEvalModel) +} + +pub fn eval_reasoning_effort() -> Result, EvalHarnessError> { + let Ok(value) = env::var("AETHER_EVAL_REASONING_EFFORT") else { + return Ok(None); + }; + ReasoningEffort::parse(value.trim()).map_err(|error| EvalHarnessError::InvalidEvalReasoningEffort { value, error }) +} + +pub fn batteries_included_settings() -> Result { + let harnesses = [HarnessIntegration::Agents]; + let model = eval_model()?; + let effort = eval_reasoning_effort()?; + let mut settings = build_batteries_included_settings("Eval", model, effort, InitScope::User, &harnesses); + let template_root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../aether-cli/src/init/templates"); + settings.inline_resources(&template_root)?; + + if let Some(source) = HarnessIntegration::Agents.prompt_source() { + settings.prompts.push(source); + } + + Ok(settings) +} diff --git a/crates/internal-evals/tests/batched_edits.rs b/crates/internal-evals/tests/batched_edits.rs index 261635c05..8ae8b2fc3 100644 --- a/crates/internal-evals/tests/batched_edits.rs +++ b/crates/internal-evals/tests/batched_edits.rs @@ -1,12 +1,10 @@ -mod common; -use aether_evals::{Agent, Task, Transcript, Workspace}; -use common::{EvalHarnessError, create_aether_agent}; +use aether_evals::{Task, Transcript, Workspace}; +use internal_evals::{EvalAgent, EvalHarnessError}; #[tokio::test] async fn edit_file_multi_point_revision_in_single_call_eval() -> Result<(), EvalHarnessError> { let workspace = Workspace::from_files([("config.txt", &file_contents(&["host = localhost", "port = 8080", "debug = false"]))])?; - let (_container, agent) = create_aether_agent(&workspace).await?; let prompt = lines(&[ "Use the coding MCP tools to update config.txt.", "Read the file first, then call coding__edit_file EXACTLY ONCE, passing all three changes together in the edits array:", @@ -14,8 +12,9 @@ async fn edit_file_multi_point_revision_in_single_call_eval() -> Result<(), Eval "- set port to 443", "- set debug to true", ]); + let (_container, stream) = EvalAgent::new().run(&workspace, Task::new(prompt.clone())).await?; - let trace = Transcript::from_stream(agent.run(Task::new(prompt.clone()))).await?; + let trace = Transcript::from_stream(stream).await?; assert_single_edit_call(&trace, "coding__edit_file"); assert_eq!( @@ -28,7 +27,6 @@ async fn edit_file_multi_point_revision_in_single_call_eval() -> Result<(), Eval #[tokio::test] async fn edit_plan_multi_point_revision_in_single_call_eval() -> Result<(), EvalHarnessError> { let workspace = Workspace::empty()?; - let (_container, agent) = create_aether_agent(&workspace).await?; let prompt = lines(&[ "Use the plan MCP tools.", "First call plan__write_plan with planName 'feature' and this exact body:", @@ -39,8 +37,9 @@ async fn edit_plan_multi_point_revision_in_single_call_eval() -> Result<(), Eval "- change 'Step one: scaffold' to 'Step one: design'", "- change 'Step two: wire it up' to 'Step two: implement'", ]); + let (_container, stream) = EvalAgent::new().run(&workspace, Task::new(prompt.clone())).await?; - let trace = Transcript::from_stream(agent.run(Task::new(prompt.clone()))).await?; + let trace = Transcript::from_stream(stream).await?; assert_single_edit_call(&trace, "plan__edit_plan"); assert_eq!( diff --git a/crates/internal-evals/tests/coding_edit_file.rs b/crates/internal-evals/tests/coding_edit_file.rs index c5e0c1ddb..2e0bbb730 100644 --- a/crates/internal-evals/tests/coding_edit_file.rs +++ b/crates/internal-evals/tests/coding_edit_file.rs @@ -1,22 +1,20 @@ -mod common; - use std::fs::read_to_string; -use aether_evals::{Agent, Task, Transcript, Workspace}; -use common::{EvalHarnessError, create_aether_agent}; +use aether_evals::{Task, Transcript, Workspace}; +use internal_evals::{EvalAgent, EvalHarnessError}; #[tokio::test] async fn edit_file_replaces_first_match_by_default_eval() -> Result<(), EvalHarnessError> { let initial_notes = file_contents(&["alpha", "alpha"]); let workspace = Workspace::from_files([("notes.txt", &initial_notes)])?; - let (_container, agent) = create_aether_agent(&workspace).await?; let prompt = lines(&[ "Use the coding MCP tools to update notes.txt.", "Read the file first, then call coding__edit_file exactly once to replace only the first 'alpha' with 'beta'.", "Do not replace the second 'alpha'.", ]); + let (_container, stream) = EvalAgent::new().run(&workspace, Task::new(prompt.clone())).await?; - let trace = Transcript::from_stream(agent.run(Task::new(prompt.clone()))).await?; + let trace = Transcript::from_stream(stream).await?; assert_read_then_single_edit(&trace); assert_eq!(read_file(&workspace, "notes.txt")?, file_contents(&["beta", "alpha"])); @@ -27,13 +25,13 @@ async fn edit_file_replaces_first_match_by_default_eval() -> Result<(), EvalHarn async fn edit_file_replace_all_updates_every_match_eval() -> Result<(), EvalHarnessError> { let initial_tasks = file_contents(&["todo: one", "todo: two", "todo: three"]); let workspace = Workspace::from_files([("tasks.md", &initial_tasks)])?; - let (_container, agent) = create_aether_agent(&workspace).await?; let prompt = lines(&[ "Use the coding MCP tools to update tasks.md.", "Read the file first, then call coding__edit_file exactly once, using a single replace edit with replaceAll enabled, to change every 'todo' marker to 'done'.", ]); + let (_container, stream) = EvalAgent::new().run(&workspace, Task::new(prompt.clone())).await?; - let trace = Transcript::from_stream(agent.run(Task::new(prompt.clone()))).await?; + let trace = Transcript::from_stream(stream).await?; let contents = read_file(&workspace, "tasks.md")?; assert_read_then_single_edit(&trace); @@ -46,7 +44,6 @@ async fn edit_file_replace_all_updates_every_match_eval() -> Result<(), EvalHarn async fn edit_file_handles_multiline_exact_replacement_eval() -> Result<(), EvalHarnessError> { let initial_lib = file_contents(&["pub fn greet() {", " println!(\"hello\");", "}", "", "pub fn keep() {}"]); let workspace = Workspace::from_files([("src/lib.rs", &initial_lib)])?; - let (_container, agent) = create_aether_agent(&workspace).await?; let prompt = lines(&[ "Use the coding MCP tools to update src/lib.rs.", "Read the file first, then call coding__edit_file exactly once to replace the entire greet function with:", @@ -57,8 +54,9 @@ async fn edit_file_handles_multiline_exact_replacement_eval() -> Result<(), Eval "", "Preserve pub fn keep() unchanged.", ]); + let (_container, stream) = EvalAgent::new().run(&workspace, Task::new(prompt.clone())).await?; - let trace = Transcript::from_stream(agent.run(Task::new(prompt.clone()))).await?; + let trace = Transcript::from_stream(stream).await?; let contents = read_file(&workspace, "src/lib.rs")?; assert_read_then_single_edit(&trace); @@ -71,14 +69,14 @@ async fn edit_file_handles_multiline_exact_replacement_eval() -> Result<(), Eval async fn edit_file_pattern_not_found_leaves_file_unchanged_eval() -> Result<(), EvalHarnessError> { let initial_config = file_contents(&["mode = \"safe\""]); let workspace = Workspace::from_files([("config.toml", &initial_config)])?; - let (_container, agent) = create_aether_agent(&workspace).await?; let prompt = lines(&[ "Use the coding MCP tools on config.toml.", "Read the file first, then intentionally call coding__edit_file exactly once with a single replace edit whose oldString is 'mode = \"missing\"' and newString is 'mode = \"unsafe\"'.", "This old string is not present; report the tool error and leave the file unchanged.", ]); + let (_container, stream) = EvalAgent::new().run(&workspace, Task::new(prompt.clone())).await?; - let trace = Transcript::from_stream(agent.run(Task::new(prompt.clone()))).await?; + let trace = Transcript::from_stream(stream).await?; let contents = read_file(&workspace, "config.toml")?; assert_read_then_single_edit(&trace); diff --git a/crates/internal-evals/tests/coding_find.rs b/crates/internal-evals/tests/coding_find.rs index fe7d496d2..edf79d0ea 100644 --- a/crates/internal-evals/tests/coding_find.rs +++ b/crates/internal-evals/tests/coding_find.rs @@ -1,9 +1,7 @@ -mod common; - use std::fs::read_to_string; -use aether_evals::{Agent, Task, Transcript, Workspace}; -use common::{EvalHarnessError, create_aether_agent}; +use aether_evals::{Task, Transcript, Workspace}; +use internal_evals::{EvalAgent, EvalHarnessError}; use serde_json::Value; #[tokio::test] @@ -14,14 +12,14 @@ async fn find_bare_readme_pattern_matches_nested_basenames_eval() -> Result<(), .with_file_contents("docs/guide.md", "not a readme\n") .workspace()?; - let (_container, agent) = create_aether_agent(&workspace).await?; let prompt = lines(&[ "Use the coding MCP tools, not shell commands.", "Read every README file in this workspace, including README files in nested directories.", "Write find-report.txt summarizing the path and contents of each README you read.", ]); + let (_container, stream) = EvalAgent::new().run(&workspace, Task::new(prompt)).await?; - let trace = Transcript::from_stream(agent.run(Task::new(prompt))).await?; + let trace = Transcript::from_stream(stream).await?; assert!(trace.tool_called("coding__find")); assert_eq!(trace.tool_call_count("coding__bash"), 0); @@ -42,14 +40,14 @@ async fn find_slash_pattern_is_relative_to_workspace_root_eval() -> Result<(), E .with_file("crates/cli/src/main.rs") .with_file("examples/demo.rs") .workspace()?; - let (_container, agent) = create_aether_agent(&workspace).await?; let prompt = lines(&[ "Use the coding MCP tools to inventory this Rust workspace.", "List only Rust source files that are inside the crates directory; ignore Rust files at the workspace root or under examples.", "Write crates-rust-files.txt with the paths you found, one path per line.", ]); + let (_container, stream) = EvalAgent::new().run(&workspace, Task::new(prompt)).await?; - let trace = Transcript::from_stream(agent.run(Task::new(prompt))).await?; + let trace = Transcript::from_stream(stream).await?; assert!(trace.tool_called("coding__find")); let report = read_file(&workspace, "crates-rust-files.txt")?; @@ -67,15 +65,15 @@ async fn find_hidden_case_insensitive_limited_search_eval() -> Result<(), EvalHa .with_file("CONFIG/SETTINGS.JSON") .with_file("notes/settings.toml") .workspace()?; - let (_container, agent) = create_aether_agent(&workspace).await?; let prompt = lines(&[ "Use exactly one coding__find call and no shell commands.", "Find settings JSON files in this workspace, including files in hidden directories and files whose names use different casing.", "Only return one matching path from the search, and report whether the result list was truncated.", "Write settings-find-summary.txt with two lines: count=, truncated=.", ]); + let (_container, stream) = EvalAgent::new().run(&workspace, Task::new(prompt)).await?; - let trace = Transcript::from_stream(agent.run(Task::new(prompt))).await?; + let trace = Transcript::from_stream(stream).await?; assert_eq!(trace.tool_call_count("coding__find"), 1); assert_eq!(trace.tool_call_count("coding__bash"), 0); From ff17392072990791c032f69a646e03c7f767f099 Mon Sep 17 00:00:00 2001 From: Josh Carver Date: Tue, 30 Jun 2026 06:15:09 -0700 Subject: [PATCH 5/7] rebase me --- crates/internal-evals/tests/common/mod.rs | 29 ----------------------- 1 file changed, 29 deletions(-) delete mode 100644 crates/internal-evals/tests/common/mod.rs diff --git a/crates/internal-evals/tests/common/mod.rs b/crates/internal-evals/tests/common/mod.rs deleted file mode 100644 index 23590bc57..000000000 --- a/crates/internal-evals/tests/common/mod.rs +++ /dev/null @@ -1,29 +0,0 @@ -use aether_evals::{ - CONTAINER_AETHER_HOME, Container, ContainerError, DockerAgent, Image, Workspace, default_eval_env_vars, -}; -use thiserror::Error; - -#[derive(Debug, Error)] -pub enum EvalHarnessError { - #[error("workspace setup failed: {0}")] - Workspace(#[from] aether_evals::WorkspaceError), - - #[error("transcript collection failed: {0}")] - Transcript(#[from] aether_evals::TranscriptError), - - #[error("container setup failed: {0}")] - Container(#[from] ContainerError), - - #[error("file IO failed: {0}")] - Io(#[from] std::io::Error), -} - -pub async fn create_aether_agent(workspace: &Workspace) -> Result<(Container, DockerAgent), EvalHarnessError> { - let container = Container::builder(Image::new("aether-sandbox", "latest")) - .with_env_vars(default_eval_env_vars()) - .with_ephemeral_mount(CONTAINER_AETHER_HOME) - .start(workspace) - .await?; - let agent = DockerAgent::new(container.clone(), vec!["/usr/local/bin/aether-eval-agent".to_string()]); - Ok((container, agent)) -} From 7ca36531b08339aafdfeb291e6df97571a08a128 Mon Sep 17 00:00:00 2001 From: Josh Carver Date: Tue, 30 Jun 2026 06:16:01 -0700 Subject: [PATCH 6/7] refactor(internal-evals): Tweak eval's Dockerfile to build aether internally --- crates/internal-evals/examples/Dockerfile | 19 +++++++++++++++---- .../internal-evals/examples/aether-eval-agent | 4 ++++ justfile | 3 +-- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/crates/internal-evals/examples/Dockerfile b/crates/internal-evals/examples/Dockerfile index 325eff141..05838e4dd 100644 --- a/crates/internal-evals/examples/Dockerfile +++ b/crates/internal-evals/examples/Dockerfile @@ -7,15 +7,26 @@ # Build it with `just build-sandbox`, or point an eval file at this file with # `"docker": { "file": "Dockerfile", "context": "../../.." }` (paths relative to the eval file). -FROM archlinux:latest +FROM rust:bookworm AS builder -RUN pacman -Syu --noconfirm ca-certificates git dbus \ - && pacman -Scc --noconfirm +RUN apt-get update \ + && apt-get install -y --no-install-recommends cmake libdbus-1-dev pkg-config \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /repo +COPY . . +RUN cargo build -p aether-agent-cli --bin aether + +FROM debian:bookworm-slim + +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates git dbus \ + && rm -rf /var/lib/apt/lists/* # Install your project's toolchain here, e.g.: # RUN apt-get update && apt-get install -y python3 python3-pip -COPY target/debug/aether /usr/local/bin/aether +COPY --from=builder /repo/target/debug/aether /usr/local/bin/aether COPY .aether /repo/.aether COPY AGENTS.md /repo/AGENTS.md COPY mcp.json /repo/mcp.json diff --git a/crates/internal-evals/examples/aether-eval-agent b/crates/internal-evals/examples/aether-eval-agent index 8a77192d7..cf3d4f3f9 100755 --- a/crates/internal-evals/examples/aether-eval-agent +++ b/crates/internal-evals/examples/aether-eval-agent @@ -6,6 +6,10 @@ cp -R /repo/.aether /workspace/.aether cp /repo/AGENTS.md /workspace/AGENTS.md cp /repo/mcp.json /workspace/mcp.json +if [ "${AETHER_EVAL_OPTIONS_JSON:-}" ]; then + exec aether headless --options-json "$AETHER_EVAL_OPTIONS_JSON" +fi + if [ "${AETHER_EVAL_AGENT:-}" ]; then exec aether headless \ --output json \ diff --git a/justfile b/justfile index de67270f0..b5d42441a 100644 --- a/justfile +++ b/justfile @@ -107,9 +107,8 @@ sweep DAYS="7": sweep-installed: cargo sweep --time 1 -# Build the sandbox image from the eval example Dockerfile (copies the aether debug binary) +# Build the sandbox image from the eval example Dockerfile build-sandbox TAG="aether-sandbox:latest": - cargo build -p aether-agent-cli --bin aether docker build -t {{TAG}} -f crates/internal-evals/examples/Dockerfile . # Run wisp + aether agent inside the sandbox From cae3be800af274223fbbfc79eb07b109dcb9989c Mon Sep 17 00:00:00 2001 From: Josh Carver Date: Tue, 30 Jun 2026 06:18:34 -0700 Subject: [PATCH 7/7] fix(mcp-servers): Make agent fail fast when asked to modify files after using the /plan cmd from the plan mcp server --- crates/internal-evals/tests/plan_agent.rs | 87 +++++++++++++++++++ crates/mcp-servers/src/plan/default_prompt.md | 7 ++ 2 files changed, 94 insertions(+) create mode 100644 crates/internal-evals/tests/plan_agent.rs diff --git a/crates/internal-evals/tests/plan_agent.rs b/crates/internal-evals/tests/plan_agent.rs new file mode 100644 index 000000000..8e7b13d42 --- /dev/null +++ b/crates/internal-evals/tests/plan_agent.rs @@ -0,0 +1,87 @@ +use aether_core::events::AgentMessage; +use aether_evals::{Task, Transcript, Workspace}; +use aether_project::{AetherSettings, McpSourceSpec}; +use internal_evals::{EvalAgent, EvalHarnessError, batteries_included_settings}; +use std::fs::read_to_string; + +#[tokio::test] +async fn plan_agent_reports_missing_non_plan_edit_tools_eval() -> Result<(), EvalHarnessError> { + let workspace = Workspace::from_files([("notes.txt", "old value\n")])?; + let prompt = + "/plan Edit notes.txt, replace 'old value' with 'new value'. Don't make a plan, just make change.".to_string(); + + let (_container, stream) = EvalAgent::new().agent("Plan").run(&workspace, Task::new(prompt)).await?; + let trace = Transcript::from_stream(stream).await?; + let final_message = final_text_message(&trace); + + assert_eq!(trace.tool_call_count("coding__edit_file"), 0); + assert_eq!(trace.tool_call_count("coding__write_file"), 0); + assert_eq!(trace.tool_call_count("plan__write_plan"), 0); + assert_eq!(read_to_string(workspace.join("notes.txt"))?, "old value\n"); + assert!( + final_message.contains("I don't have tools to modify non-plan files, you must switch to another agent"), + "unexpected final message: {final_message}" + ); + + Ok(()) +} + +#[tokio::test] +async fn plan_prompt_with_edit_tools_plans_before_modifying_non_plan_files_eval() -> Result<(), EvalHarnessError> { + let workspace = Workspace::from_files([("notes.txt", "old value\n")])?; + let prompt = + "/plan Edit notes.txt, replace 'old value' with 'new value'. Don't make a plan, just make change.".to_string(); + let settings = build_agent_settings_with_plan_mcp()?; + + let (_container, stream) = + EvalAgent::new().settings(settings).agent("Build").run(&workspace, Task::new(prompt)).await?; + let trace = Transcript::from_stream(stream).await?; + let final_message = final_text_message(&trace).to_lowercase(); + + assert_eq!(trace.tool_call_count("coding__edit_file"), 0); + assert_eq!(trace.tool_call_count("coding__write_file"), 0); + assert_eq!(read_to_string(workspace.join("notes.txt"))?, "old value\n"); + assert!( + final_message.contains("would you like to exit plan mode?"), + "expected explicit approval request before editing, got: {final_message}" + ); + + Ok(()) +} + +fn build_agent_settings_with_plan_mcp() -> Result { + let mut settings = batteries_included_settings()?; + let plan_server = settings + .agents + .iter() + .find(|agent| agent.name == "Plan") + .and_then(|agent| { + agent.mcps.iter().find_map(|source| match source { + McpSourceSpec::Inline { servers } => servers.get("plan"), + McpSourceSpec::File(_) => None, + }) + }) + .expect("Plan agent should include plan MCP") + .clone(); + + let build_agent = settings.agents.iter_mut().find(|agent| agent.name == "Build").expect("Build agent should exist"); + let build_mcp = build_agent.mcps.first_mut().expect("Build agent should include MCPs"); + let McpSourceSpec::Inline { servers } = build_mcp else { + panic!("Build agent MCPs should be inline"); + }; + servers.insert("plan".to_string(), plan_server); + + Ok(settings) +} + +fn final_text_message(trace: &Transcript) -> &str { + trace + .messages() + .iter() + .rev() + .find_map(|message| match message { + AgentMessage::Text { chunk, .. } => Some(chunk.as_str()), + _ => None, + }) + .expect("expected final text message") +} diff --git a/crates/mcp-servers/src/plan/default_prompt.md b/crates/mcp-servers/src/plan/default_prompt.md index c68adc9ce..f9c0d4062 100644 --- a/crates/mcp-servers/src/plan/default_prompt.md +++ b/crates/mcp-servers/src/plan/default_prompt.md @@ -56,6 +56,13 @@ A markdown table that lists: The user may ask you to revise or update a plan based on feedback. Use the `edit_plan` tool with the plan's `planName` to update the plan file and call the `submit_plan` tool again. +## Non-Plan Files + +If asked to edit, write, delete, or otherwise modify a non-plan file: + +1. If you have tools for creating or modifying non-plan files (e.g. `create_file`, `edit_file` etc), you may do so. But you must ensure the user has explicitly approved your plan or instructed you to exit plan mode before proceeding. If the user hasn't approved the plan and/or instructed you to exit plan mode, stop and output exactly: "I can't modify files in plan mode, would you like to exit plan mode?". +2. If you do not have tools for creating or modifying non-plan files, stop and output exactly "I don't have tools to modify non-plan files, you must switch to another agent". + ## Task The task to plan is: $ARGUMENTS