diff --git a/Cargo.lock b/Cargo.lock index 1a417a92..ddb7e146 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3411,9 +3411,13 @@ dependencies = [ name = "internal-evals" version = "0.1.0" dependencies = [ + "aether-agent-cli", "aether-agent-core", "aether-evals", + "aether-llm", "aether-project", + "async-stream", + "futures", "serde_json", "tempfile", "testcontainers", diff --git a/crates/aether-cli/src/acp/error.rs b/crates/aether-cli/src/acp/error.rs index 24109653..665d7c4f 100644 --- a/crates/aether-cli/src/acp/error.rs +++ b/crates/aether-cli/src/acp/error.rs @@ -17,15 +17,3 @@ pub enum SessionError { #[error("active agent runtime is not running")] ActiveRuntimeNotRunning, } - -#[derive(Error, Debug)] -pub(super) enum SlashCommandError { - #[error("command channel error: {0}")] - CommandChannel(String), - #[error("MCP operation failed: {0}")] - McpOperation(String), - #[error("slash command '/{0}' not found")] - NotFound(String), - #[error("prompt result contains no text content")] - NoTextContent, -} diff --git a/crates/aether-cli/src/acp/slash_commands.rs b/crates/aether-cli/src/acp/slash_commands.rs index 5799bfd9..feb3fc98 100644 --- a/crates/aether-cli/src/acp/slash_commands.rs +++ b/crates/aether-cli/src/acp/slash_commands.rs @@ -2,11 +2,14 @@ use acp_utils::server::AcpServerError; use agent_client_protocol::schema::{self as acp, SessionId}; use agent_client_protocol::{Client, ConnectionTo}; use llm::ContentBlock; -use std::collections::HashSet; use tracing::{error, info}; use super::agent_runtime::AgentRuntime; -use super::error::{SessionError, SlashCommandError}; +use crate::acp::error::SessionError; +pub(crate) use crate::slash_commands::dedupe_commands_by_name; +use crate::slash_commands::{ + SlashCommandError, find_prompt_name, parse_slash_command, parse_slash_command_arguments, prompt_result_text, +}; pub(crate) async fn expand_slash_command_in_content( runtime: &AgentRuntime, @@ -15,31 +18,24 @@ pub(crate) async fn expand_slash_command_in_content( if let Some(ContentBlock::Text { text }) = content.first() && text.starts_with('/') { - let expanded = expand_slash_command_if_needed(runtime, text.clone()).await; + let expanded = expand_slash_command_text(runtime, text.clone()).await; content[0] = ContentBlock::text(expanded); } content } -async fn expand_slash_command_if_needed(runtime: &AgentRuntime, text: String) -> String { - let Some(slash_command_text) = text.strip_prefix('/') else { +async fn expand_slash_command_text(runtime: &AgentRuntime, text: String) -> String { + let Some(slash_command) = parse_slash_command(&text) else { return text; }; - let (command_name, args_text) = if let Some(space_idx) = slash_command_text.find(char::is_whitespace) { - let (cmd, args) = slash_command_text.split_at(space_idx); - (cmd, args.trim()) - } else { - (slash_command_text, "") - }; - - match expand_slash_command(runtime, command_name, args_text).await { + match expand_slash_command(runtime, slash_command.command_name, slash_command.args_text).await { Ok(expanded) => { - info!("Expanded slash command '{}' -> {} chars", command_name, expanded.len()); + info!("Expanded slash command -> {} chars", expanded.len()); expanded } Err(e) => { - error!("Failed to expand slash command '{}': {}", command_name, e); + error!("Failed to expand slash command: {}", e); text } } @@ -51,57 +47,20 @@ async fn expand_slash_command( args_text: &str, ) -> Result { let arguments = parse_slash_command_arguments(args_text); + let prompts = runtime.list_prompts().await.map_err(slash_command_error)?; + let prompt_name = find_prompt_name(&prompts, command_name)?; + let prompt_result = runtime.get_prompt(prompt_name, arguments).await.map_err(slash_command_error)?; - let prompts = runtime.list_prompts().await.map_err(|error| match error { - SessionError::CommandChannel(message) => SlashCommandError::CommandChannel(message), - other => SlashCommandError::McpOperation(other.to_string()), - })?; - - let matching_prompt = prompts - .iter() - .find(|p| p.name.split("__").last().unwrap_or("") == command_name) - .ok_or_else(|| SlashCommandError::NotFound(command_name.to_string()))?; - - let namespaced_name = matching_prompt.name.clone(); + prompt_result_text(&prompt_result) +} - let prompt_result = runtime.get_prompt(namespaced_name.clone(), arguments).await.map_err(|error| match error { +fn slash_command_error(error: super::error::SessionError) -> SlashCommandError { + match error { SessionError::CommandChannel(message) => SlashCommandError::CommandChannel(message), other => SlashCommandError::McpOperation(other.to_string()), - })?; - - prompt_result - .messages - .first() - .and_then(|message| match &message.content { - rmcp::model::PromptMessageContent::Text { text } => Some(text.clone()), - _ => None, - }) - .ok_or(SlashCommandError::NoTextContent) -} - -/// Parse slash command arguments into a map with both positional and special variables. -/// -/// Creates an argument map with: -/// - "ARGUMENTS": The full argument string -/// - "1", "2", "3", etc.: Individual positional arguments (1-based) -fn parse_slash_command_arguments(args_text: &str) -> Option> { - if args_text.is_empty() { - None - } else { - let mut arg_map = serde_json::Map::new(); - arg_map.insert("ARGUMENTS".to_string(), serde_json::Value::String(args_text.to_string())); - for (i, arg) in args_text.split_whitespace().enumerate() { - arg_map.insert((i + 1).to_string(), serde_json::Value::String(arg.to_string())); - } - Some(arg_map) } } -pub(crate) fn dedupe_commands_by_name(commands: Vec) -> Vec { - let mut seen_names = HashSet::new(); - commands.into_iter().filter(|command| seen_names.insert(command.name.clone())).collect() -} - pub(crate) fn send_available_commands( connection: &ConnectionTo, acp_session_id: SessionId, @@ -117,28 +76,3 @@ pub(crate) fn send_available_commands( error!("Failed to send available commands update: {:?}", e); } } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_argument_parsing() { - let arg_map = parse_slash_command_arguments("do a thing that has spaces").expect("Expected Some"); - let expected = serde_json::Map::from_iter([ - ("ARGUMENTS".to_string(), serde_json::Value::String("do a thing that has spaces".to_string())), - ("1".to_string(), serde_json::Value::String("do".to_string())), - ("2".to_string(), serde_json::Value::String("a".to_string())), - ("3".to_string(), serde_json::Value::String("thing".to_string())), - ("4".to_string(), serde_json::Value::String("that".to_string())), - ("5".to_string(), serde_json::Value::String("has".to_string())), - ("6".to_string(), serde_json::Value::String("spaces".to_string())), - ]); - assert_eq!(arg_map, expected); - } - - #[test] - fn test_empty_arguments_returns_none() { - assert!(parse_slash_command_arguments("").is_none()); - } -} diff --git a/crates/aether-cli/src/headless/mod.rs b/crates/aether-cli/src/headless/mod.rs index 69ca62b6..9a56fbfc 100644 --- a/crates/aether-cli/src/headless/mod.rs +++ b/crates/aether-cli/src/headless/mod.rs @@ -6,6 +6,8 @@ use aether_project::AetherSettings; use error::CliError; use llm::{ProviderConnectionOverride, ProviderConnectionOverrides}; use mcp_utils::client::McpConfig; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use std::io::{IsTerminal, Read as _, stdin}; use std::path::{Path, PathBuf}; @@ -20,7 +22,7 @@ use crate::settings_args::SettingsSourceArgs; use aether_auth::OAuthCredentialStorage; use std::sync::Arc; -#[derive(Clone, Copy, PartialEq, Eq, Debug, clap::ValueEnum, serde::Deserialize, schemars::JsonSchema)] +#[derive(Clone, Copy, PartialEq, Eq, Debug, clap::ValueEnum, Deserialize, Serialize, JsonSchema)] #[clap(rename_all = "snake_case")] #[serde(rename_all = "snake_case")] pub enum CliEventKind { @@ -54,7 +56,7 @@ pub struct RunConfig { pub oauth_credential_store: Arc, } -#[derive(Clone, Debug, Default, serde::Deserialize, schemars::JsonSchema)] +#[derive(Clone, Debug, Default, Deserialize, Serialize, JsonSchema)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct HeadlessOptions { #[serde(default, skip_serializing_if = "Option::is_none")] @@ -89,7 +91,7 @@ pub async fn run_headless(args: HeadlessArgs) -> Result { #[derive(clap::Args)] pub struct HeadlessArgs { - #[arg(long = "options-json", value_name = "JSON")] + #[arg(long = "options-json", value_name = "JSON", hide = true)] pub options_json: Option, /// Prompt to send (reads stdin if omitted and stdin is not a TTY) diff --git a/crates/aether-cli/src/headless/run.rs b/crates/aether-cli/src/headless/run.rs index 0d82fb0f..df33531f 100644 --- a/crates/aether-cli/src/headless/run.rs +++ b/crates/aether-cli/src/headless/run.rs @@ -1,13 +1,19 @@ use aether_core::core::Prompt; use aether_core::events::{AgentMessage, Command}; +use aether_core::mcp::run_mcp_task::McpCommand; +use rmcp::model::{GetPromptResult, Prompt as McpPrompt}; use std::io; use std::process::ExitCode; -use tokio::sync::mpsc; +use tokio::sync::{mpsc, oneshot}; +use tracing::error; use super::error::CliError; use super::{CliEventKind, RunConfig}; use crate::output::OutputFormat; use crate::runtime::RuntimeBuilder; +use crate::slash_commands::{ + SlashCommandError, find_prompt_name, parse_slash_command, parse_slash_command_arguments, prompt_result_text, +}; pub async fn run(config: RunConfig) -> Result { setup_tracing(config.verbose); @@ -23,15 +29,71 @@ pub async fn run(config: RunConfig) -> Result { .build_ready(vec![]) .await?; + let prompt = expand_prompt(&agent.mcp_tx, config.prompt).await; + agent .agent_tx - .send(Command::text(&config.prompt)) + .send(Command::text(&prompt)) .await .map_err(|e| CliError::AgentError(format!("Failed to send prompt: {e}")))?; Ok(stream_output(agent.agent_rx, config.output, &config.events).await) } +async fn expand_prompt(mcp_tx: &mpsc::Sender, prompt: String) -> String { + let Some(slash_command) = parse_slash_command(&prompt) else { + return prompt; + }; + + match expand_slash_command(mcp_tx, slash_command.command_name, slash_command.args_text).await { + Ok(expanded) => expanded, + Err(error) => { + error!("Failed to expand slash command: {error}"); + prompt + } + } +} + +async fn expand_slash_command( + mcp_tx: &mpsc::Sender, + command_name: &str, + args_text: &str, +) -> Result { + let arguments = parse_slash_command_arguments(args_text); + let prompts = list_prompts(mcp_tx).await?; + let prompt_name = find_prompt_name(&prompts, command_name)?; + let prompt_result = get_prompt(mcp_tx, prompt_name, arguments).await?; + prompt_result_text(&prompt_result) +} + +async fn list_prompts(mcp_tx: &mpsc::Sender) -> Result, SlashCommandError> { + let (tx, rx) = oneshot::channel(); + mcp_tx + .send(McpCommand::ListPrompts { tx }) + .await + .map_err(|e| SlashCommandError::CommandChannel(format!("failed to send ListPrompts command: {e}")))?; + + rx.await + .map_err(|e| SlashCommandError::CommandChannel(format!("failed to receive prompts: {e}")))? + .map_err(SlashCommandError::McpOperation) +} + +async fn get_prompt( + mcp_tx: &mpsc::Sender, + name: String, + arguments: Option>, +) -> Result { + let (tx, rx) = oneshot::channel(); + mcp_tx + .send(McpCommand::GetPrompt { name, arguments, tx }) + .await + .map_err(|e| SlashCommandError::CommandChannel(format!("failed to send GetPrompt command: {e}")))?; + + rx.await + .map_err(|e| SlashCommandError::CommandChannel(format!("failed to receive prompt: {e}")))? + .map_err(SlashCommandError::McpOperation) +} + async fn stream_output( mut rx: mpsc::Receiver, format: OutputFormat, diff --git a/crates/aether-cli/src/init/build_settings.rs b/crates/aether-cli/src/init/build_settings.rs index 18e268a5..ec4b3f70 100644 --- a/crates/aether-cli/src/init/build_settings.rs +++ b/crates/aether-cli/src/init/build_settings.rs @@ -3,7 +3,7 @@ use super::harness::HarnessIntegration; use super::recommendations::{ProviderRecommendations, recommended_for_provider}; use aether_core::agent_spec::{ToolFilter, ToolMatcher}; use aether_project::{AetherSettings, AgentConfig, McpSourceSpec, PromptSource}; -use llm::catalog::Provider; +use llm::{ReasoningEffort, catalog::Provider}; use mcp_utils::client::{InMemoryServerConfig, InMemoryType, McpServerConfig}; const SYSTEM_PATH: &str = "SYSTEM.md"; @@ -38,6 +38,96 @@ pub fn supported_providers() -> impl Iterator { Provider::ALL.iter().copied().filter(|p| recommended_for_provider(*p).is_some()) } +pub fn build_batteries_included_settings( + display: &str, + model: String, + reasoning_effort: Option, + scope: InitScope, + harnesses: &[HarnessIntegration], +) -> AetherSettings { + let plan = build_batteries_included_plan_agent(display, model.clone(), reasoning_effort, harnesses); + let build = build_batteries_included_build_agent(display, model.clone(), reasoning_effort, harnesses); + let explore = build_batteries_included_explore_agent(model, reasoning_effort, scope, harnesses); + + AetherSettings { + prompts: batteries_prompts(scope, harnesses), + agents: vec![plan, build, explore], + ..AetherSettings::default() + } +} + +pub fn build_batteries_included_plan_agent( + display: &str, + model: String, + reasoning_effort: Option, + harnesses: &[HarnessIntegration], +) -> AgentConfig { + let coding = coding_args(harnesses); + let skills = skills_args(harnesses); + AgentConfig { + name: "Plan".to_string(), + description: format!("{display} planner (read-only except plan files)"), + model, + reasoning_effort, + user_invocable: true, + mcps: vec![mcps(vec![ + ("plan", vec![]), + ("coding", coding), + ("skills", skills), + ("subagents", vec![]), + ("tasks", vec![]), + ("survey", vec![]), + ])], + tools: read_only_coding_tools(), + ..AgentConfig::default() + } +} + +pub fn build_batteries_included_build_agent( + display: &str, + model: String, + reasoning_effort: Option, + harnesses: &[HarnessIntegration], +) -> AgentConfig { + let coding = coding_args(harnesses); + let skills = skills_args(harnesses); + AgentConfig { + name: "Build".to_string(), + description: format!("{display} implementor"), + model, + reasoning_effort, + user_invocable: true, + mcps: vec![mcps(vec![ + ("coding", coding), + ("skills", skills), + ("subagents", vec![]), + ("tasks", vec![]), + ("survey", vec![]), + ])], + ..AgentConfig::default() + } +} + +pub fn build_batteries_included_explore_agent( + model: String, + reasoning_effort: Option, + scope: InitScope, + harnesses: &[HarnessIntegration], +) -> AgentConfig { + let coding = coding_args(harnesses); + AgentConfig { + name: "Explore".to_string(), + description: "Explores codebases to find relevant files, patterns, and integration points".to_string(), + model, + reasoning_effort, + agent_invocable: true, + prompts: vec![PromptSource::file(scope.asset_path(EXPLORER_AGENTS_PATH))], + mcps: vec![mcps(vec![("coding", coding)])], + tools: read_only_coding_tools(), + ..AgentConfig::default() + } +} + pub(crate) fn build_preset( preset: Preset, provider: Provider, @@ -80,54 +170,26 @@ fn batteries_included_preset( harnesses: &[HarnessIntegration], ) -> ResolvedPreset { let display = provider.display_name(); - let coding = coding_args(harnesses); - let skills = skills_args(harnesses); - - let plan = AgentConfig { - name: "Plan".to_string(), - description: format!("{display} planner (read-only except plan files)"), - model: recs.plan.model.to_string(), - reasoning_effort: recs.plan.reasoning_effort, - user_invocable: true, - mcps: vec![mcps(vec![ - ("plan", vec![]), - ("coding", coding.clone()), - ("skills", skills.clone()), - ("subagents", vec![]), - ("tasks", vec![]), - ("survey", vec![]), - ])], - tools: read_only_coding_tools(), - ..AgentConfig::default() - }; + let plan = build_batteries_included_plan_agent( + display, + recs.plan.model.to_string(), + recs.plan.reasoning_effort, + harnesses, + ); - let build = AgentConfig { - name: "Build".to_string(), - description: format!("{display} implementor"), - model: recs.build.model.to_string(), - reasoning_effort: recs.build.reasoning_effort, - user_invocable: true, - mcps: vec![mcps(vec![ - ("coding", coding.clone()), - ("skills", skills), - ("subagents", vec![]), - ("tasks", vec![]), - ("survey", vec![]), - ])], - ..AgentConfig::default() - }; + let build = build_batteries_included_build_agent( + display, + recs.build.model.to_string(), + recs.build.reasoning_effort, + harnesses, + ); - let explore = AgentConfig { - name: "Explore".to_string(), - description: "Explores codebases to find relevant files, patterns, and integration points".to_string(), - model: recs.explore.model.to_string(), - reasoning_effort: recs.explore.reasoning_effort, - agent_invocable: true, - prompts: vec![PromptSource::file(scope.asset_path(EXPLORER_AGENTS_PATH))], - mcps: vec![mcps(vec![("coding", coding)])], - tools: read_only_coding_tools(), - ..AgentConfig::default() - }; + let explore = build_batteries_included_explore_agent( + recs.explore.model.to_string(), + recs.explore.reasoning_effort, + scope, + harnesses, + ); ResolvedPreset { files: &[ diff --git a/crates/aether-cli/src/init/mod.rs b/crates/aether-cli/src/init/mod.rs index 31e535e3..fdc1e037 100644 --- a/crates/aether-cli/src/init/mod.rs +++ b/crates/aether-cli/src/init/mod.rs @@ -3,7 +3,10 @@ mod harness; mod recommendations; mod tui_runner; use aether_project::user_settings_path; -pub use build_settings::Preset; +pub use build_settings::{ + Preset, build_batteries_included_build_agent, build_batteries_included_explore_agent, + build_batteries_included_plan_agent, build_batteries_included_settings, +}; pub use harness::HarnessIntegration; use llm::catalog::Provider; use recommendations::recommended_for_provider; diff --git a/crates/aether-cli/src/lib.rs b/crates/aether-cli/src/lib.rs index 93f1a37e..cf449bce 100644 --- a/crates/aether-cli/src/lib.rs +++ b/crates/aether-cli/src/lib.rs @@ -15,6 +15,7 @@ pub mod sandbox; pub mod settings; pub mod settings_args; pub mod show_prompt; +pub(crate) mod slash_commands; pub mod workspace; pub use acp::map_mcp_prompt_to_available_command; diff --git a/crates/aether-cli/src/output.rs b/crates/aether-cli/src/output.rs index 17c77b4e..cda1643e 100644 --- a/crates/aether-cli/src/output.rs +++ b/crates/aether-cli/src/output.rs @@ -1,4 +1,7 @@ -#[derive(Clone, Copy, PartialEq, Eq, Debug, clap::ValueEnum, serde::Deserialize, schemars::JsonSchema)] +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Copy, PartialEq, Eq, Debug, clap::ValueEnum, Deserialize, Serialize, JsonSchema)] #[serde(rename_all = "lowercase")] pub enum OutputFormat { Text, diff --git a/crates/aether-cli/src/slash_commands.rs b/crates/aether-cli/src/slash_commands.rs new file mode 100644 index 00000000..e33fa8e8 --- /dev/null +++ b/crates/aether-cli/src/slash_commands.rs @@ -0,0 +1,96 @@ +use agent_client_protocol::schema::AvailableCommand; +use rmcp::model::{GetPromptResult, Prompt as McpPrompt, PromptMessageContent}; +use std::collections::HashSet; +use thiserror::Error; + +pub(crate) struct SlashCommand<'a> { + pub(crate) command_name: &'a str, + pub(crate) args_text: &'a str, +} + +#[derive(Error, Debug)] +pub(crate) enum SlashCommandError { + #[error("command channel error: {0}")] + CommandChannel(String), + #[error("MCP operation failed: {0}")] + McpOperation(String), + #[error("slash command '/{0}' not found")] + NotFound(String), + #[error("prompt result contains no text content")] + NoTextContent, +} + +pub(crate) fn parse_slash_command(text: &str) -> Option> { + let slash_command_text = text.strip_prefix('/')?; + + let (command_name, args_text) = if let Some(space_idx) = slash_command_text.find(char::is_whitespace) { + let (cmd, args) = slash_command_text.split_at(space_idx); + (cmd, args.trim()) + } else { + (slash_command_text, "") + }; + + Some(SlashCommand { command_name, args_text }) +} + +pub(crate) fn dedupe_commands_by_name(commands: Vec) -> Vec { + let mut seen_names = HashSet::new(); + commands.into_iter().filter(|command| seen_names.insert(command.name.clone())).collect() +} + +pub(crate) fn find_prompt_name(prompts: &[McpPrompt], command_name: &str) -> Result { + prompts + .iter() + .find(|p| p.name.split("__").last().unwrap_or("") == command_name) + .map(|prompt| prompt.name.clone()) + .ok_or_else(|| SlashCommandError::NotFound(command_name.to_string())) +} + +pub(crate) fn prompt_result_text(prompt_result: &GetPromptResult) -> Result { + prompt_result + .messages + .first() + .and_then(|message| match &message.content { + PromptMessageContent::Text { text } => Some(text.clone()), + _ => None, + }) + .ok_or(SlashCommandError::NoTextContent) +} + +pub(crate) fn parse_slash_command_arguments(args_text: &str) -> Option> { + if args_text.is_empty() { + None + } else { + let mut arg_map = serde_json::Map::new(); + arg_map.insert("ARGUMENTS".to_string(), serde_json::Value::String(args_text.to_string())); + for (i, arg) in args_text.split_whitespace().enumerate() { + arg_map.insert((i + 1).to_string(), serde_json::Value::String(arg.to_string())); + } + Some(arg_map) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_argument_parsing() { + let arg_map = parse_slash_command_arguments("do a thing that has spaces").expect("Expected Some"); + let expected = serde_json::Map::from_iter([ + ("ARGUMENTS".to_string(), serde_json::Value::String("do a thing that has spaces".to_string())), + ("1".to_string(), serde_json::Value::String("do".to_string())), + ("2".to_string(), serde_json::Value::String("a".to_string())), + ("3".to_string(), serde_json::Value::String("thing".to_string())), + ("4".to_string(), serde_json::Value::String("that".to_string())), + ("5".to_string(), serde_json::Value::String("has".to_string())), + ("6".to_string(), serde_json::Value::String("spaces".to_string())), + ]); + assert_eq!(arg_map, expected); + } + + #[test] + fn test_empty_arguments_returns_none() { + assert!(parse_slash_command_arguments("").is_none()); + } +} diff --git a/crates/internal-evals/Cargo.toml b/crates/internal-evals/Cargo.toml index 40484074..ce9f19a5 100644 --- a/crates/internal-evals/Cargo.toml +++ b/crates/internal-evals/Cargo.toml @@ -10,12 +10,18 @@ dist = false [lints] workspace = true -[dev-dependencies] -aether-core = { package = "aether-agent-core", path = "../aether-core" } +[dependencies] +aether-cli = { package = "aether-agent-cli", path = "../aether-cli" } aether-evals = { path = "../aether-evals" } aether-project = { path = "../aether-project" } +async-stream = { workspace = true } +futures = { workspace = true } +llm = { package = "aether-llm", path = "../llm" } serde_json = { workspace = true } +thiserror = { workspace = true } + +[dev-dependencies] +aether-core = { package = "aether-agent-core", path = "../aether-core" } tempfile = { workspace = true } testcontainers = { workspace = true } -thiserror = { workspace = true } tokio = { workspace = true } diff --git a/crates/internal-evals/examples/Dockerfile b/crates/internal-evals/examples/Dockerfile index 325eff14..05838e4d 100644 --- a/crates/internal-evals/examples/Dockerfile +++ b/crates/internal-evals/examples/Dockerfile @@ -7,15 +7,26 @@ # Build it with `just build-sandbox`, or point an eval file at this file with # `"docker": { "file": "Dockerfile", "context": "../../.." }` (paths relative to the eval file). -FROM archlinux:latest +FROM rust:bookworm AS builder -RUN pacman -Syu --noconfirm ca-certificates git dbus \ - && pacman -Scc --noconfirm +RUN apt-get update \ + && apt-get install -y --no-install-recommends cmake libdbus-1-dev pkg-config \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /repo +COPY . . +RUN cargo build -p aether-agent-cli --bin aether + +FROM debian:bookworm-slim + +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates git dbus \ + && rm -rf /var/lib/apt/lists/* # Install your project's toolchain here, e.g.: # RUN apt-get update && apt-get install -y python3 python3-pip -COPY target/debug/aether /usr/local/bin/aether +COPY --from=builder /repo/target/debug/aether /usr/local/bin/aether COPY .aether /repo/.aether COPY AGENTS.md /repo/AGENTS.md COPY mcp.json /repo/mcp.json diff --git a/crates/internal-evals/examples/aether-eval-agent b/crates/internal-evals/examples/aether-eval-agent index 8a77192d..cf3d4f3f 100755 --- a/crates/internal-evals/examples/aether-eval-agent +++ b/crates/internal-evals/examples/aether-eval-agent @@ -6,6 +6,10 @@ cp -R /repo/.aether /workspace/.aether cp /repo/AGENTS.md /workspace/AGENTS.md cp /repo/mcp.json /workspace/mcp.json +if [ "${AETHER_EVAL_OPTIONS_JSON:-}" ]; then + exec aether headless --options-json "$AETHER_EVAL_OPTIONS_JSON" +fi + if [ "${AETHER_EVAL_AGENT:-}" ]; then exec aether headless \ --output json \ diff --git a/crates/internal-evals/src/lib.rs b/crates/internal-evals/src/lib.rs new file mode 100644 index 00000000..e4db24fa --- /dev/null +++ b/crates/internal-evals/src/lib.rs @@ -0,0 +1,139 @@ +use aether_cli::{ + headless::HeadlessOptions, + init::{HarnessIntegration, InitScope, build_batteries_included_settings}, + output::OutputFormat, +}; +use aether_evals::{ + Agent, AgentRunResult, CONTAINER_AETHER_HOME, Container, ContainerError, DockerAgent, Image, Task, Workspace, + default_eval_env_vars, +}; +use aether_project::{AetherSettings, SettingsError}; +use async_stream::try_stream; +use futures::{Stream, StreamExt}; +use llm::ReasoningEffort; +use std::{env, path::Path}; +use thiserror::Error; + +pub struct EvalAgent { + settings: Option, + agent: Option, + output: OutputFormat, +} + +#[derive(Debug, Error)] +pub enum EvalHarnessError { + #[error("workspace setup failed: {0}")] + Workspace(#[from] aether_evals::WorkspaceError), + + #[error("transcript collection failed: {0}")] + Transcript(#[from] aether_evals::TranscriptError), + + #[error("container setup failed: {0}")] + Container(#[from] ContainerError), + + #[error("file IO failed: {0}")] + Io(#[from] std::io::Error), + + #[error("AETHER_EVAL_MODEL must be set for this eval")] + MissingEvalModel, + + #[error("invalid AETHER_EVAL_REASONING_EFFORT value '{value}': {error}")] + InvalidEvalReasoningEffort { value: String, error: String }, + + #[error("settings setup failed: {0}")] + Settings(#[from] SettingsError), + + #[error("failed to serialize generated headless options: {0}")] + HeadlessOptionsSerialize(#[from] serde_json::Error), +} + +impl EvalAgent { + pub fn new() -> Self { + Self { settings: None, agent: Some("Build".to_string()), output: OutputFormat::Json } + } + + pub fn settings(mut self, settings: AetherSettings) -> Self { + self.settings = Some(settings); + self + } + + pub fn agent(mut self, agent: impl Into) -> Self { + self.agent = Some(agent.into()); + self + } + + pub async fn run( + self, + workspace: &Workspace, + task: Task, + ) -> Result<(Container, impl Stream + Send), EvalHarnessError> { + let container = Container::builder(Image::new("aether-sandbox", "latest")) + .with_env_vars(default_eval_env_vars()) + .with_ephemeral_mount(CONTAINER_AETHER_HOME) + .start(workspace) + .await?; + + let settings = match self.settings { + Some(settings) => settings, + None => batteries_included_settings()?, + }; + + let options = HeadlessOptions { + prompt: Some(task.prompt().to_string()), + settings: Some(settings), + agent: self.agent, + output: Some(self.output), + ..HeadlessOptions::default() + }; + let options_json = serde_json::to_string(&options)?; + + let agent = DockerAgent::new(container.clone(), vec!["/usr/local/bin/aether-eval-agent".to_string()]) + .with_env_var("AETHER_EVAL_OPTIONS_JSON", options_json); + + let stream = try_stream! { + let messages = agent.run(task); + futures::pin_mut!(messages); + while let Some(message) = messages.next().await { + yield message?; + } + }; + + Ok((container, stream)) + } +} + +impl Default for EvalAgent { + fn default() -> Self { + Self::new() + } +} + +pub fn eval_model() -> Result { + env::var("AETHER_EVAL_MODEL") + .ok() + .map(|model| model.trim().to_string()) + .filter(|model| !model.is_empty()) + .ok_or(EvalHarnessError::MissingEvalModel) +} + +pub fn eval_reasoning_effort() -> Result, EvalHarnessError> { + let Ok(value) = env::var("AETHER_EVAL_REASONING_EFFORT") else { + return Ok(None); + }; + ReasoningEffort::parse(value.trim()).map_err(|error| EvalHarnessError::InvalidEvalReasoningEffort { value, error }) +} + +pub fn batteries_included_settings() -> Result { + let harnesses = [HarnessIntegration::Agents]; + let model = eval_model()?; + let effort = eval_reasoning_effort()?; + let mut settings = build_batteries_included_settings("Eval", model, effort, InitScope::User, &harnesses); + let template_root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../aether-cli/src/init/templates"); + settings.inline_resources(&template_root)?; + + if let Some(source) = HarnessIntegration::Agents.prompt_source() { + settings.prompts.push(source); + } + + Ok(settings) +} diff --git a/crates/internal-evals/tests/batched_edits.rs b/crates/internal-evals/tests/batched_edits.rs index 261635c0..8ae8b2fc 100644 --- a/crates/internal-evals/tests/batched_edits.rs +++ b/crates/internal-evals/tests/batched_edits.rs @@ -1,12 +1,10 @@ -mod common; -use aether_evals::{Agent, Task, Transcript, Workspace}; -use common::{EvalHarnessError, create_aether_agent}; +use aether_evals::{Task, Transcript, Workspace}; +use internal_evals::{EvalAgent, EvalHarnessError}; #[tokio::test] async fn edit_file_multi_point_revision_in_single_call_eval() -> Result<(), EvalHarnessError> { let workspace = Workspace::from_files([("config.txt", &file_contents(&["host = localhost", "port = 8080", "debug = false"]))])?; - let (_container, agent) = create_aether_agent(&workspace).await?; let prompt = lines(&[ "Use the coding MCP tools to update config.txt.", "Read the file first, then call coding__edit_file EXACTLY ONCE, passing all three changes together in the edits array:", @@ -14,8 +12,9 @@ async fn edit_file_multi_point_revision_in_single_call_eval() -> Result<(), Eval "- set port to 443", "- set debug to true", ]); + let (_container, stream) = EvalAgent::new().run(&workspace, Task::new(prompt.clone())).await?; - let trace = Transcript::from_stream(agent.run(Task::new(prompt.clone()))).await?; + let trace = Transcript::from_stream(stream).await?; assert_single_edit_call(&trace, "coding__edit_file"); assert_eq!( @@ -28,7 +27,6 @@ async fn edit_file_multi_point_revision_in_single_call_eval() -> Result<(), Eval #[tokio::test] async fn edit_plan_multi_point_revision_in_single_call_eval() -> Result<(), EvalHarnessError> { let workspace = Workspace::empty()?; - let (_container, agent) = create_aether_agent(&workspace).await?; let prompt = lines(&[ "Use the plan MCP tools.", "First call plan__write_plan with planName 'feature' and this exact body:", @@ -39,8 +37,9 @@ async fn edit_plan_multi_point_revision_in_single_call_eval() -> Result<(), Eval "- change 'Step one: scaffold' to 'Step one: design'", "- change 'Step two: wire it up' to 'Step two: implement'", ]); + let (_container, stream) = EvalAgent::new().run(&workspace, Task::new(prompt.clone())).await?; - let trace = Transcript::from_stream(agent.run(Task::new(prompt.clone()))).await?; + let trace = Transcript::from_stream(stream).await?; assert_single_edit_call(&trace, "plan__edit_plan"); assert_eq!( diff --git a/crates/internal-evals/tests/coding_edit_file.rs b/crates/internal-evals/tests/coding_edit_file.rs index c5e0c1dd..2e0bbb73 100644 --- a/crates/internal-evals/tests/coding_edit_file.rs +++ b/crates/internal-evals/tests/coding_edit_file.rs @@ -1,22 +1,20 @@ -mod common; - use std::fs::read_to_string; -use aether_evals::{Agent, Task, Transcript, Workspace}; -use common::{EvalHarnessError, create_aether_agent}; +use aether_evals::{Task, Transcript, Workspace}; +use internal_evals::{EvalAgent, EvalHarnessError}; #[tokio::test] async fn edit_file_replaces_first_match_by_default_eval() -> Result<(), EvalHarnessError> { let initial_notes = file_contents(&["alpha", "alpha"]); let workspace = Workspace::from_files([("notes.txt", &initial_notes)])?; - let (_container, agent) = create_aether_agent(&workspace).await?; let prompt = lines(&[ "Use the coding MCP tools to update notes.txt.", "Read the file first, then call coding__edit_file exactly once to replace only the first 'alpha' with 'beta'.", "Do not replace the second 'alpha'.", ]); + let (_container, stream) = EvalAgent::new().run(&workspace, Task::new(prompt.clone())).await?; - let trace = Transcript::from_stream(agent.run(Task::new(prompt.clone()))).await?; + let trace = Transcript::from_stream(stream).await?; assert_read_then_single_edit(&trace); assert_eq!(read_file(&workspace, "notes.txt")?, file_contents(&["beta", "alpha"])); @@ -27,13 +25,13 @@ async fn edit_file_replaces_first_match_by_default_eval() -> Result<(), EvalHarn async fn edit_file_replace_all_updates_every_match_eval() -> Result<(), EvalHarnessError> { let initial_tasks = file_contents(&["todo: one", "todo: two", "todo: three"]); let workspace = Workspace::from_files([("tasks.md", &initial_tasks)])?; - let (_container, agent) = create_aether_agent(&workspace).await?; let prompt = lines(&[ "Use the coding MCP tools to update tasks.md.", "Read the file first, then call coding__edit_file exactly once, using a single replace edit with replaceAll enabled, to change every 'todo' marker to 'done'.", ]); + let (_container, stream) = EvalAgent::new().run(&workspace, Task::new(prompt.clone())).await?; - let trace = Transcript::from_stream(agent.run(Task::new(prompt.clone()))).await?; + let trace = Transcript::from_stream(stream).await?; let contents = read_file(&workspace, "tasks.md")?; assert_read_then_single_edit(&trace); @@ -46,7 +44,6 @@ async fn edit_file_replace_all_updates_every_match_eval() -> Result<(), EvalHarn async fn edit_file_handles_multiline_exact_replacement_eval() -> Result<(), EvalHarnessError> { let initial_lib = file_contents(&["pub fn greet() {", " println!(\"hello\");", "}", "", "pub fn keep() {}"]); let workspace = Workspace::from_files([("src/lib.rs", &initial_lib)])?; - let (_container, agent) = create_aether_agent(&workspace).await?; let prompt = lines(&[ "Use the coding MCP tools to update src/lib.rs.", "Read the file first, then call coding__edit_file exactly once to replace the entire greet function with:", @@ -57,8 +54,9 @@ async fn edit_file_handles_multiline_exact_replacement_eval() -> Result<(), Eval "", "Preserve pub fn keep() unchanged.", ]); + let (_container, stream) = EvalAgent::new().run(&workspace, Task::new(prompt.clone())).await?; - let trace = Transcript::from_stream(agent.run(Task::new(prompt.clone()))).await?; + let trace = Transcript::from_stream(stream).await?; let contents = read_file(&workspace, "src/lib.rs")?; assert_read_then_single_edit(&trace); @@ -71,14 +69,14 @@ async fn edit_file_handles_multiline_exact_replacement_eval() -> Result<(), Eval async fn edit_file_pattern_not_found_leaves_file_unchanged_eval() -> Result<(), EvalHarnessError> { let initial_config = file_contents(&["mode = \"safe\""]); let workspace = Workspace::from_files([("config.toml", &initial_config)])?; - let (_container, agent) = create_aether_agent(&workspace).await?; let prompt = lines(&[ "Use the coding MCP tools on config.toml.", "Read the file first, then intentionally call coding__edit_file exactly once with a single replace edit whose oldString is 'mode = \"missing\"' and newString is 'mode = \"unsafe\"'.", "This old string is not present; report the tool error and leave the file unchanged.", ]); + let (_container, stream) = EvalAgent::new().run(&workspace, Task::new(prompt.clone())).await?; - let trace = Transcript::from_stream(agent.run(Task::new(prompt.clone()))).await?; + let trace = Transcript::from_stream(stream).await?; let contents = read_file(&workspace, "config.toml")?; assert_read_then_single_edit(&trace); diff --git a/crates/internal-evals/tests/coding_find.rs b/crates/internal-evals/tests/coding_find.rs index fe7d496d..edf79d0e 100644 --- a/crates/internal-evals/tests/coding_find.rs +++ b/crates/internal-evals/tests/coding_find.rs @@ -1,9 +1,7 @@ -mod common; - use std::fs::read_to_string; -use aether_evals::{Agent, Task, Transcript, Workspace}; -use common::{EvalHarnessError, create_aether_agent}; +use aether_evals::{Task, Transcript, Workspace}; +use internal_evals::{EvalAgent, EvalHarnessError}; use serde_json::Value; #[tokio::test] @@ -14,14 +12,14 @@ async fn find_bare_readme_pattern_matches_nested_basenames_eval() -> Result<(), .with_file_contents("docs/guide.md", "not a readme\n") .workspace()?; - let (_container, agent) = create_aether_agent(&workspace).await?; let prompt = lines(&[ "Use the coding MCP tools, not shell commands.", "Read every README file in this workspace, including README files in nested directories.", "Write find-report.txt summarizing the path and contents of each README you read.", ]); + let (_container, stream) = EvalAgent::new().run(&workspace, Task::new(prompt)).await?; - let trace = Transcript::from_stream(agent.run(Task::new(prompt))).await?; + let trace = Transcript::from_stream(stream).await?; assert!(trace.tool_called("coding__find")); assert_eq!(trace.tool_call_count("coding__bash"), 0); @@ -42,14 +40,14 @@ async fn find_slash_pattern_is_relative_to_workspace_root_eval() -> Result<(), E .with_file("crates/cli/src/main.rs") .with_file("examples/demo.rs") .workspace()?; - let (_container, agent) = create_aether_agent(&workspace).await?; let prompt = lines(&[ "Use the coding MCP tools to inventory this Rust workspace.", "List only Rust source files that are inside the crates directory; ignore Rust files at the workspace root or under examples.", "Write crates-rust-files.txt with the paths you found, one path per line.", ]); + let (_container, stream) = EvalAgent::new().run(&workspace, Task::new(prompt)).await?; - let trace = Transcript::from_stream(agent.run(Task::new(prompt))).await?; + let trace = Transcript::from_stream(stream).await?; assert!(trace.tool_called("coding__find")); let report = read_file(&workspace, "crates-rust-files.txt")?; @@ -67,15 +65,15 @@ async fn find_hidden_case_insensitive_limited_search_eval() -> Result<(), EvalHa .with_file("CONFIG/SETTINGS.JSON") .with_file("notes/settings.toml") .workspace()?; - let (_container, agent) = create_aether_agent(&workspace).await?; let prompt = lines(&[ "Use exactly one coding__find call and no shell commands.", "Find settings JSON files in this workspace, including files in hidden directories and files whose names use different casing.", "Only return one matching path from the search, and report whether the result list was truncated.", "Write settings-find-summary.txt with two lines: count=, truncated=.", ]); + let (_container, stream) = EvalAgent::new().run(&workspace, Task::new(prompt)).await?; - let trace = Transcript::from_stream(agent.run(Task::new(prompt))).await?; + let trace = Transcript::from_stream(stream).await?; assert_eq!(trace.tool_call_count("coding__find"), 1); assert_eq!(trace.tool_call_count("coding__bash"), 0); diff --git a/crates/internal-evals/tests/common/mod.rs b/crates/internal-evals/tests/common/mod.rs deleted file mode 100644 index 23590bc5..00000000 --- a/crates/internal-evals/tests/common/mod.rs +++ /dev/null @@ -1,29 +0,0 @@ -use aether_evals::{ - CONTAINER_AETHER_HOME, Container, ContainerError, DockerAgent, Image, Workspace, default_eval_env_vars, -}; -use thiserror::Error; - -#[derive(Debug, Error)] -pub enum EvalHarnessError { - #[error("workspace setup failed: {0}")] - Workspace(#[from] aether_evals::WorkspaceError), - - #[error("transcript collection failed: {0}")] - Transcript(#[from] aether_evals::TranscriptError), - - #[error("container setup failed: {0}")] - Container(#[from] ContainerError), - - #[error("file IO failed: {0}")] - Io(#[from] std::io::Error), -} - -pub async fn create_aether_agent(workspace: &Workspace) -> Result<(Container, DockerAgent), EvalHarnessError> { - let container = Container::builder(Image::new("aether-sandbox", "latest")) - .with_env_vars(default_eval_env_vars()) - .with_ephemeral_mount(CONTAINER_AETHER_HOME) - .start(workspace) - .await?; - let agent = DockerAgent::new(container.clone(), vec!["/usr/local/bin/aether-eval-agent".to_string()]); - Ok((container, agent)) -} diff --git a/crates/internal-evals/tests/plan_agent.rs b/crates/internal-evals/tests/plan_agent.rs new file mode 100644 index 00000000..8e7b13d4 --- /dev/null +++ b/crates/internal-evals/tests/plan_agent.rs @@ -0,0 +1,87 @@ +use aether_core::events::AgentMessage; +use aether_evals::{Task, Transcript, Workspace}; +use aether_project::{AetherSettings, McpSourceSpec}; +use internal_evals::{EvalAgent, EvalHarnessError, batteries_included_settings}; +use std::fs::read_to_string; + +#[tokio::test] +async fn plan_agent_reports_missing_non_plan_edit_tools_eval() -> Result<(), EvalHarnessError> { + let workspace = Workspace::from_files([("notes.txt", "old value\n")])?; + let prompt = + "/plan Edit notes.txt, replace 'old value' with 'new value'. Don't make a plan, just make change.".to_string(); + + let (_container, stream) = EvalAgent::new().agent("Plan").run(&workspace, Task::new(prompt)).await?; + let trace = Transcript::from_stream(stream).await?; + let final_message = final_text_message(&trace); + + assert_eq!(trace.tool_call_count("coding__edit_file"), 0); + assert_eq!(trace.tool_call_count("coding__write_file"), 0); + assert_eq!(trace.tool_call_count("plan__write_plan"), 0); + assert_eq!(read_to_string(workspace.join("notes.txt"))?, "old value\n"); + assert!( + final_message.contains("I don't have tools to modify non-plan files, you must switch to another agent"), + "unexpected final message: {final_message}" + ); + + Ok(()) +} + +#[tokio::test] +async fn plan_prompt_with_edit_tools_plans_before_modifying_non_plan_files_eval() -> Result<(), EvalHarnessError> { + let workspace = Workspace::from_files([("notes.txt", "old value\n")])?; + let prompt = + "/plan Edit notes.txt, replace 'old value' with 'new value'. Don't make a plan, just make change.".to_string(); + let settings = build_agent_settings_with_plan_mcp()?; + + let (_container, stream) = + EvalAgent::new().settings(settings).agent("Build").run(&workspace, Task::new(prompt)).await?; + let trace = Transcript::from_stream(stream).await?; + let final_message = final_text_message(&trace).to_lowercase(); + + assert_eq!(trace.tool_call_count("coding__edit_file"), 0); + assert_eq!(trace.tool_call_count("coding__write_file"), 0); + assert_eq!(read_to_string(workspace.join("notes.txt"))?, "old value\n"); + assert!( + final_message.contains("would you like to exit plan mode?"), + "expected explicit approval request before editing, got: {final_message}" + ); + + Ok(()) +} + +fn build_agent_settings_with_plan_mcp() -> Result { + let mut settings = batteries_included_settings()?; + let plan_server = settings + .agents + .iter() + .find(|agent| agent.name == "Plan") + .and_then(|agent| { + agent.mcps.iter().find_map(|source| match source { + McpSourceSpec::Inline { servers } => servers.get("plan"), + McpSourceSpec::File(_) => None, + }) + }) + .expect("Plan agent should include plan MCP") + .clone(); + + let build_agent = settings.agents.iter_mut().find(|agent| agent.name == "Build").expect("Build agent should exist"); + let build_mcp = build_agent.mcps.first_mut().expect("Build agent should include MCPs"); + let McpSourceSpec::Inline { servers } = build_mcp else { + panic!("Build agent MCPs should be inline"); + }; + servers.insert("plan".to_string(), plan_server); + + Ok(settings) +} + +fn final_text_message(trace: &Transcript) -> &str { + trace + .messages() + .iter() + .rev() + .find_map(|message| match message { + AgentMessage::Text { chunk, .. } => Some(chunk.as_str()), + _ => None, + }) + .expect("expected final text message") +} diff --git a/crates/mcp-servers/src/plan/default_prompt.md b/crates/mcp-servers/src/plan/default_prompt.md index c68adc9c..f9c0d406 100644 --- a/crates/mcp-servers/src/plan/default_prompt.md +++ b/crates/mcp-servers/src/plan/default_prompt.md @@ -56,6 +56,13 @@ A markdown table that lists: The user may ask you to revise or update a plan based on feedback. Use the `edit_plan` tool with the plan's `planName` to update the plan file and call the `submit_plan` tool again. +## Non-Plan Files + +If asked to edit, write, delete, or otherwise modify a non-plan file: + +1. If you have tools for creating or modifying non-plan files (e.g. `create_file`, `edit_file` etc), you may do so. But you must ensure the user has explicitly approved your plan or instructed you to exit plan mode before proceeding. If the user hasn't approved the plan and/or instructed you to exit plan mode, stop and output exactly: "I can't modify files in plan mode, would you like to exit plan mode?". +2. If you do not have tools for creating or modifying non-plan files, stop and output exactly "I don't have tools to modify non-plan files, you must switch to another agent". + ## Task The task to plan is: $ARGUMENTS diff --git a/justfile b/justfile index de67270f..b5d42441 100644 --- a/justfile +++ b/justfile @@ -107,9 +107,8 @@ sweep DAYS="7": sweep-installed: cargo sweep --time 1 -# Build the sandbox image from the eval example Dockerfile (copies the aether debug binary) +# Build the sandbox image from the eval example Dockerfile build-sandbox TAG="aether-sandbox:latest": - cargo build -p aether-agent-cli --bin aether docker build -t {{TAG}} -f crates/internal-evals/examples/Dockerfile . # Run wisp + aether agent inside the sandbox