diff --git a/crates/libsy/src/algorithms/util/tool_signals.rs b/crates/libsy/src/algorithms/util/tool_signals.rs index a98c11b8ea..b350e8ab66 100644 --- a/crates/libsy/src/algorithms/util/tool_signals.rs +++ b/crates/libsy/src/algorithms/util/tool_signals.rs @@ -12,7 +12,7 @@ #![allow(dead_code)] -use std::collections::HashSet; +use std::collections::HashMap; use std::path::Path; use std::sync::LazyLock; @@ -439,6 +439,13 @@ struct ObservedToolCall<'a> { code_semantic: Option, } +#[derive(Clone, Copy, Default)] +struct ToolResultContext { + is_retrieval: bool, + search_no_match: bool, + codex_session: bool, +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum MutationKind { Write, @@ -626,6 +633,54 @@ fn command_is_retrieval(command: &Value) -> bool { } } +// Only a final search can explain exit 1 as "no matches". Conditional execution +// may have stopped before the search, so leave those statuses uninterpreted. +fn command_ends_with_search(command: &Value) -> bool { + match command { + Value::String(command) => { + if quoted_chars(command).any(|(_, c, quote)| quote.is_none() && c == '&') { + return false; + } + shell_segments(command) + .last() + .and_then(shlex::split) + .is_some_and(|words| { + search_command(&words.iter().map(String::as_str).collect::>(), 0) + }) + } + Value::Array(words) => search_command( + &words.iter().filter_map(Value::as_str).collect::>(), + 0, + ), + _ => false, + } +} + +fn search_command(words: &[&str], depth: usize) -> bool { + if depth > MAX_COMMAND_DEPTH { + return false; + } + let words = skip_shell_assignments(words); + match words.split_first() { + Some((program, args)) => match program_name(program) { + "env" => search_command( + skip_shell_options(args, &["-u", "--unset", "-C", "--chdir"]), + depth + 1, + ), + "rg" | "grep" => true, + _ => false, + }, + None => false, + } +} + +fn codex_session_id(header: &str) -> Option<&str> { + header.lines().find_map(|line| { + line.strip_prefix("Process running with session ID ") + .filter(|id| !id.is_empty() && id.bytes().all(|c| c.is_ascii_digit())) + }) +} + // Keep shell parsing bounded on the request thread. Larger commands retain // ordinary error scanning, just like unrecognized shell syntax. const MAX_RETRIEVAL_COMMAND_BYTES: usize = 16 * 1024; @@ -677,10 +732,7 @@ fn has_short_flag(word: &str, flags: &[char]) -> bool { // Bounds recursion through nested wrappers and `sh -c` scripts. const MAX_COMMAND_DEPTH: usize = 20; -fn retrieval_command(mut words: &[&str], depth: usize) -> bool { - if depth > MAX_COMMAND_DEPTH { - return false; - } +fn skip_shell_assignments<'a>(mut words: &'a [&'a str]) -> &'a [&'a str] { while words.first().is_some_and(|word| { word.split_once('=').is_some_and(|(name, _)| { !name.is_empty() @@ -691,6 +743,14 @@ fn retrieval_command(mut words: &[&str], depth: usize) -> bool { }) { words = &words[1..]; } + words +} + +fn retrieval_command(words: &[&str], depth: usize) -> bool { + if depth > MAX_COMMAND_DEPTH { + return false; + } + let words = skip_shell_assignments(words); let Some((&program, args)) = words.split_first() else { return false; }; @@ -1066,8 +1126,8 @@ fn extract_tool_signals_with_window_and_semantics( let namespaces = tool_namespaces(&request.llm_request.extensions); let mut tool_texts: Vec<(String, bool)> = Vec::new(); let mut tool_calls: Vec = Vec::new(); - // IDs whose latest call is a retrieval tool. - let mut retrieval_calls: HashSet<&str> = HashSet::new(); + let mut result_contexts: HashMap<&str, ToolResultContext> = HashMap::new(); + let mut sessions: HashMap = HashMap::new(); let mut tool_result_count = 0usize; let mut assistant_turn_count = 0usize; @@ -1077,7 +1137,6 @@ fn extract_tool_signals_with_window_and_semantics( } for block in &message.content { match block { - _ if index < start => continue, ContentBlock::ToolCall(call) => { // Responses namespaced tools arrive as `__`. let bare_name = namespaces @@ -1093,7 +1152,7 @@ fn extract_tool_signals_with_window_and_semantics( let command_field = command_of(arguments); // Codex uses exec.input for JavaScript. OpenClaw's exec.command // remains a shell call, even when the names are identical. - let code_semantic = (bare_name.unwrap_or(&call.name) == "exec" + let mut code_semantic = (bare_name.unwrap_or(&call.name) == "exec" && arguments.get("command").is_none() && arguments.get("cmd").is_none()) .then(|| arguments.get("input").and_then(Value::as_str)) @@ -1120,15 +1179,48 @@ fn extract_tool_signals_with_window_and_semantics( } _ => call.name.as_str(), }; - let is_retrieval = + let mut is_retrieval = code_semantic.is_none() && is_retrieval_tool(name, command_field); + let mut context = ToolResultContext { + is_retrieval, + search_no_match: is_retrieval + && command_field.is_some_and(command_ends_with_search), + codex_session: name == "exec_command", + }; + if name == "write_stdin" { + let session_id = arguments.get("session_id").map(|id| { + id.as_str() + .map(str::to_owned) + .unwrap_or_else(|| id.to_string()) + }); + if let Some(id) = session_id { + if arguments + .get("chars") + .and_then(Value::as_str) + .unwrap_or("") + .is_empty() + { + context = sessions.get(&id).copied().unwrap_or_default(); + is_retrieval = context.is_retrieval; + // Polling observes an existing command; it does not + // repeat a write or edit performed by that command. + code_semantic = Some(if is_retrieval { + ToolSemantic::Observe + } else { + ToolSemantic::Unknown + }); + } else { + // Input can change what an interactive command does. + sessions.remove(&id); + } + } + } if !call.id.is_empty() { // A reused ID links to its latest call. - if is_retrieval { - retrieval_calls.insert(call.id.as_str()); - } else { - retrieval_calls.remove(call.id.as_str()); - } + result_contexts.insert(call.id.as_str(), context); + } + if index < start { + continue; } tool_calls.push(ObservedToolCall { name: call.name.clone(), @@ -1139,8 +1231,6 @@ fn extract_tool_signals_with_window_and_semantics( }); } ContentBlock::ToolResult(result) => { - // Before the empty-text filter: empty results still count. - tool_result_count += 1; let is_error = result.is_error == Some(true); let text = result .content @@ -1148,9 +1238,27 @@ fn extract_tool_signals_with_window_and_semantics( .filter_map(text_of) .collect::>() .join("\n"); - let is_retrieval_result = - retrieval_calls.contains(result.tool_call_id.as_str()); - if is_retrieval_result { + let context = result_contexts + .get(result.tool_call_id.as_str()) + .copied() + .unwrap_or_default(); + // Never interpret a session marker inside retrieved file contents. + let status = text + .split_once("\nOutput:\n") + .map_or(text.as_str(), |(header, _)| header); + if context.codex_session + && !is_error + && let Some(id) = codex_session_id(status) + { + sessions.insert(id.to_owned(), context); + } + // Old calls can establish session context, but their activity + // and results stay outside the current user-turn counters. + if index < start { + continue; + } + tool_result_count += 1; + if context.is_retrieval { // Shell reads can return bare JSON from a file. Hermes wraps // terminal output with output and exit_code fields. let shell_result = serde_json::Deserializer::from_str(&text) @@ -1162,9 +1270,6 @@ fn extract_tool_signals_with_window_and_semantics( }); // Codex reports the exit status in a header before `Output:`. // An exit line after it is part of what was read. - let status = text - .split_once("\nOutput:\n") - .map_or(text.as_str(), |(header, _)| header); // Hermes reports it in fields, so its output is not scanned. let shell_failed = match &shell_result { Some(value) => { @@ -1176,7 +1281,20 @@ fn extract_tool_signals_with_window_and_semantics( } None => has_nonzero_exit_status(&status.to_lowercase()), }; - if is_missing_file(&text) || (!is_error && !shell_failed) { + let exit_code = shell_result + .as_ref() + .and_then(|value| value["exit_code"].as_i64()) + .or_else(|| { + status.lines().find_map(|line| { + line.strip_prefix("Process exited with code ")? + .parse::() + .ok() + }) + }); + let no_match = context.search_no_match + && exit_code == Some(1) + && classify_text(&text).0 <= SOFT; + if is_missing_file(&text) || (!is_error && (!shell_failed || no_match)) { // Keep the window slot after dropping retrieved content. if !text.is_empty() { tool_texts.push((String::new(), false)); @@ -1191,7 +1309,8 @@ fn extract_tool_signals_with_window_and_semantics( } // Built-in tool history stays opaque so it can be replayed unchanged. ContentBlock::Unknown { provider, raw } - if provider.as_str() == WireFormat::OpenAiResponses.as_str() + if index >= start + && provider.as_str() == WireFormat::OpenAiResponses.as_str() && raw.get("type").and_then(Value::as_str) == Some("apply_patch_call_output") => { @@ -1204,7 +1323,8 @@ fn extract_tool_signals_with_window_and_semantics( tool_texts.push((text.to_owned(), is_error)); } ContentBlock::Unknown { provider, raw } - if provider.as_str() == WireFormat::OpenAiResponses.as_str() + if index >= start + && provider.as_str() == WireFormat::OpenAiResponses.as_str() && raw.get("type").and_then(Value::as_str) == Some("shell_call_output") => { tool_result_count += 1; @@ -2256,6 +2376,158 @@ mod tests { assert_eq!(read("Chunk ID: abc\nProcess exited with code 1"), CRITICAL); } + #[test] + fn search_no_match_is_an_observation_not_a_failure() { + for command in [ + json!("rg missing ."), + json!("grep missing file.txt"), + json!("LC_ALL=C rg missing ."), + json!("env LC_ALL=C rg missing ."), + json!(["env", "LC_ALL=C", "rg", "missing", "."]), + json!(["rg", "needle; cat file", "."]), + json!("rg version poetry.lock; rg limits poetry.lock | sed -n '1,80p'; rg headers ."), + ] { + for output in [ + "Process exited with code 1\nOutput:\n", + r#"{"output":"","exit_code":1,"error":null}"#, + ] { + let request = call_with_result(exec_command(json!({"cmd": command})), tr(output)); + let signal = ToolSignals::from_request(&request, None); + assert_eq!(signal.severity, 0.0, "{command}: {output}"); + assert_eq!(signal.read_count, 1); + } + } + } + + #[test] + fn search_detection_preserves_argv_boundaries() { + let request = call_with_result( + exec_command(json!({"cmd": ["cat", "log; rg needle ."]})), + tr("Process exited with code 1\nOutput:\nPermission denied"), + ); + assert_eq!(ToolSignals::from_request(&request, None).severity, SOFT); + } + + #[test] + fn search_errors_and_non_search_exit_one_remain_failures() { + for (command, output) in [ + ( + "rg missing .", + "Process exited with code 2\nOutput:\nrg: invalid regex", + ), + ( + "rg missing .", + "Process exited with code 1\nOutput:\nMemoryError", + ), + ( + "rg missing .", + r#"{"output":"","exit_code":1,"error":"execution failed"}"#, + ), + ( + "cat file.txt", + "Process exited with code 1\nOutput:\npermission denied", + ), + ( + "pytest; rg missing .", + "Process exited with code 1\nOutput:\n", + ), + ( + "false && rg missing .", + "Process exited with code 1\nOutput:\n", + ), + ] { + let request = call_with_result(exec_command(json!({"cmd": command})), tr(output)); + assert!( + ToolSignals::from_request(&request, None).severity > 0.0, + "{command}: {output}" + ); + } + let mut request = call_with_result( + exec_command(json!({"cmd": "rg missing ."})), + tr("Process exited with code 1\nOutput:\n"), + ); + if let ContentBlock::ToolResult(result) = &mut request.llm_request.messages[1].content[0] { + result.is_error = Some(true); + } + assert_eq!(ToolSignals::from_request(&request, None).severity, HARD); + } + + fn polled_command(command: &str, chars: &str, output: &str) -> Request { + let mut request = call_with_result( + exec_command(json!({"cmd": command})), + tr("Process running with session ID 123\nOutput:\n"), + ); + let mut poll = tc("write_stdin"); + if let ContentBlock::ToolCall(call) = &mut poll.content[0] { + call.id = "poll-1".into(); + call.arguments = json!({"session_id":123, "chars":chars}); + } + let mut result = tr(output); + if let ContentBlock::ToolResult(result) = &mut result.content[0] { + result.tool_call_id = "poll-1".into(); + } + request.llm_request.messages.extend([poll, result]); + request + } + + #[test] + fn codex_poll_preserves_read_context_without_hiding_execution_errors() { + let output = "Process exited with code 0\nOutput:\nMemoryError\n5 passed in 0.12s"; + let request = polled_command("cat run.log", "", output); + let signal = ToolSignals::from_request(&request, Some(1)); + assert_eq!(signal.severity, 0.0); + assert_eq!(signal.recent_read_count, 1); + assert!(!signal.tests_passed); + for (command, chars) in [("python script.py", ""), ("cat run.log", "x")] { + let signal = + ToolSignals::from_request(&polled_command(command, chars, output), Some(1)); + assert_eq!(signal.severity, CRITICAL); + } + let failed = polled_command( + "cat run.log", + "", + "Process exited with code 2\nOutput:\nMemoryError", + ); + assert_eq!( + ToolSignals::from_request(&failed, Some(1)).severity, + CRITICAL + ); + let no_match = polled_command("rg missing .", "", "Process exited with code 1\nOutput:\n"); + assert_eq!(ToolSignals::from_request(&no_match, Some(1)).severity, 0.0); + let tests = polled_command( + "yarn test", + "", + "Process exited with code 0\nOutput:\n5 passed in 0.12s", + ); + assert!(ToolSignals::from_request(&tests, Some(1)).tests_passed); + let write = polled_command( + "echo done > result.txt", + "", + "Process exited with code 0\nOutput:\n", + ); + assert_eq!( + ToolSignals::from_request(&write, Some(1)).recent_write_count, + 0 + ); + } + + #[test] + fn codex_poll_keeps_context_across_user_turns_without_old_activity() { + let mut request = polled_command( + "cat run.log", + "", + "Process exited with code 0\nOutput:\nMemoryError", + ); + request + .llm_request + .messages + .insert(2, Message::text(Role::User, "Check the command")); + let signal = ToolSignalProcessor::default().for_user_turn(&request); + assert_eq!(signal.severity, 0.0); + assert_eq!(signal.read_count, 1); + assert_eq!(signal.tool_result_count, 1); + } + #[test] fn hermes_read_status_comes_from_exit_code() { let read = |exit_code: i64| {