diff --git a/CHANGELOG.md b/CHANGELOG.md index f3c5bebe08..60517ca0c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,8 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - The standalone `switchyard-server` now binds to `127.0.0.1` by default. Existing deployments that accept remote clients must add `--host 0.0.0.0`. The provided Docker image and `dev-server/switchyard.service` set this explicitly. +- **VGR escalates sooner after a tool error** - chat steps skip verification, + and steps typed as chat skip the efficient attempt (`chat_after_tool_error`). - **TypeScript composite stage profiles** - `CompositeRouter` options take `stage.efficient_first` and `stage.capable_first` profiles in place of `efficient_first_threshold`, `capable_first_threshold`, and diff --git a/crates/libsy/src/algorithms/vgr/decide.rs b/crates/libsy/src/algorithms/vgr/decide.rs index 186084a61b..108c44525d 100644 --- a/crates/libsy/src/algorithms/vgr/decide.rs +++ b/crates/libsy/src/algorithms/vgr/decide.rs @@ -9,7 +9,7 @@ use super::{Branch, Capabilities}; /// Version of these rules and dials, carried on every decision record. -pub(super) const POLICY_VERSION: &str = "1.2"; +pub(super) const POLICY_VERSION: &str = "1.3"; const READOUT: f64 = 0.9; const DELIBERATION: f64 = 0.5; diff --git a/crates/libsy/src/algorithms/vgr/runtime.rs b/crates/libsy/src/algorithms/vgr/runtime.rs index 9fe1713754..e9f5d3be8d 100644 --- a/crates/libsy/src/algorithms/vgr/runtime.rs +++ b/crates/libsy/src/algorithms/vgr/runtime.rs @@ -171,6 +171,23 @@ impl Classifier for VgrClassifier { let Some(trial) = self.breaker.attempt() else { return Ok(self.escalate(driver, "breaker_open", None, request, state, None)); }; + let typed = if text::ToolRecord::from_request(request).errors > 0 { + let context = RequestContext { + driver, + request, + started, + }; + Some(self.type_task(&context).await) + } else { + None + }; + if typed == Some(Some(TaskType::Chat)) { + let reason = "chat_after_tool_error"; + return Ok(self.escalate(driver, reason, Some(Branch::Chat), request, state, None)); + } + if self.config.bounded_efficient && self.remaining(started).is_none() { + return Ok(self.escalate(driver, "efficient_timed_out", None, request, state, None)); + } let attempted = self .within_budget(started, async { let response = driver @@ -255,7 +272,10 @@ impl Classifier for VgrClassifier { } let attempt = rungs::response_text(&buffered.agg).unwrap_or_default(); - let task_type = self.type_task(&context).await; + let task_type = match typed { + Some(task_type) => task_type, + None => self.type_task(&context).await, + }; let caps = derive_capabilities( request, &attempt, @@ -451,7 +471,10 @@ impl VgrClassifier { let run = (branch == Branch::Agentic).then(|| decide::agentic_run(caps, confirmed)); let (Some(judged), false) = ( caps.transcript.as_deref(), - branch == Branch::Unknown || run == Some(AgenticRun::Vetoed), + branch == Branch::Unknown + || run == Some(AgenticRun::Vetoed) + || (matches!(branch, Branch::Chat | Branch::DefaultVerified) + && caps.tools.is_some_and(|tools| tools.errors > 0)), ) else { return signals; }; diff --git a/crates/libsy/src/algorithms/vgr/runtime/tests.rs b/crates/libsy/src/algorithms/vgr/runtime/tests.rs index fe9599559f..767b2d8fa6 100644 --- a/crates/libsy/src/algorithms/vgr/runtime/tests.rs +++ b/crates/libsy/src/algorithms/vgr/runtime/tests.rs @@ -472,7 +472,7 @@ async fn decisions_attach_the_reason_and_exclude_efficient_capable_fallbacks() - assert_eq!(evidence["source"], "vgr"); assert_eq!(evidence["reason_code"], reason); assert_eq!(evidence["verdict"], served); - assert_eq!(evidence["policy_version"], "1.2"); + assert_eq!(evidence["policy_version"], "1.3"); if reason == "decided" { assert_eq!(evidence["branch"], "DefaultVerified"); } @@ -1376,3 +1376,33 @@ async fn a_failed_capable_verifier_still_offers_the_efficient_answer() -> Result assert_eq!(fallback.llm_request.messages, task().llm_request.messages); Ok(()) } + +#[tokio::test] +async fn a_step_that_cannot_commit_over_a_tool_error_skips_local_checks() -> Result<()> { + let failed = session(vec![ + Message::text(Role::User, "What is happening today?"), + call("c1"), + result("c1", r#"{"error": "search-only backend"}"#), + ]); + // A typed answer with tool use in its turn is judged as chat, which needs a clean record. + for (typed, expected) in [ + ("Answer", vec!["judge", "efficient", "capable"]), + ("Chat", vec!["judge", "capable"]), + ] { + let calls = Arc::new(parking_lot::Mutex::new(Vec::new())); + let seen = Arc::clone(&calls); + let (selected, _) = test_drive_with_models( + route(|config| config.task_typing = true)?, + failed.clone(), + category_models(Category::Any, &["efficient", "capable"]), + move |target: ModelId, _| { + seen.lock().push(target.to_string()); + std::future::ready(Ok(reply(typed))) + }, + ) + .await?; + assert_eq!(selected.as_str(), "capable"); + assert_eq!(*calls.lock(), expected, "typed {typed}"); + } + Ok(()) +} diff --git a/docs/routing_algorithms/vgr_routing.md b/docs/routing_algorithms/vgr_routing.md index 34abc9f920..8c196c4184 100644 --- a/docs/routing_algorithms/vgr_routing.md +++ b/docs/routing_algorithms/vgr_routing.md @@ -190,10 +190,13 @@ headers take precedence. Hosts that omit reporting retain message-shape detectio ## Verification and capable-tier outages -Policy version `1.2` allows the default verification branch to accept a clean +Policy version `1.3` allows the default verification branch to accept a clean efficient attempt when the capable judge approves both the evidence and the answer. A negative or unavailable evidence verdict skips the answer check. +After a failed tool result, chat steps escalate without verification, and steps +typed as chat escalate before any efficient attempt. + Typed factual answers are verified on the answer itself, including later user turns without tool work. A readout of at least 0.9 can serve the efficient answer; lower scores require capable-tier verification. Scored verdicts