From 0f6d2f168a592b5c45ee531a13af602cd5b9fe47 Mon Sep 17 00:00:00 2001 From: adhaile Date: Fri, 13 Nov 2026 10:17:29 -0800 Subject: [PATCH 1/2] fix(vgr): skip checks that can't change the result after a tool error A chat step can't serve the efficient attempt once a tool result in the conversation has failed, yet VGR still drafted it locally and ran the readout, deliberation, its verdict and a capable-tier check before escalating. After a failed tool result, VGR now types the step before the efficient attempt and escalates a step typed as chat right away, with reason chat_after_tool_error. Chat and default-verified steps over a tool error also return from gather() before any check. Policy version 1.3. Signed-off-by: adhaile --- CHANGELOG.md | 2 ++ crates/libsy/src/algorithms/vgr/decide.rs | 2 +- crates/libsy/src/algorithms/vgr/runtime.rs | 24 ++++++++++++-- .../libsy/src/algorithms/vgr/runtime/tests.rs | 32 ++++++++++++++++++- docs/routing_algorithms/vgr_routing.md | 5 ++- 5 files changed, 60 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f3c5bebe08..60517ca0c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,8 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - The standalone `switchyard-server` now binds to `127.0.0.1` by default. Existing deployments that accept remote clients must add `--host 0.0.0.0`. The provided Docker image and `dev-server/switchyard.service` set this explicitly. +- **VGR escalates sooner after a tool error** - chat steps skip verification, + and steps typed as chat skip the efficient attempt (`chat_after_tool_error`). - **TypeScript composite stage profiles** - `CompositeRouter` options take `stage.efficient_first` and `stage.capable_first` profiles in place of `efficient_first_threshold`, `capable_first_threshold`, and diff --git a/crates/libsy/src/algorithms/vgr/decide.rs b/crates/libsy/src/algorithms/vgr/decide.rs index 186084a61b..108c44525d 100644 --- a/crates/libsy/src/algorithms/vgr/decide.rs +++ b/crates/libsy/src/algorithms/vgr/decide.rs @@ -9,7 +9,7 @@ use super::{Branch, Capabilities}; /// Version of these rules and dials, carried on every decision record. -pub(super) const POLICY_VERSION: &str = "1.2"; +pub(super) const POLICY_VERSION: &str = "1.3"; const READOUT: f64 = 0.9; const DELIBERATION: f64 = 0.5; diff --git a/crates/libsy/src/algorithms/vgr/runtime.rs b/crates/libsy/src/algorithms/vgr/runtime.rs index 9fe1713754..8d13f0ee77 100644 --- a/crates/libsy/src/algorithms/vgr/runtime.rs +++ b/crates/libsy/src/algorithms/vgr/runtime.rs @@ -171,6 +171,20 @@ impl Classifier for VgrClassifier { let Some(trial) = self.breaker.attempt() else { return Ok(self.escalate(driver, "breaker_open", None, request, state, None)); }; + let typed = if text::ToolRecord::from_request(request).errors > 0 { + let context = RequestContext { + driver, + request, + started, + }; + Some(self.type_task(&context).await) + } else { + None + }; + if typed == Some(Some(TaskType::Chat)) { + let reason = "chat_after_tool_error"; + return Ok(self.escalate(driver, reason, Some(Branch::Chat), request, state, None)); + } let attempted = self .within_budget(started, async { let response = driver @@ -255,7 +269,10 @@ impl Classifier for VgrClassifier { } let attempt = rungs::response_text(&buffered.agg).unwrap_or_default(); - let task_type = self.type_task(&context).await; + let task_type = match typed { + Some(task_type) => task_type, + None => self.type_task(&context).await, + }; let caps = derive_capabilities( request, &attempt, @@ -451,7 +468,10 @@ impl VgrClassifier { let run = (branch == Branch::Agentic).then(|| decide::agentic_run(caps, confirmed)); let (Some(judged), false) = ( caps.transcript.as_deref(), - branch == Branch::Unknown || run == Some(AgenticRun::Vetoed), + branch == Branch::Unknown + || run == Some(AgenticRun::Vetoed) + || (matches!(branch, Branch::Chat | Branch::DefaultVerified) + && caps.tools.is_some_and(|tools| tools.errors > 0)), ) else { return signals; }; diff --git a/crates/libsy/src/algorithms/vgr/runtime/tests.rs b/crates/libsy/src/algorithms/vgr/runtime/tests.rs index fe9599559f..767b2d8fa6 100644 --- a/crates/libsy/src/algorithms/vgr/runtime/tests.rs +++ b/crates/libsy/src/algorithms/vgr/runtime/tests.rs @@ -472,7 +472,7 @@ async fn decisions_attach_the_reason_and_exclude_efficient_capable_fallbacks() - assert_eq!(evidence["source"], "vgr"); assert_eq!(evidence["reason_code"], reason); assert_eq!(evidence["verdict"], served); - assert_eq!(evidence["policy_version"], "1.2"); + assert_eq!(evidence["policy_version"], "1.3"); if reason == "decided" { assert_eq!(evidence["branch"], "DefaultVerified"); } @@ -1376,3 +1376,33 @@ async fn a_failed_capable_verifier_still_offers_the_efficient_answer() -> Result assert_eq!(fallback.llm_request.messages, task().llm_request.messages); Ok(()) } + +#[tokio::test] +async fn a_step_that_cannot_commit_over_a_tool_error_skips_local_checks() -> Result<()> { + let failed = session(vec![ + Message::text(Role::User, "What is happening today?"), + call("c1"), + result("c1", r#"{"error": "search-only backend"}"#), + ]); + // A typed answer with tool use in its turn is judged as chat, which needs a clean record. + for (typed, expected) in [ + ("Answer", vec!["judge", "efficient", "capable"]), + ("Chat", vec!["judge", "capable"]), + ] { + let calls = Arc::new(parking_lot::Mutex::new(Vec::new())); + let seen = Arc::clone(&calls); + let (selected, _) = test_drive_with_models( + route(|config| config.task_typing = true)?, + failed.clone(), + category_models(Category::Any, &["efficient", "capable"]), + move |target: ModelId, _| { + seen.lock().push(target.to_string()); + std::future::ready(Ok(reply(typed))) + }, + ) + .await?; + assert_eq!(selected.as_str(), "capable"); + assert_eq!(*calls.lock(), expected, "typed {typed}"); + } + Ok(()) +} diff --git a/docs/routing_algorithms/vgr_routing.md b/docs/routing_algorithms/vgr_routing.md index 34abc9f920..8c196c4184 100644 --- a/docs/routing_algorithms/vgr_routing.md +++ b/docs/routing_algorithms/vgr_routing.md @@ -190,10 +190,13 @@ headers take precedence. Hosts that omit reporting retain message-shape detectio ## Verification and capable-tier outages -Policy version `1.2` allows the default verification branch to accept a clean +Policy version `1.3` allows the default verification branch to accept a clean efficient attempt when the capable judge approves both the evidence and the answer. A negative or unavailable evidence verdict skips the answer check. +After a failed tool result, chat steps escalate without verification, and steps +typed as chat escalate before any efficient attempt. + Typed factual answers are verified on the answer itself, including later user turns without tool work. A readout of at least 0.9 can serve the efficient answer; lower scores require capable-tier verification. Scored verdicts From 71bfca3c93cef138a74c76aadb1086d691475f4e Mon Sep 17 00:00:00 2001 From: adhaile Date: Fri, 13 Nov 2026 10:33:19 -0800 Subject: [PATCH 2/2] fix(vgr): don't count a deadline spent on typing as an efficient failure After a tool error, typing runs before the efficient attempt and shares its deadline. When typing spent the budget, the efficient call was never made but the breaker still recorded a failure. Escalate with efficient_timed_out before the attempt instead. Signed-off-by: adhaile --- crates/libsy/src/algorithms/vgr/runtime.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/libsy/src/algorithms/vgr/runtime.rs b/crates/libsy/src/algorithms/vgr/runtime.rs index 8d13f0ee77..e9f5d3be8d 100644 --- a/crates/libsy/src/algorithms/vgr/runtime.rs +++ b/crates/libsy/src/algorithms/vgr/runtime.rs @@ -185,6 +185,9 @@ impl Classifier for VgrClassifier { let reason = "chat_after_tool_error"; return Ok(self.escalate(driver, reason, Some(Branch::Chat), request, state, None)); } + if self.config.bounded_efficient && self.remaining(started).is_none() { + return Ok(self.escalate(driver, "efficient_timed_out", None, request, state, None)); + } let attempted = self .within_budget(started, async { let response = driver