Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
- The standalone `switchyard-server` now binds to `127.0.0.1` by default.
Existing deployments that accept remote clients must add `--host 0.0.0.0`.
The provided Docker image and `dev-server/switchyard.service` set this explicitly.
- **VGR escalates sooner after a tool error** - chat steps skip verification,
and steps typed as chat skip the efficient attempt (`chat_after_tool_error`).
- **TypeScript composite stage profiles** - `CompositeRouter` options take
`stage.efficient_first` and `stage.capable_first` profiles in place of
`efficient_first_threshold`, `capable_first_threshold`, and
Expand Down
2 changes: 1 addition & 1 deletion crates/libsy/src/algorithms/vgr/decide.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
use super::{Branch, Capabilities};

/// Version of these rules and dials, carried on every decision record.
pub(super) const POLICY_VERSION: &str = "1.2";
pub(super) const POLICY_VERSION: &str = "1.3";

const READOUT: f64 = 0.9;
const DELIBERATION: f64 = 0.5;
Expand Down
27 changes: 25 additions & 2 deletions crates/libsy/src/algorithms/vgr/runtime.rs
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,23 @@ impl Classifier for VgrClassifier {
let Some(trial) = self.breaker.attempt() else {
return Ok(self.escalate(driver, "breaker_open", None, request, state, None));
};
let typed = if text::ToolRecord::from_request(request).errors > 0 {
let context = RequestContext {
driver,
request,
started,
};
Some(self.type_task(&context).await)
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
} else {
None
};
if typed == Some(Some(TaskType::Chat)) {
let reason = "chat_after_tool_error";
return Ok(self.escalate(driver, reason, Some(Branch::Chat), request, state, None));
}
if self.config.bounded_efficient && self.remaining(started).is_none() {
return Ok(self.escalate(driver, "efficient_timed_out", None, request, state, None));
}
let attempted = self
.within_budget(started, async {
let response = driver
Expand Down Expand Up @@ -255,7 +272,10 @@ impl Classifier for VgrClassifier {
}

let attempt = rungs::response_text(&buffered.agg).unwrap_or_default();
let task_type = self.type_task(&context).await;
let task_type = match typed {
Some(task_type) => task_type,
None => self.type_task(&context).await,
};
let caps = derive_capabilities(
request,
&attempt,
Expand Down Expand Up @@ -451,7 +471,10 @@ impl VgrClassifier {
let run = (branch == Branch::Agentic).then(|| decide::agentic_run(caps, confirmed));
let (Some(judged), false) = (
caps.transcript.as_deref(),
branch == Branch::Unknown || run == Some(AgenticRun::Vetoed),
branch == Branch::Unknown
|| run == Some(AgenticRun::Vetoed)
|| (matches!(branch, Branch::Chat | Branch::DefaultVerified)
&& caps.tools.is_some_and(|tools| tools.errors > 0)),
) else {
return signals;
};
Expand Down
32 changes: 31 additions & 1 deletion crates/libsy/src/algorithms/vgr/runtime/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -472,7 +472,7 @@ async fn decisions_attach_the_reason_and_exclude_efficient_capable_fallbacks() -
assert_eq!(evidence["source"], "vgr");
assert_eq!(evidence["reason_code"], reason);
assert_eq!(evidence["verdict"], served);
assert_eq!(evidence["policy_version"], "1.2");
assert_eq!(evidence["policy_version"], "1.3");
if reason == "decided" {
assert_eq!(evidence["branch"], "DefaultVerified");
}
Expand Down Expand Up @@ -1376,3 +1376,33 @@ async fn a_failed_capable_verifier_still_offers_the_efficient_answer() -> Result
assert_eq!(fallback.llm_request.messages, task().llm_request.messages);
Ok(())
}

#[tokio::test]
async fn a_step_that_cannot_commit_over_a_tool_error_skips_local_checks() -> Result<()> {
let failed = session(vec![
Message::text(Role::User, "What is happening today?"),
call("c1"),
result("c1", r#"{"error": "search-only backend"}"#),
]);
// A typed answer with tool use in its turn is judged as chat, which needs a clean record.
for (typed, expected) in [
("Answer", vec!["judge", "efficient", "capable"]),
("Chat", vec!["judge", "capable"]),
] {
let calls = Arc::new(parking_lot::Mutex::new(Vec::new()));
let seen = Arc::clone(&calls);
let (selected, _) = test_drive_with_models(
route(|config| config.task_typing = true)?,
failed.clone(),
category_models(Category::Any, &["efficient", "capable"]),
move |target: ModelId, _| {
seen.lock().push(target.to_string());
std::future::ready(Ok(reply(typed)))
},
)
.await?;
assert_eq!(selected.as_str(), "capable");
assert_eq!(*calls.lock(), expected, "typed {typed}");
}
Ok(())
}
5 changes: 4 additions & 1 deletion docs/routing_algorithms/vgr_routing.md
Original file line number Diff line number Diff line change
Expand Up @@ -190,10 +190,13 @@ headers take precedence. Hosts that omit reporting retain message-shape detectio

## Verification and capable-tier outages

Policy version `1.2` allows the default verification branch to accept a clean
Policy version `1.3` allows the default verification branch to accept a clean
efficient attempt when the capable judge approves both the evidence and the
answer. A negative or unavailable evidence verdict skips the answer check.

After a failed tool result, chat steps escalate without verification, and steps
typed as chat escalate before any efficient attempt.

Typed factual answers are verified on the answer itself, including
later user turns without tool work. A readout of at least 0.9 can serve the
efficient answer; lower scores require capable-tier verification. Scored verdicts
Expand Down
Loading