From f1edb57ea6c657aef4fe8cf1acc9847f5c593305 Mon Sep 17 00:00:00 2001 From: Josh Cox <6137404+joshuacox@users.noreply.github.com> Date: Fri, 9 Oct 2026 06:39:42 -0500 Subject: [PATCH 1/2] fix(stage-router): honor classifier.classify_trigger in the stage cascade Signed-off-by: Josh Cox <6137404+joshuacox@users.noreply.github.com> --- crates/libsy/src/algorithms/llm_class.rs | 2 +- crates/libsy/src/algorithms/stage.rs | 197 +++++++++++++----- crates/switchyard-runner/tests/stage_knobs.rs | 18 +- docs/reference/toml_schema.md | 11 +- .../stage_router_routing.md | 4 +- 5 files changed, 164 insertions(+), 68 deletions(-) diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs index ff86e89c89..7040a10c50 100644 --- a/crates/libsy/src/algorithms/llm_class.rs +++ b/crates/libsy/src/algorithms/llm_class.rs @@ -629,7 +629,7 @@ impl JudgePolicy for CustomPolicyRuntime { } /// Builds the affinity router a trigger calls for, if any. -fn affinity_router( +pub(crate) fn affinity_router( trigger: ClassifyTrigger, message_hash_fallback: bool, ) -> Option> { diff --git a/crates/libsy/src/algorithms/stage.rs b/crates/libsy/src/algorithms/stage.rs index edb3a7d1df..39fbf48e3b 100644 --- a/crates/libsy/src/algorithms/stage.rs +++ b/crates/libsy/src/algorithms/stage.rs @@ -18,8 +18,10 @@ use std::sync::Arc; use async_trait::async_trait; use super::fall_through::FallThrough; -use super::llm_class::{LlmClassifierConfig, LlmTaskClassifier, TaskClassifierConfig}; -use super::util::affinity::{ClassifyTrigger, has_new_user_turn}; +use super::llm_class::{ + LlmClassifierConfig, LlmTaskClassifier, TaskClassifierConfig, affinity_router, +}; +use super::util::affinity::has_new_user_turn; use super::util::stage::{ DecisionSource, HandoffNoteConfig, PickerMode, StageClassifier, Tier, fall_open_tier, record_routing_decision, @@ -141,11 +143,12 @@ impl Classifier for FallOpen { pub struct LlmFallback { /// Overrides the runtime judge for this stage configuration. pub target: Option, - /// Judge configuration. Its `recent_turn_window` counts trailing messages, while - /// this router's `recent_window` counts tool results. - /// Note: `classify_trigger` and `message_hash_fallback` have no effect here. The judge - /// runs as a cascade classifier on every ambiguous turn, not as a standalone algorithm, - /// so no verdict is retained. + /// Judge configuration. `recent_turn_window` is worth setting to this router's + /// `recent_window` so the judge reads the same span the signal scorer scored. + /// `classify_trigger` and `message_hash_fallback` apply here: a retaining + /// trigger keeps the judge's verdict for the rest of the user turn or the + /// session, so a later undecided turn reuses it instead of calling the + /// judge again. pub config: TaskClassifierConfig, } @@ -216,9 +219,9 @@ impl StageRouter { /// when configured, is called through the runtime `judge` model. /// /// Errors if either threshold in `config` is outside `[0.0, 1.0]`, or if the - /// hold counts are outside `-1..=u32::MAX`. Also rejects a - /// fallback judge configured to retain verdicts: it runs on every ambiguous - /// turn, so `classify_trigger` and `message_hash_fallback` would do nothing. + /// hold counts are outside `-1..=u32::MAX`. A fallback judge configured + /// with a retaining `classify_trigger` keeps its verdict for the rest of + /// the user turn or the session. pub fn new(config: StageRouterConfig) -> Result { Ok(Self { route: build_stage_route(config, false)?, @@ -247,17 +250,6 @@ pub(crate) fn build_stage_route( config: StageRouterConfig, reset_on_user_turn: bool, ) -> Result { - if let Some(fallback) = &config.llm_fallback - && (fallback.config.classify_trigger != ClassifyTrigger::EveryRequest - || fallback.config.message_hash_fallback) - { - return Err(LibsyError::AlgorithmError { - message: "classify_trigger and message_hash_fallback have no effect on the \ - stage fallback judge, which runs on every ambiguous turn. Remove \ - them, or use a composite route to judge once per user turn" - .to_string(), - }); - } for (name, threshold) in [ ("confidence_threshold", config.confidence_threshold), ("capable_first_threshold", config.capable_first_threshold), @@ -307,6 +299,18 @@ pub(crate) fn build_stage_route( .with_processor(classifier.clone()) .with_classifier(classifier); if let Some(fallback) = config.llm_fallback { + // A retaining trigger keeps the judge's verdict, so the cascade needs + // the affinity component the standalone route builds for itself. Both + // roles share one `Arc`, and the classifier sits ahead of the judge so + // a retained verdict short-circuits the judge call. + if let Some(affinity) = affinity_router( + fallback.config.classify_trigger, + fallback.config.message_hash_fallback, + ) { + router = router + .with_processor(affinity.clone()) + .with_classifier(affinity); + } router = router.with_classifier(Arc::new(SourceStamp { inner: Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Capability { config: fallback.config, @@ -342,10 +346,11 @@ mod tests { use parking_lot::Mutex; use super::*; + use crate::algorithms::util::affinity::ClassifyTrigger; use crate::algorithms::util::stage::{clear_fall_open, set_fall_open}; use crate::algorithms::util::tier_fixtures::{JUDGE, Recorder, turn_request}; use crate::core::testing::test_drive_with_models; - use switchyard_protocol::{Category, ModelId}; + use switchyard_protocol::{Category, Message, ModelId, Role}; /// A classifier that always picks `target`, standing in for a cascade member. struct Fixed(&'static str); @@ -477,32 +482,6 @@ mod tests { )); } - #[test] - fn rejects_retaining_triggers_on_the_fallback_judge() { - for (trigger, message_hash_fallback) in [ - (ClassifyTrigger::UserTurn, false), - (ClassifyTrigger::NewSession, false), - (ClassifyTrigger::UserTurn, true), - ] { - let mut config = config(); - config.llm_fallback = Some(LlmFallback { - target: None, - config: TaskClassifierConfig { - classify_trigger: trigger, - message_hash_fallback, - ..Default::default() - }, - }); - assert!( - matches!( - StageRouter::new(config), - Err(LibsyError::AlgorithmError { .. }) - ), - "{trigger:?} with message_hash_fallback = {message_hash_fallback}" - ); - } - } - #[test] fn builds() -> Result<()> { let router = StageRouter::new(config())?; @@ -966,6 +945,128 @@ mod tests { Ok(()) } + fn config_with_judge_trigger( + recorder: &Arc, + p_solve: f64, + trigger: ClassifyTrigger, + ) -> StageRouterConfig { + let mut c = config_with_judge(recorder, p_solve); + // The capable hold also keeps a tier across requests. It is off here + // so only the trigger's retention can pin the verdict. + c.capable_hold_turns = 0; + if let Some(fallback) = c.llm_fallback.as_mut() { + fallback.config.classify_trigger = trigger; + } + c + } + + fn judge_call_count(recorder: &Arc) -> usize { + recorder + .calls + .lock() + .iter() + .filter(|c| c.target == JUDGE) + .count() + } + + /// The same turn shape with a new user message on the end, so the turn the + /// request belongs to is a new one. + fn next_user_turn_request() -> Request { + let mut request = turn_request(false); + request + .llm_request + .messages + .push(Message::text(Role::User, "now rewrite the parser")); + request + } + + #[tokio::test] + async fn a_user_turn_trigger_pins_the_judges_verdict_within_one_turn() -> Result<()> { + let recorder = Arc::new(Recorder::default()); + let router = recording_router(config_with_judge_trigger( + &recorder, + 0.1, + ClassifyTrigger::UserTurn, + ))?; + + test_drive_with_models( + router.clone(), + turn_request(false), + runtime_models(), + recorder.serve(), + ) + .await?; + // The judge would now pick the other tier, but a tool step inside the + // same user turn reuses the retained verdict instead of paying for a + // second judge call. + *recorder.judge_p_solve.lock() = 0.9; + test_drive_with_models( + router.clone(), + turn_request(false), + runtime_models(), + recorder.serve(), + ) + .await?; + + let routed = recorder.routed(); + assert_eq!(routed[0].target, "strong"); + assert_eq!(routed[1].target, "strong"); + assert_eq!( + judge_call_count(&recorder), + 1, + "one user turn should pay for one judge call" + ); + + // A new user message starts a new turn, so the judge runs again. + test_drive_with_models( + router.clone(), + next_user_turn_request(), + runtime_models(), + recorder.serve(), + ) + .await?; + + assert_eq!(recorder.routed()[2].target, "weak"); + assert_eq!(judge_call_count(&recorder), 2); + Ok(()) + } + + #[tokio::test] + async fn a_new_session_trigger_pins_the_judges_verdict_for_the_session() -> Result<()> { + let recorder = Arc::new(Recorder::default()); + let router = recording_router(config_with_judge_trigger( + &recorder, + 0.1, + ClassifyTrigger::NewSession, + ))?; + + test_drive_with_models( + router.clone(), + turn_request(false), + runtime_models(), + recorder.serve(), + ) + .await?; + *recorder.judge_p_solve.lock() = 0.9; + test_drive_with_models( + router.clone(), + next_user_turn_request(), + runtime_models(), + recorder.serve(), + ) + .await?; + + let routed = recorder.routed(); + assert_eq!(routed[0].target, "strong"); + assert_eq!(routed[1].target, "strong"); + assert_eq!( + judge_call_count(&recorder), + 1, + "one session should pay for one judge call" + ); + Ok(()) + } + #[tokio::test] async fn a_judge_that_cannot_tell_lands_on_the_picker_default() -> Result<()> { let recorder = Arc::new(Recorder::default()); diff --git a/crates/switchyard-runner/tests/stage_knobs.rs b/crates/switchyard-runner/tests/stage_knobs.rs index d660eab1c7..eb96a8ea8c 100644 --- a/crates/switchyard-runner/tests/stage_knobs.rs +++ b/crates/switchyard-runner/tests/stage_knobs.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Stage fallback-judge knobs that cannot take effect are rejected at load. +//! Stage fallback-judge knobs: retaining triggers keep the judge's verdict. use switchyard_runner::Runner; @@ -30,13 +30,6 @@ confidence_threshold = 0.5 const CLASSIFIER: &str = "[routes.stage.classifier]\ntarget = \"judge\"\nbase_threshold = 0.5\n"; -fn load_error(config: &str) -> String { - match Runner::from_toml(config) { - Ok(_) => panic!("expected a load error for:\n{config}"), - Err(error) => error.to_string(), - } -} - #[test] fn accepts_the_default_classifier_trigger() { assert!(Runner::from_toml(&format!("{CONFIG}{CLASSIFIER}")).is_ok()); @@ -45,16 +38,15 @@ fn accepts_the_default_classifier_trigger() { } #[test] -fn rejects_classifier_knobs_that_have_no_effect() { +fn accepts_retaining_classifier_triggers() { for setting in [ "classify_trigger = \"user_turn\"", "classify_trigger = \"new_session\"", - "message_hash_fallback = true", + "classify_trigger = \"user_turn\"\nmessage_hash_fallback = true", ] { - let error = load_error(&format!("{CONFIG}{CLASSIFIER}{setting}\n")); assert!( - error.contains("have no effect on the stage fallback judge"), - "{setting}: {error}" + Runner::from_toml(&format!("{CONFIG}{CLASSIFIER}{setting}\n")).is_ok(), + "{setting} should load: a retaining trigger keeps the judge's verdict" ); } } diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index 308492d312..a2d093e543 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -485,6 +485,8 @@ the signals work and for tuning. | `tool_semantics.new` | No | `[]` | Exact ASCII case-insensitive domain tool names that demonstrate forward activity without favoring either tier. | | `handoff_notes` | No | unset | Table of notes appended to the forwarded request when the signals pick a tier. Keys below. | | `classifier` | No | unset | Table for the optional capability judge consulted on ambiguous turns. Keys below. | +| `classifier.classify_trigger` | No | `every_request` | When the judge runs on turns the signals leave undecided. `user_turn` keeps the judge's verdict across the tool calls in one user turn, and `new_session` keeps it for the session. Keeping a verdict needs a session ID, or `classifier.message_hash_fallback`. | +| `classifier.message_hash_fallback` | No | `false` | Keeps the judge's verdict against a hash of the first user message when a request carries no session ID. Needs a retaining `classifier.classify_trigger`. | | `subagents` | No | unset | Nested `passthrough` or custom `llm_classifier` policy used only for delegated sub-agent work. See [Sub-Agent-Aware Routing](../routing_algorithms/subagent_routing.md). | #### `handoff_notes` keys @@ -498,8 +500,9 @@ the signals work and for tuning. #### `classifier` keys The judge runs when the signals are ambiguous or the request has no tool -history. It runs as a step inside the stage cascade, not as a standalone route, -so it is consulted on every ambiguous turn and never retains a verdict. +history. It runs as a step inside the stage cascade, not as a standalone route. +With a retaining `classify_trigger`, its verdict is kept for the rest of the +user turn or the session instead of being recomputed on every ambiguous turn. | Key | Required | Default | Meaning | |---|:---:|---|---| @@ -508,8 +511,8 @@ so it is consulted on every ambiguous turn and never retains a verdict. | `classifier.threshold_step` | No | `0.0` | Finite, non-negative amount added once for uncertain or unmatched verdicts and twice for unsupported verdicts. `base_threshold + 2 * threshold_step` must be at most `1`. | | `classifier.fail_open` | No | `true` | When `true`, judge client failures and deadlines let the turn fall open to the picker default and record fail-open evidence. When `false`, they stop the request. | | `classifier.decision` | No | defaults for a decision target | Decision model settings. A classifier target in `decision_targets` uses the defaults when this table is absent. It cannot be combined with `base_threshold`, `threshold_step`, `prompt`, `response_format_type`, or `max_output_tokens`; the load error is `decision cannot use LLM judge settings`. See [Decision model classifier](../routing_algorithms/composite_routing.md#decision-model-classifier). | -| `classifier.classify_trigger` | No | `every_request` | Only the default is accepted here. The stage fallback judge runs on every ambiguous turn and retains no verdict, so `user_turn` and `new_session` are rejected at load. Use a [composite](#composite) route to judge once per user turn. | -| `classifier.message_hash_fallback` | No | `false` | Rejected at load on a `stage_router` route. The fallback judge retains nothing to key on. | +| `classifier.classify_trigger` | No | `every_request` | When the judge runs on turns the signals leave undecided. `user_turn` keeps the judge's verdict across the tool calls in one user turn, and `new_session` keeps it for the session. Keeping a verdict needs a session ID, or `classifier.message_hash_fallback`. | +| `classifier.message_hash_fallback` | No | `false` | Keeps the judge's verdict against a hash of the first user message when a request carries no session ID. Needs a retaining `classifier.classify_trigger`. | | `classifier.recent_turn_window` | No | unset | LLM judges default to task messages. Decision judges default to 10 recent messages plus the opening task. An explicit value counts trailing messages, while the route window counts tool results. | | `classifier.prompt` | No | packaged prompt | Replaces the capability judge prompt. The verdict schema is unchanged. | | `classifier.response_format_type` | No | `json_schema` | Structured-output mode for the LLM judge. Use `json_object` when the provider does not support JSON Schema; Switchyard adds the schema to the prompt and validates the verdict locally. Not with `classifier.decision`. | diff --git a/docs/routing_algorithms/stage_router_routing.md b/docs/routing_algorithms/stage_router_routing.md index af6638605d..ed7bb410e7 100644 --- a/docs/routing_algorithms/stage_router_routing.md +++ b/docs/routing_algorithms/stage_router_routing.md @@ -270,8 +270,8 @@ required columns. | `classifier.threshold_step` | `0.0` | Added once for uncertain or unmatched verdicts, twice for unsupported. | | `classifier.fail_open` | `true` | On judge failure, fall open to the picker default instead of failing the request. | | `classifier.decision` | unset | Use a decision model instead of an LLM judge. Excludes the LLM judge keys below. | -| `classifier.classify_trigger` | `every_request` | Only the default is accepted. The judge runs on every ambiguous turn. | -| `classifier.message_hash_fallback` | `false` | Rejected at load. The judge retains no verdict. | +| `classifier.classify_trigger` | `every_request` | When the judge runs on turns the signals leave undecided. `user_turn` keeps the judge's verdict across the tool calls in one user turn, and `new_session` keeps it for the session. Keeping a verdict needs a session ID, or `classifier.message_hash_fallback`. | +| `classifier.message_hash_fallback` | `false` | Keeps the judge's verdict against a hash of the first user message when a request carries no session ID. Needs a retaining `classifier.classify_trigger`. | | `classifier.recent_turn_window` | unset | Trailing messages the judge sees after the opening task. A different unit from the route's `recent_turn_window`, which counts tool results. | | `classifier.prompt` | packaged | Replaces the judge prompt. | | `classifier.response_format_type` | `json_schema` | `json_object` for providers without JSON Schema. | From f28f705a4f9082af72f2cf4092d306075277e9bf Mon Sep 17 00:00:00 2001 From: Josh Cox <6137404+joshuacox@users.noreply.github.com> Date: Sat, 31 Oct 2026 12:31:27 -0500 Subject: [PATCH 2/2] fix(stage-router): latch only the judge verdict into affinity Signed-off-by: Josh Cox <6137404+joshuacox@users.noreply.github.com> --- crates/libsy/src/algorithms/stage.rs | 147 ++++++++++++++++++++++++--- 1 file changed, 133 insertions(+), 14 deletions(-) diff --git a/crates/libsy/src/algorithms/stage.rs b/crates/libsy/src/algorithms/stage.rs index 39fbf48e3b..5bbcdca995 100644 --- a/crates/libsy/src/algorithms/stage.rs +++ b/crates/libsy/src/algorithms/stage.rs @@ -21,7 +21,7 @@ use super::fall_through::FallThrough; use super::llm_class::{ LlmClassifierConfig, LlmTaskClassifier, TaskClassifierConfig, affinity_router, }; -use super::util::affinity::has_new_user_turn; +use super::util::affinity::{AffinityRouter, has_new_user_turn}; use super::util::stage::{ DecisionSource, HandoffNoteConfig, PickerMode, StageClassifier, Tier, fall_open_tier, record_routing_decision, @@ -109,6 +109,45 @@ impl Classifier for SourceStamp { } } +/// Latches a judge win into affinity, and nothing else. +/// +/// Affinity's processor role latches every decision it is replayed, and the +/// cascade's post-decision replay does not say who won, so registering +/// affinity as a processor here would pin signal and fall-open wins too. The +/// judge wrapper replays its own win through the same processor logic, which +/// leaves the standalone and subagent routes' latch-everything behavior +/// untouched. +struct JudgeVerdictLatch { + inner: Arc, + affinity: Arc, +} + +#[async_trait] +impl Classifier for JudgeVerdictLatch { + async fn score( + &self, + state: &mut State, + request: &mut Request, + driver: &Driver, + ) -> Result<(Classification, Option)> { + let (classification, served) = self.inner.score(state, request, driver).await?; + if let Some(winner) = classification.argmax(false)? { + self.affinity + .process( + state, + Event::Decision { + request, + selected_model_id: &winner.target, + category: winner.category.clone(), + driver, + }, + ) + .await?; + } + Ok((classification, served)) + } +} + /// Closes the cascade at zero confidence: a fallback, not a judgement. struct FallOpen { default_tier: Tier, @@ -300,25 +339,32 @@ pub(crate) fn build_stage_route( .with_classifier(classifier); if let Some(fallback) = config.llm_fallback { // A retaining trigger keeps the judge's verdict, so the cascade needs - // the affinity component the standalone route builds for itself. Both - // roles share one `Arc`, and the classifier sits ahead of the judge so - // a retained verdict short-circuits the judge call. - if let Some(affinity) = affinity_router( - fallback.config.classify_trigger, - fallback.config.message_hash_fallback, - ) { - router = router - .with_processor(affinity.clone()) - .with_classifier(affinity); - } - router = router.with_classifier(Arc::new(SourceStamp { + // the affinity component the standalone route builds for itself. The + // classifier sits ahead of the judge so a retained verdict + // short-circuits the judge call, but affinity is never a processor + // here: the post-decision replay would latch signal and fall-open + // wins too. The judge wrapper below latches only the judge's win. + let trigger = fallback.config.classify_trigger; + let hash_fallback = fallback.config.message_hash_fallback; + let judge = Arc::new(SourceStamp { inner: Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Capability { config: fallback.config, })?), source: DecisionSource::LlmClassifier, judge: fallback.target, skip_user_turn: reset_on_user_turn, - })); + }); + let judge: Arc = match affinity_router(trigger, hash_fallback) { + Some(affinity) => { + router = router.with_classifier(affinity.clone()); + Arc::new(JudgeVerdictLatch { + inner: judge, + affinity, + }) + } + None => judge, + }; + router = router.with_classifier(judge); } // Nothing behind this, so no turn is left unrouted. router = router.with_classifier(Arc::new(SourceStamp { @@ -1067,6 +1113,79 @@ mod tests { Ok(()) } + #[tokio::test] + async fn a_signal_decision_does_not_latch_for_later_turns() -> Result<()> { + let recorder = Arc::new(Recorder::default()); + let router = recording_router(config_with_judge_trigger( + &recorder, + 0.9, + ClassifyTrigger::NewSession, + ))?; + + // A failed tool result decides the turn on signals alone: strong, + // and the judge is never called. + test_drive_with_models( + router.clone(), + turn_request(true), + runtime_models(), + recorder.serve(), + ) + .await?; + assert_eq!(recorder.routed()[0].target, "strong"); + assert_eq!(judge_call_count(&recorder), 0); + + // The next turn's signals leave it undecided. Had the signal win + // latched, affinity would serve "strong" and the judge would never + // run; instead the judge decides this turn. + test_drive_with_models( + router.clone(), + turn_request(false), + runtime_models(), + recorder.serve(), + ) + .await?; + assert_eq!(judge_call_count(&recorder), 1); + assert_eq!(recorder.routed()[1].target, "weak"); + Ok(()) + } + + #[tokio::test] + async fn a_fall_open_win_does_not_latch_for_later_turns() -> Result<()> { + let recorder = Arc::new(Recorder::default()); + let router = recording_router(config_with_judge_trigger( + &recorder, + 42.0, + ClassifyTrigger::NewSession, + ))?; + + // The judge cannot tell, so the turn lands on the picker default — + // and the judge call is still recorded. + test_drive_with_models( + router.clone(), + turn_request(false), + runtime_models(), + recorder.serve(), + ) + .await?; + assert_eq!(recorder.routed()[0].target, "weak"); + assert_eq!(judge_call_count(&recorder), 1); + + // The next turn's signals leave it undecided too. Had the fall-open + // win latched, affinity would serve "weak" and the judge would never + // run; instead the judge decides this turn. + *recorder.judge_p_solve.lock() = 0.1; + test_drive_with_models( + router.clone(), + turn_request(false), + runtime_models(), + recorder.serve(), + ) + .await?; + assert_eq!(judge_call_count(&recorder), 2); + assert_eq!(recorder.routed()[1].target, "strong"); + Ok(()) + } + #[tokio::test] async fn a_judge_that_cannot_tell_lands_on_the_picker_default() -> Result<()> { let recorder = Arc::new(Recorder::default());