From 3f8a65b15a33e9d37c7b842871708708a35b976c Mon Sep 17 00:00:00 2001 From: Alex Fournier Date: Fri, 13 Nov 2026 12:06:22 -0700 Subject: [PATCH] fix(relay): preserve upstream HTTP error statuses Signed-off-by: Alex Fournier --- Cargo.lock | 8 +-- Cargo.toml | 2 +- crates/switchyard-nemo-relay-plugin/README.md | 9 ++-- .../relay-plugin.toml | 2 +- .../src/runtime.rs | 54 ++++++++++++++++--- docs/integrations/nemo_relay.md | 22 +++++--- tests/relay_plugin/test_package_bundle.py | 2 +- 7 files changed, 75 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f9411ca107..45c4b7d656 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1349,9 +1349,9 @@ dependencies = [ [[package]] name = "nemo-relay-plugin" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb9225443357599ceb56134c1bcdae5c4845bdb7a8b03be2b5a60fd27cfca270" +checksum = "e56be447b525b1af7f6be868e59e4ee14c66c1018755461cc65db25642c6f007" dependencies = [ "futures", "nemo-relay-types", @@ -1363,9 +1363,9 @@ dependencies = [ [[package]] name = "nemo-relay-types" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc2c1819818ca76215d1a6955ae0269ebafb6904bf198f26f0dc443ae1c155ac" +checksum = "8f520f781c702ea318f83c536827ef545cdc379b44146cb7266553dcf7fc35ab" dependencies = [ "bitflags", "chrono", diff --git a/Cargo.toml b/Cargo.toml index 29103b281b..526a306ba1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,7 +43,7 @@ http = "1" httpdate = "1" jsonschema = { version = "0.49.4", default-features = false } jsonptr = { version = "0.8.1", default-features = false, features = ["std", "json", "resolve"] } -nemo-relay-plugin = "0.10.0" +nemo-relay-plugin = "0.10.1" parking_lot = "0.12" prefill-router = { path = "crates/prefill-router", version = "0.3.0" } rand = "0.10" diff --git a/crates/switchyard-nemo-relay-plugin/README.md b/crates/switchyard-nemo-relay-plugin/README.md index 4b5819dbe6..7c03860571 100644 --- a/crates/switchyard-nemo-relay-plugin/README.md +++ b/crates/switchyard-nemo-relay-plugin/README.md @@ -10,13 +10,14 @@ algorithm construction, retry policy, and route validation. ## Install -The plugin uses `nemo-relay-plugin` 0.10.0 and requires NeMo Relay -`>=0.10.0, <1.0.0`. Relay 0.10 changes the native callback layout, so older -hosts cannot load this build. The manifest's native plugin API label stays `1`. +The plugin uses `nemo-relay-plugin` 0.10.1 and requires NeMo Relay +`>=0.10.1, <1.0.0`. Relay 0.10 changed the native callback layout; 0.10.1 adds +the HTTP-error status handling used by this build. The manifest's native plugin +API label stays `1`. Relay 0.8.x and 0.9.0 can lose upstream error status and details when this native plugin is enabled, including for models outside its configured routes. This is -a known issue on those older hosts. This build requires Relay 0.10 or later. +a known issue on those older hosts. This build requires Relay 0.10.1 or later. See the [upstream error compatibility note](../../docs/integrations/nemo_relay.md#upstream-error-compatibility). ### Install a released bundle diff --git a/crates/switchyard-nemo-relay-plugin/relay-plugin.toml b/crates/switchyard-nemo-relay-plugin/relay-plugin.toml index f658383022..7bd3cc8587 100644 --- a/crates/switchyard-nemo-relay-plugin/relay-plugin.toml +++ b/crates/switchyard-nemo-relay-plugin/relay-plugin.toml @@ -8,7 +8,7 @@ id = "nvidia.switchyard" kind = "rust_dynamic" [compat] -relay = ">=0.10.0, <1.0.0" +relay = ">=0.10.1, <1.0.0" native_api = "1" [defaults] diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rs index ae834353f7..37f0ba5274 100644 --- a/crates/switchyard-nemo-relay-plugin/src/runtime.rs +++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs @@ -302,14 +302,16 @@ impl SwitchyardRuntime { &metadata, route, ); - self.route_execution_error_mark( - &mut events, - &error.execution_error_summary(), - None, - outcome_fields, - ); + let summary = error.execution_error_summary(); + self.route_execution_error_mark(&mut events, &summary, None, outcome_fields); + // Preserve the status without exposing the provider's error body. + let message = "Switchyard route execution failed"; + let message = summary + .upstream_status + .and_then(|status| nemo_relay_plugin::http_error_message(status, message).ok()) + .unwrap_or_else(|| message.to_owned()); Execution { - result: Err("Switchyard route execution failed".into()), + result: Err(message), events, } } @@ -955,6 +957,7 @@ mod tests { "target": { "format": format, "base_url": base_url, + "max_retries": 0, } }, "targets": { @@ -1168,6 +1171,43 @@ seed = 17 assert!(saw_decision); } + #[tokio::test] + async fn failed_routes_preserve_upstream_status_without_provider_body() { + for status in [401, 429, 503] { + let server = MockServer::start().await; + let private_body = "private provider error details"; + Mock::given(method("POST")) + .and(path("/v1/chat/completions")) + .respond_with(ResponseTemplate::new(status).set_body_string(private_body)) + .expect(1) + .mount(&server) + .await; + let runtime = runtime_for_target("openai_chat", &format!("{}/v1", server.uri())); + let request = runtime + .decode_request( + WireFormat::OpenAiChat, + RelayRequest { + headers: Map::new(), + content: json!({ + "model": "switchyard/default", + "messages": [{"role": "user", "content": "hello"}], + }), + }, + false, + ) + .unwrap(); + let execution = runtime + .execute_buffered(WireFormat::OpenAiChat, request) + .await; + let error = execution.result.unwrap_err(); + assert_eq!( + error, + format!("HTTP {status}: Switchyard route execution failed") + ); + assert!(!format!("{error} {:?}", execution.events).contains(private_body)); + } + } + // Routing succeeds before answer candidates exhaust, so the error keeps its metadata. #[tokio::test] async fn failed_answer_keeps_routing_outcome_in_error_mark() { diff --git a/docs/integrations/nemo_relay.md b/docs/integrations/nemo_relay.md index ea29559704..6d88227e64 100644 --- a/docs/integrations/nemo_relay.md +++ b/docs/integrations/nemo_relay.md @@ -18,11 +18,21 @@ a configured Switchyard route. For example, if the route is `switchyard/core` and its target is `azure/openai/gpt-5.5`, requesting the target name directly still delegates the request to Relay. -This plugin build uses `nemo-relay-plugin` 0.10.0 and requires Relay -`>=0.10.0, <1.0.0`. Older hosts cannot load it because Relay 0.10 changes the -native callback layout. If you use an older plugin build on Relay 0.8.x or -0.9.0, send unmanaged traffic through a separate Relay instance with the plugin -disabled. Upgrading Switchyard alone does not upgrade the Relay host. +This plugin build uses `nemo-relay-plugin` 0.10.1 and requires Relay +`>=0.10.1, <1.0.0`. Relay 0.10 changed the native callback layout; 0.10.1 adds +the HTTP-error status handling used by this build. If you use an older plugin +build on Relay 0.8.x or 0.9.0, send unmanaged traffic through a separate Relay +instance with the plugin disabled. Upgrading Switchyard alone does not upgrade +the Relay host. + +For managed routes, known upstream HTTP errors retain their status through +Relay's HTTP-error API. The client receives the generic message +`Switchyard route execution failed`; provider error bodies remain private. +Failures without a known HTTP error status keep Relay's existing error handling. + +This preserves the HTTP status for buffered responses. Relay's streaming path +can send HTTP 200 before route execution finishes, so later failures cannot +change that status and retain the existing stream-error behavior. ## Why Use Switchyard with NeMo Relay? @@ -171,7 +181,7 @@ and how it !!! note "Relay compatibility" - The plugin requires `relay = ">=0.10.0, <1.0.0"` and native plugin API `1`. + The plugin requires `relay = ">=0.10.1, <1.0.0"` and native plugin API `1`. The packaged [`relay-plugin.toml`](../../crates/switchyard-nemo-relay-plugin/relay-plugin.toml) is the source of truth. diff --git a/tests/relay_plugin/test_package_bundle.py b/tests/relay_plugin/test_package_bundle.py index 6500af4e93..f2a8035c2c 100644 --- a/tests/relay_plugin/test_package_bundle.py +++ b/tests/relay_plugin/test_package_bundle.py @@ -58,7 +58,7 @@ def test_materializes_and_archives_supported_formats(self) -> None: self.assertEqual({path.name for path in output.iterdir()}, expected) manifest = (output / "relay-plugin.toml").read_text(encoding="utf-8") self.assertIn(f'artifact = "{library.name}"', manifest) - self.assertIn('relay = ">=0.10.0, <1.0.0"', manifest) + self.assertIn('relay = ">=0.10.1, <1.0.0"', manifest) self.assertIn(hashlib.sha256(library.read_bytes()).hexdigest(), manifest) self.assertNotIn("", manifest) self.assertNotIn("", manifest)