Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 58 additions & 0 deletions crates/switchyard-translation/src/codecs/common.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,11 @@

use serde_json::{Map, Value};

use crate::diagnostic::TranslationDiagnostic;
use crate::error::{Result, TranslationError};
use crate::llm::{ContentBlock, LlmRequest, Tool, ToolChoice};
use crate::policy::TranslationPolicy;
use crate::util::push_lossy;

// Internal provenance survives mutations that invalidate exact request replay.
pub(crate) const ANTHROPIC_REQUEST_KEY: &str = "switchyard_anthropic_request";
Expand All @@ -24,6 +27,61 @@ pub(crate) fn is_anthropic_request(request: &LlmRequest) -> bool {
request.extensions.fields.get(ANTHROPIC_REQUEST_KEY) == Some(&Value::Bool(true))
}

// Budget share of `max_tokens` at or above which each effort applies, in percent. The bands stop
// at `high` because every OpenAI reasoning model accepts low, medium and high.
const BUDGET_EFFORT_BANDS: [(u64, &str); 2] = [(80, "high"), (50, "medium")];

/// Returns the OpenAI reasoning effort for an Anthropic request.
///
/// An explicit `output_config.effort` wins. Otherwise a legacy `enabled` budget maps to an effort
/// band by its share of `max_tokens`. The budget mapping is approximate, so it records a
/// lossy-conversion diagnostic.
pub(crate) fn openai_effort_for_request(
request: &LlmRequest,
diagnostics: &mut Vec<TranslationDiagnostic>,
policy: &TranslationPolicy,
) -> Result<Option<String>> {
if request.reasoning.effort.is_some() || !is_anthropic_request(request) {
return Ok(request.reasoning.effort.clone());
}
let Some(thinking) = request.reasoning.raw.as_ref() else {
return Ok(None);
};
match thinking.get("type").and_then(Value::as_str) {
Some("enabled") => {
let budget = thinking.get("budget_tokens").and_then(Value::as_u64);
let (Some(budget), Some(max_tokens)) = (budget, request.output.max_output_tokens)
else {
return Ok(None);
};
if max_tokens == 0 {
return Ok(None);
}
let effort = budget_effort(budget, max_tokens);
push_lossy(
diagnostics,
policy,
format!(
"Anthropic thinking budget_tokens {budget} of max_tokens {max_tokens} \
approximated as reasoning effort {effort}"
),
)?;
Ok(Some(effort.to_string()))
}
_ => Ok(None),
}
}

fn budget_effort(budget: u64, max_tokens: u64) -> &'static str {
let budget = u128::from(budget) * 100;
for (percent, effort) in BUDGET_EFFORT_BANDS {
if budget >= u128::from(max_tokens) * u128::from(percent) {
return effort;
}
}
"low"
}

/// Converts an OpenAI allowed-tools policy to a restricted function list and mode.
pub(crate) fn allowed_function_tools(
request: &LlmRequest,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@ use switchyard_protocol::MAX_IR_TEMPERATURE;

use super::MAX_TEMPERATURE;
use crate::codecs::common::{
first_nonempty_string, is_known_role_name, provider_extensions, reasoning_text_from_blocks,
reasoning_text_from_details, scale_temperature, text_from_blocks,
first_nonempty_string, is_known_role_name, openai_effort_for_request, provider_extensions,
reasoning_text_from_blocks, reasoning_text_from_details, scale_temperature, text_from_blocks,
};
use crate::codecs::openai_media::{
ImagePayload, file_payload, file_source_text, image_payload, image_source_text,
Expand Down Expand Up @@ -258,11 +258,8 @@ impl FormatCodec for OpenAiChatCodec {
if request.stream {
body.insert("stream".to_string(), Value::Bool(true));
}
if let Some(effort) = &request.reasoning.effort {
body.insert(
"reasoning_effort".to_string(),
Value::String(effort.clone()),
);
if let Some(effort) = openai_effort_for_request(request, &mut diagnostics, policy)? {
body.insert("reasoning_effort".to_string(), Value::String(effort));
}
if let Some(format) = encode_openai_format(&request.output, &mut diagnostics, policy)? {
body.insert("response_format".to_string(), format);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@ use switchyard_protocol::MAX_IR_TEMPERATURE;
use super::MAX_TEMPERATURE;
use crate::codecs::common::{
collect_responses_reasoning_text, encrypted_reasoning_data, encrypted_reasoning_item_id,
is_anthropic_request, is_known_role_name, provider_extensions, reasoning_text_from_blocks,
scale_temperature, text_from_blocks,
is_anthropic_request, is_known_role_name, openai_effort_for_request, provider_extensions,
reasoning_text_from_blocks, scale_temperature, text_from_blocks,
};
use crate::codecs::openai_chat::{decode_file_source, decode_image_source};
use crate::codecs::openai_media::{
Expand Down Expand Up @@ -263,7 +263,7 @@ impl FormatCodec for OpenAiResponsesCodec {
);
}
// An Anthropic request's raw reasoning is its `thinking` object, which Responses
// does not accept. Keep only whether thinking is disabled.
// does not accept. Keep whether thinking is disabled and the effort derived from it.
let mut reasoning = if is_anthropic_request(request) {
let mut reasoning = Map::new();
if request
Expand All @@ -286,7 +286,7 @@ impl FormatCodec for OpenAiResponsesCodec {
.cloned()
.unwrap_or_default()
};
if let Some(effort) = &request.reasoning.effort {
if let Some(effort) = openai_effort_for_request(request, &mut diagnostics, policy)? {
reasoning.insert("effort".to_string(), json!(effort));
}
if !reasoning.is_empty() {
Expand Down
120 changes: 119 additions & 1 deletion crates/switchyard-translation/tests/request_translation.rs
Original file line number Diff line number Diff line change
Expand Up @@ -703,7 +703,7 @@ fn anthropic_thinking_to_responses_uses_normalized_effort() -> TestResult {
(
json!({"type": "enabled", "budget_tokens": 2048}),
None,
None,
Some(json!({"effort": "medium"})),
),
(
json!({"type": "disabled"}),
Expand Down Expand Up @@ -790,6 +790,124 @@ fn anthropic_reconstruction_rejects_fallbacks_with_credit_token() {
));
}

#[test]
fn anthropic_thinking_budget_maps_to_openai_effort_bands() -> TestResult {
let engine = TranslationEngine::default();
let policy = normalized_policy();
for (budget, effort) in [
(10000, "high"),
(9500, "high"),
(8000, "high"),
(7999, "medium"),
(5000, "medium"),
(4999, "low"),
(1024, "low"),
] {
let body = json!({
"model": "route",
"max_tokens": 10000,
"messages": [{"role": "user", "content": "hi"}],
"thinking": {"type": "enabled", "budget_tokens": budget}
});
let chat = engine.translate_request(
WireFormat::AnthropicMessages,
WireFormat::OpenAiChat,
&body,
&policy,
)?;
assert_eq!(
chat.body["reasoning_effort"],
json!(effort),
"budget {budget}"
);
assert!(
chat.diagnostics
.iter()
.any(|diagnostic| diagnostic.code == "lossy_conversion"),
"budget {budget} records the approximation"
);
let responses = engine.translate_request(
WireFormat::AnthropicMessages,
WireFormat::OpenAiResponses,
&body,
&policy,
)?;
assert_eq!(
responses.body["reasoning"],
json!({"effort": effort}),
"budget {budget}"
);
}
Ok(())
}

#[test]
fn anthropic_thinking_controls_to_chat_effort() -> TestResult {
let engine = TranslationEngine::default();
let policy = normalized_policy();
for (thinking, output_config, expected) in [
(json!({"type": "disabled"}), None, None),
(
json!({"type": "adaptive"}),
Some(json!({"effort": "low"})),
Some("low"),
),
(json!({"type": "adaptive"}), None, None),
(
json!({"type": "enabled", "budget_tokens": 9000}),
Some(json!({"effort": "low"})),
Some("low"),
),
] {
let mut body = json!({
"model": "route",
"max_tokens": 10000,
"messages": [{"role": "user", "content": "hi"}],
"thinking": thinking
});
if let Some(output_config) = output_config {
body["output_config"] = output_config;
}
let output = engine.translate_request(
WireFormat::AnthropicMessages,
WireFormat::OpenAiChat,
&body,
&policy,
)?;
assert_eq!(
output.body.get("reasoning_effort").and_then(Value::as_str),
expected,
"{body}"
);
assert!(output.diagnostics.is_empty(), "{body}");
}
Ok(())
}

#[test]
fn anthropic_thinking_budget_is_rejected_under_strict_lossy_policy() {
let engine = TranslationEngine::default();
let policy = TranslationPolicy {
lossy_conversion_policy: LossyConversionPolicy::Reject,
..normalized_policy()
};
let body = json!({
"model": "route",
"max_tokens": 10000,
"messages": [{"role": "user", "content": "hi"}],
"thinking": {"type": "enabled", "budget_tokens": 4000}
});
let error = engine
.translate_request(
WireFormat::AnthropicMessages,
WireFormat::OpenAiChat,
&body,
&policy,
)
.expect_err("approximating a thinking budget is lossy");
assert!(matches!(error, TranslationError::LossyConversion(_)));
}

#[test]
fn anthropic_reconstruction_preserves_thinking() -> TestResult {
let engine = TranslationEngine::default();
Expand Down
Loading