Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 20 additions & 2 deletions crates/switchyard-translation/src/codecs/openai_chat/stream.rs
Original file line number Diff line number Diff line change
Expand Up @@ -176,8 +176,26 @@ fn decode_openai_chat_stream(
for tool_call in tool_calls {
if let Some(tool_call) = tool_call.as_object() {
let function = tool_call.get("function").and_then(Value::as_object);
let index =
tool_call.get("index").and_then(Value::as_u64).unwrap_or(0) as usize;
let call_id = tool_call.get("id").and_then(Value::as_str);
// Some backends (e.g. Gemini's OpenAI-compatible endpoint) omit `index`.
// Without it, a new call id starts a new slot; id-less deltas continue
// the current one.
let index = match tool_call.get("index").and_then(Value::as_u64) {
Some(index) => index as usize,
None => match (&state.chat_tool_slot, call_id) {
(None, _) => 0,
(Some((slot, _)), None) => *slot,
(Some((slot, Some(last))), Some(id)) if last == id => *slot,
(Some((slot, _)), Some(_)) => slot + 1,
},
};
if let Some(id) = call_id {
state.chat_tool_slot = Some((index, Some(id.to_string())));
} else if state.chat_tool_slot.as_ref().map(|(slot, _)| *slot)
!= Some(index)
{
state.chat_tool_slot = Some((index, None));
}
if let Some(name) = function
.and_then(|function| function.get("name"))
.and_then(Value::as_str)
Expand Down
4 changes: 4 additions & 0 deletions crates/switchyard-translation/src/codecs/stream.rs
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,10 @@ pub struct StreamTranslationState {
pub(crate) tool_states: BTreeMap<usize, StreamToolState>,
#[serde(default)]
pub(crate) pending_chat_tool_names: BTreeMap<usize, String>,
/// Slot and id of the most recent Chat tool call, used to number deltas from backends
/// that omit `tool_calls[].index`.
#[serde(default)]
pub(crate) chat_tool_slot: Option<(usize, Option<String>)>,
#[serde(default)]
pub(crate) active_anthropic_tool: Option<usize>,
#[serde(default)]
Expand Down
33 changes: 33 additions & 0 deletions crates/switchyard-translation/tests/stream_translation.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3656,3 +3656,36 @@ fn responses_custom_tool_calls_decode_from_the_finished_item() -> TestResult {
}
Ok(())
}

// Some OpenAI-compatible backends (Gemini) stream tool calls without `tool_calls[].index`.
// Each new call id must open its own slot instead of being merged into slot 0.
#[test]
fn chat_stream_tool_calls_without_index_get_separate_slots() -> TestResult {
let engine = TranslationEngine::default();
let mut state =
StreamTranslationState::new(WireFormat::OpenAiChat, WireFormat::AnthropicMessages);
let mut events = Vec::new();
for event in [
json!({"choices": [{"delta": {"tool_calls": [{"id": "call_a",
"function": {"name": "alpha", "arguments": "{\"x\":"}}]}}]}),
json!({"choices": [{"delta": {"tool_calls": [{
"function": {"arguments": "1}"}}]}}]}),
json!({"choices": [{"delta": {"tool_calls": [{"id": "call_b",
"function": {"name": "beta", "arguments": "{}"}}]}}]}),
json!({"choices": [{"delta": {}, "finish_reason": "tool_calls"}]}),
] {
events.extend(engine.translate_event(
&mut state,
WireFormat::OpenAiChat,
WireFormat::AnthropicMessages,
&event,
)?);
}
events.extend(engine.finish_stream(&mut state, WireFormat::AnthropicMessages)?);
let names: Vec<_> = events
.iter()
.filter_map(|event| event.get("content_block")?.get("name")?.as_str())
.collect();
assert_eq!(names, vec!["alpha", "beta"]);
Ok(())
}
Loading