Skip to content

fix(voice): keep the agent thinking while a tool is still running #6908

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
VihaanAgarwal wants to merge 1 commit into
base: main
Choose a base branch
from
+50 −3
Open

fix(voice): keep the agent thinking while a tool is still running #6908

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions livekit-agents/livekit/agents/voice/agent_activity.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2766,7 +2766,11 @@ def _to_instructions(v: Instructions | str) -> Instructions:

def _on_pipeline_reply_done(self, _: asyncio.Task[None]) -> None:
if not self._speech_q and (not self._current_speech or self._current_speech.done()):
self._session._update_agent_state("listening")
# a speech awaiting its tool executions keeps the agent busy: stay in
# "thinking" so the user-away timer isn't armed mid-tool (#6904)
self._session._update_agent_state(
"thinking" if self._background_speeches else "listening"
)
if self._audio_recognition:
self._audio_recognition._on_end_of_agent_speech(
ignore_user_transcript_until=time.time()
Expand Down Expand Up @@ -2999,7 +3003,9 @@ def _on_first_frame(fut: asyncio.Future[float] | asyncio.Future[None]) -> None:
self._session._conversation_item_added(msg)

if self._session.agent_state == "speaking":
self._session._update_agent_state("listening")
self._session._update_agent_state(
"thinking" if self._background_speeches else "listening"
)
if self._audio_recognition:
self._audio_recognition._on_end_of_agent_speech(
ignore_user_transcript_until=time.time()
Expand Down Expand Up @@ -3495,7 +3501,7 @@ async def _next_segment() -> _SpeechSegment | None:
speech_handle._item_added([msg])
current_span.set_attribute(trace_types.ATTR_RESPONSE_TEXT, forwarded_text)

if not speech_handle.interrupted and len(tool_output.output) > 0:
if not speech_handle.interrupted and (len(tool_output.output) > 0 or not exe_task.done()):

@longcw longcw Aug 20, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this branch also calls _on_end_of_agent_speech and _restore_interruption_by_audio_activity

@longcw longcw Aug 20, 2026 •
edited
Loading

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe move it to the other branch, but may need to take care if there is no tool reply after exe_task, the agent state should be back to listening

elif self._session.agent_state == "speaking":
    # a tool still running keeps the agent busy: "listening" here would arm the
    # user-away timer mid-tool (#6904)
    still_running = not speech_handle.interrupted and not exe_task.done()
    self._session._update_agent_state("thinking" if still_running else "listening")

self._session._update_agent_state("thinking")
if self._audio_recognition:
self._audio_recognition._on_end_of_agent_speech(
Expand Down
41 changes: 41 additions & 0 deletions tests/test_agent_session.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
MetricsCollectedEvent,
ModelSettings,
NotGivenOr,
RunContext,
TurnHandlingOptions,
UserInputTranscribedEvent,
UserStateChangedEvent,
Expand Down Expand Up @@ -396,6 +397,46 @@ def _record_agent_speech_end(
assert chat_ctx_items[6].text_content == "The weather in Tokyo is sunny today."


async def test_slow_tool_keeps_agent_thinking_after_filler() -> None:
actions = FakeActions()
actions.add_user_speech(0.5, 2.5, "Look up my order")
actions.add_llm(
content="Let me check.",
tool_calls=[FunctionToolCall(name="lookup_order", arguments="{}", call_id="1")],
)
actions.add_tts(2.0)
actions.add_tts(1.0, input="Just a moment.")
actions.add_llm(content="Order 42 is on the way.", input="order 42 shipped")
actions.add_tts(1.0)

class SlowToolAgent(Agent):
def __init__(self) -> None:
super().__init__(instructions="You are a helpful assistant.")

@function_tool
async def lookup_order(self, context: RunContext) -> str:
async with context.with_filler("Just a moment.", delay=0.5):
await asyncio.sleep(8.0)
return "order 42 shipped"

session = create_session(actions, extra_kwargs={"user_away_timeout": 3.0})

agent_state_events: list[AgentStateChangedEvent] = []
user_state_events: list[UserStateChangedEvent] = []
session.on("agent_state_changed", agent_state_events.append)
session.on("user_state_changed", user_state_events.append)

await asyncio.wait_for(
run_session(session, SlowToolAgent(), drain_delay=10), timeout=SESSION_TIMEOUT
)

# both the reply and the filler end while the tool is still running, so each
# must hand back "thinking", not "listening" (which arms the user-away timer)
speaking_ends = [ev.new_state for ev in agent_state_events if ev.old_state == "speaking"]
assert speaking_ends == ["thinking", "thinking", "listening"]
assert all(ev.new_state != "away" for ev in user_state_events)


@pytest.mark.parametrize(
"resume_false_interruption, expected_interruption_time",
[
Expand Down