fix(voice): deadline-based user_away_timeout ignores transcript-less noise (#6030)

livekit-agents/livekit/agents/voice/agent_session.py

@@ -235,6 +235,14 @@ class ExpressiveOptions(TypedDict, total=False):
     speech_steering=DEFAULT_SPEECH_STEERING_OPTIONS,
 )
 
+# Activity signal for pausing/refreshing ``user_away_timeout``:
+# - presence: cancel while user or agent is speaking (VAD/STT speech-started);
+#   transcript-less flips keep the existing deadline (#6030).
+# - conversation: cancel only while the agent is active; ignore user VAD speaking;
+#   final transcripts still refresh the deadline mid-utterance (preferred for
+#   noisy telephony / turn_detection="stt").
+UserAwayOn = Literal["presence", "conversation"]
+
 
 def _append_instructions(template: Instructions | str, extra: str) -> Instructions:
     # concatenate the *raw* template text so any {placeholders} survive until render()
@@ -285,6 +293,8 @@ class AgentSessionOptions:
     """sparse endpointing keys the user provided explicitly"""
     max_tool_steps: int
     user_away_timeout: float | None
+    user_away_on: UserAwayOn
+    """See ``UserAwayOn`` — ``"presence"`` (default) or ``"conversation"``."""
     transcription_timeout: float | None
     min_consecutive_speech_delay: float
     use_tts_aligned_transcript: bool | None
@@ -389,6 +399,7 @@ def __init__(
         aec_warmup_duration: NotGivenOr[float | None] = NOT_GIVEN,
         ivr_detection: bool = False,
         user_away_timeout: float | None = 15.0,
+        user_away_on: UserAwayOn = "presence",
         transcription_timeout: float | None = None,
         session_close_transcript_timeout: float = 2.0,
         # Runtime settings
@@ -476,8 +487,18 @@ def __init__(
             loop (asyncio.AbstractEventLoop, optional): Event loop to bind the
                 session to. Falls back to :pyfunc:`asyncio.get_event_loop()`.
             user_away_timeout (float, optional): If set, set the user state as
-                "away" after this amount of time after user and agent are silent.
-                Defaults to ``15.0`` s, set to ``None`` to disable.
+                "away" after this amount of mutual silence. The deadline is
+                refreshed by meaningful activity (agent leaving idle, or a final
+                user transcript). Defaults to ``15.0`` s, set to ``None`` to disable.
+            user_away_on (Literal["presence", "conversation"], optional): Which
+                activity cancels/refreshes the away countdown. ``"presence"``
+                (default) pauses while the user or agent is speaking; transcript-less
+                user speaking↔listening flips keep the existing deadline so noise
+                cannot defer "away". ``"conversation"`` ignores user VAD speaking and
+                only pauses while the agent is active; final user transcripts still
+                refresh the deadline mid-utterance so a long answer cannot trip
+                "away" before end-of-speech — preferred for noisy telephony /
+                ``turn_detection="stt"`` (#6030).
             transcription_timeout (float, optional): If set, emit a
                 ``user_transcription_timeout`` event when VAD detects user speech
                 during the user's turn but no non-empty final transcript arrives
@@ -568,6 +589,7 @@ def __init__(
             endpointing_overrides=endpointing_overrides,
             max_tool_steps=max_tool_steps,
             user_away_timeout=user_away_timeout,
+            user_away_on=user_away_on,
             transcription_timeout=transcription_timeout,
             min_consecutive_speech_delay=min_consecutive_speech_delay,
             tts_text_transforms=(
@@ -670,6 +692,13 @@ def __init__(
         self._user_state: UserState = "listening"
         self._agent_state: AgentState = "initializing"
         self._user_away_timer: asyncio.TimerHandle | None = None
+        # absolute deadline for "away"; preserved across transcript-less user
+        # speaking↔listening flips so telephony noise cannot defer the timeout
+        self._user_away_deadline: float | None = None
+        # set when a final transcript is seen during the current user speech
+        # segment; speaking→listening then refreshes the deadline (genuine speech)
+        # instead of re-arming a stale remaining=0 window
+        self._user_away_speech_had_transcript: bool = False
 
         self._userdata: Userdata_T | None = userdata if is_given(userdata) else None
         self._closing_task: asyncio.Task[None] | None = None
@@ -1856,7 +1885,15 @@ async def _forward_video_task(self) -> None:
 
                 self._activity.push_video(frame)
 
-    def _set_user_away_timer(self) -> None:
+    def _set_user_away_timer(self, *, reset: bool = True) -> None:
+        """Arm the away timer.
+
+        Args:
+            reset: When True, push the away deadline to ``now + user_away_timeout``
+                (meaningful activity). When False, re-arm with whatever time
+                remains on the existing deadline — used for transcript-less user
+                state flips so background noise cannot defer "away".
+        """
         self._cancel_user_away_timer()
         if self._opts.user_away_timeout is None:
             return
@@ -1869,9 +1906,12 @@ def _set_user_away_timer(self) -> None:
             # skip the timer before user join the room
             return
 
-        self._user_away_timer = self._loop.call_later(
-            self._opts.user_away_timeout, self._update_user_state, "away"
-        )
+        now = time.time()
+        if reset or self._user_away_deadline is None:
+            self._user_away_deadline = now + self._opts.user_away_timeout
+
+        remaining = max(0.0, self._user_away_deadline - now)
+        self._user_away_timer = self._loop.call_later(remaining, self._update_user_state, "away")
 
     def _cancel_user_away_timer(self) -> None:
         if self._user_away_timer is not None:
@@ -1949,7 +1989,12 @@ def _update_agent_state(
                 self._aec_warmup_remaining,
             )
 
-        if state == "listening" and self._user_state == "listening":
+        # conversation mode treats user VAD "speaking" as idle for away purposes
+        # (noise must not block re-arm after the agent finishes talking)
+        user_idle_for_away = self._user_state == "listening" or (
+            self._opts.user_away_on == "conversation" and self._user_state == "speaking"
+        )
+        if state == "listening" and user_idle_for_away:
             self._set_user_away_timer()
         else:
             self._cancel_user_away_timer()
@@ -1993,10 +2038,27 @@ def _update_user_state(
             self._user_speaking_span.end(end_time=last_speaking_time_ns)
             self._user_speaking_span = None
 
+        if state == "speaking":
+            # new speech segment — only a final transcript during this segment
+            # counts as meaningful activity for deadline refresh on EOS
+            self._user_away_speech_had_transcript = False
+
         if state == "listening" and self._agent_state == "listening":
-            self._set_user_away_timer()
+            # genuine speech (saw a final transcript) refreshes the full window;
+            # transcript-less noise flips preserve the existing deadline (#6030)
+            self._set_user_away_timer(reset=self._user_away_speech_had_transcript)
+            self._user_away_speech_had_transcript = False
+        elif state == "speaking" and self._opts.user_away_on == "conversation":
+            # conversation mode: raw VAD/STT speaking must not pause the countdown
+            pass
         else:
             self._cancel_user_away_timer()
+            if state == "away":
+                self._user_away_deadline = None
+            if state == "listening":
+                # agent not idle — discard segment activity; agent return-to-listening
+                # re-arms with a full window via ``_update_agent_state``
+                self._user_away_speech_had_transcript = False
 
         old_state = self._user_state
         self._user_state = state
@@ -2022,13 +2084,30 @@ def _user_input_transcribed(self, ev: UserInputTranscribedEvent) -> None:
             # a transcript means stt recovered; reset its error tolerance
             self._stt_error_counts = 0
 
+        if ev.is_final and ev.transcript:
+            # mark activity even while VAD still reports speaking — EOS uses this
+            # to refresh the away deadline instead of a stale remaining=0 window
+            self._user_away_speech_had_transcript = True
+
+            # conversation mode keeps the countdown running during user speaking;
+            # a final transcript proves presence, so push the deadline mid-utterance
+            # rather than letting a long answer trip "away" before EOS.
+            if (
+                self._opts.user_away_on == "conversation"
+                and self.user_state == "speaking"
+                and self._agent_state == "listening"
+            ):
+                self._set_user_away_timer(reset=True)
+
         if ev.is_final and self.user_state != "speaking":
             if self.user_state == "away":
-                # reset user state from away to listening in case VAD has a miss detection
+                # reset user state from away to listening in case VAD has a miss detection.
+                # ``_update_user_state`` re-arms a full window when the agent is listening
+                # (deadline was cleared on away); must not arm while the agent is speaking.
                 self._update_user_state("listening")
             elif self.user_state == "listening" and self._agent_state == "listening":
                 # VAD may have missed speech; STT still saw activity, so refresh away timeout
-                self._set_user_away_timer()
+                self._set_user_away_timer(reset=True)
 
         self.emit("user_input_transcribed", ev)
 

livekit-agents/livekit/agents/voice/remote_session.py

@@ -409,6 +409,7 @@ def _serialize_options(opts: AgentSessionOptions) -> dict[str, str]:
         "interruption": str(dict(opts.interruption)),
         "max_tool_steps": str(opts.max_tool_steps),
         "user_away_timeout": str(opts.user_away_timeout),
+        "user_away_on": str(opts.user_away_on),
         "transcription_timeout": str(opts.transcription_timeout),
         "preemptive_generation": str(dict(opts.preemptive_generation)),
         "min_consecutive_speech_delay": str(opts.min_consecutive_speech_delay),

livekit-agents/livekit/agents/voice/report.py

@@ -61,6 +61,7 @@ def to_dict(self) -> dict:
                 "max_endpointing_delay": self.options.endpointing["max_delay"],
                 "max_tool_steps": self.options.max_tool_steps,
                 "user_away_timeout": self.options.user_away_timeout,
+                "user_away_on": self.options.user_away_on,
                 "min_consecutive_speech_delay": self.options.min_consecutive_speech_delay,
                 "preemptive_generation": dict(self.options.preemptive_generation),
                 "recording_options": dict(self.options.recording_options),