From 12829f9795aba7221cc7d157933ab4abbd7f9587 Mon Sep 17 00:00:00 2001 From: "rosetta-livekit-bot[bot]" <282703043+rosetta-livekit-bot[bot]@users.noreply.github.com> Date: Fri, 14 Aug 2026 13:09:51 +0000 Subject: [PATCH] fix: preserve adaptive interruption across tool calls --- .../adaptive-interruption-tool-calls.md | 5 + .../interruption/interruption_stream.test.ts | 44 +++++ .../interruption/interruption_stream.ts | 2 +- agents/src/voice/agent_activity.test.ts | 4 +- agents/src/voice/agent_activity.ts | 29 ++-- .../agent_activity_tool_output_commit.test.ts | 88 ++++++++++ agents/src/voice/audio_recognition.ts | 162 +++++++++++------- .../realtime_adaptive_interruption.test.ts | 77 +++++---- agents/src/voice/remote_session.ts | 1 + 9 files changed, 300 insertions(+), 112 deletions(-) create mode 100644 .changeset/adaptive-interruption-tool-calls.md create mode 100644 agents/src/inference/interruption/interruption_stream.test.ts diff --git a/.changeset/adaptive-interruption-tool-calls.md b/.changeset/adaptive-interruption-tool-calls.md new file mode 100644 index 000000000..61f946b2b --- /dev/null +++ b/.changeset/adaptive-interruption-tool-calls.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents': patch +--- + +Preserve adaptive interruption boundaries when agent playout pauses or enters a tool-call thinking gap. diff --git a/agents/src/inference/interruption/interruption_stream.test.ts b/agents/src/inference/interruption/interruption_stream.test.ts new file mode 100644 index 000000000..3e83daea1 --- /dev/null +++ b/agents/src/inference/interruption/interruption_stream.test.ts @@ -0,0 +1,44 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { describe, expect, it, vi } from 'vitest'; +import type { InterruptionMetrics } from '../../metrics/base.js'; +import { MockWebSocket } from './_mock_ws.js'; +import { AdaptiveInterruptionDetector } from './interruption_detector.js'; +import { InterruptionStreamBase, InterruptionStreamSentinel } from './interruption_stream.js'; + +vi.mock('ws', async () => { + const { MockWebSocket } = await import('./_mock_ws.js'); + return { default: MockWebSocket, WebSocket: MockWebSocket }; +}); + +describe('InterruptionStreamBase metrics', () => { + it('does not count agent-ended overlap as a backchannel', async () => { + MockWebSocket.instances.length = 0; + const detector = new AdaptiveInterruptionDetector({ + apiKey: 'test-key', + apiSecret: 'test-secret', + baseUrl: 'http://localhost:9999', + }); + const stream = new InterruptionStreamBase(detector, {}); + const metrics: InterruptionMetrics[] = []; + detector.on('metrics_collected', (event) => metrics.push(event)); + const reader = stream.stream().getReader(); + + try { + await vi.waitFor(() => expect(MockWebSocket.instances).toHaveLength(1)); + MockWebSocket.instances[0]!.simulateOpen(); + const event = reader.read(); + await stream.pushFrame(InterruptionStreamSentinel.agentSpeechStarted()); + await stream.pushFrame(InterruptionStreamSentinel.overlapSpeechStarted(0, Date.now())); + await stream.pushFrame(InterruptionStreamSentinel.overlapSpeechEnded(Date.now(), true)); + await event; + + expect(metrics).toHaveLength(1); + expect(metrics[0]).toMatchObject({ numInterruptions: 0, numBackchannels: 0 }); + } finally { + await reader.cancel(); + await stream.close(); + } + }); +}); diff --git a/agents/src/inference/interruption/interruption_stream.ts b/agents/src/inference/interruption/interruption_stream.ts index 8b086ceba..3d6e28109 100644 --- a/agents/src/inference/interruption/interruption_stream.ts +++ b/agents/src/inference/interruption/interruption_stream.ts @@ -337,7 +337,7 @@ export class InterruptionStreamBase { predictionDuration: chunk.predictionDurationInS * 1000, detectionDelay: chunk.detectionDelayInS * 1000, numInterruptions: chunk.isInterruption ? 1 : 0, - numBackchannels: chunk.isInterruption ? 0 : 1, + numBackchannels: !chunk.isInterruption && !chunk.agentEnded ? 1 : 0, numRequests: chunk.numRequests, metadata: { modelProvider: this.model.provider, diff --git a/agents/src/voice/agent_activity.test.ts b/agents/src/voice/agent_activity.test.ts index e7a206f22..f54e2735e 100644 --- a/agents/src/voice/agent_activity.test.ts +++ b/agents/src/voice/agent_activity.test.ts @@ -452,7 +452,7 @@ describe('AgentActivity - mainTask', () => { } }); - it('does not deadlock cancelling a paused speech whose generation never finishes', async () => { + it('does not end paused agent speech twice when cancelling it', async () => { const handle = SpeechHandle.create({ allowInterruptions: true }); handle._authorizeGeneration(); @@ -501,7 +501,7 @@ describe('AgentActivity - mainTask', () => { expect(result).toBe('resolved'); expect(handle.interrupted).toBe(true); expect(fakeActivity.pausedSpeech).toBeUndefined(); - expect(fakeActivity.audioRecognition.onEndOfAgentSpeech).toHaveBeenCalledOnce(); + expect(fakeActivity.audioRecognition.onEndOfAgentSpeech).not.toHaveBeenCalled(); }); }); diff --git a/agents/src/voice/agent_activity.ts b/agents/src/voice/agent_activity.ts index 5eeecd1be..aa0851c5a 100644 --- a/agents/src/voice/agent_activity.ts +++ b/agents/src/voice/agent_activity.ts @@ -1695,7 +1695,6 @@ export class AgentActivity implements RecognitionHooks { if (this.audioRecognition) { this.audioRecognition.onEndOfAgentSpeech( options?.ignoreUserTranscriptUntil ?? Date.now(), - { paused: true }, ); } if (this.isInterruptionDetectionEnabled) { @@ -1719,9 +1718,7 @@ export class AgentActivity implements RecognitionHooks { ignoreUserTranscriptUntil: ev.overlapStartedAt || ev.detectedAt, }); if (this.audioRecognition) { - this.audioRecognition.onEndOfAgentSpeech(ev.overlapStartedAt || ev.detectedAt, { - paused: this.pausedSpeech !== undefined, - }); + this.audioRecognition.onEndOfAgentSpeech(ev.overlapStartedAt || ev.detectedAt); } } @@ -3485,6 +3482,12 @@ export class AgentActivity implements RecognitionHooks { if (!speechHandle.interrupted && toolOutput.output.length > 0) { this.agentSession._updateAgentState('thinking'); + if (this.audioRecognition) { + this.audioRecognition.onEndOfAgentSpeech(Date.now()); + } + if (this.isInterruptionDetectionEnabled) { + this.restoreInterruptionByAudioActivity(); + } } else if (this.agentSession.agentState === 'speaking') { this.agentSession._updateAgentState('listening'); if (this.audioRecognition) { @@ -4069,6 +4072,12 @@ export class AgentActivity implements RecognitionHooks { if (toolOutput.output.length > 0) { this.agentSession._updateAgentState('thinking'); + if (this.audioRecognition) { + this.audioRecognition.onEndOfAgentSpeech(Date.now()); + } + if (this.isInterruptionDetectionEnabled) { + this.restoreInterruptionByAudioActivity(); + } } else if (this.agentSession.agentState === 'speaking') { this.agentSession._updateAgentState('listening'); if (this.audioRecognition) { @@ -4850,7 +4859,7 @@ export class AgentActivity implements RecognitionHooks { otelContext: this.pausedSpeech.handle._agentTurnContext, }); if (this.audioRecognition && this.pausedSpeech.agentState === 'speaking') { - this.audioRecognition.onStartOfAgentSpeech(Date.now(), { resumed: true }); + this.audioRecognition.onStartOfAgentSpeech(Date.now()); } if (this.isInterruptionDetectionEnabled) { this.disableVadInterruptionSoon(); @@ -4932,16 +4941,6 @@ export class AgentActivity implements RecognitionHooks { return; } - // The pause withheld end-of-agent-speech for a resume. Interrupting ends the turn instead; - // audio stopped when it was paused, so no playout is left to wait for. - if (interrupt && this.audioRecognition) { - void this.audioRecognition - .onEndOfAgentSpeech(Date.now()) - .catch((error) => - this.logger.warn({ error }, 'failed to report end of agent speech on pause cancel'), - ); - } - if ( interrupt && !this.pausedSpeech.handle.interrupted && diff --git a/agents/src/voice/agent_activity_tool_output_commit.test.ts b/agents/src/voice/agent_activity_tool_output_commit.test.ts index d35cf167e..afb94af01 100644 --- a/agents/src/voice/agent_activity_tool_output_commit.test.ts +++ b/agents/src/voice/agent_activity_tool_output_commit.test.ts @@ -1,13 +1,18 @@ // SPDX-FileCopyrightText: 2026 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 +import { AudioFrame } from '@livekit/rtc-node'; +import { ReadableStream } from 'node:stream/web'; import { describe, expect, it, vi } from 'vitest'; import type { ChatContext } from '../llm/chat_context.js'; import { tool } from '../llm/tool_context.js'; +import { FakeSTT } from '../stt/testing/fake_stt.js'; import { Future } from '../utils.js'; import { Agent } from './agent.js'; import { AgentSession } from './agent_session.js'; +import { AudioRecognition } from './audio_recognition.js'; import { RUNNING_TOOL_PLACEHOLDER } from './generation.js'; +import { AudioOutput } from './io.js'; import { FakeLLM } from './testing/fake_llm.js'; type PostToolContextObservation = { @@ -49,7 +54,90 @@ class ContextInspectingLLM extends FakeLLM { } } +class ImmediateOutput extends AudioOutput { + constructor() { + super(24_000); + } + + override async captureFrame(frame: AudioFrame): Promise { + const segmentCount = this.capturedPlayoutSegments; + await super.captureFrame(frame); + if (this.capturedPlayoutSegments > segmentCount) { + this.onPlaybackStarted(Date.now()); + } + } + + override flush(): void { + super.flush(); + if (this.pendingPlayoutSegments > 0) { + this.onPlaybackFinished({ playbackPosition: 0.02, interrupted: false }); + } + } + + override clearBuffer(): void { + if (this.pendingPlayoutSegments > 0) { + this.onPlaybackFinished({ playbackPosition: 0, interrupted: true }); + } + } +} + +class FrameAgent extends Agent { + constructor() { + super({ + instructions: 'test', + tools: { + lookup: tool({ + description: 'Look up a value', + execute: async () => 'forecast', + }), + }, + }); + } + + override async ttsNode(): Promise> { + return new ReadableStream({ + start(controller) { + controller.enqueue(new AudioFrame(new Int16Array(480), 24_000, 1, 480)); + controller.close(); + }, + }); + } +} + describe('AgentActivity tool output commit ordering', () => { + it('ends active speech after entering the tool-call thinking state', async () => { + const llm = new FakeLLM([ + { + input: 'look it up', + content: 'Let me check.', + toolCalls: [{ name: 'lookup', args: {} }], + }, + { input: '"forecast"', content: 'The forecast is clear.' }, + ]); + const session = new AgentSession({ llm, stt: new FakeSTT() }); + session.output.audio = new ImmediateOutput(); + const speechEndStates: string[] = []; + const onEndOfAgentSpeech = AudioRecognition.prototype.onEndOfAgentSpeech; + const speechEndSpy = vi + .spyOn(AudioRecognition.prototype, 'onEndOfAgentSpeech') + .mockImplementation(async function (this: AudioRecognition, ignoreUntil: number) { + speechEndStates.push(session.agentState); + await onEndOfAgentSpeech.call(this, ignoreUntil); + }); + + await session.start({ agent: new FrameAgent() }); + try { + const speech = session.generateReply({ userInput: 'look it up' }); + await speech.waitForPlayout(); + + expect(speechEndStates[0]).toBe('thinking'); + expect(speechEndStates.slice(1).every((state) => state === 'listening')).toBe(true); + } finally { + speechEndSpy.mockRestore(); + await session.close(); + } + }); + it('invalidates a stale preemptive generation when late EOU interrupts the post-tool reply', async () => { const llm = new ContextInspectingLLM([ { diff --git a/agents/src/voice/audio_recognition.ts b/agents/src/voice/audio_recognition.ts index 9e1160bfa..6eb680651 100644 --- a/agents/src/voice/audio_recognition.ts +++ b/agents/src/voice/audio_recognition.ts @@ -26,6 +26,8 @@ import type { AdaptiveInterruptionDetector } from '../inference/interruption/int import { InterruptionStreamSentinel } from '../inference/interruption/interruption_stream.js'; import { type InterruptionSentinel, + type OverlapSpeechEnded, + type OverlapSpeechStarted, type OverlappingSpeechEvent, } from '../inference/interruption/types.js'; import type { LanguageCode } from '../language.js'; @@ -332,6 +334,7 @@ export class AudioRecognition { private userTurnStart: number | undefined; private userTurnCommitted = false; private speaking = false; + private activeUserSpeakingSpan?: Span; private vadSpeechStarted = false; private sampleRate?: number; @@ -383,6 +386,7 @@ export class AudioRecognition { // An overlap is open right now, awaiting a verdict; several can occur within one turn. private overlapOpen = false; private interruptionStreamChannel?: StreamChannel; + private interruptionSentinelWrite: Promise = Promise.resolve(); private closed = false; // backchannel boundary for adaptive interruption suppression @@ -740,7 +744,13 @@ export class AudioRecognition { this.backchannelBoundaryCallback = undefined; } - async onStartOfAgentSpeech(startedAt: number, options?: { resumed?: boolean }) { + /** + * Mark the start of active agent speech. + * + * This lifecycle follows audible playout, not the generation. Resuming paused playout starts + * a new active-speech interval. + */ + async onStartOfAgentSpeech(startedAt: number) { this.isAgentSpeaking = true; this.agentSpeechStartedAt = startedAt; this.endpointing.onStartOfAgentSpeech(startedAt); @@ -755,20 +765,27 @@ export class AudioRecognition { ); } - // A resume re-enters the same agent turn; restarting would discard the open overlap. - if (!options?.resumed) { - return this.trySendInterruptionSentinel(InterruptionStreamSentinel.agentSpeechStarted()); + const sentinels: InterruptionSentinel[] = [InterruptionStreamSentinel.agentSpeechStarted()]; + + if (this.speaking) { + const overlapStarted = this.startOverlapInference(0, startedAt, this.activeUserSpeakingSpan); + if (overlapStarted) { + sentinels.push(overlapStarted); + } } + await this.trySendInterruptionSentinel(sentinels); } - async onEndOfAgentSpeech(ignoreUserTranscriptUntil: number, options?: { paused?: boolean }) { + /** + * Mark the end of active agent speech. + * + * This can occur while the generation remains active, such as when playout is paused. + */ + async onEndOfAgentSpeech(ignoreUserTranscriptUntil: number) { this.cancelBackchannelBoundary(); const now = Date.now(); const wasAgentSpeaking = this.isAgentSpeaking; - // Capture before the assignment below; the overlap-end notification only fires when no - // overlap had been registered during this agent speech. - const priorIgnoreUserTranscriptUntil = this.ignoreUserTranscriptUntil; if (wasAgentSpeaking) { this.endpointing.onEndOfAgentSpeech(now); } @@ -789,63 +806,86 @@ export class AudioRecognition { // before the agent finished speaking (premature corrections) are surfaced. this.ignoreUserTranscriptUntil = ignoreUntil - endCooldown; } - // Clear before awaiting the sentinel so STT events arriving while the sentinel is in - // flight are not buffered. + // Python's detector channel writes are synchronous. Clear before awaiting the equivalent JS + // writes so STT events arriving while boundaries are sent are not buffered as active speech. this.isAgentSpeaking = false; - - if (!options?.paused) { - const inputOpen = await this.trySendInterruptionSentinel( - InterruptionStreamSentinel.agentSpeechEnded(), - ); - if (!inputOpen) { - // Python has no early return here and always reaches the clear below. The stream the - // overlap belonged to is gone, so clear before bailing rather than leaking the flag. - this.overlapOpen = false; - return; - } - } - + const sentinels: InterruptionSentinel[] = []; if (wasAgentSpeaking) { - // Notify overlap end after the agent-speech-ended sentinel resets the inference stream - // so it does not emit a synthetic `isInterruption: false` event following a real - // interruption. - if (!options?.paused && priorIgnoreUserTranscriptUntil === undefined) { - this.onEndOfOverlapSpeech(Date.now(), undefined, true); + // Close any unresolved overlap before resetting the detector. + const overlapEnded = this.closeOverlap(now, undefined, true); + if (overlapEnded) { + sentinels.push(overlapEnded); } - await this.flushHeldTranscripts(endCooldown); } - if (!options?.paused) { - // The sentinel sent above resets the detector stream, dropping any open overlap. + sentinels.push(InterruptionStreamSentinel.agentSpeechEnded()); + const inputOpen = await this.trySendInterruptionSentinel(sentinels); + if (!inputOpen) { this.overlapOpen = false; + return; + } + + if (wasAgentSpeaking) { + await this.flushHeldTranscripts(endCooldown); } } /** Start interruption inference when agent is speaking and overlap speech starts. */ async onStartOfOverlapSpeech(speechDuration: number, startedAt: number, userSpeakingSpan?: Span) { + this.activeUserSpeakingSpan = userSpeakingSpan; if (this.isAgentSpeaking) { if (!this.endpointing.overlapping) { this.endpointing.onStartOfSpeech(startedAt, true); } this.turnBackchannelOverAgent = false; - this.overlapInCurrentTurn = true; - this.overlapOpen = true; - this.trySendInterruptionSentinel( - InterruptionStreamSentinel.overlapSpeechStarted( - speechDuration, - startedAt, - userSpeakingSpan, - ), + const overlapStarted = this.startOverlapInference( + speechDuration, + startedAt, + userSpeakingSpan, ); + if (overlapStarted) { + await this.trySendInterruptionSentinel(overlapStarted); + } } } + private startOverlapInference( + speechDuration: number, + startedAt: number, + userSpeakingSpan?: Span, + ): OverlapSpeechStarted | undefined { + if (!this.isInterruptionEnabled || !this.isAgentSpeaking) { + return undefined; + } + this.overlapInCurrentTurn = true; + this.overlapOpen = true; + return InterruptionStreamSentinel.overlapSpeechStarted( + speechDuration, + startedAt, + userSpeakingSpan, + ); + } + /** End interruption inference when overlap speech ends. */ async onEndOfOverlapSpeech(endedAt: number, userSpeakingSpan?: Span, agentEnded = false) { + if (!agentEnded) { + this.activeUserSpeakingSpan = undefined; + } + const overlapEnded = this.closeOverlap(endedAt, userSpeakingSpan, agentEnded); + if (overlapEnded) { + return this.trySendInterruptionSentinel(overlapEnded); + } + } + + private closeOverlap( + endedAt: number, + userSpeakingSpan?: Span, + agentEnded = false, + ): OverlapSpeechEnded | undefined { // The overlap ends once, on the first of a verdict, the user stopping, the agent stopping, // or teardown, so a call can arrive with it already closed. if (!this.isInterruptionEnabled || !this.overlapOpen) { - return; + return undefined; } this.overlapOpen = false; if ( @@ -857,9 +897,7 @@ export class AudioRecognition { userSpeakingSpan.setAttribute(traceTypes.ATTR_IS_INTERRUPTION, 'false'); } - return this.trySendInterruptionSentinel( - InterruptionStreamSentinel.overlapSpeechEnded(endedAt, agentEnded), - ); + return InterruptionStreamSentinel.overlapSpeechEnded(endedAt, agentEnded); } /** @@ -1019,23 +1057,31 @@ export class AudioRecognition { } private async trySendInterruptionSentinel( - frame: AudioFrame | InterruptionSentinel, + frame: AudioFrame | InterruptionSentinel | InterruptionSentinel[], ): Promise { - if ( - this.isInterruptionEnabled && - this.interruptionStreamChannel && - !this.interruptionStreamChannel.closed - ) { - try { - await this.interruptionStreamChannel.write(frame); - return true; - } catch (e: unknown) { - this.logger.warn( - `could not forward interruption sentinel: ${e instanceof Error ? e.message : String(e)}`, - ); + const frames = Array.isArray(frame) ? frame : [frame]; + let sent = false; + const write = async () => { + if ( + this.isInterruptionEnabled && + this.interruptionStreamChannel && + !this.interruptionStreamChannel.closed + ) { + try { + for (const item of frames) { + await this.interruptionStreamChannel.write(item); + } + sent = true; + } catch (e: unknown) { + this.logger.warn( + `could not forward interruption sentinel: ${e instanceof Error ? e.message : String(e)}`, + ); + } } - } - return false; + }; + this.interruptionSentinelWrite = this.interruptionSentinelWrite.then(write, write); + await this.interruptionSentinelWrite; + return sent; } private ensureUserTurnSpan(startTime?: number): Span { diff --git a/agents/src/voice/realtime_adaptive_interruption.test.ts b/agents/src/voice/realtime_adaptive_interruption.test.ts index 96d225d89..72fa62820 100644 --- a/agents/src/voice/realtime_adaptive_interruption.test.ts +++ b/agents/src/voice/realtime_adaptive_interruption.test.ts @@ -227,6 +227,7 @@ type RecognitionInternals = { type RecognitionStreamInternals = AudioRecognition & { overlapOpen: boolean; + speaking: boolean; trySendInterruptionSentinel: ReturnType; }; @@ -274,9 +275,15 @@ function recognitionWithInterruptionStream(): { agentSpeechStartedAt: undefined, endpointing: { overlapping: false, - onStartOfAgentSpeech: vi.fn(), - onEndOfAgentSpeech: vi.fn(), - onStartOfSpeech: vi.fn(), + onStartOfAgentSpeech: vi.fn(() => { + recognition.endpointing.overlapping = false; + }), + onEndOfAgentSpeech: vi.fn(() => { + recognition.endpointing.overlapping = false; + }), + onStartOfSpeech: vi.fn(() => { + recognition.endpointing.overlapping = true; + }), }, backchannelBoundary: undefined, backchannelBoundaryTimer: undefined, @@ -284,6 +291,7 @@ function recognitionWithInterruptionStream(): { ignoreUserTranscriptUntil: undefined, overlapInCurrentTurn: false, overlapOpen: false, + speaking: false, turnBackchannelOverAgent: false, transcriptBuffer: [], hooks: { @@ -291,10 +299,12 @@ function recognitionWithInterruptionStream(): { onBackchannelConfirmed: vi.fn(), }, logger: { trace: vi.fn() }, - trySendInterruptionSentinel: vi.fn(async (item: InterruptionSentinel) => { - sent.push(item); - return true; - }), + trySendInterruptionSentinel: vi.fn( + async (item: InterruptionSentinel | InterruptionSentinel[]) => { + sent.push(...(Array.isArray(item) ? item : [item])); + return true; + }, + ), }); return { recognition, sent }; } @@ -342,40 +352,50 @@ describe('AudioRecognition realtime adaptive backchannel verdicts', () => { expect(recognition.hooks.onBackchannelConfirmed).not.toHaveBeenCalled(); }); - it('keeps overlap inference alive while agent speech is paused', async () => { + it('closes overlap before resetting inference when agent speech ends', async () => { const { recognition, sent } = recognitionWithInterruptionStream(); await recognition.onStartOfAgentSpeech(Date.now()); await recognition.onStartOfOverlapSpeech(0, Date.now()); sent.length = 0; - await recognition.onEndOfAgentSpeech(Date.now(), { paused: true }); + await recognition.onEndOfAgentSpeech(Date.now()); - expect(sent).toEqual([]); + expect(sent.map((item) => item.type)).toEqual(['overlap-speech-ended', 'agent-speech-ended']); + expect(sent[0]).toMatchObject({ agentEnded: true }); + expect(recognition.overlapOpen).toBe(false); }); - it('lets user speech ending close an overlap while agent speech is paused', async () => { + it('does not close overlap again when user speech ends after agent speech', async () => { const { recognition, sent } = recognitionWithInterruptionStream(); await recognition.onStartOfAgentSpeech(Date.now()); await recognition.onStartOfOverlapSpeech(0, Date.now()); - await recognition.onEndOfAgentSpeech(Date.now(), { paused: true }); + await recognition.onEndOfAgentSpeech(Date.now()); sent.length = 0; await recognition.onEndOfOverlapSpeech(Date.now()); - expect(sent.map((item) => item.type)).toEqual(['overlap-speech-ended']); - expect(sent[0]).toMatchObject({ agentEnded: false }); + expect(sent).toEqual([]); }); - it('does not restart the detector when paused speech resumes', async () => { + it('restarts the detector when paused speech resumes', async () => { const { recognition, sent } = recognitionWithInterruptionStream(); await recognition.onStartOfAgentSpeech(Date.now()); - await recognition.onStartOfOverlapSpeech(0, Date.now()); - await recognition.onEndOfAgentSpeech(Date.now(), { paused: true }); + const userStartedAt = Date.now(); + recognition.speaking = true; + await recognition.onStartOfOverlapSpeech(0, userStartedAt); + await recognition.onEndOfAgentSpeech(Date.now()); sent.length = 0; - await recognition.onStartOfAgentSpeech(Date.now(), { resumed: true }); + const resumedAt = Date.now(); + await recognition.onStartOfAgentSpeech(resumedAt); - expect(sent).toEqual([]); + expect(sent.map((item) => item.type)).toEqual([ + 'agent-speech-started', + 'overlap-speech-started', + ]); + expect(sent[1]).toMatchObject({ speechDuration: 0, startedAt: resumedAt }); + expect(recognition.endpointing.onStartOfSpeech).toHaveBeenCalledOnce(); + expect(recognition.endpointing.onStartOfSpeech).toHaveBeenCalledWith(userStartedAt, true); }); it('does not close an overlap again after a verdict resolves it', async () => { @@ -390,18 +410,6 @@ describe('AudioRecognition realtime adaptive backchannel verdicts', () => { expect(sent).toEqual([]); }); - it('tears down inference when interrupted paused speech ends', async () => { - const { recognition, sent } = recognitionWithInterruptionStream(); - await recognition.onStartOfAgentSpeech(Date.now()); - await recognition.onStartOfOverlapSpeech(0, Date.now()); - await recognition.onEndOfAgentSpeech(Date.now(), { paused: true }); - sent.length = 0; - - await recognition.onEndOfAgentSpeech(Date.now()); - - expect(sent.map((item) => item.type)).toEqual(['agent-speech-ended']); - }); - it('still tears down inference at the real end of agent speech', async () => { const { recognition, sent } = recognitionWithInterruptionStream(); await recognition.onStartOfAgentSpeech(Date.now()); @@ -410,10 +418,7 @@ describe('AudioRecognition realtime adaptive backchannel verdicts', () => { await recognition.onEndOfAgentSpeech(Date.now()); - // The synthetic overlap end must follow the sentinel, while the overlap is still open. - // Clearing `overlapOpen` any earlier makes onEndOfOverlapSpeech bail and silently drops - // this event, so assert the exact sequence rather than just containment. - expect(sent.map((item) => item.type)).toEqual(['agent-speech-ended', 'overlap-speech-ended']); - expect(sent[1]).toMatchObject({ agentEnded: true }); + expect(sent.map((item) => item.type)).toEqual(['overlap-speech-ended', 'agent-speech-ended']); + expect(sent[0]).toMatchObject({ agentEnded: true }); }); }); diff --git a/agents/src/voice/remote_session.ts b/agents/src/voice/remote_session.ts index 40da3e5da..c45e49c0b 100644 --- a/agents/src/voice/remote_session.ts +++ b/agents/src/voice/remote_session.ts @@ -849,6 +849,7 @@ export class SessionHost { }; private onOverlappingSpeech = (event: OverlappingSpeechEvent): void => { + // TODO(AGT-3180): Forward agentEnded when the remote-session protocol supports it. const value = new pb.AgentSessionEvent_OverlappingSpeech({ isInterruption: event.isInterruption, detectionDelay: event.detectionDelayInS,