feat(desktop): hands-free wake word to command the assistant during ambient listening

desktop/macos/Desktop/Sources/AppState/AppState+ListenEvents.swift

@@ -54,12 +54,40 @@
       // Extract speaker_id from backend (e.g. "SPEAKER_00" → 0)
       let speakerId = segment.speaker_id ?? 0
 
+      // Omi speaks into a room Omi is also recording, so ambient capture returns the
+      // assistant's own voice attributed to the primary speaker. Every consumer below
+      // then acts on it as if a person had spoken: barge-in halts the very playback that
+      // produced it (observed live, three times in one six-turn session, which is why a
+      // long answer stops partway), the wake word can be commanded by an answer carrying
+      // the wake phrase, and the conversation record and memory extraction gain speech
+      // nobody said. One guard here, where all of them route through.
+      var segment = segment
+      switch VoicePlaybackEchoPolicy.classify(
+        transcript: segment.text,
+        spokenWords: FloatingBarVoicePlaybackService.shared.recentlySpokenWords
+      ) {
+      case .keep:
+        break
+      case .drop:
+        log("Transcription [ECHO]: dropped Omi's own playback heard back: \(segment.text.prefix(60))")
+        continue
+      case .keepResidue(let spoken):
+        // The user talked over the end of the playback. There is no pause to close the
+        // window on while Omi is speaking, so both land in one segment; keeping only the
+        // part Omi did not say is what lets a barge-in survive.
+        log("Transcription [ECHO]: kept the user's words from a segment Omi spoke over: \(spoken.prefix(60))")
+        segment.text = spoken
+      }
+
       // Barge-in interruption: if the user speaks while voice playback is active,
       // halt playback immediately so Omi never talks over the user.
       if VoiceBargeInPolicy.shouldInterrupt(
         isUser: segment.is_user,
         speaker: speakerId,
         text: segment.text,
+        previouslyHeard: segment.id.flatMap { id in
+          speakerSegments.first(where: { $0.segmentId == id })?.text
+        },
         isSpeaking: FloatingBarVoicePlaybackService.shared.isSpeaking
       ) {
         log("Transcription [BARGE-IN]: User spoke mid-playback; interrupting voice output")
@@ -81,6 +109,8 @@
         translations: translations
       )
 
+      WakeWordService.shared.observe(newSeg)
+
       // Upsert: if we already have a segment with this ID, update it; otherwise append
       if let segId = segment.id,
         let existingIdx = speakerSegments.firstIndex(where: { $0.segmentId == segId })
@@ -316,7 +346,7 @@
   }
 
   /// Handle message events from Python backend `/v4/listen`
   func handleListenEvent(_ event: TranscriptionService.ListenEvent) {
     switch event.type {
     case "service_status":
       let status = event.raw["status"] as? String ?? "unknown"

desktop/macos/Desktop/Sources/AppState/AppState+Transcription.swift

@@ -1,4 +1,4 @@
 @preconcurrency import AVFoundation
 import Combine
 import SwiftUI
 @preconcurrency import UserNotifications
@@ -75,7 +75,8 @@
       sttSession.beginRecording(
         audioSource: effectiveSource,
         isAppleSilicon: Self.isAppleSilicon,
-        debugForceCloud: debugForceCloud
+        debugForceCloud: debugForceCloud,
+        wakeWordNeedsRecognizableName: AssistantSettings.shared.wakeWordNeedsRecognizableName
       )
       let clientConversationId = UUID().uuidString.lowercased()
       currentClientConversationId = sttSession.useLocalSTT ? nil : clientConversationId

desktop/macos/Desktop/Sources/AppState/STTSessionState.swift

@@ -36,12 +36,32 @@ struct STTSessionState: Equatable {
   }
 
   /// Resolve which STT path to use for a new recording.
+  ///
+  /// `wakeWordNeedsRecognizableName` is the opt-in the wake word needs to work. Ambient
+  /// transcription is on-device by default, and the manager that path runs on takes a
+  /// language hint and nothing else — `AsrManager.transcribe(_:decoderState:language:)` has
+  /// no keyword or vocabulary parameter, so it cannot be told that "Omi" is a word.
+  /// FluidAudio does ship term biasing, but only on `SlidingWindowAsrManager`
+  /// (`configureVocabularyBoosting(vocabulary:ctcModels:config:)`), which is a different
+  /// streaming architecture and pulls a second set of CTC models. That is the upgrade path
+  /// out of this flag; it is not a parameter we can pass today. Measured
+  /// on one machine, same script and voices, only the lane changed: the phrase was usable
+  /// in 12 of 20 utterances on-device against 19 of 20 on the cloud lane, which reaches
+  /// `/v4/listen` — and that path prepends "Omi" to the STT keyword vocabulary server-side
+  /// (`backend/utils/listen_session_bootstrap.py`), so the recognizer is told the name.
+  /// Seven of the eight on-device misses came back as "Only", which cannot be accepted as
+  /// a rendering — it opens ordinary sentences.
+  ///
+  /// Off by default. It trades on-device transcription for cloud transcription while the
+  /// wake word is enabled, which is a privacy and cost decision, not a technical one.
   func resolveMode(
     audioSource: AudioSource,
     isAppleSilicon: Bool,
-    debugForceCloud: Bool
+    debugForceCloud: Bool,
+    wakeWordNeedsRecognizableName: Bool = false
   ) -> ResolvedMode {
-    let forceCloud = !sessionForceLocal && (debugForceCloud || appRunForceCloud)
+    let forceCloud =
+      !sessionForceLocal && (debugForceCloud || appRunForceCloud || wakeWordNeedsRecognizableName)
     if audioSource == .bleDevice || !isAppleSilicon || forceCloud {
       return .cloud
     }
@@ -51,12 +71,14 @@ struct STTSessionState: Equatable {
   mutating func beginRecording(
     audioSource: AudioSource,
     isAppleSilicon: Bool,
-    debugForceCloud: Bool
+    debugForceCloud: Bool,
+    wakeWordNeedsRecognizableName: Bool = false
   ) {
     activeMode = resolveMode(
       audioSource: audioSource,
       isAppleSilicon: isAppleSilicon,
-      debugForceCloud: debugForceCloud
+      debugForceCloud: debugForceCloud,
+      wakeWordNeedsRecognizableName: wakeWordNeedsRecognizableName
     )
   }
 

desktop/macos/Desktop/Sources/FloatingControlBar/FloatingBarVoicePlaybackService.swift

@@ -1,4 +1,4 @@
 @preconcurrency import AVFoundation
 import CryptoKit
 import Foundation
 import OmiSupport
@@ -117,6 +117,65 @@
   private var activeRealtimeSlowToolAcknowledgement: RealtimeSlowToolAcknowledgementKind?
   private var activeRealtimeSlowToolAcknowledgementTransport: String?
 
+  /// What this service has recently said, so ambient capture can recognise Omi's own voice
+  /// coming back through the microphone instead of treating it as the user
+  /// (`VoicePlaybackEchoPolicy`). Kept for a window *after* playback ends, because the
+  /// transcript of the last words arrives about a second behind the audio.
+  private var spokenWordHistory: [(word: String, at: Date)] = []
+  /// Last text recorded from an engine outside this service, so a streamed reply that
+  /// repeats or extends itself is entered once rather than once per event.
+  private var lastExternallySpokenText: String?
+  private static let spokenHistoryWordCap = 300
+
+  /// Each word expires on its own clock rather than the history being kept alive as a
+  /// block. Extending one shared deadline on every chunk left several turns of speech
+  /// matchable at once, and a few hundred words of ordinary English will align with almost
+  /// any short sentence — live, "Sorry my mistake it's taking" was deleted that way. An
+  /// echo arrives a second or two behind the audio, so this only has to outlive that.
+  private static let spokenWordLifetime: TimeInterval = 15
+
+  /// Playback from the last few seconds, which is all an echo can be an echo of.
+  var recentlySpokenWords: [String] {
+    let cutoff = Date().addingTimeInterval(-Self.spokenWordLifetime)
+    return spokenWordHistory.filter { $0.at > cutoff }.map(\.word)
+  }
+
+  /// Record speech this app produced through some other engine.
+  ///
+  /// The realtime session plays its own audio rather than routing through this service, so
+  /// nothing here would know what was said — and the microphone hears it identically. Left
+  /// unrecorded, the model's own answer comes back through ambient capture and is attributed
+  /// to the user, which is the exact defect `VoicePlaybackEchoPolicy` exists to prevent.
+  /// What matters to the echo check is that Omi said it out loud, not which engine spoke it.
+  /// Arrives as a stream, and providers differ on whether each event is the new fragment or
+  /// the whole reply so far. Recording every event verbatim would enter the same words over
+  /// and over, and a padded history matches more of what a person says — the direction that
+  /// deletes real speech. Only the part that extends what was already recorded is kept.
+  func recordExternallySpokenText(_ text: String) {
+    let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
+    guard !trimmed.isEmpty else { return }
+    defer { lastExternallySpokenText = trimmed }
+    guard let previous = lastExternallySpokenText else { return recordSpokenText(trimmed) }
+    if trimmed == previous { return }
+    if trimmed.hasPrefix(previous) {
+      let addition = String(trimmed.dropFirst(previous.count))
+      return recordSpokenText(addition)
+    }
+    recordSpokenText(trimmed)
+  }
+
+  private func recordSpokenText(_ text: String) {
+    let words = VoicePlaybackEchoPolicy.words(text)
+    guard !words.isEmpty else { return }
+    let now = Date()
+    let cutoff = now.addingTimeInterval(-Self.spokenWordLifetime)
+    spokenWordHistory.removeAll { $0.at <= cutoff }
+    spokenWordHistory.append(contentsOf: words.map { (word: $0, at: now) })
+    if spokenWordHistory.count > Self.spokenHistoryWordCap {
+      spokenWordHistory.removeFirst(spokenWordHistory.count - Self.spokenHistoryWordCap)
+    }
+  }
+
   /// QueryTracer for the in-flight query, handed in by the floating-bar window.
   /// Used to bracket the `tts_start` span (first real chunk → first audio out).
   var tracer: QueryTracer?
@@ -689,6 +748,7 @@
       }
       audioPlayer = player
       activePlayerFallbackText = fallbackText
+      recordSpokenText(fallbackText)
       if let lease = activePTTLease {
         _ = VoiceTurnCoordinator.shared.noteOutputProgress(lease)
       }
@@ -823,6 +883,7 @@
       leaseID: activePTTLease?.id,
       utterance: utterance)
     speechSynthesizer.speak(utterance)
+    recordSpokenText(text)
     tracer?.end("tts_start")
   }
 

desktop/macos/Desktop/Sources/FloatingControlBar/FloatingControlBarState.swift

@@ -1,4 +1,4 @@
 import Combine
 import SwiftUI
 import VoiceTurnDomain
 
@@ -361,6 +361,15 @@
   @Published private(set) var localAnswerOverride: ChatMessage? = nil
   @Published var lastConversationActivityAt: Date? = nil
   @Published var activeAgentChatPillID: UUID? = nil
+  /// This query answers in the notch: spoken, with the existing response glow, and without
+  /// growing the bar into a response panel.
+  ///
+  /// Set for a wake-word command, where hands and eyes are elsewhere and a panel covering a
+  /// fifth of the screen is the opposite of what was asked for. It has to survive the whole
+  /// query rather than just its start: the answer's arrival presents `.mainResponse` again
+  /// on its own, which is why suppressing only the initial expansion left the panel showing.
+  @Published var answersQuietly: Bool = false
+
   @Published var conversationSurface: FloatingConversationSurface = .closed
   private var activeAIDraftKey = ChatDraftKey.floatingMain
   private var isRestoringAIDraft = false