diff --git a/.changes/audio-engine-error-mapping b/.changes/audio-engine-error-mapping new file mode 100644 index 000000000..56f911842 --- /dev/null +++ b/.changes/audio-engine-error-mapping @@ -0,0 +1 @@ +patch type="changed" "Microphone permission and audio session failures now throw TrackCreateException / AudioSessionException instead of AudioProcessingException" diff --git a/.changes/native-audio-session-preset b/.changes/native-audio-session-preset new file mode 100644 index 000000000..5c3677a01 --- /dev/null +++ b/.changes/native-audio-session-preset @@ -0,0 +1 @@ +patch type="fixed" "iOS: audio session is configured for recording even when capture starts before connect (pre-connect audio, pre-join mic), fixing audio engine error -9001" diff --git a/lib/src/audio/audio_engine_error.dart b/lib/src/audio/audio_engine_error.dart new file mode 100644 index 000000000..aeaad2e80 --- /dev/null +++ b/lib/src/audio/audio_engine_error.dart @@ -0,0 +1,48 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import 'package:flutter/services.dart' show PlatformException; + +import 'package:meta/meta.dart'; + +import '../exceptions.dart'; + +/// Error codes the native plugin uses for audio device module failures with a +/// known cause. Anything else keeps the caller-specific fallback code. +@internal +const String audioEngineErrorCodeDeviceAccessDenied = 'deviceAccessDenied'; +@internal +const String audioEngineErrorCodeAudioSessionInvalidCategory = 'audioSessionInvalidCategory'; +@internal +const String audioEngineErrorCodeAudioSessionConfigureFailed = 'audioSessionConfigureFailed'; + +/// Maps a [PlatformException] from an audio device module call to the +/// [LiveKitException] describing its cause, or `null` when the code is not one +/// of the known audio engine failures and the caller should apply its own +/// mapping. +@internal +LiveKitException? audioEngineExceptionFrom(PlatformException error) { + final native = error.message?.trim() ?? ''; + String message(String fallback) => native.isEmpty ? fallback : native; + switch (error.code) { + case audioEngineErrorCodeDeviceAccessDenied: + return TrackCreateException(message('Microphone permission is not granted')); + case audioEngineErrorCodeAudioSessionInvalidCategory: + return AudioSessionException(message('Audio session category does not support recording')); + case audioEngineErrorCodeAudioSessionConfigureFailed: + return AudioSessionException(message('Failed to configure the audio session')); + default: + return null; + } +} diff --git a/lib/src/audio/audio_manager.dart b/lib/src/audio/audio_manager.dart index be25e83e2..9ab66d2f8 100644 --- a/lib/src/audio/audio_manager.dart +++ b/lib/src/audio/audio_manager.dart @@ -14,6 +14,8 @@ import 'dart:async'; +import 'package:flutter/services.dart' show PlatformException; + import 'package:meta/meta.dart'; import '../logger.dart'; @@ -21,6 +23,7 @@ import '../support/native.dart'; import '../support/platform.dart'; import 'android_audio_session_adapter.dart'; import 'audio_engine_availability.dart'; +import 'audio_engine_error.dart'; import 'audio_processing_state.dart'; import 'audio_session.dart'; import 'audio_session_policy.dart'; @@ -214,16 +217,23 @@ class AudioManager { /// cross-platform code. /// /// Throws if the native side rejects the change, so callers never assume - /// the engine is gated when it is not. + /// the engine is gated when it is not. Enabling input requires microphone + /// permission, which is not requested here: a [TrackCreateException] is + /// thrown when it is missing, and an [AudioSessionException] when the audio + /// session does not permit recording. /// /// Experimental: this API may change in a future release. @experimental Future setEngineAvailability(AudioEngineAvailability availability) async { if (!lkPlatformIsApple()) return; - await Native.setEngineAvailability( - isInputAvailable: availability.isInputAvailable, - isOutputAvailable: availability.isOutputAvailable, - ); + try { + await Native.setEngineAvailability( + isInputAvailable: availability.isInputAvailable, + isOutputAvailable: availability.isOutputAvailable, + ); + } on PlatformException catch (error) { + throw audioEngineExceptionFrom(error) ?? error; + } } /// Selects whether LiveKit manages the platform audio session automatically. @@ -309,6 +319,7 @@ class AudioManager { automatic: true, selectCategoryByEngineState: true, forceSpeakerOutput: policy.forceSpeakerOutput, + preferSpeakerOutput: policy.preferSpeakerOutput, ); } else { // Manual mode: re-apply the fixed Apple config. Non-forced receiver vs @@ -359,6 +370,7 @@ class AudioManager { automatic: _isAutomaticConfigurationEnabled, selectCategoryByEngineState: _isAutomaticConfigurationEnabled, forceSpeakerOutput: policy.forceSpeakerOutput, + preferSpeakerOutput: policy.preferSpeakerOutput, ); } diff --git a/lib/src/exceptions.dart b/lib/src/exceptions.dart index bae9a4819..8e30b4030 100644 --- a/lib/src/exceptions.dart +++ b/lib/src/exceptions.dart @@ -82,6 +82,16 @@ class TrackCreateException extends LiveKitException { TrackCreateException([String msg = 'Failed to create track']) : super._(msg); } +/// The platform audio session could not be configured for, or does not permit, +/// the requested audio operation (Apple platforms). +/// Common reasons: +/// - Recording was started while the app-managed audio session +/// (`AudioSessionManagementMode.manual`) has a category without input. +/// - The system rejected the audio session configuration. +class AudioSessionException extends LiveKitException { + AudioSessionException([String msg = 'Audio session error']) : super._(msg); +} + /// Failed to publish a local track. /// Common reasons: /// - Token does not have track publish permission. diff --git a/lib/src/support/native.dart b/lib/src/support/native.dart index ff5f75802..b73750cc5 100644 --- a/lib/src/support/native.dart +++ b/lib/src/support/native.dart @@ -50,6 +50,7 @@ class Native { bool automatic = false, bool selectCategoryByEngineState = false, bool forceSpeakerOutput = false, + bool preferSpeakerOutput = true, }) async { try { final result = await channel.invokeMethod( @@ -59,6 +60,9 @@ class Native { 'automatic': automatic, 'selectCategoryByEngineState': selectCategoryByEngineState, 'forceSpeakerOutput': forceSpeakerOutput, + // Lets the native built-in recording preset pick the same mode the + // Dart policy would, for engine starts that happen before any push. + 'preferSpeakerOutput': preferSpeakerOutput, }, ); return result == true; diff --git a/lib/src/track/local/audio.dart b/lib/src/track/local/audio.dart index 47b21e910..051eb00f2 100644 --- a/lib/src/track/local/audio.dart +++ b/lib/src/track/local/audio.dart @@ -20,6 +20,7 @@ import 'package:collection/collection.dart'; import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc; import 'package:meta/meta.dart'; +import '../../audio/audio_engine_error.dart'; import '../../events.dart'; import '../../internal/events.dart'; import '../../logger.dart'; @@ -91,10 +92,14 @@ class LocalAudioTrack extends LocalTrack with AudioTrack, LocalAudioManagementMi // processing options are applied before WebRTC opens the microphone. await Native.startLocalRecording(currentOptions.processing.toMap()); } on PlatformException catch (error) { - throw track_options.AudioProcessingException( - _audioProcessingFailureReason(error.code), - error.message ?? '', - ); + // Missing microphone permission or an audio session that does not + // permit recording are not audio processing failures, so they surface + // as their own exception types. + throw audioEngineExceptionFrom(error) ?? + track_options.AudioProcessingException( + _audioProcessingFailureReason(error.code), + error.message ?? '', + ); } } } diff --git a/shared_swift/LiveKitPlugin.swift b/shared_swift/LiveKitPlugin.swift index a4f1b6b48..05eb65ea5 100644 --- a/shared_swift/LiveKitPlugin.swift +++ b/shared_swift/LiveKitPlugin.swift @@ -360,10 +360,12 @@ public class LiveKitPlugin: NSObject, FlutterPlugin { let automatic = args["automatic"] as? Bool ?? false let selectCategoryByEngineState = args["selectCategoryByEngineState"] as? Bool ?? false let forceSpeakerOutput = args["forceSpeakerOutput"] as? Bool ?? false + let preferSpeakerOutput = args["preferSpeakerOutput"] as? Bool ?? true audioEngineObserver?.updatePolicy(configuration, automaticManagementEnabled: automatic, selectCategoryByEngineState: selectCategoryByEngineState, - forceSpeakerOutput: forceSpeakerOutput) + forceSpeakerOutput: forceSpeakerOutput, + preferSpeakerOutput: preferSpeakerOutput) let shouldApplyNow = !automatic || (audioEngineObserver?.isSessionActive ?? false) guard shouldApplyNow else { @@ -524,11 +526,8 @@ public class LiveKitPlugin: NSObject, FlutterPlugin { if admResult == 0 { result(nil) } else { - result(FlutterError( - code: "setEngineAvailability", - message: "Audio engine returned error code: \(admResult)", - details: nil - )) + result(LiveKitPlugin.flutterError(forAudioEngineResult: admResult, + fallbackCode: "setEngineAvailability")) } } } @@ -606,11 +605,10 @@ public class LiveKitPlugin: NSObject, FlutterPlugin { if admResult == 0 { result(nil) } else { - result(FlutterError( - code: "applyFailed", - message: "Audio engine returned error code: \(admResult)", - details: nil - )) + // Permission and audio session failures get their own codes so + // Dart does not report them as audio processing failures. + result(LiveKitPlugin.flutterError(forAudioEngineResult: admResult, + fallbackCode: "applyFailed")) } } } @@ -786,14 +784,46 @@ public class LiveKitPlugin: NSObject, FlutterPlugin { } } -#if !os(macOS) -@available(iOS 13.0, *) extension LiveKitPlugin { /// SDK-side audio engine error code (mirrors client-sdk-swift): returned /// from a delegate callback to make WebRTC abort / roll back the engine /// operation when the audio session cannot be configured. static let kAudioEngineErrorFailedToConfigureAudioSession = -4100 + /// Error codes originating from the WebRTC AudioEngineDevice. Keep in sync + /// with `audio_engine_device.h` in the webrtc-sdk fork. + static let kAudioEngineErrorInsufficientDevicePermission = -9000 + static let kAudioEngineErrorAudioSessionInvalidCategory = -9001 + + /// Maps a non-zero audio device module result to a `FlutterError` whose code + /// the Dart side can act on. Codes with a known cause get their own error + /// code, mirroring client-sdk-swift's `checkAdmResult`. Anything else falls + /// back to `fallbackCode` with the raw value in the message. + static func flutterError(forAudioEngineResult result: Int, fallbackCode: String) -> FlutterError { + switch result { + case kAudioEngineErrorInsufficientDevicePermission: + return FlutterError(code: "deviceAccessDenied", + message: "Microphone permission is not granted (audio engine error \(result))", + details: result) + case kAudioEngineErrorAudioSessionInvalidCategory: + return FlutterError(code: "audioSessionInvalidCategory", + message: "Audio session category does not support recording (audio engine error \(result))", + details: result) + case kAudioEngineErrorFailedToConfigureAudioSession: + return FlutterError(code: "audioSessionConfigureFailed", + message: "Failed to configure the audio session (audio engine error \(result))", + details: result) + default: + return FlutterError(code: fallbackCode, + message: "Audio engine returned error code: \(result)", + details: result) + } + } +} + +#if !os(macOS) +@available(iOS 13.0, *) +extension LiveKitPlugin { /// Applies an `RTCAudioSessionConfiguration` to the shared `RTCAudioSession`. /// Returns `nil` on success or the thrown error. Safe to call on any thread. static func applyAudioSessionConfiguration(_ configuration: RTCAudioSessionConfiguration, @@ -869,6 +899,13 @@ class LKAudioEngineObserver: NSObject, RTCAudioDeviceModuleDelegate { private weak var channel: FlutterMethodChannel? #if !os(macOS) + // Policy pushed from Dart, if any. It is an override: when nothing has been + // pushed yet (recording before the room connects, or an engine start driven + // from native before the Flutter side exists) the observer resolves a + // built-in preset from engine state instead, so the engine never enables + // against the app-default soloAmbient category. Mirrors the Swift SDK's + // AudioSessionEngineObserver, which derives the session from engine state + // alone. private var cachedConfiguration: RTCAudioSessionConfiguration? // When true, the category is chosen from the live engine state at apply time // (playAndRecord while recording, playback for playout-only) rather than @@ -878,6 +915,10 @@ class LKAudioEngineObserver: NSObject, RTCAudioDeviceModuleDelegate { // override or manual mode, where the config is applied verbatim. private var selectCategoryByEngineState = false private var forceSpeakerOutput = false + // Speaker preference for the built-in recording preset (videoChat routes to + // the speaker, voiceChat to the receiver). Defaults to true like the Dart + // AudioManager, so the preset matches the policy Dart pushes on connect. + private var preferSpeakerOutput = true private var isAutomaticManagementEnabled = true // False when an external call system (CallKit) owns session activation: // configurations are applied without activating, and the session is never @@ -916,13 +957,15 @@ class LKAudioEngineObserver: NSObject, RTCAudioDeviceModuleDelegate { func updatePolicy(_ configuration: RTCAudioSessionConfiguration, automaticManagementEnabled: Bool, selectCategoryByEngineState: Bool, - forceSpeakerOutput: Bool) { + forceSpeakerOutput: Bool, + preferSpeakerOutput: Bool) { let cachedConfiguration = copyConfiguration(configuration) lock.lock() self.cachedConfiguration = cachedConfiguration self.isAutomaticManagementEnabled = automaticManagementEnabled self.selectCategoryByEngineState = selectCategoryByEngineState self.forceSpeakerOutput = forceSpeakerOutput + self.preferSpeakerOutput = preferSpeakerOutput lock.unlock() } @@ -972,9 +1015,19 @@ class LKAudioEngineObserver: NSObject, RTCAudioDeviceModuleDelegate { /// category would leave playAndRecord-only mode/options (e.g. videoChat, /// allowBluetooth) that are invalid for the playback category. Mirrors the /// Swift SDK's `.playback` preset (playback + spokenAudio + mixWithOthers). + /// + /// With no pushed config and automatic management on, the built-in + /// recording preset stands in for the Dart policy, so the result is the + /// same as if the default policy had been pushed. Returns `nil` only in + /// manual mode with nothing pushed, where the app owns the session. private func effectiveConfigurationLocked(isRecordingEnabled: Bool) -> RTCAudioSessionConfiguration? { - guard let configuration = cachedConfiguration else { return nil } - guard selectCategoryByEngineState, !isRecordingEnabled else { return configuration } + let usesDefaultPreset = cachedConfiguration == nil + guard let configuration = cachedConfiguration + ?? (isAutomaticManagementEnabled ? defaultRecordingConfigurationLocked() : nil) + else { return nil } + // The default preset is always resolved by engine state, like the Dart + // automatic-mode push it stands in for. + guard usesDefaultPreset || selectCategoryByEngineState, !isRecordingEnabled else { return configuration } let playback = copyConfiguration(configuration) playback.category = AVAudioSession.Category.playback.rawValue @@ -983,6 +1036,21 @@ class LKAudioEngineObserver: NSObject, RTCAudioDeviceModuleDelegate { return playback } + /// Built-in playAndRecord preset used until Dart pushes a policy. Must be + /// called with `lock` held. + /// + /// Keep in sync with the automatic-mode branch of + /// `ResolvedAudioSessionPolicy.appleConfiguration` in + /// `lib/src/audio/audio_session_policy.dart`, so a later push of the default + /// policy (on connect) does not change the live session. + private func defaultRecordingConfigurationLocked() -> RTCAudioSessionConfiguration { + let configuration = RTCAudioSessionConfiguration.webRTC() + configuration.category = AVAudioSession.Category.playAndRecord.rawValue + configuration.categoryOptions = [.allowBluetooth, .allowBluetoothA2DP, .allowAirPlay] + configuration.mode = (preferSpeakerOutput ? AVAudioSession.Mode.videoChat : AVAudioSession.Mode.voiceChat).rawValue + return configuration + } + private func copyConfiguration(_ configuration: RTCAudioSessionConfiguration) -> RTCAudioSessionConfiguration { let copy = RTCAudioSessionConfiguration() copy.category = configuration.category diff --git a/test/audio/audio_session_test.dart b/test/audio/audio_session_test.dart index d02b54d58..79be9a5df 100644 --- a/test/audio/audio_session_test.dart +++ b/test/audio/audio_session_test.dart @@ -17,9 +17,11 @@ import 'package:flutter/services.dart'; import 'package:flutter_test/flutter_test.dart'; import 'package:livekit_client/src/audio/android_audio_session_adapter.dart'; +import 'package:livekit_client/src/audio/audio_engine_error.dart'; import 'package:livekit_client/src/audio/audio_manager.dart'; import 'package:livekit_client/src/audio/audio_session.dart'; import 'package:livekit_client/src/audio/audio_session_policy.dart'; +import 'package:livekit_client/src/exceptions.dart'; import 'package:livekit_client/src/support/native.dart'; import 'package:livekit_client/src/support/native_audio.dart' as native_audio; import 'package:livekit_client/src/support/webrtc_initialize_options.dart'; @@ -654,6 +656,7 @@ void main() { automatic: true, selectCategoryByEngineState: true, forceSpeakerOutput: true, + preferSpeakerOutput: true, ); expect(result, isTrue); @@ -662,6 +665,26 @@ void main() { calls.single.arguments, containsPair('forceSpeakerOutput', true), ); + expect( + calls.single.arguments, + containsPair('preferSpeakerOutput', true), + ); + }); + + test('passes speaker preference to native so its built-in preset matches the Dart policy', () async { + await Native.configureAudio( + native_audio.NativeAudioConfiguration( + appleAudioCategory: AppleAudioCategory.playAndRecord, + appleAudioMode: AppleAudioMode.voiceChat, + ), + automatic: true, + selectCategoryByEngineState: true, + preferSpeakerOutput: false, + ); + + expect(calls.single.method, 'configureNativeAudio'); + expect(calls.single.arguments, containsPair('preferSpeakerOutput', false)); + expect(calls.single.arguments, containsPair('forceSpeakerOutput', false)); }); test('returns platform unavailable when audio processing channel is missing', () async { @@ -873,4 +896,34 @@ void main() { ); }); }); + + group('audioEngineExceptionFrom', () { + test('maps missing microphone permission to TrackCreateException', () { + final error = audioEngineExceptionFrom( + PlatformException(code: audioEngineErrorCodeDeviceAccessDenied, message: 'no mic'), + ); + + expect(error, isA()); + expect(error!.message, 'no mic'); + }); + + test('maps audio session failures to AudioSessionException', () { + final invalidCategory = audioEngineExceptionFrom( + PlatformException(code: audioEngineErrorCodeAudioSessionInvalidCategory), + ); + final configureFailed = audioEngineExceptionFrom( + PlatformException(code: audioEngineErrorCodeAudioSessionConfigureFailed, message: ' detail '), + ); + + expect(invalidCategory, isA()); + expect(invalidCategory!.message, 'Audio session category does not support recording'); + expect(configureFailed, isA()); + expect(configureFailed!.message, 'detail'); + }); + + test('leaves other codes to the caller', () { + expect(audioEngineExceptionFrom(PlatformException(code: 'applyFailed')), isNull); + expect(audioEngineExceptionFrom(PlatformException(code: 'setEngineAvailability')), isNull); + }); + }); }