| 1 | import AVFoundation |
| 2 | import Foundation |
| 3 | import LiveKitWebRTC |
| 4 | |
| 5 | /// Something other than a microphone to capture from: the interop |
| 6 | /// harness's synthetic voice, as SyntheticVideo is its camera. Called on |
| 7 | /// the audio engine's render thread with mono float samples in -1...1. |
| 8 | /// On Mac only, where the engine module can run without any device. |
| 9 | public protocol AudioInputSource: AnyObject, Sendable { |
| 10 | func render(_ samples: UnsafeMutablePointer<Float>, count: Int, sampleRate: Double) |
| 11 | } |
| 12 | |
| 13 | /// Which of WebRTC's processing stages run on the microphone, decided in |
| 14 | /// one place so the rule can be read and tested apart from WebRTC. |
| 15 | /// |
| 16 | /// The rule the plan holds: our suppressor never runs beside another one. |
| 17 | /// Not beside WebRTC's own noise suppressor, and not beside Apple's Voice |
| 18 | /// Isolation. And WebRTC's gain control, which runs *before* our |
| 19 | /// suppressor, is off whenever ours is on; SpeechGain lifts the voice |
| 20 | /// after the noise has gone instead. |
| 21 | public struct ProcessingPlan: Equatable, Sendable { |
| 22 | public enum Where: Equatable, Sendable { case off, software, platform } |
| 23 | /// Our neural suppressor, in the capture post-processing hook. |
| 24 | public var suppressor: Bool |
| 25 | public var echoCancellation: Where |
| 26 | public var noiseSuppression: Where |
| 27 | public var gainControl: Where |
| 28 | /// WebRTC's high-pass filter, always: it is cheap and only takes out |
| 29 | /// rumble below the voice. |
| 30 | public var highPass = true |
| 31 | |
| 32 | /// - Parameters: |
| 33 | /// - iPhone: the phone runs on Apple's voice-processing unit, whose |
| 34 | /// echo canceller and noise suppressor are one switch; the Mac runs |
| 35 | /// WebRTC's software processing. |
| 36 | /// - reduceNoise: the person's toggle. |
| 37 | /// - rate: the capture rate WebRTC reported, or nil before it has. |
| 38 | /// - voiceIsolation: Apple's Voice Isolation is the active mic mode. |
| 39 | public static func plan(iPhone: Bool, reduceNoise: Bool, rate: Int?, voiceIsolation: Bool) |
| 40 | -> ProcessingPlan |
| 41 | { |
| 42 | let rateOK = rate.map { $0 == CaptureDenoiser.supportedRate } ?? true |
| 43 | let ours = reduceNoise && rateOK && !voiceIsolation |
| 44 | if iPhone { |
| 45 | // Apple's echo canceller cannot be had without Apple's suppressor |
| 46 | // beside it; that pairing is what the phone has always run. Its gain |
| 47 | // control is separate, and is what comes off. |
| 48 | return ProcessingPlan( |
| 49 | suppressor: ours, echoCancellation: .platform, noiseSuppression: .platform, |
| 50 | gainControl: ours ? .off : .platform) |
| 51 | } |
| 52 | let classicNS = reduceNoise && !rateOK |
| 53 | return ProcessingPlan( |
| 54 | suppressor: ours, echoCancellation: .software, |
| 55 | noiseSuppression: classicNS ? .software : .off, |
| 56 | gainControl: ours ? .off : .software) |
| 57 | } |
| 58 | } |
| 59 | |
| 60 | /// What the audio readout says about processing. |
| 61 | public struct ProcessingReport: Equatable, Sendable { |
| 62 | public var suppression: SuppressionState |
| 63 | public var echoCancellation: String |
| 64 | public var noiseSuppression: String |
| 65 | public var gainControl: String |
| 66 | /// Set when WebRTC refused the options asked of it. |
| 67 | public var refused: String? |
| 68 | /// False until the microphone is actually capturing. WebRTC opens it |
| 69 | /// only once there is someone to send to, so a person alone in a call |
| 70 | /// has no microphone running yet, and what the processing module says |
| 71 | /// before then is its construction-time default (16 kHz, everything |
| 72 | /// off) rather than anything about their microphone. |
| 73 | public var started = true |
| 74 | |
| 75 | /// One or two plain sentences. |
| 76 | public var summary: String { |
| 77 | guard started else { |
| 78 | return "The microphone has not started yet. It starts when someone else is in the call, and this will say what is running on it then." |
| 79 | } |
| 80 | var lines: [String] = [] |
| 81 | switch suppression { |
| 82 | case .off: lines.append("Background noise reduction is off.") |
| 83 | case .running(let engine, let cpu): |
| 84 | lines.append( |
| 85 | "Reducing background noise with \(engine) (\(String(format: "%.1f", cpu * 100))% of real time on the audio thread).") |
| 86 | case .idle(let engine): lines.append("Reducing background noise with \(engine) when you speak.") |
| 87 | case .unsupportedRate(let rate): |
| 88 | lines.append( |
| 89 | "The microphone runs at \(rate / 1000) kHz, which the noise reducer cannot; WebRTC's own is standing in.") |
| 90 | case .standingAside: |
| 91 | lines.append("Voice Isolation is on, so Calls leaves noise reduction to it.") |
| 92 | } |
| 93 | lines.append( |
| 94 | "Echo cancellation: \(echoCancellation). Noise suppression: \(noiseSuppression). Gain: \(gainControl).") |
| 95 | if let refused { lines.append("WebRTC refused the requested processing: \(refused)") } |
| 96 | return lines.joined(separator: " ") |
| 97 | } |
| 98 | } |
| 99 | |
| 100 | /// How a call's audio is built: which audio device module WebRTC runs on, |
| 101 | /// what processing runs on the microphone, and — on Mac — what the app is |
| 102 | /// told about the devices it can use. |
| 103 | /// |
| 104 | /// Kept out of Media.swift so that file only changes where the factory is |
| 105 | /// made. |
| 106 | @MainActor final class CallAudio { |
| 107 | let factory: RTCPeerConnectionFactory |
| 108 | /// The factory's audio device module, fetched once, off the main |
| 109 | /// thread. Fetching it, and every property and call on it, is a blocking |
| 110 | /// hop to WebRTC's worker thread, which is held for the whole of an |
| 111 | /// engine start, a Bluetooth switch included (1,544 ms measured on the |
| 112 | /// main thread with the switch simulated). So it is only ever asked on |
| 113 | /// the audio queue; the main thread reads `state`. |
| 114 | let module: RTCAudioDeviceModule |
| 115 | /// The module's state for the main thread to read: never by asking the |
| 116 | /// module, whose every property waits out an engine start (ModuleState). |
| 117 | let state: ModuleState |
| 118 | /// Nil for the iPhone tests' silent device, which takes the constructor |
| 119 | /// that has no processing module. |
| 120 | let processing: RTCDefaultAudioProcessingModule? |
| 121 | let denoiser: CaptureDenoiser |
| 122 | private weak var track: RTCAudioTrack? |
| 123 | private var rate: Int? |
| 124 | private var voiceIsolation = false |
| 125 | private var refused: String? |
| 126 | private(set) var plan: ProcessingPlan |
| 127 | private var buffersAtStart: UInt64 = 0 |
| 128 | var reduceNoise: Bool { |
| 129 | didSet { if reduceNoise != oldValue { apply() } } |
| 130 | } |
| 131 | |
| 132 | #if os(macOS) |
| 133 | private let devices: EngineWatch |
| 134 | /// Where the engine is put (AudioDevices.swift); read by the tests. |
| 135 | var enginePin: AudioEnginePin { devices.pin } |
| 136 | #endif |
| 137 | |
| 138 | /// What building a CallAudio takes, made off the main thread (`make`): |
| 139 | /// the peer connection factory with its audio device module and |
| 140 | /// processing module, and the capture hook. Making the factory alone |
| 141 | /// took 100–135 ms on an M1 Max. |
| 142 | struct Parts: @unchecked Sendable { |
| 143 | let factory: RTCPeerConnectionFactory |
| 144 | let module: RTCAudioDeviceModule |
| 145 | let state: ModuleState |
| 146 | let processing: RTCDefaultAudioProcessingModule? |
| 147 | let denoiser: CaptureDenoiser |
| 148 | #if os(macOS) |
| 149 | let devices: EngineWatch |
| 150 | #endif |
| 151 | } |
| 152 | |
| 153 | /// Builds the parts. Nonisolated, and slow: `make` runs it on the audio |
| 154 | /// queue. `routes` is only held (weakly) by the module's observer, which |
| 155 | /// reaches it on the main queue. |
| 156 | nonisolated static func parts( |
| 157 | routes: AudioRoutes, defaults: UserDefaults, audioDevice: RTCAudioDevice?, input: AudioInputSource?, |
| 158 | denoiser: CaptureDenoiser? |
| 159 | ) -> Parts { |
| 160 | let encoder = RTCDefaultVideoEncoderFactory() |
| 161 | // WebRTC's own, except in the gate's decoder-wedge scenarios. |
| 162 | let decoder: RTCVideoDecoderFactory = |
| 163 | DecoderWedge.enabled ? DecoderWedge.factory() : RTCDefaultVideoDecoderFactory() |
| 164 | let denoiser = denoiser ?? CaptureDenoiser(suppressor: CallAudio.makeSuppressor()) |
| 165 | #if os(iOS) |
| 166 | if let audioDevice { |
| 167 | let factory = RTCPeerConnectionFactory( |
| 168 | encoderFactory: encoder, decoderFactory: decoder, audioDevice: audioDevice) |
| 169 | let module = factory.audioDeviceModule |
| 170 | return Parts( |
| 171 | factory: factory, module: module, state: ModuleState(factory: factory, module: module), processing: nil, |
| 172 | denoiser: denoiser) |
| 173 | } |
| 174 | let processing = RTCDefaultAudioProcessingModule( |
| 175 | config: nil, capturePostProcessingDelegate: nil, renderPreProcessingDelegate: nil) |
| 176 | // The platform default on iPhone is the voice-processing I/O unit it |
| 177 | // has always been. |
| 178 | let factory = RTCPeerConnectionFactory( |
| 179 | audioDeviceModuleType: .platformDefault, bypassVoiceProcessing: false, |
| 180 | encoderFactory: encoder, decoderFactory: decoder, audioProcessingModule: processing) |
| 181 | let module = factory.audioDeviceModule |
| 182 | return Parts( |
| 183 | factory: factory, module: module, state: ModuleState(factory: factory, module: module), |
| 184 | processing: processing, denoiser: denoiser) |
| 185 | #else |
| 186 | let processing = RTCDefaultAudioProcessingModule( |
| 187 | config: nil, capturePostProcessingDelegate: nil, renderPreProcessingDelegate: nil) |
| 188 | // AVAudioEngine rather than the CoreAudio module: it is the one with a |
| 189 | // device list and a way to choose from it. |
| 190 | let factory = RTCPeerConnectionFactory( |
| 191 | audioDeviceModuleType: .audioEngine, bypassVoiceProcessing: false, |
| 192 | encoderFactory: encoder, decoderFactory: decoder, audioProcessingModule: processing) |
| 193 | let module = factory.audioDeviceModule |
| 194 | // Apple's voice processing stays off on Mac. Left allowed, the engine |
| 195 | // module resolves echo cancellation, noise suppression and gain to |
| 196 | // Apple's coupled voice-processing unit instead of WebRTC's own — |
| 197 | // a different pipeline from the one the Mac app has always had, and |
| 198 | // one whose noise suppression cannot be separated from its echo |
| 199 | // canceller. WebRTC's software processing is what ran before this |
| 200 | // module, and is what runs now. |
| 201 | _ = module.setPlatformVoiceProcessingAllowed(false) |
| 202 | let pin = AudioEnginePin() |
| 203 | let devices = EngineWatch( |
| 204 | routes: routes, module: module, input: input, pin: pin, |
| 205 | channels: InputChannelMapper( |
| 206 | defaults: defaults, layout: pin.inputLayout, explicitInput: { pin.choices.input })) |
| 207 | module.observer = devices |
| 208 | if input != nil { _ = module.setManualRenderingMode(true) } |
| 209 | return Parts( |
| 210 | factory: factory, module: module, state: ModuleState(factory: factory, module: module), |
| 211 | processing: processing, denoiser: denoiser, devices: devices) |
| 212 | #endif |
| 213 | } |
| 214 | |
| 215 | /// Builds a CallAudio with its slow half on the audio queue: the main |
| 216 | /// thread only attaches the result. What a call and the audio test use. |
| 217 | static func make( |
| 218 | routes: AudioRoutes, audioDevice: RTCAudioDevice?, input: AudioInputSource? = nil, |
| 219 | reduceNoise: Bool = true, denoiser: CaptureDenoiser? = nil |
| 220 | ) async -> CallAudio { |
| 221 | #if os(macOS) |
| 222 | let defaults = routes.defaults |
| 223 | #else |
| 224 | let defaults = UserDefaults.standard |
| 225 | #endif |
| 226 | let parts = await AudioWork.run { |
| 227 | CallAudio.parts(routes: routes, defaults: defaults, audioDevice: audioDevice, input: input, denoiser: denoiser) |
| 228 | } |
| 229 | return CallAudio(parts: parts, routes: routes, reduceNoise: reduceNoise) |
| 230 | } |
| 231 | |
| 232 | /// `audioDevice` is the iPhone tests' silent device. It takes the old |
| 233 | /// constructor, which has no audio device module to speak of. |
| 234 | /// |
| 235 | /// `input`, on Mac, replaces the microphone and the speaker both: the |
| 236 | /// engine runs in manual rendering mode, touching no device at all. |
| 237 | /// |
| 238 | /// Builds on the calling thread; the app uses `make`. |
| 239 | convenience init( |
| 240 | routes: AudioRoutes, audioDevice: RTCAudioDevice?, input: AudioInputSource? = nil, |
| 241 | reduceNoise: Bool = true |
| 242 | ) { |
| 243 | #if os(macOS) |
| 244 | let defaults = routes.defaults |
| 245 | #else |
| 246 | let defaults = UserDefaults.standard |
| 247 | #endif |
| 248 | self.init( |
| 249 | parts: CallAudio.parts(routes: routes, defaults: defaults, audioDevice: audioDevice, input: input, denoiser: nil), |
| 250 | routes: routes, reduceNoise: reduceNoise) |
| 251 | } |
| 252 | |
| 253 | /// Attaches built parts: cheap, and on the main actor. |
| 254 | init(parts: Parts, routes: AudioRoutes, reduceNoise: Bool) { |
| 255 | factory = parts.factory |
| 256 | module = parts.module |
| 257 | state = parts.state |
| 258 | processing = parts.processing |
| 259 | denoiser = parts.denoiser |
| 260 | self.reduceNoise = reduceNoise |
| 261 | #if os(iOS) |
| 262 | plan = ProcessingPlan.plan(iPhone: true, reduceNoise: reduceNoise, rate: nil, voiceIsolation: false) |
| 263 | #else |
| 264 | plan = ProcessingPlan.plan(iPhone: false, reduceNoise: reduceNoise, rate: nil, voiceIsolation: false) |
| 265 | devices = parts.devices |
| 266 | // Attached in manual rendering too, so the interop harness drives the |
| 267 | // same routing code as the app; no device is opened either way. |
| 268 | let watch = devices |
| 269 | routes.attach(module, state: state, pin: devices.pin, inputRate: { watch.lastInputRate }) |
| 270 | #endif |
| 271 | guard let processing else { return } |
| 272 | // The listener first, then the delegate: attaching the delegate can |
| 273 | // initialise it at once, and that report used to go nowhere. |
| 274 | denoiser.rateChanged = { [weak self] rate in |
| 275 | DispatchQueue.main.async { |
| 276 | MainActor.assumeIsolated { |
| 277 | guard let self, self.rate != rate else { return } |
| 278 | self.rate = rate |
| 279 | self.apply() |
| 280 | } |
| 281 | } |
| 282 | } |
| 283 | // Set after the factory exists: handed to the module's initialiser, the |
| 284 | // delegate is never called (seen in the spike, M150). The property is |
| 285 | // weak; `denoiser` is what keeps it alive. |
| 286 | processing.capturePostProcessingDelegate = denoiser |
| 287 | upgraded = CallAudio.upgrade(denoiser) |
| 288 | denoiser.enabled = plan.suppressor |
| 289 | } |
| 290 | |
| 291 | /// Whether this CallAudio's suppressor is (or is becoming) DeepFilterNet, |
| 292 | /// so a fresh spare is loaded for the next one when it is done. |
| 293 | private var upgraded = false |
| 294 | |
| 295 | deinit { |
| 296 | #if canImport(DeepFilter) |
| 297 | if upgraded { DeepFilterPreload.finished() } |
| 298 | #endif |
| 299 | // The factory goes on the audio queue, after everything queued for |
| 300 | // it: a stop or a close queued there must never find its module gone |
| 301 | // (it did, in the harness: a crash in `stopRecording`). Its teardown, |
| 302 | // which stops the engine, then happens there too, not on the main |
| 303 | // thread. |
| 304 | let held: (RTCPeerConnectionFactory, RTCAudioDeviceModule) = (factory, module) |
| 305 | AudioWork.post { withExtendedLifetime(held) {} } |
| 306 | } |
| 307 | |
| 308 | /// Whether the microphone is capturing. The module opens it when the |
| 309 | /// first connection starts sending, not when the track is made. Either |
| 310 | /// the module says it is recording, or the capture hook has been handed |
| 311 | /// audio since this call's track was made (which is also how the |
| 312 | /// interop gate's manual rendering shows up). |
| 313 | var capturing: Bool { |
| 314 | #if os(macOS) |
| 315 | return state.read().isRecording || denoiser.buffersSeen > buffersAtStart |
| 316 | #else |
| 317 | // Not asked on iPhone: its module is the platform one, whose |
| 318 | // recording state has not been checked against a device here. |
| 319 | return true |
| 320 | #endif |
| 321 | } |
| 322 | |
| 323 | /// Whether the capture hook has been handed audio since this call's |
| 324 | /// track was made: the microphone meter's "capturing" on iPhone, where |
| 325 | /// `capturing` cannot tell a microphone that has not started (alone in a |
| 326 | /// call) from one that has, and a warning about silence from a |
| 327 | /// microphone nobody has opened would be wrong. |
| 328 | var hookFlowing: Bool { denoiser.buffersSeen > buffersAtStart } |
| 329 | |
| 330 | /// Set while this CallAudio asked the module to record by itself (the |
| 331 | /// audio test before a call); undone by `stopRecordingAlone`. |
| 332 | private var recordingAlone = false |
| 333 | /// Whether the module really was started by this CallAudio's ask, as |
| 334 | /// opposed to already recording. Only read and written on the audio |
| 335 | /// queue, where the start and the stop run in order. |
| 336 | private final class Started: @unchecked Sendable { var byUs = false } |
| 337 | private var started = Started() |
| 338 | |
| 339 | /// Opens the microphone outside a call, as a call's first sending stream |
| 340 | /// does: the module's `initRecording` then `startRecording`, which is |
| 341 | /// exactly what WebRTC's AudioState::AddSendingStream calls. |
| 342 | /// |
| 343 | /// The audio test before a call needs this. Its in-memory loopback is |
| 344 | /// never connected, and WebRTC only starts a send stream on a |
| 345 | /// connection whose transport has been writable (BaseChannel:: |
| 346 | /// IsReadyToSendMedia_w, `was_ever_writable_`), so up to 0.3.7 the test |
| 347 | /// never opened the microphone on a real device: an empty meter and |
| 348 | /// "We can't hear anything", for every microphone (Paul's AirPods, |
| 349 | /// 2026-09-26). The interop harness did not see it: in manual rendering |
| 350 | /// the module runs its input whenever its output runs. |
| 351 | /// |
| 352 | /// On the audio queue, never the main thread (0.3.9): `initRecording` |
| 353 | /// waits on WebRTC's worker thread while the engine starts, and with a |
| 354 | /// Bluetooth headset that includes its move to the hands-free profile, |
| 355 | /// one to two seconds. In 0.3.8 the main thread waited with it. |
| 356 | /// |
| 357 | /// - Parameter ask: false only for the harness's control run, which |
| 358 | /// shows that without it the module is never asked to record. |
| 359 | /// - Returns: 0, or the module's first error (-9000: no microphone |
| 360 | /// permission, as the module sees it). |
| 361 | @discardableResult |
| 362 | func startRecordingAlone(ask: Bool = true) async -> Int { |
| 363 | guard ask else { |
| 364 | AudioLog.info("test: not asking the module to record (control run)") |
| 365 | return 0 |
| 366 | } |
| 367 | let module = self.module |
| 368 | // Set before the wait, so a stop asked for during it is queued, and |
| 369 | // runs after the start, and closes what it opened. |
| 370 | recordingAlone = true |
| 371 | let started = self.started |
| 372 | let began = CaptureDenoiser.now() |
| 373 | let (initialised, result, recording, already) = await AudioWork.run { () -> (Int, Int, Bool, Bool) in |
| 374 | guard !module.isRecording else { return (0, 0, true, true) } |
| 375 | let initialised = module.initRecording() |
| 376 | let result = initialised == 0 ? module.startRecording() : 0 |
| 377 | started.byUs = true |
| 378 | return (initialised, result, module.isRecording, false) |
| 379 | } |
| 380 | if already { |
| 381 | AudioLog.info("test: the module is already recording") |
| 382 | } else { |
| 383 | AudioLog.info( |
| 384 | "test: asked the module to record (init \(initialised), start \(result)) in \(Int((CaptureDenoiser.now() - began) * 1000)) ms off the main thread; recording \(recording)" |
| 385 | ) |
| 386 | } |
| 387 | return initialised != 0 ? initialised : result |
| 388 | } |
| 389 | |
| 390 | /// A module error, in words, for the audio test. |
| 391 | static func recordingError(_ code: Int) -> String { |
| 392 | switch code { |
| 393 | case -9000, -4005, -2003: return AudioSelfTest.noPermission |
| 394 | case -4010, -2000, -2001, -2002: |
| 395 | return "The microphone would not start: macOS says it isn’t available (error \(code))." |
| 396 | case -2004: return "The microphone would not start: another app is using it (error \(code))." |
| 397 | default: return "The microphone would not start (error \(code))." |
| 398 | } |
| 399 | } |
| 400 | |
| 401 | /// Closes what `startRecordingAlone` opened, on the audio queue after |
| 402 | /// it: never waited for here (the engine's stop can take as long as its |
| 403 | /// start, a headset moving back to A2DP). |
| 404 | func stopRecordingAlone() { |
| 405 | guard recordingAlone else { return } |
| 406 | recordingAlone = false |
| 407 | let module = self.module |
| 408 | let factory = self.factory |
| 409 | let started = self.started |
| 410 | self.started = Started() |
| 411 | AudioWork.post { |
| 412 | // The module is only as alive as its factory. |
| 413 | withExtendedLifetime(factory) {} |
| 414 | guard started.byUs else { return } |
| 415 | started.byUs = false |
| 416 | let result = module.stopRecording() |
| 417 | AudioLog.info("test: stopped the module's recording (\(result))") |
| 418 | } |
| 419 | } |
| 420 | |
| 421 | /// Whether the module has been asked to enable its microphone since |
| 422 | /// this CallAudio was made (its "will enable" hook ran with recording |
| 423 | /// on). Asked, not granted: without microphone permission the module |
| 424 | /// refuses right after. The harness's proof that the test opens the |
| 425 | /// microphone on a real device. |
| 426 | var recordingRequested: Bool { |
| 427 | #if os(macOS) |
| 428 | return devices.recordingRequested |
| 429 | #else |
| 430 | return true |
| 431 | #endif |
| 432 | } |
| 433 | |
| 434 | /// The suppressor a call starts on. See `upgrade`. |
| 435 | nonisolated static func makeSuppressor() -> NoiseSuppressor { RNNoiseSuppressor() } |
| 436 | |
| 437 | /// DeepFilterNet 3 on Mac, where the app carries its model: loaded off |
| 438 | /// the main thread, then swapped in. RNNoise otherwise, and always on |
| 439 | /// iPhone (DeepFilterNet's cost there has not been measured on a phone). |
| 440 | /// |
| 441 | /// The model preloaded after launch is taken when there is one |
| 442 | /// (DeepFilterPreload), so the suppressor is DeepFilterNet from the |
| 443 | /// first buffer. Returns whether it is, or will be, DeepFilterNet. |
| 444 | private static func upgrade(_ denoiser: CaptureDenoiser) -> Bool { |
| 445 | #if canImport(DeepFilter) |
| 446 | if denoiser.engineName == "DeepFilterNet 3" { return true } |
| 447 | if let ready = DeepFilterPreload.take() { |
| 448 | denoiser.replace(with: ready) |
| 449 | return true |
| 450 | } |
| 451 | guard let model = DeepFilterSuppressor.model else { return false } |
| 452 | DispatchQueue.global(qos: .userInitiated).async { |
| 453 | if let deep = DeepFilterSuppressor(model: model) { denoiser.replace(with: deep) } |
| 454 | } |
| 455 | return true |
| 456 | #else |
| 457 | return false |
| 458 | #endif |
| 459 | } |
| 460 | |
| 461 | /// The microphone track, made here so its processing options are set |
| 462 | /// before it is ever sent. |
| 463 | func makeTrack() -> RTCAudioTrack { |
| 464 | let track = factory.audioTrack(withTrackId: "microphone") |
| 465 | self.track = track |
| 466 | buffersAtStart = denoiser.buffersSeen |
| 467 | apply() |
| 468 | return track |
| 469 | } |
| 470 | |
| 471 | /// Re-reads Apple's mic mode and reapplies the plan if anything moved. |
| 472 | /// Called on the audio-health tick. |
| 473 | func refresh() { |
| 474 | let isolating = AVCaptureDevice.activeMicrophoneMode == .voiceIsolation |
| 475 | if isolating != voiceIsolation { |
| 476 | voiceIsolation = isolating |
| 477 | apply() |
| 478 | } |
| 479 | } |
| 480 | |
| 481 | private func apply() { |
| 482 | #if os(iOS) |
| 483 | let iPhone = true |
| 484 | #else |
| 485 | let iPhone = false |
| 486 | #endif |
| 487 | plan = ProcessingPlan.plan( |
| 488 | iPhone: iPhone, reduceNoise: reduceNoise, rate: rate, voiceIsolation: voiceIsolation) |
| 489 | denoiser.enabled = reduceNoise |
| 490 | denoiser.standAside = voiceIsolation |
| 491 | guard let track, processing != nil else { return } |
| 492 | // Set on the audio queue: setting a track's processing waits out an |
| 493 | // engine start (1,061 ms measured on the main thread with a Bluetooth |
| 494 | // switch simulated), and a rate change — which calls this — arrives |
| 495 | // exactly then. Queued ahead of anything that sends the track, so the |
| 496 | // options are still in place before it is ever sent. |
| 497 | let plan = self.plan |
| 498 | let factory = self.factory |
| 499 | applied += 1 |
| 500 | let generation = applied |
| 501 | AudioWork.post { |
| 502 | withExtendedLifetime(factory) {} |
| 503 | var refused: String? |
| 504 | let result = track.setAudioProcessingOptions(CallAudio.options(plan)) |
| 505 | refused = result.isSuccess ? nil : result.message |
| 506 | // Platform modes can be refused where Apple's unit cannot give them |
| 507 | // (a simulator, a route without it). Then let WebRTC choose where each |
| 508 | // enabled stage runs; what was off stays off, so the plan's rule — |
| 509 | // never two suppressors, never gain before ours — still holds. |
| 510 | if !result.isSuccess { |
| 511 | let fallback = track.setAudioProcessingOptions(CallAudio.options(plan, automatic: true)) |
| 512 | if !fallback.isSuccess { refused = fallback.message } |
| 513 | } |
| 514 | DispatchQueue.main.async { |
| 515 | MainActor.assumeIsolated { [weak self] in |
| 516 | guard let self, self.applied == generation else { return } |
| 517 | self.refused = refused |
| 518 | } |
| 519 | } |
| 520 | } |
| 521 | } |
| 522 | private var applied = 0 |
| 523 | |
| 524 | static func options(_ plan: ProcessingPlan, automatic: Bool = false) -> RTCAudioProcessingOptions { |
| 525 | func component(_ place: ProcessingPlan.Where) -> RTCAudioProcessingComponentOptions { |
| 526 | switch place { |
| 527 | case .off: return RTCAudioProcessingComponentOptions(enabled: false) |
| 528 | case .software: |
| 529 | return RTCAudioProcessingComponentOptions(enabled: true, mode: automatic ? .automatic : .software) |
| 530 | case .platform: |
| 531 | return RTCAudioProcessingComponentOptions(enabled: true, mode: automatic ? .automatic : .platform) |
| 532 | } |
| 533 | } |
| 534 | return RTCAudioProcessingOptions( |
| 535 | echoCancellationOptions: component(plan.echoCancellation), |
| 536 | noiseSuppressionOptions: component(plan.noiseSuppression), |
| 537 | autoGainControlOptions: component(plan.gainControl), |
| 538 | highPassFilterOptions: component(plan.highPass ? .software : .off)) |
| 539 | } |
| 540 | |
| 541 | /// For "Copy audio diagnostics": the module, the engine, the hook. |
| 542 | func diagnostics() -> [String] { |
| 543 | var lines: [String] = [] |
| 544 | let module = state.read() |
| 545 | #if os(macOS) |
| 546 | lines.append( |
| 547 | "module: manual rendering \(module.isManualRenderingMode), engine running \(module.isEngineRunning), playout initialised \(module.isPlayoutInitialized), playing \(module.isPlaying), recording initialised \(module.isRecordingInitialized), recording \(module.isRecording), microphone muted \(module.isMicrophoneMuted)\(module.stale ? " (as last read: the module is busy)" : "")" |
| 548 | ) |
| 549 | let choices = enginePin.choices |
| 550 | lines.append( |
| 551 | "engine pin: chosen input \(choices.input ?? "default"), output \(choices.output ?? "default"); last applied \(enginePin.lastApplied?.description ?? "none yet"); would apply now \(enginePin.target().description)" |
| 552 | ) |
| 553 | lines.append("recording requested since made: \(devices.recordingRequested)") |
| 554 | lines.append(contentsOf: devices.history()) |
| 555 | #else |
| 556 | lines.append("module: playing \(module.isPlaying), recording \(module.isRecording)") |
| 557 | #endif |
| 558 | let hook = denoiser.format |
| 559 | lines.append( |
| 560 | "capture hook: \(hook.initialised ? "initialised" : "not initialised") at \(hook.rate) Hz, \(hook.channels) ch; buffers \(denoiser.buffersSeen) in all, \(denoiser.buffersSeen - buffersAtStart) since this microphone track; frames suppressed \(denoiser.framesProcessed)" |
| 561 | ) |
| 562 | lines.append("suppressor: \(denoiser.engineName), \(String(describing: denoiser.state)); reduce background noise \(reduceNoise)") |
| 563 | lines.append("plan: \(String(describing: plan))") |
| 564 | lines.append("processing: \(report().summary)") |
| 565 | return lines |
| 566 | } |
| 567 | |
| 568 | /// What is actually running, as WebRTC reports it — not what was asked. |
| 569 | func report() -> ProcessingReport { |
| 570 | guard let state = state.read().processing else { |
| 571 | return ProcessingReport( |
| 572 | suppression: denoiser.state, echoCancellation: "unknown", noiseSuppression: "unknown", |
| 573 | gainControl: "unknown", refused: refused, started: capturing) |
| 574 | } |
| 575 | func says(_ component: RTCAudioProcessingComponentState) -> String { |
| 576 | switch component.effective { |
| 577 | case .disabled: return "off" |
| 578 | case .software: return "WebRTC" |
| 579 | case .platform: return "Apple" |
| 580 | case .softwareAndPlatform: return "WebRTC and Apple" |
| 581 | default: return "unknown" |
| 582 | } |
| 583 | } |
| 584 | var gain = says(state.autoGainControl) |
| 585 | if plan.suppressor, gain == "off" { gain = "Calls, after noise reduction" } |
| 586 | return ProcessingReport( |
| 587 | suppression: denoiser.state, echoCancellation: says(state.echoCancellation), |
| 588 | noiseSuppression: says(state.noiseSuppression), gainControl: gain, refused: refused, |
| 589 | started: capturing) |
| 590 | } |
| 591 | } |
| 592 | |
| 593 | #if os(macOS) |
| 594 | /// The module's observer. The device-list callback keeps the pickers |
| 595 | /// current; "will enable" is where the engine is put on the chosen |
| 596 | /// devices, before the module reads a format from it (AudioDevices.swift); |
| 597 | /// configuring the input is where a synthetic source is wired in, or — |
| 598 | /// for a real microphone — where the chosen input of a multi-channel one |
| 599 | /// is mapped (InputChannel.swift). Starting and stopping are watched so a |
| 600 | /// device the engine cannot start on is noticed rather than left silent. |
| 601 | final class EngineWatch: NSObject, RTCAudioDeviceModuleDelegate, @unchecked Sendable { |
| 602 | weak var routes: AudioRoutes? |
| 603 | weak var module: RTCAudioDeviceModule? |
| 604 | let input: AudioInputSource? |
| 605 | let pin: AudioEnginePin |
| 606 | let channels: InputChannelMapper |
| 607 | private var node: AVAudioSourceNode? |
| 608 | private let lock = NSLock() |
| 609 | /// Starts and stops seen, so a start that is followed by neither a |
| 610 | /// running engine nor a deliberate stop is known to have failed. |
| 611 | private var starts = 0 |
| 612 | private var stops = 0 |
| 613 | private var requested = false |
| 614 | /// The input format of the last engine start, for the rate watch. |
| 615 | private var inputRate: Double? |
| 616 | /// The last few engine events, for the diagnostics. |
| 617 | private var events: [String] = [] |
| 618 | |
| 619 | var recordingRequested: Bool { |
| 620 | lock.lock() |
| 621 | defer { lock.unlock() } |
| 622 | return requested |
| 623 | } |
| 624 | |
| 625 | /// The input rate the engine last started with; nil when it last |
| 626 | /// started for playout only. |
| 627 | var lastInputRate: Double? { |
| 628 | lock.lock() |
| 629 | defer { lock.unlock() } |
| 630 | return inputRate |
| 631 | } |
| 632 | |
| 633 | func history() -> [String] { |
| 634 | lock.lock() |
| 635 | defer { lock.unlock() } |
| 636 | return events.isEmpty ? ["engine: no events yet"] : events.map { "engine: \($0)" } |
| 637 | } |
| 638 | |
| 639 | private func note(_ event: String) { |
| 640 | lock.lock() |
| 641 | events.append("\(AudioDiagnosticsStore.stamp(Date())) \(event)") |
| 642 | if events.count > 12 { events.removeFirst(events.count - 12) } |
| 643 | lock.unlock() |
| 644 | } |
| 645 | init( |
| 646 | routes: AudioRoutes, module: RTCAudioDeviceModule, input: AudioInputSource?, pin: AudioEnginePin, |
| 647 | channels: InputChannelMapper |
| 648 | ) { |
| 649 | self.routes = routes |
| 650 | self.module = module |
| 651 | self.input = input |
| 652 | self.pin = pin |
| 653 | self.channels = channels |
| 654 | } |
| 655 | |
| 656 | func audioDeviceModuleDidUpdateDevices(_ audioDeviceModule: RTCAudioDeviceModule) { |
| 657 | DispatchQueue.main.async { [weak self] in |
| 658 | MainActor.assumeIsolated { self?.routes?.refresh() } |
| 659 | } |
| 660 | } |
| 661 | |
| 662 | func audioDeviceModule( |
| 663 | _ audioDeviceModule: RTCAudioDeviceModule, willEnableEngine engine: AVAudioEngine, |
| 664 | isPlayoutEnabled: Bool, isRecordingEnabled: Bool, isVoiceProcessingEnabled: Bool |
| 665 | ) -> Int { |
| 666 | if isRecordingEnabled { |
| 667 | lock.lock() |
| 668 | requested = true |
| 669 | lock.unlock() |
| 670 | // The harness's stand-in for a Bluetooth headset changing profile, |
| 671 | // which holds up the engine's start here, on WebRTC's worker |
| 672 | // thread. Zero in the app. |
| 673 | let delay = AudioStartProbe.engineStartDelay |
| 674 | if delay > 0, input == nil { |
| 675 | AudioLog.info("harness: holding the engine's start \(Int(delay * 1000)) ms, as a Bluetooth switch does") |
| 676 | Thread.sleep(forTimeInterval: delay) |
| 677 | } |
| 678 | } |
| 679 | note("will enable: playout \(isPlayoutEnabled), recording \(isRecordingEnabled), voice processing \(isVoiceProcessingEnabled)") |
| 680 | guard input == nil, !engine.isInManualRenderingMode else { |
| 681 | AudioLog.info( |
| 682 | "engine will enable (playout \(isPlayoutEnabled), recording \(isRecordingEnabled)): manual rendering, no device") |
| 683 | return 0 |
| 684 | } |
| 685 | let outcome = pin.apply(to: engine, recording: isRecordingEnabled) |
| 686 | switch outcome { |
| 687 | case .applied(let target): |
| 688 | AudioLog.info( |
| 689 | "engine will enable (playout \(isPlayoutEnabled), recording \(isRecordingEnabled)): on \(target.description)") |
| 690 | case .failed(let target, let why): |
| 691 | AudioLog.error( |
| 692 | "engine will enable: could not put it on \(target.description): \(why)") |
| 693 | DispatchQueue.main.async { [weak self] in |
| 694 | MainActor.assumeIsolated { self?.routes?.engineFailed(target, reason: why) } |
| 695 | } |
| 696 | } |
| 697 | return 0 |
| 698 | } |
| 699 | func audioDeviceModule( |
| 700 | _ audioDeviceModule: RTCAudioDeviceModule, |
| 701 | didReceiveSpeechActivityEvent speechActivityEvent: RTCSpeechActivityEvent |
| 702 | ) {} |
| 703 | func audioDeviceModule(_ audioDeviceModule: RTCAudioDeviceModule, didCreateEngine engine: AVAudioEngine) |
| 704 | -> Int |
| 705 | { 0 } |
| 706 | func audioDeviceModule( |
| 707 | _ audioDeviceModule: RTCAudioDeviceModule, willStartEngine engine: AVAudioEngine, |
| 708 | isPlayoutEnabled: Bool, isRecordingEnabled: Bool |
| 709 | ) -> Int { |
| 710 | // By now the engine has connected the input and set the unit's |
| 711 | // client format, so the map is sized to what it will run with. |
| 712 | // Only with the microphone in use: asking for `inputNode` makes one. |
| 713 | guard input == nil, !engine.isInManualRenderingMode else { return 0 } |
| 714 | if isRecordingEnabled { channels.apply(to: engine.inputNode) } |
| 715 | let unit = isRecordingEnabled ? engine.inputNode.audioUnit : engine.outputNode.audioUnit |
| 716 | if isPlayoutEnabled, let unit { pin.mapOutput(on: unit) } |
| 717 | let actual = unit.flatMap(pin.actualInput) |
| 718 | let target = pin.lastApplied ?? .systemDefault |
| 719 | let output = engine.outputNode.outputFormat(forBus: 0) |
| 720 | let device = unit.flatMap(CoreAudioDevices.current(of:)) |
| 721 | let on = device.flatMap(CoreAudioDevices.uid) ?? "unknown" |
| 722 | let rate = device.map(CoreAudioDevices.sampleRate) ?? 0 |
| 723 | let line: String |
| 724 | if isRecordingEnabled { |
| 725 | let format = engine.inputNode.inputFormat(forBus: 0) |
| 726 | line = |
| 727 | "engine starting on \(target.description) (unit on \(on), \(Int(rate)) Hz): microphone \(actual ?? "unknown"), \(format.channelCount) ch at \(Int(format.sampleRate)) Hz; output \(output.channelCount) ch at \(Int(output.sampleRate)) Hz" |
| 728 | lock.lock() |
| 729 | inputRate = format.sampleRate |
| 730 | lock.unlock() |
| 731 | } else { |
| 732 | line = |
| 733 | "engine starting on \(target.description) (unit on \(on), \(Int(rate)) Hz) for playout only; output \(output.channelCount) ch at \(Int(output.sampleRate)) Hz" |
| 734 | lock.lock() |
| 735 | inputRate = nil |
| 736 | lock.unlock() |
| 737 | } |
| 738 | AudioLog.info(line) |
| 739 | note(line) |
| 740 | lock.lock() |
| 741 | starts += 1 |
| 742 | let seen = (starts, stops) |
| 743 | lock.unlock() |
| 744 | DispatchQueue.main.async { [weak self] in |
| 745 | MainActor.assumeIsolated { |
| 746 | if isRecordingEnabled { self?.routes?.engineStarted(input: actual) } |
| 747 | } |
| 748 | } |
| 749 | // The module retries a start a few times and then gives up without |
| 750 | // telling its observer. Looked at once it has had time: an engine |
| 751 | // that is not running, with no stop since, did not start. |
| 752 | // Asked on the audio queue: the module's answer waits out any start |
| 753 | // in progress. |
| 754 | AudioWork.queue.asyncAfter(deadline: .now() + 2) { [weak self] in |
| 755 | guard let self, let module = self.module else { return } |
| 756 | self.lock.lock() |
| 757 | let now = (self.starts, self.stops) |
| 758 | self.lock.unlock() |
| 759 | guard now == seen, !module.isEngineRunning else { return } |
| 760 | AudioLog.error("engine did not start on \(target.description)") |
| 761 | self.note("did not start on \(target.description)") |
| 762 | DispatchQueue.main.async { [weak self] in |
| 763 | MainActor.assumeIsolated { |
| 764 | self?.routes?.engineFailed(target, reason: "the audio engine would not start on it") |
| 765 | } |
| 766 | } |
| 767 | } |
| 768 | return 0 |
| 769 | } |
| 770 | func audioDeviceModule( |
| 771 | _ audioDeviceModule: RTCAudioDeviceModule, didStopEngine engine: AVAudioEngine, |
| 772 | isPlayoutEnabled: Bool, isRecordingEnabled: Bool |
| 773 | ) -> Int { |
| 774 | note("stopped (playout \(isPlayoutEnabled), recording \(isRecordingEnabled) next)") |
| 775 | stopped() |
| 776 | return 0 |
| 777 | } |
| 778 | func audioDeviceModule( |
| 779 | _ audioDeviceModule: RTCAudioDeviceModule, didDisableEngine engine: AVAudioEngine, |
| 780 | isPlayoutEnabled: Bool, isRecordingEnabled: Bool |
| 781 | ) -> Int { 0 } |
| 782 | func audioDeviceModule( |
| 783 | _ audioDeviceModule: RTCAudioDeviceModule, willReleaseEngine engine: AVAudioEngine |
| 784 | ) -> Int { |
| 785 | note("released") |
| 786 | stopped() |
| 787 | return 0 |
| 788 | } |
| 789 | private func stopped() { |
| 790 | lock.lock() |
| 791 | stops += 1 |
| 792 | lock.unlock() |
| 793 | } |
| 794 | func audioDeviceModule( |
| 795 | _ audioDeviceModule: RTCAudioDeviceModule, engine: AVAudioEngine, |
| 796 | configureInputFromSource source: AVAudioNode?, toDestination destination: AVAudioNode, |
| 797 | format: AVAudioFormat, context: [AnyHashable: Any] |
| 798 | ) -> Int { |
| 799 | // A real microphone: map the chosen input and connect nothing. With |
| 800 | // nothing connected to the mixer the module makes its own default |
| 801 | // connection, input node to mixer in mono (audio_engine_device.mm, |
| 802 | // "Nothing connected to input mixer, connecting input node"). |
| 803 | guard let input else { |
| 804 | channels.apply(to: source) |
| 805 | return 0 |
| 806 | } |
| 807 | guard let mono = AVAudioFormat(standardFormatWithSampleRate: format.sampleRate, channels: 1) |
| 808 | else { return 0 } |
| 809 | if let node { engine.detach(node) } |
| 810 | let rate = format.sampleRate |
| 811 | let node = AVAudioSourceNode(format: mono) { _, _, frames, buffers in |
| 812 | for buffer in UnsafeMutableAudioBufferListPointer(buffers) { |
| 813 | guard let data = buffer.mData else { continue } |
| 814 | input.render(data.assumingMemoryBound(to: Float.self), count: Int(frames), sampleRate: rate) |
| 815 | } |
| 816 | return noErr |
| 817 | } |
| 818 | self.node = node |
| 819 | engine.attach(node) |
| 820 | let mixer = |
| 821 | engine.isInManualRenderingMode |
| 822 | ? engine.mainMixerNode |
| 823 | : (context[kLKRTCAudioEngineInputMixerNodeKey] as? AVAudioMixerNode) ?? engine.mainMixerNode |
| 824 | engine.connect(node, to: mixer, format: mono) |
| 825 | return 0 |
| 826 | } |
| 827 | func audioDeviceModule( |
| 828 | _ audioDeviceModule: RTCAudioDeviceModule, engine: AVAudioEngine, |
| 829 | configureOutputFromSource source: AVAudioNode, toDestination destination: AVAudioNode?, |
| 830 | format: AVAudioFormat, context: [AnyHashable: Any] |
| 831 | ) -> Int { 0 } |
| 832 | } |
| 833 | #endif |
| 834 | |
| 835 | /// The audio device module's state, for the main thread to read. |
| 836 | /// |
| 837 | /// Every property of LiveKit's module is a blocking hop to WebRTC's worker |
| 838 | /// thread, and the worker holds on to it for the whole of an engine start |
| 839 | /// or rebuild — a Bluetooth headset's profile switch included. Measured |
| 840 | /// with that switch simulated at 1.5 s: `isRecording`, `isEngineRunning`, |
| 841 | /// `isPlaying`, the device lists and the factory's processing state each |
| 842 | /// held the main thread for 0.9 to 1.2 s. The meters read it ten times a |
| 843 | /// second. |
| 844 | /// |
| 845 | /// So the main thread never asks the module. It asks this, which reads the |
| 846 | /// module on a queue of its own and waits at most a few milliseconds for |
| 847 | /// the answer; if the module is busy it gets the last answer, marked stale, |
| 848 | /// and the fresh one is there for the next look. |
| 849 | final class ModuleState: @unchecked Sendable { |
| 850 | struct Snapshot: @unchecked Sendable { |
| 851 | var isRecording = false |
| 852 | var isPlaying = false |
| 853 | var isEngineRunning = false |
| 854 | var isManualRenderingMode = false |
| 855 | var isPlayoutInitialized = false |
| 856 | var isRecordingInitialized = false |
| 857 | var isMicrophoneMuted = false |
| 858 | var processing: RTCAudioProcessingState? |
| 859 | /// The module did not answer in time: this is its last answer. |
| 860 | var stale = false |
| 861 | } |
| 862 | |
| 863 | private weak var factory: RTCPeerConnectionFactory? |
| 864 | private weak var module: RTCAudioDeviceModule? |
| 865 | private let queue = DispatchQueue(label: "net.amadan.oficina.calls.audio-state", qos: .userInitiated) |
| 866 | private let lock = NSLock() |
| 867 | private var latest = Snapshot() |
| 868 | private var reading: DispatchGroup? |
| 869 | /// How long a read on the main thread may wait for the module. |
| 870 | static let patience: TimeInterval = 0.005 |
| 871 | |
| 872 | /// Reads once, here and now: made off the main thread with the parts. |
| 873 | init(factory: RTCPeerConnectionFactory, module: RTCAudioDeviceModule) { |
| 874 | self.factory = factory |
| 875 | self.module = module |
| 876 | latest = ModuleState.snapshot(factory: factory, module: module) |
| 877 | } |
| 878 | |
| 879 | func read(patience: TimeInterval = ModuleState.patience) -> Snapshot { |
| 880 | lock.lock() |
| 881 | let group: DispatchGroup |
| 882 | if let reading { |
| 883 | group = reading |
| 884 | } else { |
| 885 | group = DispatchGroup() |
| 886 | group.enter() |
| 887 | reading = group |
| 888 | queue.async { [self] in |
| 889 | let fresh = factory.flatMap { f in module.map { ModuleState.snapshot(factory: f, module: $0) } } |
| 890 | lock.lock() |
| 891 | if let fresh { latest = fresh } |
| 892 | reading = nil |
| 893 | lock.unlock() |
| 894 | group.leave() |
| 895 | } |
| 896 | } |
| 897 | lock.unlock() |
| 898 | let answered = group.wait(timeout: .now() + patience) == .success |
| 899 | lock.lock() |
| 900 | defer { lock.unlock() } |
| 901 | var out = latest |
| 902 | out.stale = !answered |
| 903 | return out |
| 904 | } |
| 905 | |
| 906 | private static func snapshot(factory: RTCPeerConnectionFactory, module: RTCAudioDeviceModule) -> Snapshot { |
| 907 | var s = Snapshot() |
| 908 | s.isRecording = module.isRecording |
| 909 | s.isPlaying = module.isPlaying |
| 910 | #if os(macOS) |
| 911 | s.isEngineRunning = module.isEngineRunning |
| 912 | s.isManualRenderingMode = module.isManualRenderingMode |
| 913 | s.isPlayoutInitialized = module.isPlayoutInitialized |
| 914 | s.isRecordingInitialized = module.isRecordingInitialized |
| 915 | s.isMicrophoneMuted = module.isMicrophoneMuted |
| 916 | #endif |
| 917 | s.processing = factory.audioProcessingState |
| 918 | return s |
| 919 | } |
| 920 | } |
| 921 | |
amadan