| 1 | import AVFoundation |
| 2 | import Foundation |
| 3 | import LiveKitWebRTC |
| 4 | |
| 5 | #if os(macOS) |
| 6 | import AudioToolbox |
| 7 | import CoreAudio |
| 8 | #endif |
| 9 | |
| 10 | // "Test audio": the microphone, the speaker, and a recording played back, |
| 11 | // all through the path a call uses. |
| 12 | // |
| 13 | // Where the microphone is measured. The meter and the verdict read the |
| 14 | // capture hook's own meters (CaptureDenoiser.level and .inputLevel), and |
| 15 | // the recording is taken from its sink: the audio after WebRTC's audio |
| 16 | // processing (echo canceller, high-pass filter) and after our noise |
| 17 | // reduction and gain — the audio a call encodes and sends, and nothing |
| 18 | // nearer the microphone than that. |
| 19 | // |
| 20 | // How the microphone is started outside a call. A CallAudio is built |
| 21 | // exactly as a call builds one (LiveKit's AVAudioEngine module on Mac, the |
| 22 | // platform module on iPhone, WebRTC's processing module with the |
| 23 | // CaptureDenoiser as its capture post-processing delegate, the same |
| 24 | // ProcessingPlan and the person's Reduce background noise setting), a |
| 25 | // microphone track is made on it as a call makes one, and the track is |
| 26 | // sent from one peer connection to another in this process |
| 27 | // (SelfTestLoopback): negotiated in memory, never connected, nothing |
| 28 | // leaving the device. That starts the microphone the way a call does, and |
| 29 | // the capture runs through the processing module and the hook exactly as |
| 30 | // it would in a call. Asking the module to record on its own |
| 31 | // (`initAndStartRecording`) was tried first and is not enough: the module |
| 32 | // runs, but the hook sees no buffers. The path is checked end to end by |
| 33 | // `CallsInterop --audio-selftest`, which runs it in manual rendering with |
| 34 | // the synthetic voice and requires the recording to be non-silent (and a |
| 35 | // silent input to be called silent). |
| 36 | // |
| 37 | // During a call nothing is started or stopped: the test reads the running |
| 38 | // call's meters and does not touch its audio. It does not play anything |
| 39 | // either — a sound played beside a call is heard by its microphone and |
| 40 | // sent to everyone, so the speaker test and the recording are for before |
| 41 | // a call, and say so. |
| 42 | |
| 43 | /// Where the test's microphone audio comes from. The standalone capture |
| 44 | /// outside a call, the call's own inside one, and a fake in the tests. |
| 45 | @MainActor protocol AudioSelfTestCapture: AnyObject { |
| 46 | var denoiser: CaptureDenoiser { get } |
| 47 | var microphoneName: String { get } |
| 48 | /// The device the microphone actually is, with how it is connected and |
| 49 | /// its rate now: "Paul's AirPods Pro (Bluetooth, 24 kHz)". Empty where |
| 50 | /// that is not known. |
| 51 | var microphoneDetail: String { get } |
| 52 | var speakerName: String { get } |
| 53 | /// The chosen speaker, as the module lists it (a CoreAudio UID on Mac). |
| 54 | var speakerID: String? { get } |
| 55 | var inCall: Bool { get } |
| 56 | /// False in a call whose microphone WebRTC has not opened yet. |
| 57 | var captureExpected: Bool { get } |
| 58 | var muted: Bool { get } |
| 59 | var reduceNoise: Bool { get } |
| 60 | /// Whether the capture must stop while the test plays something. On |
| 61 | /// iPhone the voice-processing unit ducks other audio while it runs. |
| 62 | var stopsForPlayback: Bool { get } |
| 63 | func permitted() async -> Bool |
| 64 | /// A Bluetooth microphone that starting will move to its hands-free |
| 65 | /// profile, by name: said while it switches. Nil for anything else. |
| 66 | func switchingMicrophone() async -> String? |
| 67 | func start() async throws |
| 68 | func stop() |
| 69 | /// The meters since the last read: before the suppressor, and after. |
| 70 | func levels() -> (input: CaptureLevel.Reading, output: CaptureLevel.Reading) |
| 71 | /// For "Copy audio diagnostics": titled blocks of lines. |
| 72 | func diagnostics() -> [(String, [String])] |
| 73 | } |
| 74 | |
| 75 | extension AudioSelfTestCapture { |
| 76 | func switchingMicrophone() async -> String? { nil } |
| 77 | } |
| 78 | |
| 79 | /// Plays mono audio to the chosen speaker. Public so the interop harness |
| 80 | /// can stand in a player that only checks what it was given. |
| 81 | @MainActor public protocol AudioSelfTestPlayer: AnyObject { |
| 82 | /// `samples` in -1...1. Returns once they have played, or on `stop`. |
| 83 | func play(_ samples: [Float], sampleRate: Double, speaker: String?, meter: OutputMeter) async throws |
| 84 | func stop() |
| 85 | } |
| 86 | |
| 87 | public enum SpeakerTestState: Equatable, Sendable { |
| 88 | case idle, playing, played |
| 89 | case failed(String) |
| 90 | case unavailableInCall |
| 91 | } |
| 92 | |
| 93 | public enum RecordingTestState: Equatable, Sendable { |
| 94 | case idle, recording, playing |
| 95 | /// `silent`: the recording itself was below the silence line. |
| 96 | case played(silent: Bool) |
| 97 | case failed(String) |
| 98 | case unavailableInCall |
| 99 | } |
| 100 | |
| 101 | /// What the last playback of a recording was, without the audio: for the |
| 102 | /// interop harness, which asserts on it. |
| 103 | public struct RecordingSummary: Equatable, Sendable { |
| 104 | public var seconds: Double |
| 105 | public var rate: Int |
| 106 | public var rms: Float |
| 107 | } |
| 108 | |
| 109 | @MainActor public final class AudioSelfTest: ObservableObject { |
| 110 | @Published public private(set) var micStep = 0 |
| 111 | @Published public private(set) var micVerdict: MicVerdict = .listening |
| 112 | /// Said instead of the verdict when the microphone cannot be tested at |
| 113 | /// all: no permission, muted in the call, or it would not start. |
| 114 | @Published public private(set) var micProblem: String? |
| 115 | @Published public private(set) var outputStep = 0 |
| 116 | @Published public private(set) var speaker: SpeakerTestState |
| 117 | @Published public private(set) var recording: RecordingTestState |
| 118 | /// 0...1 in tenths while recording. |
| 119 | @Published public private(set) var recordingProgress = 0.0 |
| 120 | @Published public private(set) var microphoneName = "" |
| 121 | /// "Paul's AirPods Pro (Bluetooth, 24 kHz)": what the verdict names. |
| 122 | @Published public private(set) var microphoneDetail = "" |
| 123 | @Published public private(set) var speakerName = "" |
| 124 | /// Where the microphone is on its way to running: the window opens at |
| 125 | /// once and says "Starting microphone…" until it is. |
| 126 | @Published public private(set) var startPhase: MicStartPhase = .idle |
| 127 | public private(set) var lastRecording: RecordingSummary? |
| 128 | |
| 129 | public let recordSeconds: Double |
| 130 | /// The standalone test's devices, for its pickers. Nil in a call, whose |
| 131 | /// device panel already has them. |
| 132 | public let routes: AudioRoutes? |
| 133 | public var inCall: Bool { capture.inCall } |
| 134 | public var reduceNoise: Bool { capture.reduceNoise } |
| 135 | /// Whether a recording exists right now. The harness checks it is gone |
| 136 | /// after playback. |
| 137 | public var holdsRecording: Bool { recorder.recorded > 0 } |
| 138 | public var suppressorName: String { capture.denoiser.engineName } |
| 139 | public var framesSuppressed: UInt64 { capture.denoiser.framesProcessed } |
| 140 | |
| 141 | private let capture: AudioSelfTestCapture |
| 142 | private let player: AudioSelfTestPlayer |
| 143 | private let clock: () -> Double |
| 144 | private let recorder: SelfTestRecorder |
| 145 | private let outputMeter = OutputMeter() |
| 146 | private var judge = MicJudge() |
| 147 | private var start = MicStart() |
| 148 | private var capturing = false |
| 149 | private var closed: Bool { start.phase == .cancelled } |
| 150 | /// Set while this test is playing something, when the microphone's |
| 151 | /// readings are its own sound and are not judged. |
| 152 | private var playingSince: Double? |
| 153 | private var ticker: Task<Void, Never>? |
| 154 | private var ticks = 0 |
| 155 | private var lastVerdict: MicVerdict? |
| 156 | |
| 157 | init( |
| 158 | capture: AudioSelfTestCapture, player: AudioSelfTestPlayer, routes: AudioRoutes?, |
| 159 | recordSeconds: Double = 5, clock: @escaping () -> Double = CaptureDenoiser.now |
| 160 | ) { |
| 161 | self.capture = capture |
| 162 | self.player = player |
| 163 | self.routes = routes |
| 164 | self.recordSeconds = recordSeconds |
| 165 | self.clock = clock |
| 166 | recorder = SelfTestRecorder(seconds: recordSeconds) |
| 167 | speaker = capture.inCall ? .unavailableInCall : .idle |
| 168 | recording = capture.inCall ? .unavailableInCall : .idle |
| 169 | refreshNames() |
| 170 | } |
| 171 | |
| 172 | /// Before a call: a capture of its own, on the person's remembered |
| 173 | /// devices and their Reduce background noise setting. |
| 174 | public static func standalone() -> AudioSelfTest { |
| 175 | let routes = AudioRoutes() |
| 176 | let capture = StandaloneCapture(routes: routes, input: nil, reduceNoise: Call.rememberedReduceNoise) |
| 177 | return AudioSelfTest(capture: capture, player: EnginePlayer(), routes: routes) |
| 178 | } |
| 179 | |
| 180 | #if os(macOS) |
| 181 | /// For the interop harness: the standalone test's capture on the real |
| 182 | /// audio device module (device rendering, no synthetic voice), started |
| 183 | /// as the test starts it, and whether that asked the module to record. |
| 184 | /// Asking is what is checked, because a test process has no microphone |
| 185 | /// permission and the module refuses right after being asked; without |
| 186 | /// the ask (0.3.7) the microphone was never opened on any Mac. |
| 187 | /// |
| 188 | /// - Parameter askToRecord: false is the control: the 0.3.7 start, |
| 189 | /// which must come back not asked. |
| 190 | public static func deviceStartProbe(askToRecord: Bool) async -> (asked: Bool, detail: String) { |
| 191 | let routes = AudioRoutes(defaults: UserDefaults(suiteName: "calls-selftest-probe-\(UUID().uuidString)")!) |
| 192 | let capture = StandaloneCapture(routes: routes, input: nil, reduceNoise: true, askToRecord: askToRecord) |
| 193 | do { |
| 194 | try await capture.start() |
| 195 | } catch { |
| 196 | // Refused (in a test process, for want of permission): what |
| 197 | // matters is whether it was asked. |
| 198 | let asked = capture.audio?.recordingRequested ?? false |
| 199 | return (asked, "recording requested \(asked); the module refused: \(error.localizedDescription)") |
| 200 | } |
| 201 | let deadline = Date().addingTimeInterval(3) |
| 202 | while capture.audio?.recordingRequested != true, Date() < deadline { |
| 203 | try? await Task.sleep(for: .milliseconds(100)) |
| 204 | } |
| 205 | let asked = capture.audio?.recordingRequested ?? false |
| 206 | let recording = capture.audio?.module.isRecording ?? false |
| 207 | capture.stop() |
| 208 | await AudioWork.drain() |
| 209 | return (asked, "recording requested \(asked), module recording \(recording) (no permission refuses it)") |
| 210 | } |
| 211 | #endif |
| 212 | |
| 213 | /// In a call: its meters, read and nothing else. Standalone if the call |
| 214 | /// has no media up. |
| 215 | public static func during(_ call: Call) -> AudioSelfTest { |
| 216 | guard let capture = LiveCallCapture(call) else { return standalone() } |
| 217 | return AudioSelfTest(capture: capture, player: EnginePlayer(), routes: nil) |
| 218 | } |
| 219 | |
| 220 | #if os(macOS) |
| 221 | /// The interop harness's: the module in manual rendering with `input` |
| 222 | /// as the microphone, and devices remembered nowhere real. |
| 223 | public static func harness( |
| 224 | input: AudioInputSource, player: AudioSelfTestPlayer, reduceNoise: Bool, recordSeconds: Double = 5 |
| 225 | ) -> AudioSelfTest { |
| 226 | let routes = AudioRoutes(defaults: UserDefaults(suiteName: "calls-selftest-\(UUID().uuidString)")!) |
| 227 | let capture = StandaloneCapture(routes: routes, input: input, reduceNoise: reduceNoise) |
| 228 | return AudioSelfTest(capture: capture, player: player, routes: routes, recordSeconds: recordSeconds) |
| 229 | } |
| 230 | #endif |
| 231 | |
| 232 | // MARK: Lifecycle |
| 233 | |
| 234 | /// Starts the microphone (outside a call) and the meters. |
| 235 | /// |
| 236 | /// Nothing here waits on the main thread: the audio is built, the |
| 237 | /// microphone started and any Bluetooth headset switched on the audio |
| 238 | /// queue (AudioWork), and the window says what is happening meanwhile |
| 239 | /// (`startPhase`). Closing it partway cancels cleanly: a start that |
| 240 | /// finishes after the close is stopped at once, so the microphone is |
| 241 | /// never left open. |
| 242 | public func open() async { |
| 243 | guard start.open() else { return } |
| 244 | startPhase = start.phase |
| 245 | let began = clock() |
| 246 | let granted = await capture.permitted() |
| 247 | guard start.permission(granted) else { |
| 248 | startPhase = start.phase |
| 249 | if !closed { micProblem = AudioSelfTest.noPermission } |
| 250 | return |
| 251 | } |
| 252 | startPhase = start.phase |
| 253 | ticker = Task { [weak self] in |
| 254 | while !Task.isCancelled { |
| 255 | try? await Task.sleep(for: .milliseconds(100)) |
| 256 | guard let self, !Task.isCancelled else { return } |
| 257 | self.tick() |
| 258 | } |
| 259 | } |
| 260 | await startCapture() |
| 261 | guard !closed else { return } |
| 262 | refreshNames() |
| 263 | AudioLog.info( |
| 264 | "test opened \(inCall ? "in a call" : "before a call") in \(Int((clock() - began) * 1000)) ms: microphone \(microphoneName) = \(microphoneDetail.isEmpty ? "unknown" : microphoneDetail); capturing \(capturing)") |
| 265 | } |
| 266 | |
| 267 | /// Stops everything this test started and throws any recording away. |
| 268 | /// Called when the view closes; safe to call twice, and at any point of |
| 269 | /// the start. |
| 270 | public func close() { |
| 271 | guard !closed else { return } |
| 272 | let live = start.close() |
| 273 | startPhase = start.phase |
| 274 | ticker?.cancel() |
| 275 | ticker = nil |
| 276 | player.stop() |
| 277 | if !capture.inCall { capture.denoiser.sink = nil } |
| 278 | if capturing || live { capture.stop() } |
| 279 | capturing = false |
| 280 | recorder.discard() |
| 281 | } |
| 282 | |
| 283 | private func startCapture() async { |
| 284 | let switching = await capture.switchingMicrophone() |
| 285 | start.microphone(bluetoothSwitching: switching) |
| 286 | startPhase = start.phase |
| 287 | var failure: Error? |
| 288 | do { try await capture.start() } catch { failure = error } |
| 289 | switch start.finished(ok: failure == nil) { |
| 290 | case .keep: |
| 291 | startPhase = start.phase |
| 292 | capturing = true |
| 293 | micProblem = nil |
| 294 | judge.start(at: clock()) |
| 295 | // Whatever was measured before this start is not this microphone's. |
| 296 | _ = capture.levels() |
| 297 | case .stop: |
| 298 | // Closed while it started: what it opened is closed now. |
| 299 | capture.stop() |
| 300 | AudioLog.info("test: closed while the microphone started; stopped it") |
| 301 | case .failed: |
| 302 | startPhase = start.phase |
| 303 | capturing = false |
| 304 | guard !closed, let failure else { return } |
| 305 | let said = failure.localizedDescription |
| 306 | micProblem = said.hasPrefix("The microphone") || said == AudioSelfTest.noPermission |
| 307 | ? said : "The microphone would not start: \(said)" |
| 308 | AudioLog.error("test: \(micProblem ?? said)") |
| 309 | } |
| 310 | } |
| 311 | |
| 312 | /// A different microphone was chosen: judge it afresh. |
| 313 | public func microphoneChanged() { |
| 314 | judge.start(at: clock()) |
| 315 | refreshNames() |
| 316 | } |
| 317 | |
| 318 | private func refreshNames() { |
| 319 | if microphoneName != capture.microphoneName { microphoneName = capture.microphoneName } |
| 320 | let detail = capture.microphoneDetail |
| 321 | if microphoneDetail != detail { microphoneDetail = detail } |
| 322 | if speakerName != capture.speakerName { speakerName = capture.speakerName } |
| 323 | } |
| 324 | |
| 325 | /// What "Copy audio diagnostics" copies from the test. |
| 326 | public func diagnosticsText() -> String { |
| 327 | var test = [ |
| 328 | "where: \(inCall ? "during a call (reads the call's microphone)" : "before a call (its own microphone)")", |
| 329 | "microphone: \(microphoneName) = \(microphoneDetail.isEmpty ? "unknown" : microphoneDetail)", |
| 330 | "speaker: \(speakerName)", |
| 331 | "capturing: \(capturing); verdict \(String(describing: micVerdict)); meter step \(micStep)", |
| 332 | "says: \(micSentence)", |
| 333 | "reduce background noise: \(reduceNoise)", |
| 334 | "recording test: \(String(describing: recording)); speaker test: \(String(describing: speaker))", |
| 335 | ] |
| 336 | if let problem = micProblem { test.append("problem: \(problem)") } |
| 337 | return AudioDiagnosticsReport.make( |
| 338 | context: inCall ? "Test audio in a call" : "Test audio before a call", |
| 339 | sections: [("Test audio", test)] + capture.diagnostics()) |
| 340 | } |
| 341 | |
| 342 | // MARK: The 10 Hz step |
| 343 | |
| 344 | /// One read of the meters. Publishes only what changed, so a view is |
| 345 | /// redrawn at most ten times a second and usually less. |
| 346 | func tick() { |
| 347 | let (input, output) = capture.levels() |
| 348 | let out = outputMeter.take() |
| 349 | let playing = playingSince != nil |
| 350 | set(\.outputStep, playing ? MeterSteps.step(out) : 0) |
| 351 | if case .recording = recording { |
| 352 | set(\.recordingProgress, (min(recorder.recorded / recordSeconds, 1) * 10).rounded(.down) / 10) |
| 353 | } |
| 354 | guard capturing, !playing else { |
| 355 | set(\.micStep, 0) |
| 356 | return |
| 357 | } |
| 358 | if capture.inCall, capture.muted { |
| 359 | set(\.micStep, 0) |
| 360 | set(\.micProblem, AudioSelfTest.mutedInCall) |
| 361 | judge.start(at: clock()) |
| 362 | return |
| 363 | } |
| 364 | if micProblem == AudioSelfTest.mutedInCall { micProblem = nil } |
| 365 | set(\.micStep, output.buffers > 0 ? MeterSteps.step(output.rms) : 0) |
| 366 | let verdict = judge.observe( |
| 367 | input: input, output: output, at: clock(), captureExpected: capture.captureExpected) |
| 368 | set(\.micVerdict, verdict) |
| 369 | AudioDiagnosticsStore.shared.record( |
| 370 | .init( |
| 371 | at: clock(), input: input.peak, inputBuffers: input.buffers, output: output.peak, |
| 372 | outputBuffers: output.buffers, source: "test")) |
| 373 | if verdict != lastVerdict { |
| 374 | lastVerdict = verdict |
| 375 | AudioLog.info("test verdict: \(String(describing: verdict)) — \(MicJudge.sentence(verdict, device: verdictName))") |
| 376 | } |
| 377 | // The device's name is cheap; its rate means reading CoreAudio, so |
| 378 | // once a second (a Bluetooth headset changes rate as its microphone |
| 379 | // opens). |
| 380 | ticks += 1 |
| 381 | if ticks % 10 == 0 { refreshNames() } |
| 382 | } |
| 383 | |
| 384 | /// The microphone as the verdict names it: the device it resolves to, |
| 385 | /// with its connection and rate, where that is known. |
| 386 | var verdictName: String { microphoneDetail.isEmpty ? microphoneName : microphoneDetail } |
| 387 | |
| 388 | private func set<Value: Equatable>(_ key: ReferenceWritableKeyPath<AudioSelfTest, Value>, _ value: Value) { |
| 389 | if self[keyPath: key] != value { self[keyPath: key] = value } |
| 390 | } |
| 391 | |
| 392 | // MARK: Speaker |
| 393 | |
| 394 | public func playTestSound() async { |
| 395 | guard !capture.inCall, !closed, !busy else { return } |
| 396 | speaker = .playing |
| 397 | do { |
| 398 | try await playing { try await player.play(TestChime.samples(), sampleRate: TestChime.rate, speaker: capture.speakerID, meter: outputMeter) } |
| 399 | if !closed { speaker = .played } |
| 400 | } catch { |
| 401 | if !closed { speaker = .failed(error.localizedDescription) } |
| 402 | } |
| 403 | } |
| 404 | |
| 405 | // MARK: Record and play back |
| 406 | |
| 407 | /// Records `recordSeconds` from the capture hook's sink — what a call |
| 408 | /// would send — and plays it back once. The audio is held in memory |
| 409 | /// only, handed to the player, and gone when this returns. |
| 410 | public func recordAndPlayBack() async { |
| 411 | guard !capture.inCall, !closed, !busy else { return } |
| 412 | guard capturing else { |
| 413 | recording = .failed(micProblem ?? "The microphone is not running.") |
| 414 | return |
| 415 | } |
| 416 | let recorder = self.recorder |
| 417 | recorder.arm() |
| 418 | recordingProgress = 0 |
| 419 | recording = .recording |
| 420 | capture.denoiser.sink = { samples, frames, rate in recorder.append(samples, frames: frames, rate: rate) } |
| 421 | let deadline = clock() + recordSeconds + 3 |
| 422 | while !recorder.full, clock() < deadline, !closed { |
| 423 | try? await Task.sleep(for: .milliseconds(100)) |
| 424 | } |
| 425 | capture.denoiser.sink = nil |
| 426 | guard !closed else { return recorder.discard() } |
| 427 | var taken = recorder.take() |
| 428 | guard !taken.samples.isEmpty, taken.rate > 0 else { |
| 429 | recording = .failed("Nothing was recorded. \(MicJudge.sentence(.noAudio, device: verdictName))") |
| 430 | return |
| 431 | } |
| 432 | let rms = SampleLevel.rms(taken.samples) |
| 433 | lastRecording = RecordingSummary( |
| 434 | seconds: Double(taken.samples.count) / Double(taken.rate), rate: taken.rate, rms: rms) |
| 435 | recordingProgress = 1 |
| 436 | recording = .playing |
| 437 | do { |
| 438 | let samples = taken.samples |
| 439 | let rate = Double(taken.rate) |
| 440 | taken.samples = [] |
| 441 | try await playing { try await player.play(samples, sampleRate: rate, speaker: capture.speakerID, meter: outputMeter) } |
| 442 | if !closed { recording = .played(silent: rms < MicJudge.silenceRMS) } |
| 443 | } catch { |
| 444 | if !closed { recording = .failed(error.localizedDescription) } |
| 445 | } |
| 446 | } |
| 447 | |
| 448 | private var busy: Bool { |
| 449 | speaker == .playing || recording == .recording || recording == .playing |
| 450 | } |
| 451 | |
| 452 | /// Runs `body` with the microphone's readings set aside (they would be |
| 453 | /// this test's own sound), stopping the capture where it must stop, and |
| 454 | /// judging on from where it left off. |
| 455 | private func playing(_ body: () async throws -> Void) async throws { |
| 456 | let started = clock() |
| 457 | playingSince = started |
| 458 | let restart = capturing && capture.stopsForPlayback |
| 459 | if restart { |
| 460 | capture.stop() |
| 461 | capturing = false |
| 462 | } |
| 463 | var failure: Error? |
| 464 | do { try await body() } catch { failure = error } |
| 465 | playingSince = nil |
| 466 | outputStep = 0 |
| 467 | if restart, start.restart() { await startCapture() } |
| 468 | judge.resume(after: clock() - started) |
| 469 | if let failure { throw failure } |
| 470 | } |
| 471 | |
| 472 | // MARK: Words |
| 473 | |
| 474 | static let noPermission = |
| 475 | "Calls doesn’t have permission to use the microphone. Allow it in Settings, under Privacy & Security, Microphone." |
| 476 | static let mutedInCall = "You’re muted. Unmute to test your microphone." |
| 477 | public static let inCallPlayback = |
| 478 | "Not during a call: everyone in it would hear the sound too. Test your speaker and hear yourself from the home screen before a call." |
| 479 | |
| 480 | public var micSentence: String { |
| 481 | micProblem ?? start.sentence ?? MicJudge.sentence(micVerdict, device: verdictName) |
| 482 | } |
| 483 | /// Said beside a spinner while the microphone starts; nil once it has. |
| 484 | public var startingSentence: String? { micProblem == nil ? start.sentence : nil } |
| 485 | public var micAdvice: String? { micProblem == nil && !start.isStarting ? MicJudge.advice(micVerdict) : nil } |
| 486 | |
| 487 | public var speakerSentence: String? { |
| 488 | let name = speakerName.isEmpty ? "your speaker" : speakerName |
| 489 | switch speaker { |
| 490 | case .idle: return nil |
| 491 | case .playing: return "Playing a test sound through \(name)." |
| 492 | case .played: |
| 493 | return "Played a test sound through \(name). If you didn’t hear it, check the volume and the speaker chosen here." |
| 494 | case .failed(let why): return "The test sound didn’t play: \(why)" |
| 495 | case .unavailableInCall: return AudioSelfTest.inCallPlayback |
| 496 | } |
| 497 | } |
| 498 | |
| 499 | public var recordingSentence: String? { |
| 500 | switch recording { |
| 501 | case .idle: return nil |
| 502 | case .recording: return "Recording. Speak as you would in a call." |
| 503 | case .playing: return "Playing back what others would hear." |
| 504 | case .played(let silent): |
| 505 | return silent |
| 506 | ? "The recording was silent. \(MicJudge.sentence(.silent, device: verdictName))" |
| 507 | : "That is how you sound to others. The recording has been deleted." |
| 508 | case .failed(let why): return why |
| 509 | case .unavailableInCall: return nil |
| 510 | } |
| 511 | } |
| 512 | } |
| 513 | |
| 514 | extension Call { |
| 515 | /// The person's Reduce background noise setting, as a call reads it. |
| 516 | static var rememberedReduceNoise: Bool { |
| 517 | UserDefaults.standard.object(forKey: Call.reduceNoiseKey) as? Bool ?? true |
| 518 | } |
| 519 | } |
| 520 | |
| 521 | // MARK: - Captures |
| 522 | |
| 523 | /// The microphone before a call: a CallAudio of the test's own, recording |
| 524 | /// on its own. See the top of this file. |
| 525 | /// |
| 526 | /// The CallAudio is built when the test starts, not when its window is |
| 527 | /// made, and on the audio queue (CallAudio.make): the window appears at |
| 528 | /// once. The capture hook is made first, cheaply, so the test can read it |
| 529 | /// from the start. |
| 530 | @MainActor final class StandaloneCapture: AudioSelfTestCapture { |
| 531 | let routes: AudioRoutes |
| 532 | /// Nil until the start has built it. |
| 533 | private(set) var audio: CallAudio? |
| 534 | let denoiser: CaptureDenoiser |
| 535 | let reduceNoise: Bool |
| 536 | private let input: AudioInputSource? |
| 537 | private let askToRecord: Bool |
| 538 | private var track: RTCAudioTrack? |
| 539 | private var loopback: SelfTestLoopback? |
| 540 | /// Bumped by every stop, so a start in flight knows it was stopped. |
| 541 | private var attempt = 0 |
| 542 | #if os(iOS) |
| 543 | private var session: SavedSession? |
| 544 | #endif |
| 545 | |
| 546 | /// Set by the stall harness only, which has no microphone permission to |
| 547 | /// ask for (a test process), and measures the start behind the question. |
| 548 | private let assumePermitted: Bool |
| 549 | |
| 550 | init( |
| 551 | routes: AudioRoutes, input: AudioInputSource?, reduceNoise: Bool, askToRecord: Bool = true, |
| 552 | assumePermitted: Bool = false |
| 553 | ) { |
| 554 | self.routes = routes |
| 555 | self.reduceNoise = reduceNoise |
| 556 | self.askToRecord = askToRecord |
| 557 | self.assumePermitted = assumePermitted |
| 558 | self.input = input |
| 559 | denoiser = CaptureDenoiser(suppressor: CallAudio.makeSuppressor()) |
| 560 | } |
| 561 | |
| 562 | private var synthetic: Bool { input != nil } |
| 563 | var inCall: Bool { false } |
| 564 | var captureExpected: Bool { true } |
| 565 | var muted: Bool { false } |
| 566 | #if os(iOS) |
| 567 | var stopsForPlayback: Bool { true } |
| 568 | #else |
| 569 | var stopsForPlayback: Bool { false } |
| 570 | #endif |
| 571 | |
| 572 | var microphoneName: String { AudioSelfTest.inputName(routes) } |
| 573 | var microphoneDetail: String { AudioSelfTest.inputDetail(routes) } |
| 574 | var speakerName: String { routes.outputName } |
| 575 | var speakerID: String? { routes.currentOutput } |
| 576 | /// Read from CoreAudio on the audio queue: the process's first CoreAudio |
| 577 | /// read sets up its connection to the audio server, 80–180 ms measured. |
| 578 | func switchingMicrophone() async -> String? { |
| 579 | guard !synthetic else { return nil } |
| 580 | #if os(macOS) |
| 581 | let (preference, failed) = (AudioPreference.load(input: true, routes.defaults), routes.failed) |
| 582 | return await AudioWork.run { AudioRoutes.bluetoothSwitching(preference: preference, failed: failed) } |
| 583 | #else |
| 584 | return nil |
| 585 | #endif |
| 586 | } |
| 587 | |
| 588 | func diagnostics() -> [(String, [String])] { |
| 589 | #if os(macOS) |
| 590 | let routeLines = routes.diagnostics() |
| 591 | #else |
| 592 | let routeLines = ["microphone: \(microphoneName)", "speaker: \(speakerName)"] |
| 593 | #endif |
| 594 | return [ |
| 595 | ("Devices", routeLines), |
| 596 | ("Audio engine and processing", audio?.diagnostics() ?? ["the audio has not been built yet"]), |
| 597 | ] |
| 598 | } |
| 599 | |
| 600 | func permitted() async -> Bool { |
| 601 | if synthetic || assumePermitted { return true } |
| 602 | #if os(macOS) |
| 603 | switch AVCaptureDevice.authorizationStatus(for: .audio) { |
| 604 | case .authorized: return true |
| 605 | case .notDetermined: return await AVCaptureDevice.requestAccess(for: .audio) |
| 606 | default: return false |
| 607 | } |
| 608 | #else |
| 609 | switch AVAudioApplication.shared.recordPermission { |
| 610 | case .granted: return true |
| 611 | case .undetermined: return await AVAudioApplication.requestRecordPermission() |
| 612 | default: return false |
| 613 | } |
| 614 | #endif |
| 615 | } |
| 616 | |
| 617 | /// Builds the audio (the first time), sends the microphone track over |
| 618 | /// the loopback and opens the microphone: every slow step on the audio |
| 619 | /// queue. A stop during any of it wins: what was opened is closed, in |
| 620 | /// order, behind it. |
| 621 | func start() async throws { |
| 622 | let attempt = self.attempt |
| 623 | #if os(iOS) |
| 624 | if session == nil { session = try SavedSession.configureForCall() } |
| 625 | #endif |
| 626 | let audio: CallAudio |
| 627 | if let built = self.audio { |
| 628 | audio = built |
| 629 | } else { |
| 630 | #if os(macOS) |
| 631 | audio = await CallAudio.make( |
| 632 | routes: routes, audioDevice: nil, input: input, reduceNoise: reduceNoise, denoiser: denoiser) |
| 633 | #else |
| 634 | audio = await CallAudio.make(routes: routes, audioDevice: nil, reduceNoise: reduceNoise, denoiser: denoiser) |
| 635 | #endif |
| 636 | self.audio = audio |
| 637 | } |
| 638 | guard attempt == self.attempt else { throw CancellationError() } |
| 639 | // Made here, as a call makes its own: making it applies the call's |
| 640 | // processing options (CallAudio.apply). CallAudio holds it weakly. |
| 641 | let track = audio.makeTrack() |
| 642 | self.track = track |
| 643 | let loop = try await SelfTestLoopback.make(factory: audio.factory, track: track) |
| 644 | guard attempt == self.attempt else { |
| 645 | loop.close() |
| 646 | throw CancellationError() |
| 647 | } |
| 648 | loopback = loop |
| 649 | // The loopback is never connected, and WebRTC opens the microphone |
| 650 | // only for a connection that has been (CallAudio.startRecordingAlone): |
| 651 | // so the microphone is opened here, as a call's first send stream |
| 652 | // opens it. |
| 653 | let result = await audio.startRecordingAlone(ask: askToRecord) |
| 654 | guard attempt == self.attempt else { throw CancellationError() } |
| 655 | routes.refresh() |
| 656 | if result != 0, !synthetic { |
| 657 | stop() |
| 658 | throw NSError( |
| 659 | domain: "Calls", code: result, userInfo: [NSLocalizedDescriptionKey: CallAudio.recordingError(result)]) |
| 660 | } |
| 661 | } |
| 662 | |
| 663 | /// Closes the loopback and the microphone, on the audio queue behind |
| 664 | /// anything still starting; never waits. |
| 665 | func stop() { |
| 666 | attempt += 1 |
| 667 | loopback?.close() |
| 668 | loopback = nil |
| 669 | audio?.stopRecordingAlone() |
| 670 | track = nil |
| 671 | #if os(iOS) |
| 672 | session?.restore() |
| 673 | session = nil |
| 674 | #endif |
| 675 | } |
| 676 | |
| 677 | func levels() -> (input: CaptureLevel.Reading, output: CaptureLevel.Reading) { |
| 678 | (denoiser.inputLevel.take(), denoiser.level.take()) |
| 679 | } |
| 680 | |
| 681 | } |
| 682 | |
| 683 | /// The microphone during a call: the call's own hook, read only. |
| 684 | /// |
| 685 | /// The hook's two meters are the call's to read — its own microphone |
| 686 | /// meter takes from them, and a peak-hold read by two readers gives each |
| 687 | /// half the peaks — so the test measures from the hook's sink instead, |
| 688 | /// which the call does not use. That is the processed audio, so here the |
| 689 | /// "can't hear anything" line is drawn after the suppressor too. It is |
| 690 | /// still -90 dBFS: the suppressor takes at most 30 dB |
| 691 | /// (noiseAttenuationLimitDb), so a room above -60 dBFS stays above it. |
| 692 | @MainActor final class LiveCallCapture: AudioSelfTestCapture { |
| 693 | private weak var call: Call? |
| 694 | let denoiser: CaptureDenoiser |
| 695 | let reduceNoise: Bool |
| 696 | private let tapped = CaptureLevel() |
| 697 | |
| 698 | init?(_ call: Call) { |
| 699 | guard let denoiser = call.captureDenoiser else { return nil } |
| 700 | self.call = call |
| 701 | self.denoiser = denoiser |
| 702 | reduceNoise = call.reduceNoise |
| 703 | } |
| 704 | |
| 705 | var inCall: Bool { true } |
| 706 | var captureExpected: Bool { call?.captureStarted ?? false } |
| 707 | var muted: Bool { call?.muted ?? false } |
| 708 | var stopsForPlayback: Bool { false } |
| 709 | var microphoneName: String { call.map { AudioSelfTest.inputName($0.audio) } ?? "" } |
| 710 | var microphoneDetail: String { call.map { AudioSelfTest.inputDetail($0.audio) } ?? "" } |
| 711 | var speakerName: String { call?.audio.outputName ?? "" } |
| 712 | func diagnostics() -> [(String, [String])] { call?.audioDiagnosticSections() ?? [] } |
| 713 | var speakerID: String? { call?.audio.currentOutput } |
| 714 | func permitted() async -> Bool { true } |
| 715 | func start() async throws { |
| 716 | let tapped = self.tapped |
| 717 | denoiser.sink = { samples, frames, _ in tapped.record(samples, frames: frames, at: 0) } |
| 718 | } |
| 719 | func stop() { denoiser.sink = nil } |
| 720 | func levels() -> (input: CaptureLevel.Reading, output: CaptureLevel.Reading) { |
| 721 | let reading = tapped.take() |
| 722 | return (reading, reading) |
| 723 | } |
| 724 | } |
| 725 | |
| 726 | extension AudioSelfTest { |
| 727 | /// The chosen microphone's name as the pickers list it. |
| 728 | static func inputName(_ routes: AudioRoutes) -> String { |
| 729 | routes.inputs.first { $0.id == routes.currentInput }?.name ?? "" |
| 730 | } |
| 731 | |
| 732 | /// The device the microphone resolves to, with connection and rate. |
| 733 | static func inputDetail(_ routes: AudioRoutes) -> String { |
| 734 | #if os(macOS) |
| 735 | return routes.inputDescription() |
| 736 | #else |
| 737 | let session = AVAudioSession.sharedInstance() |
| 738 | guard let port = session.currentRoute.inputs.first else { return "" } |
| 739 | let kind = port.portType == .bluetoothHFP ? "Bluetooth, " : "" |
| 740 | return "\(port.portName) (\(kind)\(Int(session.sampleRate / 1000)) kHz)" |
| 741 | #endif |
| 742 | } |
| 743 | } |
| 744 | |
| 745 | #if os(iOS) |
| 746 | /// The audio session as the test found it, put back when it is done. |
| 747 | /// Outside a call nothing else is using the session, so the test sets |
| 748 | /// it up as a call would (play and record, video chat, no hands-free |
| 749 | /// Bluetooth) and, at the end, restores the category and deactivates it |
| 750 | /// if it was not active before. |
| 751 | @MainActor struct SavedSession { |
| 752 | let category: AVAudioSession.Category |
| 753 | let mode: AVAudioSession.Mode |
| 754 | let options: AVAudioSession.CategoryOptions |
| 755 | let wasActive: Bool |
| 756 | |
| 757 | static func configureForCall() throws -> SavedSession { |
| 758 | let session = RTCAudioSession.sharedInstance() |
| 759 | let saved = SavedSession( |
| 760 | category: AVAudioSession.Category(rawValue: session.category), |
| 761 | mode: AVAudioSession.Mode(rawValue: session.mode), options: session.categoryOptions, |
| 762 | wasActive: session.isActive) |
| 763 | session.lockForConfiguration() |
| 764 | defer { session.unlockForConfiguration() } |
| 765 | try session.setCategory( |
| 766 | AVAudioSession.Category.playAndRecord, mode: AVAudioSession.Mode.videoChat, |
| 767 | options: AudioRoutes.options(bluetoothInput: false)) |
| 768 | return saved |
| 769 | } |
| 770 | |
| 771 | func restore() { |
| 772 | let session = RTCAudioSession.sharedInstance() |
| 773 | session.lockForConfiguration() |
| 774 | defer { session.unlockForConfiguration() } |
| 775 | try? session.setCategory(category, mode: mode, options: options) |
| 776 | if !wasActive, session.isActive { try? session.setActive(false) } |
| 777 | } |
| 778 | } |
| 779 | #endif |
| 780 | |
| 781 | // MARK: - Playing |
| 782 | |
| 783 | /// Plays through an AVAudioEngine of its own, apart from WebRTC's. |
| 784 | /// |
| 785 | /// On Mac the engine's output unit is pointed at the chosen speaker (the |
| 786 | /// UID the call's module lists it by; the system default if it cannot be |
| 787 | /// found). It only plays outside a call, so it never shares a device with |
| 788 | /// the call's engine. On iPhone it plays on the session the test set up, |
| 789 | /// to whatever the route is — the speaker a call would use. |
| 790 | @MainActor final class EnginePlayer: AudioSelfTestPlayer { |
| 791 | private var engine: AVAudioEngine? |
| 792 | private var node: AVAudioPlayerNode? |
| 793 | private var waiting: CheckedContinuation<Void, Error>? |
| 794 | /// Which playback a completion belongs to: one from a sound that was |
| 795 | /// stopped must not end the next. |
| 796 | private var generation = 0 |
| 797 | #if os(iOS) |
| 798 | /// RTCAudioSession counts activations, and only the last balanced |
| 799 | /// deactivation lets the session go; each play is balanced in `finish`. |
| 800 | private var activated = false |
| 801 | #endif |
| 802 | |
| 803 | func play(_ samples: [Float], sampleRate: Double, speaker: String?, meter: OutputMeter) async throws { |
| 804 | stop() |
| 805 | guard !samples.isEmpty, |
| 806 | let format = AVAudioFormat(standardFormatWithSampleRate: sampleRate, channels: 1), |
| 807 | let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: AVAudioFrameCount(samples.count)), |
| 808 | let channel = buffer.floatChannelData?[0] |
| 809 | else { return } |
| 810 | buffer.frameLength = AVAudioFrameCount(samples.count) |
| 811 | samples.withUnsafeBufferPointer { channel.update(from: $0.baseAddress!, count: samples.count) } |
| 812 | defer { channel.update(repeating: 0, count: samples.count) } |
| 813 | |
| 814 | let engine = AVAudioEngine() |
| 815 | #if os(macOS) |
| 816 | if let speaker, let device = EnginePlayer.device(uid: speaker), let unit = engine.outputNode.audioUnit { |
| 817 | var id = device |
| 818 | AudioUnitSetProperty( |
| 819 | unit, kAudioOutputUnitProperty_CurrentDevice, kAudioUnitScope_Global, 0, &id, |
| 820 | UInt32(MemoryLayout<AudioDeviceID>.size)) |
| 821 | } |
| 822 | #else |
| 823 | let session = RTCAudioSession.sharedInstance() |
| 824 | session.lockForConfiguration() |
| 825 | activated = (try? session.setActive(true)) != nil |
| 826 | session.unlockForConfiguration() |
| 827 | #endif |
| 828 | let node = AVAudioPlayerNode() |
| 829 | engine.attach(node) |
| 830 | engine.connect(node, to: engine.mainMixerNode, format: format) |
| 831 | engine.mainMixerNode.installTap(onBus: 0, bufferSize: 1024, format: nil) { tapped, _ in |
| 832 | guard let data = tapped.floatChannelData?[0] else { return } |
| 833 | meter.record(data, count: Int(tapped.frameLength)) |
| 834 | } |
| 835 | self.engine = engine |
| 836 | self.node = node |
| 837 | generation += 1 |
| 838 | let playback = generation |
| 839 | // Started on the audio queue: an engine start on a Bluetooth headset |
| 840 | // waits for its route, and the main thread must not wait with it. |
| 841 | do { try await AudioWork.run { try engine.start() } } catch { |
| 842 | if generation == playback { finish() } |
| 843 | throw error |
| 844 | } |
| 845 | // Stopped while it started: `finish` has queued its stop. |
| 846 | guard generation == playback, self.engine === engine else { return } |
| 847 | try await withCheckedThrowingContinuation { (continuation: CheckedContinuation<Void, Error>) in |
| 848 | waiting = continuation |
| 849 | node.scheduleBuffer(buffer, at: nil, options: [], completionCallbackType: .dataPlayedBack) { |
| 850 | [weak self] _ in |
| 851 | DispatchQueue.main.async { |
| 852 | MainActor.assumeIsolated { |
| 853 | guard let self, self.generation == playback else { return } |
| 854 | self.finish() |
| 855 | } |
| 856 | } |
| 857 | } |
| 858 | node.play() |
| 859 | } |
| 860 | } |
| 861 | |
| 862 | private func finish() { |
| 863 | let waiting = self.waiting |
| 864 | self.waiting = nil |
| 865 | // Stopped on the audio queue, behind its start if that is still |
| 866 | // running. |
| 867 | if let engine, let node { |
| 868 | AudioWork.post { |
| 869 | engine.mainMixerNode.removeTap(onBus: 0) |
| 870 | node.stop() |
| 871 | engine.stop() |
| 872 | } |
| 873 | } |
| 874 | node = nil |
| 875 | engine = nil |
| 876 | #if os(iOS) |
| 877 | if activated { |
| 878 | activated = false |
| 879 | let session = RTCAudioSession.sharedInstance() |
| 880 | session.lockForConfiguration() |
| 881 | try? session.setActive(false) |
| 882 | session.unlockForConfiguration() |
| 883 | } |
| 884 | #endif |
| 885 | waiting?.resume() |
| 886 | } |
| 887 | |
| 888 | func stop() { |
| 889 | generation += 1 |
| 890 | finish() |
| 891 | } |
| 892 | |
| 893 | #if os(macOS) |
| 894 | /// A CoreAudio device from its UID, or nil. |
| 895 | static func device(uid: String) -> AudioDeviceID? { |
| 896 | guard !uid.isEmpty else { return nil } |
| 897 | var address = AudioObjectPropertyAddress( |
| 898 | mSelector: kAudioHardwarePropertyTranslateUIDToDevice, mScope: kAudioObjectPropertyScopeGlobal, |
| 899 | mElement: kAudioObjectPropertyElementMain) |
| 900 | var device = AudioDeviceID(kAudioObjectUnknown) |
| 901 | var size = UInt32(MemoryLayout<AudioDeviceID>.size) |
| 902 | let cf = uid as CFString |
| 903 | let status = withUnsafePointer(to: cf) { qualifier in |
| 904 | AudioObjectGetPropertyData( |
| 905 | AudioObjectID(kAudioObjectSystemObject), &address, UInt32(MemoryLayout<CFString>.size), |
| 906 | qualifier, &size, &device) |
| 907 | } |
| 908 | guard status == noErr, device != kAudioObjectUnknown else { return nil } |
| 909 | return device |
| 910 | } |
| 911 | #endif |
| 912 | } |
| 913 | |
amadan