amadan
Merge branch 'fix/test-audio-beachball' into main
20 files changed,
+1899
−307
Makefile+1 −1internal/callstest/native_selftest_test.go+54 −0native/README.md+49 −3native/apple/Sources/CallsInterop/AudioStartStallHarness.swift+121 −0native/apple/Sources/CallsInterop/Interop.swift+3 −0native/apple/Sources/CallsInterop/MainThreadWatchdog.swift+91 −0native/apple/Sources/CallsKit/Audio.swift+79 −22native/apple/Sources/CallsKit/AudioSelfTest.swift+108 −0native/apple/Sources/CallsKit/AudioSelfTestLoopback.swift+52 −49native/apple/Sources/CallsKit/AudioSelfTestModel.swift+153 −51native/apple/Sources/CallsKit/AudioSelfTestView.swift+16 −4native/apple/Sources/CallsKit/AudioStartProbe.swift+141 −0native/apple/Sources/CallsKit/AudioWork.swift+258 −0native/apple/Sources/CallsKit/Call.swift+39 −12native/apple/Sources/CallsKit/CallAudio.swift+330 −63native/apple/Sources/CallsKit/Media.swift+173 −98native/apple/Sources/CallsMac/CallsMac.swift+1 −0native/apple/Tests/CallsKitTests/AudioRoutesTests.swift+33 −1native/apple/Tests/CallsKitTests/AudioSelfTestTests.swift+196 −2native/ios/project.yml+1 −1
CI passed — run details
diff --git a/Makefile b/Makefile| index eb52588..f14d226 100644 |
| --- a/Makefile |
| +++ b/Makefile |
| @@ -173,7 +173,7 @@ native-browser: native-mac |
| # The gate runs CallsInterop, which native-mac does not build; without |
| # this it runs whatever binary a previous build left behind. |
| swift build --package-path native/apple --product CallsInterop |
| - CALLS_DEEPFILTER_MODEL=$(CURDIR)/native/build/DeepFilterNet3_onnx.tar.gz CGO_ENABLED=0 go test -tags browser -count=1 -timeout 10m -run 'TestNative(MacCallsBrowser|MacChat|MacScreenMotion|BrowserAuthentication|MacAudioSelfTest)' -v ./internal/callstest/ |
| + CALLS_DEEPFILTER_MODEL=$(CURDIR)/native/build/DeepFilterNet3_onnx.tar.gz CGO_ENABLED=0 go test -tags browser -count=1 -timeout 10m -run 'TestNative(MacCallsBrowser|MacChat|MacScreenMotion|BrowserAuthentication|MacAudioSelfTest|MacAudioStartStall)' -v ./internal/callstest/ |
| # Two apps in one call, sharing screens to each other. Separate from |
| # native-browser because two unsigned apps on one Mac do not reliably |
diff --git a/internal/callstest/native_selftest_test.go b/internal/callstest/native_selftest_test.go| index 402f50f..6586525 100644 |
| --- a/internal/callstest/native_selftest_test.go |
| +++ b/internal/callstest/native_selftest_test.go |
| @@ -69,3 +69,57 @@ func TestNativeMacAudioSelfTest(t *testing.T) { |
| } |
| } |
| } |
| + |
| +// The main thread is never held longer than 100 ms while the audio |
| +// starts: Test audio before a call (plain, with a Bluetooth headset's |
| +// profile switch simulated as a 1.5 s hold of the engine's start on |
| +// WebRTC's worker thread, and closed partway through that start), and a |
| +// call's audio as a join starts it, with the same switch. A watchdog in |
| +// CallsInterop times the main queue every 5 ms, on the real engine |
| +// module; a test process has no microphone permission, so the module |
| +// refuses at the very end, after everything that used to block. |
| +// |
| +// 0.3.8 held it 0.1–0.7 s for the plain start and about 1.5 s with the |
| +// switch (Paul's "pause, then a beachball", 2026-09-27). A control run |
| +// blocks the main thread on purpose and must be seen, so a pass means the |
| +// watchdog was looking. Five runs, five PASS lines. |
| +func TestNativeMacAudioStartStall(t *testing.T) { |
| + root, err := filepath.Abs("../..") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + binary := filepath.Join(root, "native/apple/.build/debug/CallsInterop") |
| + if _, err := os.Stat(binary); err != nil { |
| + t.Fatal("build native interop executable with swift build --package-path native/apple --product CallsInterop first") |
| + } |
| + ctx, cancel := context.WithTimeout(t.Context(), 90*time.Second) |
| + defer cancel() |
| + out, err := exec.CommandContext(ctx, binary, "--audio-start-stall").CombinedOutput() |
| + var passes, fails []string |
| + for _, line := range strings.Split(string(out), "\n") { |
| + switch { |
| + case strings.HasPrefix(line, "PASS "): |
| + passes = append(passes, line) |
| + case strings.HasPrefix(line, "FAIL "): |
| + fails = append(fails, line) |
| + } |
| + } |
| + for _, line := range append(passes, fails...) { |
| + t.Log(line) |
| + } |
| + if err != nil || len(fails) > 0 || len(passes) != 5 { |
| + t.Fatalf("audio start stall: %d passed, %d failed, exit %v\n%s", len(passes), len(fails), err, out) |
| + } |
| + for _, want := range []string{ |
| + "audio start stall control", "PASS Test audio start:", "Test audio start, Bluetooth switch simulated", |
| + "Test audio closed while its microphone starts", "Call join audio start, Bluetooth switch simulated", |
| + } { |
| + found := false |
| + for _, line := range passes { |
| + found = found || strings.Contains(line, want) |
| + } |
| + if !found { |
| + t.Errorf("no PASS line for %s", want) |
| + } |
| + } |
| +} |
diff --git a/native/README.md b/native/README.md| index a6cd8bc..459ee6f 100644 |
| --- a/native/README.md |
| +++ b/native/README.md |
| @@ -301,6 +301,35 @@ also runs the start on the real device module, with 0.3.7's start as a control |
| that must never ask for the microphone. Calls themselves were not affected: |
| their connections are connected. |
| +**Nothing in the audio start runs on the main thread** (0.3.9). Paul's |
| +0.3.8, on a MacBook Air with AirPods Pro: "after I open the Test window, |
| +there's a pause, then a beachball, then it works." A main-thread watchdog put |
| +numbers on it: building the factory and its engine module (100–135 ms) and |
| +logging the module's first device list (61–68 ms); applying the loopback's |
| +answer, which waits for the engine's playout to start (188–229 ms); and |
| +`initRecording`, which waits while the engine starts with the microphone — for |
| +a Bluetooth headset, while it moves from A2DP to its hands-free profile, one to |
| +two seconds. Every call into WebRTC's objects is a blocking hop to its own |
| +threads, and while the worker thread is starting the engine every one of them |
| +waits: the module's `isRecording`, `isPlaying` and device lists, the factory's |
| +processing state, a track's `isEnabled` (mute) and its processing options, |
| +0.9–1.2 s each with the switch simulated. So all of it runs on one serial |
| +queue (`AudioWork`), in the order asked; the main thread reads the module's |
| +state from `ModuleState`, which waits at most 5 ms and otherwise gives the last |
| +answer. The window opens at once and says **Starting microphone…** beside a |
| +spinner — **Switching Paul's AirPods Pro to microphone mode…** for a Bluetooth |
| +headset not yet at its hands-free rate, because that wait is physics, not a |
| +fault — and the meter comes alive when the microphone has started. Closing it |
| +partway cancels cleanly: the stop is queued behind the start, so what the |
| +start opens is closed (`MicStart`). A call's join builds its media the same |
| +way (`MediaEngine.make`), and its connections' descriptions, candidates, |
| +statistics, sender parameters and closes go through the same queue |
| +(`PeerCalls`). Measured by `CallsInterop --audio-start-stall` on an M1 Max, |
| +longest main-thread stall, before → after: Test audio 142–716 ms → 0–5 ms; |
| +with a 1.5 s Bluetooth switch simulated 1,536–1,657 ms → 1–7 ms; closed |
| +mid-start → 0–8 ms; a call's audio start with the switch 1,322–1,514 ms → |
| +21–43 ms. |
| + |
| The verdict names the device the microphone really is, with its connection |
| and rate ("Paul's AirPods Pro (Bluetooth, 24 kHz)"), and tells apart "no sound |
| is arriving" (no buffers at all: the microphone was asked for and nothing came |
| @@ -389,9 +418,18 @@ microphone really capturing at 16 kHz brings WebRTC's own suppressor back. |
| (`--features capi`), and copies the standard model |
| (`DeepFilterNet3_onnx.tar.gz`, 8 MB) beside it; the Mac bundle carries the |
| model. Needs rustup (https://rustup.rs). The library is linked on Mac only. |
| - The model takes about 0.3 s to load, so a call starts on RNNoise and moves |
| - to DeepFilterNet when it has loaded off the main thread; a missing model |
| - leaves it on RNNoise. `CALLS_DEEPFILTER_MODEL` points at a model file |
| + The model takes about 0.3 s to load (on an M1 Max; longer on an Air), so |
| + one is loaded three seconds after launch, at utility priority, and the |
| + first Test audio or call takes it ready-made (`DeepFilterPreload`). A |
| + loaded model is about 88 MB resident, so only one is kept spare: taking it |
| + does not load another at once (a call would then hold two), a new spare is |
| + loaded when its user is done, and a spare nobody has taken for fifteen |
| + minutes is let go. Without a spare a call starts on RNNoise and moves to |
| + DeepFilterNet when one has loaded, off the main thread, as before; a model |
| + is never handed from one user to the next, because its state holds the |
| + last audio it heard and libDF has no way to clear it. A missing model |
| + leaves it on RNNoise. (RNNoise itself is made in under a millisecond and is |
| + not preloaded.) `CALLS_DEEPFILTER_MODEL` points at a model file |
| outside a bundle (the tests and the interop harness). |
| - **RNNoise** (xiph, BSD-3; notice in `LICENSE.rnnoise`, bundled in both apps): |
| 480-sample frames at 48 kHz in WebRTC's own ±32768 scale. Its model weights |
| @@ -627,6 +665,14 @@ make native-pair |
| CGO_ENABLED=0 go test -tags browser -run TestNativeBrowserAuthentication ./internal/callstest |
| ``` |
| +`TestNativeMacAudioStartStall` (in `make native-browser`) runs |
| +`CallsInterop --audio-start-stall`: a watchdog posts to the main queue every |
| +5 ms and times the answer while Test audio starts (plainly, with a 1.5 s |
| +Bluetooth switch simulated by holding the engine's start on WebRTC's worker |
| +thread, and closed partway through) and while a call's audio starts with the |
| +same switch, all on the real engine module. Each must hold the main thread |
| +under 100 ms; a control that blocks it for 300 ms on purpose must be seen. |
| + |
| `swift test` needs `native/scripts/build-bridge.sh`, |
| `native/scripts/build-rnnoise.sh` and `native/scripts/build-deepfilter.sh` to |
| have run. Its suppressor tests feed |
diff --git a/native/apple/Sources/CallsInterop/AudioStartStallHarness.swift b/native/apple/Sources/CallsInterop/AudioStartStallHarness.swift| new file mode 100644 |
| index 0000000..e299163 |
| --- /dev/null |
| +++ b/native/apple/Sources/CallsInterop/AudioStartStallHarness.swift |
| @@ -0,0 +1,121 @@ |
| +@_spi(Fixtures) import CallsKit |
| +import Foundation |
| + |
| +/// `CallsInterop --audio-start-stall`: the main thread is never held |
| +/// longer than `limit` while the audio starts — Test audio before a call, |
| +/// and a call's audio as a join starts it — on the real engine module. |
| +/// |
| +/// 0.3.8 held it for a pause and then a beachball (Paul's AirPods, |
| +/// 2026-09-27): building the factory, applying the loopback's answer, and |
| +/// `initRecording` waiting out the headset's switch to its hands-free |
| +/// profile, all on the main thread. The switch is stood in for by holding |
| +/// the engine's start on WebRTC's worker thread for 1.5 s |
| +/// (AudioStartProbe.engineStartDelay), where a real one holds it. |
| +/// |
| +/// First a control: a deliberate 300 ms block of the main thread must be |
| +/// seen, so a pass means the watchdog was looking. Prints one PASS or FAIL |
| +/// line per run, with the longest stall; exits 0 only if all passed. |
| +@MainActor enum AudioStartStallHarness { |
| + static let limit = 0.1 |
| + static let switchDelay = 1.5 |
| + |
| + static func run() async -> Bool { |
| + let dog = MainThreadWatchdog() |
| + dog.start() |
| + try? await Task.sleep(for: .milliseconds(200)) |
| + var passed = await control(dog) |
| + |
| + passed = await testAudio(dog, label: "Test audio start", delay: 0) && passed |
| + passed = await testAudio(dog, label: "Test audio start, Bluetooth switch simulated", delay: switchDelay) && passed |
| + passed = await testAudio( |
| + dog, label: "Test audio closed while its microphone starts", delay: switchDelay, closeAfter: 0.3) && passed |
| + passed = await callJoin(dog) && passed |
| + _ = dog.stop() |
| + return passed |
| + } |
| + |
| + private static func control(_ dog: MainThreadWatchdog) async -> Bool { |
| + let label = "audio start stall control (the watchdog sees a 300 ms block)" |
| + dog.reset() |
| + Thread.sleep(forTimeInterval: 0.3) |
| + // The watchdog's posts answer once the main thread is free again. |
| + try? await Task.sleep(for: .milliseconds(50)) |
| + let seen = dog.longestSoFar |
| + if seen >= 0.25 { |
| + print("PASS \(label): saw \(ms(seen))") |
| + return true |
| + } |
| + print("FAIL \(label): saw only \(ms(seen))") |
| + return false |
| + } |
| + |
| + private static func testAudio( |
| + _ dog: MainThreadWatchdog, label: String, delay: TimeInterval, closeAfter: Double? = nil |
| + ) async -> Bool { |
| + AudioStartProbe.engineStartDelay = delay |
| + defer { AudioStartProbe.engineStartDelay = 0 } |
| + await settle() |
| + dog.reset() |
| + let report = await AudioStartProbe.testAudio(closeAfter: closeAfter) |
| + let stall = dog.longestSoFar |
| + var problems: [String] = [] |
| + if stall > limit { problems.append("main thread held \(ms(stall)), limit \(ms(limit))") } |
| + let firstShown = report.phases.first { $0.phase != .idle } |
| + if let firstShown, firstShown.at > limit { |
| + problems.append("the window said nothing for \(ms(firstShown.at))") |
| + } |
| + if firstShown == nil { problems.append("the window never said it was starting") } |
| + let starting = report.phases.first { if case .starting = $0.phase { return true } else { return false } } |
| + let ended = report.phases.last { $0.phase == .running || $0.phase == .failed || $0.phase == .cancelled } |
| + if closeAfter == nil { |
| + if !report.asked { problems.append("the module was never asked for the microphone") } |
| + if delay > 0, let starting, let ended, ended.at - starting.at < delay * 0.9 { |
| + problems.append( |
| + "'Starting microphone…' showed for \(ms(ended.at - starting.at)), not for the \(ms(delay)) switch") |
| + } |
| + } else if ended?.phase != .cancelled { |
| + problems.append("closing partway left the phase at \(String(describing: ended?.phase))") |
| + } |
| + if report.recordingAfter { problems.append("the microphone was left open") } |
| + let phases = report.phases.map { "\(String(describing: $0.phase)) at \(ms($0.at))" }.joined(separator: ", ") |
| + let detail = |
| + "longest main-thread stall \(ms(stall)); start took \(ms(report.startSeconds)); phases: \(phases); asked \(report.asked); problem: \(report.problem ?? "none"); \(report.detail)" |
| + if problems.isEmpty { |
| + print("PASS \(label): \(detail)") |
| + return true |
| + } |
| + print("FAIL \(label): \(problems.joined(separator: "; ")) [\(detail)]") |
| + return false |
| + } |
| + |
| + private static func callJoin(_ dog: MainThreadWatchdog) async -> Bool { |
| + let label = "Call join audio start, Bluetooth switch simulated" |
| + AudioStartProbe.engineStartDelay = switchDelay |
| + defer { AudioStartProbe.engineStartDelay = 0 } |
| + await settle() |
| + dog.reset() |
| + let report = await AudioStartProbe.callJoin(camera: Interop.camera) |
| + let stall = dog.longestSoFar |
| + var problems: [String] = [] |
| + if stall > limit { problems.append("main thread held \(ms(stall)), limit \(ms(limit))") } |
| + if !report.asked { problems.append("the module was never asked for the microphone") } |
| + if let problem = report.problem { problems.append(problem) } |
| + if report.recordingAfter { problems.append("the microphone was left open") } |
| + let detail = |
| + "longest main-thread stall \(ms(stall)); audio start took \(ms(report.startSeconds)); asked \(report.asked); \(report.detail)" |
| + if problems.isEmpty { |
| + print("PASS \(label): \(detail)") |
| + return true |
| + } |
| + print("FAIL \(label): \(problems.joined(separator: "; ")) [\(detail)]") |
| + return false |
| + } |
| + |
| + /// Lets the last run's queued stops and closes finish before the next |
| + /// is measured. |
| + private static func settle() async { |
| + try? await Task.sleep(for: .milliseconds(300)) |
| + } |
| + |
| + static func ms(_ seconds: Double) -> String { "\(Int((seconds * 1000).rounded())) ms" } |
| +} |
diff --git a/native/apple/Sources/CallsInterop/Interop.swift b/native/apple/Sources/CallsInterop/Interop.swift| index 8e341c6..06d4e4b 100644 |
| --- a/native/apple/Sources/CallsInterop/Interop.swift |
| +++ b/native/apple/Sources/CallsInterop/Interop.swift |
| @@ -36,6 +36,9 @@ struct Input: Decodable { |
| } catch { exit(1) } |
| return |
| } |
| + if CommandLine.arguments.count == 2, CommandLine.arguments[1] == "--audio-start-stall" { |
| + exit(await AudioStartStallHarness.run() ? 0 : 1) |
| + } |
| if CommandLine.arguments.count == 2, CommandLine.arguments[1] == "--audio-selftest" { |
| exit(await AudioSelfTestHarness.run() ? 0 : 1) |
| } |
diff --git a/native/apple/Sources/CallsInterop/MainThreadWatchdog.swift b/native/apple/Sources/CallsInterop/MainThreadWatchdog.swift| new file mode 100644 |
| index 0000000..5f66153 |
| --- /dev/null |
| +++ b/native/apple/Sources/CallsInterop/MainThreadWatchdog.swift |
| @@ -0,0 +1,91 @@ |
| +import Foundation |
| + |
| +/// Measures how long the main thread goes without answering. |
| +/// |
| +/// A thread of its own posts a block to the main queue every 5 ms and |
| +/// times how long each takes to run. A main thread that is free answers |
| +/// within a millisecond or two; one that is blocked answers only when it |
| +/// is free again, and the wait is the stall. Several posts can be waiting |
| +/// at once, so the longest single wait is the longest stall, to within |
| +/// the 5 ms period. |
| +/// |
| +/// It measures the main *queue*, which is what the main actor runs on and |
| +/// what a SwiftUI app's run loop drains: a beachball is this number above |
| +/// about a second. |
| +final class MainThreadWatchdog: @unchecked Sendable { |
| + struct Stall: Sendable { |
| + /// Seconds since the watchdog started. |
| + var at: Double |
| + var seconds: Double |
| + } |
| + |
| + private let lock = NSLock() |
| + private var running = false |
| + private var longest: Stall? |
| + private var stalls: [Stall] = [] |
| + private let started = CFAbsoluteTimeGetCurrent() |
| + private let period: Double |
| + /// Waits at or above this are kept, for the report. |
| + private let noteAbove: Double |
| + |
| + init(period: Double = 0.005, noteAbove: Double = 0.05) { |
| + self.period = period |
| + self.noteAbove = noteAbove |
| + } |
| + |
| + func start() { |
| + lock.lock() |
| + running = true |
| + lock.unlock() |
| + let thread = Thread { [self] in |
| + while isRunning { |
| + let posted = CFAbsoluteTimeGetCurrent() |
| + DispatchQueue.main.async { [self] in |
| + let now = CFAbsoluteTimeGetCurrent() |
| + record(Stall(at: posted - started, seconds: now - posted)) |
| + } |
| + Thread.sleep(forTimeInterval: period) |
| + } |
| + } |
| + thread.qualityOfService = .userInteractive |
| + thread.start() |
| + } |
| + |
| + private var isRunning: Bool { |
| + lock.lock() |
| + defer { lock.unlock() } |
| + return running |
| + } |
| + |
| + private func record(_ stall: Stall) { |
| + lock.lock() |
| + defer { lock.unlock() } |
| + guard running else { return } |
| + if stall.seconds > (longest?.seconds ?? -1) { longest = stall } |
| + if stall.seconds >= noteAbove { stalls.append(stall) } |
| + } |
| + |
| + /// Forgets what was measured; the next `longest` is from now. |
| + func reset() { |
| + lock.lock() |
| + longest = nil |
| + stalls = [] |
| + lock.unlock() |
| + } |
| + |
| + /// Stops posting and gives the longest wait since the last reset, and |
| + /// every wait above `noteAbove`. |
| + func stop() -> (longest: Double, stalls: [Stall]) { |
| + lock.lock() |
| + defer { lock.unlock() } |
| + running = false |
| + return (longest?.seconds ?? 0, stalls) |
| + } |
| + |
| + /// The longest wait so far, without stopping. |
| + var longestSoFar: Double { |
| + lock.lock() |
| + defer { lock.unlock() } |
| + return longest?.seconds ?? 0 |
| + } |
| +} |
diff --git a/native/apple/Sources/CallsKit/Audio.swift b/native/apple/Sources/CallsKit/Audio.swift| index 7e7bfe6..37bb2d7 100644 |
| --- a/native/apple/Sources/CallsKit/Audio.swift |
| +++ b/native/apple/Sources/CallsKit/Audio.swift |
| @@ -1,5 +1,6 @@ |
| import AVFoundation |
| import Foundation |
| +import os |
| #if os(macOS) |
| import CoreAudio |
| #endif |
| @@ -84,7 +85,7 @@ public struct AudioDevice: Identifiable, Hashable, Sendable { |
| /// Devices the engine could not start on, by UID, with why. Cleared |
| /// when the person chooses the device again or what is connected |
| /// changes. |
| - private var failed: [String: String] = [:] |
| + private(set) var failed: [String: String] = [:] |
| /// The microphone the engine was last seen running on, read back from |
| /// its unit — the truth when it is known. |
| private var engineInput: String? |
| @@ -184,8 +185,10 @@ public struct AudioDevice: Identifiable, Hashable, Sendable { |
| return [head] + inventory.list(input).map { AudioDevice(id: $0.uid, name: $0.name, selectable: true) } |
| } |
| - func attach(_ module: RTCAudioDeviceModule, pin: AudioEnginePin, inputRate: @escaping () -> Double?) { |
| - attach(AudioEngineControl(module, pin: pin, inputRate: inputRate)) |
| + func attach( |
| + _ module: RTCAudioDeviceModule, state: ModuleState, pin: AudioEnginePin, inputRate: @escaping () -> Double? |
| + ) { |
| + attach(AudioEngineControl(module, state: state, pin: pin, inputRate: inputRate)) |
| listenForDevices() |
| } |
| @@ -370,6 +373,24 @@ public struct AudioDevice: Identifiable, Hashable, Sendable { |
| return AudioRoutes.describe(device) |
| } |
| + /// The microphone a start would open, by name, when it is a Bluetooth |
| + /// headset still in its listening profile: opening its microphone |
| + /// moves it to the hands-free one (16 or 24 kHz), which takes it a |
| + /// second or two. Read from CoreAudio directly, so it is known before |
| + /// the call's audio is built. Nil for anything else. |
| + nonisolated static func bluetoothSwitching(preference: AudioPreference?, failed: [String: String]) -> String? { |
| + let inventory = CoreAudioDevices.inventory() |
| + let chosen = AudioRouteRules.resolve(preference, inventory, input: true, failed: failed) |
| + return bluetoothSwitching(uid: chosen.explicit ?? inventory.defaultInput, inventory) |
| + } |
| + |
| + nonisolated static func bluetoothSwitching(uid: String?, _ inventory: AudioInventory) -> String? { |
| + guard let device = inventory.devices.first(where: { $0.uid == uid }), device.transport == "bluetooth" |
| + else { return nil } |
| + // At the hands-free rates already: nothing to switch. |
| + return device.sampleRate > 0 && device.sampleRate <= 24_000 ? nil : device.name |
| + } |
| + |
| nonisolated static func describe(_ device: SystemAudioDevice) -> String { |
| var parts: [String] = [] |
| switch device.transport { |
| @@ -564,10 +585,15 @@ public struct AudioDevice: Identifiable, Hashable, Sendable { |
| self.pinned = pinned |
| } |
| - init(_ module: RTCAudioDeviceModule, pin enginePin: AudioEnginePin, inputRate: @escaping () -> Double?) { |
| + /// The module is only ever asked on the audio queue; the main thread |
| + /// reads its state from `state` (ModuleState), never from the module. |
| + init( |
| + _ module: RTCAudioDeviceModule, state: ModuleState, pin enginePin: AudioEnginePin, |
| + inputRate: @escaping () -> Double? |
| + ) { |
| inventory = CoreAudioDevices.inventory |
| self.inputRate = inputRate |
| - isRecording = { [weak module] in module?.isRecording ?? false } |
| + isRecording = { state.read().isRecording } |
| pinned = { |
| let target = enginePin.target() |
| if case .pair = target { |
| @@ -580,22 +606,45 @@ public struct AudioDevice: Identifiable, Hashable, Sendable { |
| // not what it was put on. Before an engine exists (the module makes |
| // one for the first connection) there is nothing to move: the first |
| // engine is made on the choice. |
| - pin = { [weak module] input, output in |
| + // A rebuild runs on the audio queue, so `lastApplied` lags the ask by |
| + // the time it takes; a second look in between (CoreAudio's device |
| + // list changes as the old aggregate goes) must not ask again. |
| + let asked = OSAllocatedUnfairLock<EngineTarget?>(initialState: nil) |
| + pin = { input, output in |
| _ = enginePin.set(input: input, output: output) |
| - guard let module, !module.isManualRenderingMode, module.isPlaying || module.isRecording, |
| + let module = state.read() |
| + guard !module.isManualRenderingMode, module.isPlaying || module.isRecording, |
| let applied = enginePin.lastApplied |
| else { return false } |
| - return enginePin.target() != applied |
| + let target = enginePin.target() |
| + guard target != applied else { |
| + // Caught up: the next move, even back to a target asked for |
| + // before (a device that failed, chosen again), is a new ask. |
| + asked.withLock { $0 = nil } |
| + return false |
| + } |
| + return asked.withLock { last in |
| + guard last != target else { return false } |
| + last = target |
| + return true |
| + } |
| } |
| // The module has no "rebuild" call. Taking both sides away releases |
| // its engine; giving them back makes a new one, and the pin is |
| // applied to it in the "will enable" hook. |
| + // On the audio queue, as every engine start is (AudioWork): a rebuild |
| + // onto a Bluetooth microphone waits for the headset to change |
| + // profile, and the main thread must not wait with it. |
| rebuild = { [weak module] in |
| - guard let module, !module.isManualRenderingMode else { return } |
| - let saved = module.engineAvailability |
| - _ = module.setEngineAvailability(RTCAudioEngineAvailability(isInputAvailable: false, isOutputAvailable: false)) |
| - let result = module.setEngineAvailability(saved) |
| - AudioLog.info("engine rebuilt (result \(result), running \(module.isEngineRunning))") |
| + guard let module else { return } |
| + AudioWork.post { |
| + guard !module.isManualRenderingMode else { return } |
| + let saved = module.engineAvailability |
| + _ = module.setEngineAvailability( |
| + RTCAudioEngineAvailability(isInputAvailable: false, isOutputAvailable: false)) |
| + let result = module.setEngineAvailability(saved) |
| + AudioLog.info("engine rebuilt (result \(result), running \(module.isEngineRunning))") |
| + } |
| } |
| channels = { uid in CoreAudioInput.device(forModuleID: uid).flatMap(CoreAudioInput.describe) } |
| // Taking the input away and giving it back rewires the microphone, |
| @@ -603,19 +652,27 @@ public struct AudioDevice: Identifiable, Hashable, Sendable { |
| // (Re-selecting the same device does nothing: the module ignores a |
| // state that has not changed.) |
| rebuildInput = { [weak module] in |
| - guard let module, module.isRecording, !module.isManualRenderingMode else { return } |
| - _ = module.setEngineAvailability( |
| - RTCAudioEngineAvailability(isInputAvailable: false, isOutputAvailable: true)) |
| - _ = module.setEngineAvailability( |
| - RTCAudioEngineAvailability(isInputAvailable: true, isOutputAvailable: true)) |
| + guard let module else { return } |
| + AudioWork.post { |
| + guard module.isRecording, !module.isManualRenderingMode else { return } |
| + _ = module.setEngineAvailability( |
| + RTCAudioEngineAvailability(isInputAvailable: false, isOutputAvailable: true)) |
| + _ = module.setEngineAvailability( |
| + RTCAudioEngineAvailability(isInputAvailable: true, isOutputAvailable: true)) |
| + } |
| } |
| + // Off the main thread: the module's first list is made on WebRTC's |
| + // worker thread, 61–68 ms measured, only to be logged. |
| logModuleLists = { [weak module] in |
| guard let module else { return } |
| - func line(_ list: [RTCIODevice]) -> String { |
| - list.map { "\($0.name) [\($0.deviceId)]" }.joined(separator: "; ") |
| + AudioWork.post { |
| + func line(_ list: [RTCIODevice]) -> String { |
| + list.map { "\($0.name) [\($0.deviceId)]" }.joined(separator: "; ") |
| + } |
| + AudioLog.info( |
| + "engine module lists (not used to choose): inputs \(line(module.inputDevices)); outputs \(line(module.outputDevices))" |
| + ) |
| } |
| - AudioLog.info( |
| - "engine module lists (not used to choose): inputs \(line(module.inputDevices)); outputs \(line(module.outputDevices))") |
| } |
| } |
| } |
diff --git a/native/apple/Sources/CallsKit/AudioSelfTest.swift b/native/apple/Sources/CallsKit/AudioSelfTest.swift| index 2d22e3e..0315155 100644 |
| --- a/native/apple/Sources/CallsKit/AudioSelfTest.swift |
| +++ b/native/apple/Sources/CallsKit/AudioSelfTest.swift |
| @@ -28,6 +28,114 @@ public enum MicVerdict: Equatable, Sendable { |
| case notStarted |
| } |
| +/// Where the test's microphone is on its way to running. The test's |
| +/// window opens at once; this is what it says until the microphone is |
| +/// actually capturing, which with a Bluetooth headset can be a couple of |
| +/// seconds (it has to move to its hands-free profile to open a |
| +/// microphone at all — physics, not a fault). |
| +public enum MicStartPhase: Equatable, Sendable { |
| + case idle |
| + /// Asking for microphone permission; a system prompt may be showing. |
| + case asking |
| + /// Building the audio and starting the microphone. `switching` names a |
| + /// Bluetooth microphone that is moving to its hands-free profile. |
| + case starting(switching: String?) |
| + case running |
| + /// Would not start, or no permission: the test says why elsewhere. |
| + case failed |
| + /// Closed before it had started. |
| + case cancelled |
| +} |
| + |
| +/// The start's rules, apart from any device, so they can be tested: every |
| +/// way through open, permission, start and close, and what a start that |
| +/// finishes after the window has closed must do (stop what it opened). |
| +public struct MicStart: Sendable { |
| + public private(set) var phase: MicStartPhase = .idle |
| + |
| + public init() {} |
| + |
| + /// What to do with a start that has just finished. |
| + public enum Finish: Equatable, Sendable { |
| + /// Keep it: it is the test's microphone now. |
| + case keep |
| + /// The test was closed while it started: stop it at once. |
| + case stop |
| + /// It failed; nothing to stop. |
| + case failed |
| + } |
| + |
| + /// Opening the window. False if it was already opened or closed. |
| + public mutating func open() -> Bool { |
| + guard phase == .idle else { return false } |
| + phase = .asking |
| + return true |
| + } |
| + |
| + /// The answer to the permission question. False: do not start. |
| + public mutating func permission(_ granted: Bool) -> Bool { |
| + guard phase == .asking else { return false } |
| + phase = granted ? .starting(switching: nil) : .failed |
| + return granted |
| + } |
| + |
| + /// Starting again after the test stopped the microphone to play |
| + /// something (iPhone). False once closed. |
| + public mutating func restart() -> Bool { |
| + guard phase == .running || phase == .failed else { return false } |
| + phase = .starting(switching: nil) |
| + return true |
| + } |
| + |
| + /// The audio is built and the microphone it will open is known. |
| + public mutating func microphone(bluetoothSwitching name: String?) { |
| + guard case .starting = phase else { return } |
| + phase = .starting(switching: name) |
| + } |
| + |
| + public mutating func finished(ok: Bool) -> Finish { |
| + switch phase { |
| + case .starting: |
| + phase = ok ? .running : .failed |
| + return ok ? .keep : .failed |
| + case .cancelled: |
| + return ok ? .stop : .failed |
| + default: |
| + // Not ours to judge (a start nobody asked for): stop it. |
| + return ok ? .stop : .failed |
| + } |
| + } |
| + |
| + /// The window closed. Returns whether a microphone may be open or |
| + /// opening, so the caller stops it. |
| + @discardableResult |
| + public mutating func close() -> Bool { |
| + let live: Bool |
| + switch phase { |
| + case .starting, .running: live = true |
| + default: live = false |
| + } |
| + phase = .cancelled |
| + return live |
| + } |
| + |
| + public var isStarting: Bool { |
| + if case .starting = phase { return true } |
| + return phase == .asking |
| + } |
| + |
| + /// What the window says while the microphone starts; nil otherwise. |
| + public var sentence: String? { |
| + switch phase { |
| + case .asking: return "Asking to use the microphone…" |
| + case .starting(let switching?): |
| + return "Switching \(switching) to microphone mode…" |
| + case .starting(nil): return "Starting microphone…" |
| + default: return nil |
| + } |
| + } |
| +} |
| + |
| /// Judges the microphone from the capture hook's two meters. |
| /// |
| /// "Working" is asked of `CaptureDenoiser.level`, the audio after noise |
diff --git a/native/apple/Sources/CallsKit/AudioSelfTestLoopback.swift b/native/apple/Sources/CallsKit/AudioSelfTestLoopback.swift| index 7f06aea..c929ff4 100644 |
| --- a/native/apple/Sources/CallsKit/AudioSelfTestLoopback.swift |
| +++ b/native/apple/Sources/CallsKit/AudioSelfTestLoopback.swift |
| @@ -21,70 +21,73 @@ import LiveKitWebRTC |
| /// for WebRTC to open the microphone and process every buffer, as it does |
| /// for a call waiting to connect. The receiving side hears nothing — no |
| /// packet reaches it — and its track is silenced anyway. |
| -@MainActor final class SelfTestLoopback { |
| - private var sender: RTCPeerConnection? |
| - private var receiver: RTCPeerConnection? |
| +/// |
| +/// Every call to WebRTC here is made on the audio queue (PeerCalls): |
| +/// applying the answer starts the engine's playout, 188–229 ms measured, |
| +/// and in 0.3.8 the main thread waited for it. |
| +final class SelfTestLoopback: @unchecked Sendable { |
| + private let sender: RTCPeerConnection |
| + private let receiver: RTCPeerConnection |
| + /// Held until the connections are closed: a connection must never |
| + /// outlive its factory. |
| + private var factory: RTCPeerConnectionFactory? |
| + private var closed = false |
| - init(factory: RTCPeerConnectionFactory, track: RTCAudioTrack) async throws { |
| - let config = RTCConfiguration() |
| - config.sdpSemantics = .unifiedPlan |
| - config.iceTransportPolicy = .none |
| - let constraints = RTCMediaConstraints(mandatoryConstraints: nil, optionalConstraints: nil) |
| - guard let sender = factory.peerConnection(with: config, constraints: constraints, delegate: nil), |
| - let receiver = factory.peerConnection(with: config, constraints: constraints, delegate: nil) |
| - else { throw SelfTestLoopback.failed("the audio connection could not be made") } |
| + private init(sender: RTCPeerConnection, receiver: RTCPeerConnection, factory: RTCPeerConnectionFactory) { |
| self.sender = sender |
| self.receiver = receiver |
| - do { |
| + self.factory = factory |
| + } |
| + |
| + static func make(factory: RTCPeerConnectionFactory, track: RTCAudioTrack) async throws -> SelfTestLoopback { |
| + let constraints = RTCMediaConstraints(mandatoryConstraints: nil, optionalConstraints: nil) |
| + let pair: SelfTestLoopback = try await AudioWork.run { |
| + let config = RTCConfiguration() |
| + config.sdpSemantics = .unifiedPlan |
| + config.iceTransportPolicy = .none |
| + guard let sender = factory.peerConnection(with: config, constraints: constraints, delegate: nil), |
| + let receiver = factory.peerConnection(with: config, constraints: constraints, delegate: nil) |
| + else { throw SelfTestLoopback.failed("the audio connection could not be made") } |
| + let pair = SelfTestLoopback(sender: sender, receiver: receiver, factory: factory) |
| guard sender.add(track, streamIds: ["selftest"]) != nil else { |
| + pair.close() |
| throw SelfTestLoopback.failed("the microphone could not be added") |
| } |
| - let offer = try await SelfTestLoopback.describe(sender, answer: false, constraints) |
| - try await SelfTestLoopback.set(sender, local: true, offer) |
| - try await SelfTestLoopback.set(receiver, local: false, offer) |
| - for transceiver in receiver.transceivers { |
| - (transceiver.receiver.track as? RTCAudioTrack)?.source.volume = 0 |
| - } |
| - let answer = try await SelfTestLoopback.describe(receiver, answer: true, constraints) |
| - try await SelfTestLoopback.set(receiver, local: true, answer) |
| - try await SelfTestLoopback.set(sender, local: false, answer) |
| + return pair |
| + } |
| + do { |
| + try await pair.negotiate(constraints) |
| } catch { |
| - close() |
| + pair.close() |
| throw error |
| } |
| + return pair |
| } |
| - /// Ends both; WebRTC closes the microphone with the last sending stream. |
| - func close() { |
| - sender?.close() |
| - receiver?.close() |
| - sender = nil |
| - receiver = nil |
| - } |
| - |
| - private static func describe( |
| - _ pc: RTCPeerConnection, answer: Bool, _ constraints: RTCMediaConstraints |
| - ) async throws -> RTCSessionDescription { |
| - try await withCheckedThrowingContinuation { c in |
| - let completion: @Sendable (RTCSessionDescription?, Error?) -> Void = { desc, error in |
| - if let desc { c.resume(returning: desc) } else { c.resume(throwing: error ?? CancellationError()) } |
| - } |
| - if answer { |
| - pc.answer(for: constraints, completionHandler: completion) |
| - } else { |
| - pc.offer(for: constraints, completionHandler: completion) |
| + private func negotiate(_ constraints: RTCMediaConstraints) async throws { |
| + let offer = try await PeerCalls.describe(sender, answer: false, constraints) |
| + try await PeerCalls.set(sender, local: true, offer) |
| + try await PeerCalls.set(receiver, local: false, offer) |
| + await PeerCalls.read(receiver) { receiver in |
| + for transceiver in receiver.transceivers { |
| + (transceiver.receiver.track as? RTCAudioTrack)?.source.volume = 0 |
| } |
| } |
| + let answer = try await PeerCalls.describe(receiver, answer: true, constraints) |
| + try await PeerCalls.set(receiver, local: true, answer) |
| + try await PeerCalls.set(sender, local: false, answer) |
| } |
| - private static func set(_ pc: RTCPeerConnection, local: Bool, _ desc: RTCSessionDescription) async throws { |
| - try await withCheckedThrowingContinuation { (c: CheckedContinuation<Void, Error>) in |
| - let done: @Sendable (Error?) -> Void = { error in |
| - if let error { c.resume(throwing: error) } else { c.resume() } |
| - } |
| - if local { pc.setLocalDescription(desc, completionHandler: done) } else { |
| - pc.setRemoteDescription(desc, completionHandler: done) |
| - } |
| + /// Ends both, on the audio queue after anything already asked of them; |
| + /// WebRTC closes the microphone with the last sending stream. Safe to |
| + /// call twice. |
| + func close() { |
| + AudioWork.post { [self] in |
| + guard !closed else { return } |
| + closed = true |
| + sender.close() |
| + receiver.close() |
| + factory = nil |
| } |
| } |
diff --git a/native/apple/Sources/CallsKit/AudioSelfTestModel.swift b/native/apple/Sources/CallsKit/AudioSelfTestModel.swift| index a89d3a9..4b89ea6 100644 |
| --- a/native/apple/Sources/CallsKit/AudioSelfTestModel.swift |
| +++ b/native/apple/Sources/CallsKit/AudioSelfTestModel.swift |
| @@ -61,6 +61,9 @@ import LiveKitWebRTC |
| /// iPhone the voice-processing unit ducks other audio while it runs. |
| var stopsForPlayback: Bool { get } |
| func permitted() async -> Bool |
| + /// A Bluetooth microphone that starting will move to its hands-free |
| + /// profile, by name: said while it switches. Nil for anything else. |
| + func switchingMicrophone() async -> String? |
| func start() async throws |
| func stop() |
| /// The meters since the last read: before the suppressor, and after. |
| @@ -69,6 +72,10 @@ import LiveKitWebRTC |
| func diagnostics() -> [(String, [String])] |
| } |
| +extension AudioSelfTestCapture { |
| + func switchingMicrophone() async -> String? { nil } |
| +} |
| + |
| /// Plays mono audio to the chosen speaker. Public so the interop harness |
| /// can stand in a player that only checks what it was given. |
| @MainActor public protocol AudioSelfTestPlayer: AnyObject { |
| @@ -114,6 +121,9 @@ public struct RecordingSummary: Equatable, Sendable { |
| /// "Paul's AirPods Pro (Bluetooth, 24 kHz)": what the verdict names. |
| @Published public private(set) var microphoneDetail = "" |
| @Published public private(set) var speakerName = "" |
| + /// Where the microphone is on its way to running: the window opens at |
| + /// once and says "Starting microphone…" until it is. |
| + @Published public private(set) var startPhase: MicStartPhase = .idle |
| public private(set) var lastRecording: RecordingSummary? |
| public let recordSeconds: Double |
| @@ -134,9 +144,9 @@ public struct RecordingSummary: Equatable, Sendable { |
| private let recorder: SelfTestRecorder |
| private let outputMeter = OutputMeter() |
| private var judge = MicJudge() |
| + private var start = MicStart() |
| private var capturing = false |
| - private var opened = false |
| - private var closed = false |
| + private var closed: Bool { start.phase == .cancelled } |
| /// Set while this test is playing something, when the microphone's |
| /// readings are its own sound and are not judged. |
| private var playingSince: Double? |
| @@ -185,16 +195,17 @@ public struct RecordingSummary: Equatable, Sendable { |
| } catch { |
| // Refused (in a test process, for want of permission): what |
| // matters is whether it was asked. |
| - let asked = capture.audio.recordingRequested |
| + let asked = capture.audio?.recordingRequested ?? false |
| return (asked, "recording requested \(asked); the module refused: \(error.localizedDescription)") |
| } |
| let deadline = Date().addingTimeInterval(3) |
| - while !capture.audio.recordingRequested, Date() < deadline { |
| + while capture.audio?.recordingRequested != true, Date() < deadline { |
| try? await Task.sleep(for: .milliseconds(100)) |
| } |
| - let asked = capture.audio.recordingRequested |
| - let recording = capture.audio.factory.audioDeviceModule.isRecording |
| + let asked = capture.audio?.recordingRequested ?? false |
| + let recording = capture.audio?.module.isRecording ?? false |
| capture.stop() |
| + await AudioWork.drain() |
| return (asked, "recording requested \(asked), module recording \(recording) (no permission refuses it)") |
| } |
| #endif |
| @@ -221,18 +232,24 @@ public struct RecordingSummary: Equatable, Sendable { |
| // MARK: Lifecycle |
| /// Starts the microphone (outside a call) and the meters. |
| + /// |
| + /// Nothing here waits on the main thread: the audio is built, the |
| + /// microphone started and any Bluetooth headset switched on the audio |
| + /// queue (AudioWork), and the window says what is happening meanwhile |
| + /// (`startPhase`). Closing it partway cancels cleanly: a start that |
| + /// finishes after the close is stopped at once, so the microphone is |
| + /// never left open. |
| public func open() async { |
| - guard !opened, !closed else { return } |
| - opened = true |
| - guard await capture.permitted() else { |
| - micProblem = AudioSelfTest.noPermission |
| + guard start.open() else { return } |
| + startPhase = start.phase |
| + let began = clock() |
| + let granted = await capture.permitted() |
| + guard start.permission(granted) else { |
| + startPhase = start.phase |
| + if !closed { micProblem = AudioSelfTest.noPermission } |
| return |
| } |
| - guard !closed else { return } |
| - await startCapture() |
| - refreshNames() |
| - AudioLog.info( |
| - "test opened \(inCall ? "in a call" : "before a call"): microphone \(microphoneName) = \(microphoneDetail.isEmpty ? "unknown" : microphoneDetail); capturing \(capturing)") |
| + startPhase = start.phase |
| ticker = Task { [weak self] in |
| while !Task.isCancelled { |
| try? await Task.sleep(for: .milliseconds(100)) |
| @@ -240,34 +257,52 @@ public struct RecordingSummary: Equatable, Sendable { |
| self.tick() |
| } |
| } |
| + await startCapture() |
| + guard !closed else { return } |
| + refreshNames() |
| + AudioLog.info( |
| + "test opened \(inCall ? "in a call" : "before a call") in \(Int((clock() - began) * 1000)) ms: microphone \(microphoneName) = \(microphoneDetail.isEmpty ? "unknown" : microphoneDetail); capturing \(capturing)") |
| } |
| /// Stops everything this test started and throws any recording away. |
| - /// Called when the view closes; safe to call twice. |
| + /// Called when the view closes; safe to call twice, and at any point of |
| + /// the start. |
| public func close() { |
| guard !closed else { return } |
| - closed = true |
| + let live = start.close() |
| + startPhase = start.phase |
| ticker?.cancel() |
| ticker = nil |
| player.stop() |
| if !capture.inCall { capture.denoiser.sink = nil } |
| - if capturing { capture.stop() } |
| + if capturing || live { capture.stop() } |
| capturing = false |
| recorder.discard() |
| } |
| private func startCapture() async { |
| - do { |
| - try await capture.start() |
| - guard !closed else { return capture.stop() } |
| + let switching = await capture.switchingMicrophone() |
| + start.microphone(bluetoothSwitching: switching) |
| + startPhase = start.phase |
| + var failure: Error? |
| + do { try await capture.start() } catch { failure = error } |
| + switch start.finished(ok: failure == nil) { |
| + case .keep: |
| + startPhase = start.phase |
| capturing = true |
| micProblem = nil |
| judge.start(at: clock()) |
| // Whatever was measured before this start is not this microphone's. |
| _ = capture.levels() |
| - } catch { |
| + case .stop: |
| + // Closed while it started: what it opened is closed now. |
| + capture.stop() |
| + AudioLog.info("test: closed while the microphone started; stopped it") |
| + case .failed: |
| + startPhase = start.phase |
| capturing = false |
| - let said = error.localizedDescription |
| + guard !closed, let failure else { return } |
| + let said = failure.localizedDescription |
| micProblem = said.hasPrefix("The microphone") || said == AudioSelfTest.noPermission |
| ? said : "The microphone would not start: \(said)" |
| AudioLog.error("test: \(micProblem ?? said)") |
| @@ -429,7 +464,7 @@ public struct RecordingSummary: Equatable, Sendable { |
| do { try await body() } catch { failure = error } |
| playingSince = nil |
| outputStep = 0 |
| - if restart, !closed { await startCapture() } |
| + if restart, start.restart() { await startCapture() } |
| judge.resume(after: clock() - started) |
| if let failure { throw failure } |
| } |
| @@ -443,9 +478,11 @@ public struct RecordingSummary: Equatable, Sendable { |
| "Not during a call: everyone in it would hear the sound too. Test your speaker and hear yourself from the home screen before a call." |
| public var micSentence: String { |
| - micProblem ?? MicJudge.sentence(micVerdict, device: verdictName) |
| + micProblem ?? start.sentence ?? MicJudge.sentence(micVerdict, device: verdictName) |
| } |
| - public var micAdvice: String? { micProblem == nil ? MicJudge.advice(micVerdict) : nil } |
| + /// Said beside a spinner while the microphone starts; nil once it has. |
| + public var startingSentence: String? { micProblem == nil ? start.sentence : nil } |
| + public var micAdvice: String? { micProblem == nil && !start.isStarting ? MicJudge.advice(micVerdict) : nil } |
| public var speakerSentence: String? { |
| let name = speakerName.isEmpty ? "your speaker" : speakerName |
| @@ -485,31 +522,44 @@ extension Call { |
| /// The microphone before a call: a CallAudio of the test's own, recording |
| /// on its own. See the top of this file. |
| +/// |
| +/// The CallAudio is built when the test starts, not when its window is |
| +/// made, and on the audio queue (CallAudio.make): the window appears at |
| +/// once. The capture hook is made first, cheaply, so the test can read it |
| +/// from the start. |
| @MainActor final class StandaloneCapture: AudioSelfTestCapture { |
| let routes: AudioRoutes |
| - let audio: CallAudio |
| + /// Nil until the start has built it. |
| + private(set) var audio: CallAudio? |
| + let denoiser: CaptureDenoiser |
| let reduceNoise: Bool |
| - private let synthetic: Bool |
| + private let input: AudioInputSource? |
| private let askToRecord: Bool |
| private var track: RTCAudioTrack? |
| private var loopback: SelfTestLoopback? |
| + /// Bumped by every stop, so a start in flight knows it was stopped. |
| + private var attempt = 0 |
| #if os(iOS) |
| private var session: SavedSession? |
| #endif |
| - init(routes: AudioRoutes, input: AudioInputSource?, reduceNoise: Bool, askToRecord: Bool = true) { |
| + /// Set by the stall harness only, which has no microphone permission to |
| + /// ask for (a test process), and measures the start behind the question. |
| + private let assumePermitted: Bool |
| + |
| + init( |
| + routes: AudioRoutes, input: AudioInputSource?, reduceNoise: Bool, askToRecord: Bool = true, |
| + assumePermitted: Bool = false |
| + ) { |
| self.routes = routes |
| self.reduceNoise = reduceNoise |
| self.askToRecord = askToRecord |
| - synthetic = input != nil |
| - #if os(macOS) |
| - audio = CallAudio(routes: routes, audioDevice: nil, input: input, reduceNoise: reduceNoise) |
| - #else |
| - audio = CallAudio(routes: routes, audioDevice: nil, reduceNoise: reduceNoise) |
| - #endif |
| + self.assumePermitted = assumePermitted |
| + self.input = input |
| + denoiser = CaptureDenoiser(suppressor: CallAudio.makeSuppressor()) |
| } |
| - var denoiser: CaptureDenoiser { audio.denoiser } |
| + private var synthetic: Bool { input != nil } |
| var inCall: Bool { false } |
| var captureExpected: Bool { true } |
| var muted: Bool { false } |
| @@ -523,6 +573,17 @@ extension Call { |
| var microphoneDetail: String { AudioSelfTest.inputDetail(routes) } |
| var speakerName: String { routes.outputName } |
| var speakerID: String? { routes.currentOutput } |
| + /// Read from CoreAudio on the audio queue: the process's first CoreAudio |
| + /// read sets up its connection to the audio server, 80–180 ms measured. |
| + func switchingMicrophone() async -> String? { |
| + guard !synthetic else { return nil } |
| + #if os(macOS) |
| + let (preference, failed) = (AudioPreference.load(input: true, routes.defaults), routes.failed) |
| + return await AudioWork.run { AudioRoutes.bluetoothSwitching(preference: preference, failed: failed) } |
| + #else |
| + return nil |
| + #endif |
| + } |
| func diagnostics() -> [(String, [String])] { |
| #if os(macOS) |
| @@ -530,11 +591,14 @@ extension Call { |
| #else |
| let routeLines = ["microphone: \(microphoneName)", "speaker: \(speakerName)"] |
| #endif |
| - return [("Devices", routeLines), ("Audio engine and processing", audio.diagnostics())] |
| + return [ |
| + ("Devices", routeLines), |
| + ("Audio engine and processing", audio?.diagnostics() ?? ["the audio has not been built yet"]), |
| + ] |
| } |
| func permitted() async -> Bool { |
| - if synthetic { return true } |
| + if synthetic || assumePermitted { return true } |
| #if os(macOS) |
| switch AVCaptureDevice.authorizationStatus(for: .audio) { |
| case .authorized: return true |
| @@ -550,22 +614,44 @@ extension Call { |
| #endif |
| } |
| + /// Builds the audio (the first time), sends the microphone track over |
| + /// the loopback and opens the microphone: every slow step on the audio |
| + /// queue. A stop during any of it wins: what was opened is closed, in |
| + /// order, behind it. |
| func start() async throws { |
| + let attempt = self.attempt |
| #if os(iOS) |
| if session == nil { session = try SavedSession.configureForCall() } |
| #endif |
| + let audio: CallAudio |
| + if let built = self.audio { |
| + audio = built |
| + } else { |
| + #if os(macOS) |
| + audio = await CallAudio.make( |
| + routes: routes, audioDevice: nil, input: input, reduceNoise: reduceNoise, denoiser: denoiser) |
| + #else |
| + audio = await CallAudio.make(routes: routes, audioDevice: nil, reduceNoise: reduceNoise, denoiser: denoiser) |
| + #endif |
| + self.audio = audio |
| + } |
| + guard attempt == self.attempt else { throw CancellationError() } |
| // Made here, as a call makes its own: making it applies the call's |
| // processing options (CallAudio.apply). CallAudio holds it weakly. |
| let track = audio.makeTrack() |
| self.track = track |
| - let loop = try await SelfTestLoopback(factory: audio.factory, track: track) |
| - guard self.track === track else { return loop.close() } |
| + let loop = try await SelfTestLoopback.make(factory: audio.factory, track: track) |
| + guard attempt == self.attempt else { |
| + loop.close() |
| + throw CancellationError() |
| + } |
| loopback = loop |
| // The loopback is never connected, and WebRTC opens the microphone |
| // only for a connection that has been (CallAudio.startRecordingAlone): |
| // so the microphone is opened here, as a call's first send stream |
| // opens it. |
| - let result = audio.startRecordingAlone(ask: askToRecord) |
| + let result = await audio.startRecordingAlone(ask: askToRecord) |
| + guard attempt == self.attempt else { throw CancellationError() } |
| routes.refresh() |
| if result != 0, !synthetic { |
| stop() |
| @@ -574,10 +660,13 @@ extension Call { |
| } |
| } |
| + /// Closes the loopback and the microphone, on the audio queue behind |
| + /// anything still starting; never waits. |
| func stop() { |
| + attempt += 1 |
| loopback?.close() |
| loopback = nil |
| - audio.stopRecordingAlone() |
| + audio?.stopRecordingAlone() |
| track = nil |
| #if os(iOS) |
| session?.restore() |
| @@ -586,7 +675,7 @@ extension Call { |
| } |
| func levels() -> (input: CaptureLevel.Reading, output: CaptureLevel.Reading) { |
| - (audio.denoiser.inputLevel.take(), audio.denoiser.level.take()) |
| + (denoiser.inputLevel.take(), denoiser.level.take()) |
| } |
| } |
| @@ -745,12 +834,16 @@ extension AudioSelfTest { |
| } |
| self.engine = engine |
| self.node = node |
| - do { try engine.start() } catch { |
| - finish() |
| - throw error |
| - } |
| generation += 1 |
| let playback = generation |
| + // Started on the audio queue: an engine start on a Bluetooth headset |
| + // waits for its route, and the main thread must not wait with it. |
| + do { try await AudioWork.run { try engine.start() } } catch { |
| + if generation == playback { finish() } |
| + throw error |
| + } |
| + // Stopped while it started: `finish` has queued its stop. |
| + guard generation == playback, self.engine === engine else { return } |
| try await withCheckedThrowingContinuation { (continuation: CheckedContinuation<Void, Error>) in |
| waiting = continuation |
| node.scheduleBuffer(buffer, at: nil, options: [], completionCallbackType: .dataPlayedBack) { |
| @@ -769,9 +862,15 @@ extension AudioSelfTest { |
| private func finish() { |
| let waiting = self.waiting |
| self.waiting = nil |
| - engine?.mainMixerNode.removeTap(onBus: 0) |
| - node?.stop() |
| - engine?.stop() |
| + // Stopped on the audio queue, behind its start if that is still |
| + // running. |
| + if let engine, let node { |
| + AudioWork.post { |
| + engine.mainMixerNode.removeTap(onBus: 0) |
| + node.stop() |
| + engine.stop() |
| + } |
| + } |
| node = nil |
| engine = nil |
| #if os(iOS) |
| @@ -786,7 +885,10 @@ extension AudioSelfTest { |
| waiting?.resume() |
| } |
| - func stop() { finish() } |
| + func stop() { |
| + generation += 1 |
| + finish() |
| + } |
| #if os(macOS) |
| /// A CoreAudio device from its UID, or nil. |
diff --git a/native/apple/Sources/CallsKit/AudioSelfTestView.swift b/native/apple/Sources/CallsKit/AudioSelfTestView.swift| index 7b07005..6ca7b83 100644 |
| --- a/native/apple/Sources/CallsKit/AudioSelfTestView.swift |
| +++ b/native/apple/Sources/CallsKit/AudioSelfTestView.swift |
| @@ -102,10 +102,22 @@ import SwiftUI |
| .accessibilityIdentifier("testMicrophoneDevice") |
| } |
| LevelMeter(step: test.micStep, label: "Microphone level") |
| - Label(test.micSentence, systemImage: micSymbol) |
| - .font(.callout) |
| - .fixedSize(horizontal: false, vertical: true) |
| + .opacity(test.startingSentence == nil ? 1 : 0.4) |
| + if let starting = test.startingSentence { |
| + // The window is up at once; the microphone takes a moment, and a |
| + // Bluetooth headset a second or two more. Said, not frozen. |
| + HStack(spacing: 8) { |
| + ProgressView().controlSize(.small) |
| + Text(starting).font(.callout).fixedSize(horizontal: false, vertical: true) |
| + } |
| + .accessibilityElement(children: .combine) |
| .accessibilityIdentifier("testMicrophoneResult") |
| + } else { |
| + Label(test.micSentence, systemImage: micSymbol) |
| + .font(.callout) |
| + .fixedSize(horizontal: false, vertical: true) |
| + .accessibilityIdentifier("testMicrophoneResult") |
| + } |
| if let advice = test.micAdvice { |
| Text(advice).font(.footnote).foregroundStyle(.secondary) |
| .fixedSize(horizontal: false, vertical: true) |
| @@ -181,7 +193,7 @@ import SwiftUI |
| Task { await test.recordAndPlayBack() } |
| } |
| .buttonStyle(.bordered) |
| - .disabled(busy || test.micProblem != nil) |
| + .disabled(busy || test.micProblem != nil || test.startingSentence != nil) |
| .accessibilityIdentifier("testRecord") |
| if test.recording == .recording { |
| ProgressView(value: test.recordingProgress) |
diff --git a/native/apple/Sources/CallsKit/AudioStartProbe.swift b/native/apple/Sources/CallsKit/AudioStartProbe.swift| new file mode 100644 |
| index 0000000..dd97016 |
| --- /dev/null |
| +++ b/native/apple/Sources/CallsKit/AudioStartProbe.swift |
| @@ -0,0 +1,141 @@ |
| +import Combine |
| +import Foundation |
| +import LiveKitWebRTC |
| + |
| +/// The main-thread stall harness's view into the audio start |
| +/// (`CallsInterop --audio-start-stall`). It runs the start the app runs — |
| +/// Test audio before a call, and a call's audio as a join builds it — on |
| +/// the real engine module, not manual rendering, while the harness |
| +/// watches the main thread. A test process has no microphone permission, |
| +/// so the module refuses right after it is asked; everything before that |
| +/// (building the factory, negotiating, the engine's start up to the |
| +/// microphone) runs as in the app. A Bluetooth headset's profile switch is |
| +/// stood in for by `engineStartDelay`, held on WebRTC's worker thread in |
| +/// the engine's "will enable" hook, which is where a real switch holds it. |
| +@_spi(Fixtures) public enum AudioStartProbe { |
| + /// Seconds the engine's start with a microphone is held, as a Bluetooth |
| + /// headset's switch to its hands-free profile holds it. Zero in the app. |
| + nonisolated(unsafe) public static var engineStartDelay: TimeInterval = 0 |
| + |
| + public struct Report: Sendable { |
| + /// What the test window said while it started, and when (seconds from |
| + /// the start of the run). |
| + public var phases: [(at: Double, phase: MicStartPhase)] = [] |
| + /// Seconds until `open` (or the join's audio start) returned. |
| + public var startSeconds: Double = 0 |
| + /// Whether the module was asked for the microphone. |
| + public var asked = false |
| + /// Whether the module was still recording once it had all settled. |
| + public var recordingAfter = false |
| + public var problem: String? |
| + public var detail = "" |
| + } |
| + |
| + /// Test audio before a call, on the device module: opened, and closed |
| + /// once started — or, with `closeAfter`, closed partway through. |
| + @MainActor public static func testAudio(closeAfter: Double? = nil) async -> Report { |
| + let began = CaptureDenoiser.now() |
| + var report = Report() |
| + #if os(macOS) |
| + let routes = AudioRoutes(defaults: UserDefaults(suiteName: "calls-stall-\(UUID().uuidString)")!) |
| + #else |
| + let routes = AudioRoutes() |
| + #endif |
| + let capture = StandaloneCapture(routes: routes, input: nil, reduceNoise: true, assumePermitted: true) |
| + let test = AudioSelfTest(capture: capture, player: EnginePlayer(), routes: routes) |
| + var phases: [(Double, MicStartPhase)] = [] |
| + let watch = test.$startPhase.sink { phases.append((CaptureDenoiser.now() - began, $0)) } |
| + if let closeAfter { |
| + Task { @MainActor in |
| + try? await Task.sleep(for: .milliseconds(Int(closeAfter * 1000))) |
| + test.close() |
| + } |
| + } |
| + await test.open() |
| + report.startSeconds = CaptureDenoiser.now() - began |
| + report.problem = test.micProblem |
| + test.close() |
| + // The stop is queued behind the start; let it run. |
| + await AudioWork.drain() |
| + try? await Task.sleep(for: .milliseconds(200)) |
| + await AudioWork.drain() |
| + watch.cancel() |
| + report.phases = phases |
| + report.asked = capture.audio?.recordingRequested ?? false |
| + let module = capture.audio?.module |
| + report.recordingAfter = await AudioWork.run { module?.isRecording ?? false } |
| + report.detail = "suppressor \(test.suppressorName)" |
| + return report |
| + } |
| + |
| + /// A call's audio as a join starts it: the media built as `Call.join` |
| + /// builds it (MediaEngine.make), the camera and microphone tracks made |
| + /// (`prepare`), the first sending and receiving streams negotiated on |
| + /// the call's factory through PeerCalls, as MediaPeer negotiates them, |
| + /// and then WebRTC's worker thread opening the microphone for the first |
| + /// sending stream (AudioState::AddSendingStream), as it does once a |
| + /// connection is up. While the worker is held there, the main thread |
| + /// does what a call's does: reads the meters and the readout ten times |
| + /// a second, and — half a second in — negotiates a second person's |
| + /// connection. |
| + @MainActor public static func callJoin(camera: VideoCapture) async -> Report { |
| + let began = CaptureDenoiser.now() |
| + var report = Report() |
| + #if os(macOS) |
| + let call = Call(playAudio: false, audioInput: nil) |
| + #else |
| + let call = Call(playAudio: false) |
| + #endif |
| + let media = await MediaEngine.make(call: call) |
| + do { |
| + try await media.prepare(capture: camera) |
| + } catch { |
| + report.problem = "prepare: \(error.localizedDescription)" |
| + } |
| + guard let track = media.localAudio else { |
| + report.problem = "no microphone track" |
| + return report |
| + } |
| + var loops: [SelfTestLoopback] = [] |
| + do { |
| + loops.append(try await SelfTestLoopback.make(factory: media.factory, track: track)) |
| + } catch { |
| + report.problem = "negotiation: \(error.localizedDescription)" |
| + } |
| + let module = media.audio.module |
| + let opened = DispatchSemaphore(value: 0) |
| + // WebRTC's own start, on its own thread: not the app's to call. |
| + DispatchQueue.global(qos: .userInitiated).async { |
| + if module.initRecording() == 0 { _ = module.startRecording() } |
| + opened.signal() |
| + } |
| + let deadline = CaptureDenoiser.now() + AudioStartProbe.engineStartDelay + 1 |
| + var tick = 0 |
| + while CaptureDenoiser.now() < deadline { |
| + try? await Task.sleep(for: .milliseconds(100)) |
| + tick += 1 |
| + // The call's main-thread reads each tick (sampleMic, the readout). |
| + _ = media.audio.capturing |
| + _ = media.audio.report() |
| + _ = media.audio.denoiser.level.take() |
| + if tick == 3 { _ = media.audio.diagnostics() } |
| + if tick == 5 { |
| + do { |
| + loops.append(try await SelfTestLoopback.make(factory: media.factory, track: track)) |
| + } catch { |
| + report.problem = "second negotiation: \(error.localizedDescription)" |
| + } |
| + } |
| + } |
| + await AudioWork.run { opened.wait() } |
| + report.startSeconds = CaptureDenoiser.now() - began |
| + report.asked = media.audio.recordingRequested |
| + for loop in loops { loop.close() } |
| + AudioWork.post { _ = module.stopRecording() } |
| + media.close() |
| + await AudioWork.drain() |
| + report.recordingAfter = await AudioWork.run { module.isRecording } |
| + report.detail = "\(loops.count) connections negotiated" |
| + return report |
| + } |
| +} |
diff --git a/native/apple/Sources/CallsKit/AudioWork.swift b/native/apple/Sources/CallsKit/AudioWork.swift| new file mode 100644 |
| index 0000000..ccf38ec |
| --- /dev/null |
| +++ b/native/apple/Sources/CallsKit/AudioWork.swift |
| @@ -0,0 +1,258 @@ |
| +import Foundation |
| +import LiveKitWebRTC |
| +import os |
| + |
| +// Where the audio stack's slow work runs: never on the main thread. |
| +// |
| +// Paul, 0.3.8, MacBook Air, AirPods Pro: "after I open the Test window, |
| +// there's a pause, then a beachball, then it works." Measured on an M1 Max |
| +// with a main-thread watchdog (now `CallsInterop --audio-start-stall`), |
| +// 0.3.8's test held the main thread three times on its way to a running |
| +// microphone: |
| +// |
| +// 1. Building the audio (CallAudio): 166–200 ms. WebRTC's peer connection |
| +// factory with LiveKit's engine module, 100–135 ms; the module's first |
| +// device list, read only to be logged, 61–68 ms. |
| +// 2. The in-memory loopback's answer, applied: 188–229 ms. Applying it |
| +// starts the engine's playout on WebRTC's worker thread, and |
| +// `setLocalDescription` waits for that before it returns, completion |
| +// handler or not — every RTCPeerConnection call is a blocking hop to |
| +// WebRTC's own threads. |
| +// 3. `initRecording`, which waits on the worker thread while the engine |
| +// starts with the microphone. Here, with no microphone permission, 25 |
| +// ms. With AirPods it is as long as the headset takes to move from A2DP |
| +// to its hands-free profile, often one to two seconds: with that start |
| +// simulated at 1.5 s the main thread stalled for 1.5–1.7 s. That is the |
| +// beachball; 1 and 2 are the pause before it. |
| +// |
| +// A call's join had the same shape (1.3–1.5 s with the switch simulated), |
| +// and more: while the worker thread is starting the engine, *every* call |
| +// into the engine module waits for it — `isRecording`, `isPlaying`, the |
| +// device lists, the factory's processing state, a track's |
| +// `isEnabled = false` (mute), its processing options — 0.9 to 1.2 s each, |
| +// and a call's meters read some of them ten times a second. |
| +// |
| +// None of it is the main thread's business. It all runs here, on one |
| +// serial queue, in the order it was asked for; the main actor awaits the |
| +// result and only ever sets what the window shows. What the main thread |
| +// needs to know about the module it reads from ModuleState. |
| + |
| +/// The audio stack's serial queue: the factory and its engine module, the |
| +/// engine's start and stop, and the peer connections' calls. |
| +/// |
| +/// Serial on purpose. A test closed while its microphone is still |
| +/// starting queues its stop behind the start, so the stop always sees the |
| +/// microphone it has to close; nothing is left recording. |
| +enum AudioWork { |
| + static let queue = DispatchQueue(label: "net.amadan.oficina.calls.audio", qos: .userInitiated) |
| + |
| + /// Runs `body` on the audio queue and hands its result back. |
| + static func run<T>(_ body: @escaping () -> T) async -> T { |
| + await withCheckedContinuation { (c: CheckedContinuation<T, Never>) in |
| + queue.async { c.resume(returning: body()) } |
| + } |
| + } |
| + |
| + static func run<T>(_ body: @escaping () throws -> T) async throws -> T { |
| + try await withCheckedThrowingContinuation { (c: CheckedContinuation<T, Error>) in |
| + queue.async { |
| + do { c.resume(returning: try body()) } catch { c.resume(throwing: error) } |
| + } |
| + } |
| + } |
| + |
| + /// Queues `body` and does not wait for it: a stop, a close, a log line |
| + /// that has to read the module. |
| + static func post(_ body: @escaping () -> Void) { |
| + queue.async(execute: body) |
| + } |
| + |
| + /// Waits for everything queued so far. For tests and the harness. |
| + static func drain() async { |
| + await run {} |
| + } |
| +} |
| + |
| +/// A peer connection's calls, made from the audio queue. |
| +/// |
| +/// Every RTCPeerConnection method is a blocking hop to one of WebRTC's |
| +/// own threads, completion handler or not, and applying a description |
| +/// waits for the worker thread — which may be busy starting the audio |
| +/// engine for seconds. So a call makes them from here and awaits the |
| +/// answer; the main thread never waits on WebRTC. |
| +enum PeerCalls { |
| + static func describe( |
| + _ pc: RTCPeerConnection, answer: Bool, _ constraints: RTCMediaConstraints |
| + ) async throws -> RTCSessionDescription { |
| + try await withCheckedThrowingContinuation { c in |
| + AudioWork.queue.async { |
| + let completion: @Sendable (RTCSessionDescription?, Error?) -> Void = { desc, error in |
| + if let desc { c.resume(returning: desc) } else { c.resume(throwing: error ?? CancellationError()) } |
| + } |
| + if answer { |
| + pc.answer(for: constraints, completionHandler: completion) |
| + } else { |
| + pc.offer(for: constraints, completionHandler: completion) |
| + } |
| + } |
| + } |
| + } |
| + |
| + static func set(_ pc: RTCPeerConnection, local: Bool, _ desc: RTCSessionDescription) async throws { |
| + try await withCheckedThrowingContinuation { (c: CheckedContinuation<Void, Error>) in |
| + AudioWork.queue.async { |
| + let done: @Sendable (Error?) -> Void = { error in |
| + if let error { c.resume(throwing: error) } else { c.resume() } |
| + } |
| + if local { pc.setLocalDescription(desc, completionHandler: done) } else { |
| + pc.setRemoteDescription(desc, completionHandler: done) |
| + } |
| + } |
| + } |
| + } |
| + |
| + static func add(_ pc: RTCPeerConnection, _ candidate: RTCIceCandidate) async throws { |
| + try await withCheckedThrowingContinuation { (c: CheckedContinuation<Void, Error>) in |
| + AudioWork.queue.async { |
| + pc.add(candidate) { error in if let error { c.resume(throwing: error) } else { c.resume() } } |
| + } |
| + } |
| + } |
| + |
| + static func statistics(_ pc: RTCPeerConnection) async -> RTCStatisticsReport { |
| + await withCheckedContinuation { c in |
| + AudioWork.queue.async { pc.statistics { c.resume(returning: $0) } } |
| + } |
| + } |
| + |
| + static func statistics(_ pc: RTCPeerConnection, for receiver: RTCRtpReceiver) async -> RTCStatisticsReport { |
| + await withCheckedContinuation { c in |
| + AudioWork.queue.async { pc.statistics(for: receiver) { c.resume(returning: $0) } } |
| + } |
| + } |
| + |
| + /// Anything else, read or done on the audio queue. |
| + static func read<T>(_ pc: RTCPeerConnection, _ body: @escaping (RTCPeerConnection) -> T) async -> T { |
| + await AudioWork.run { body(pc) } |
| + } |
| + |
| + /// Closes it after whatever is queued for it; not waited for. Closing |
| + /// stops the audio engine when it is the last connection, and that can |
| + /// take as long as a start. |
| + static func close(_ pc: RTCPeerConnection) { |
| + AudioWork.post { pc.close() } |
| + } |
| +} |
| + |
| +#if canImport(DeepFilter) |
| + /// DeepFilterNet's model, loaded before anyone needs it. |
| + /// |
| + /// Loading takes about a third of a second on an M1 Max (and longer on |
| + /// an Air) of one core, and a call or test that has to load it starts on |
| + /// RNNoise and moves to DeepFilterNet when it is ready. So one is loaded |
| + /// in the background a few seconds after launch, at utility priority, |
| + /// and the first test or call takes it ready-made. |
| + /// |
| + /// Memory is the cost: a loaded model is about 88 MB resident. So only |
| + /// one is kept spare; a spare nobody has taken for fifteen minutes is |
| + /// let go, and the next user loads its own as before (off the main |
| + /// thread, starting on RNNoise). Taking the spare does not replace it at |
| + /// once — a call would then hold two — but when its user is done, a new |
| + /// spare is loaded for the next one. A model is never handed from one |
| + /// user to the next: its state holds the last audio it heard, and there |
| + /// is no way to clear it. |
| + public enum DeepFilterPreload { |
| + private struct State { |
| + var spare: DeepFilterSuppressor? |
| + var loading = false |
| + var generation = 0 |
| + } |
| + // The suppressor is only ever touched by whoever holds it; the lock |
| + // guards the handing over. |
| + nonisolated(unsafe) private static var state = State() |
| + private static let lock = NSLock() |
| + private static let background = DispatchQueue(label: "net.amadan.oficina.calls.deepfilter", qos: .utility) |
| + static let idleRelease: TimeInterval = 15 * 60 |
| + |
| + /// After launch: loads a spare in `delay` seconds, if there is a model |
| + /// and no spare yet. |
| + public static func preload(after delay: TimeInterval = 3) { |
| + background.asyncAfter(deadline: .now() + delay) { load() } |
| + } |
| + |
| + /// The spare, if one is ready; nil otherwise, and the caller loads. |
| + static func take() -> DeepFilterSuppressor? { |
| + lock.lock() |
| + defer { lock.unlock() } |
| + let spare = state.spare |
| + state.spare = nil |
| + state.generation += 1 |
| + if spare != nil { AudioLog.info("DeepFilterNet: using the preloaded model") } |
| + return spare |
| + } |
| + |
| + /// A user of a model is done with it: a fresh spare, a little later. |
| + static func finished() { |
| + preload(after: 5) |
| + } |
| + |
| + /// Whether a spare is loaded (for the tests). |
| + static var ready: Bool { |
| + lock.lock() |
| + defer { lock.unlock() } |
| + return state.spare != nil |
| + } |
| + |
| + private static func load() { |
| + lock.lock() |
| + guard state.spare == nil, !state.loading, let model = DeepFilterSuppressor.model else { |
| + lock.unlock() |
| + return |
| + } |
| + state.loading = true |
| + lock.unlock() |
| + let started = CaptureDenoiser.now() |
| + let loaded = DeepFilterSuppressor(model: model) |
| + lock.lock() |
| + state.loading = false |
| + state.spare = loaded |
| + state.generation += 1 |
| + let generation = state.generation |
| + lock.unlock() |
| + guard loaded != nil else { return } |
| + AudioLog.info( |
| + "DeepFilterNet: preloaded in \(Int((CaptureDenoiser.now() - started) * 1000)) ms, kept for the next test or call") |
| + background.asyncAfter(deadline: .now() + idleRelease) { release(ifStill: generation) } |
| + } |
| + |
| + private static func release(ifStill generation: Int) { |
| + lock.lock() |
| + guard state.generation == generation, state.spare != nil else { |
| + lock.unlock() |
| + return |
| + } |
| + let spare = state.spare |
| + state.spare = nil |
| + lock.unlock() |
| + _ = spare |
| + AudioLog.info("DeepFilterNet: preloaded model unused for \(Int(idleRelease / 60)) minutes; released") |
| + } |
| + } |
| +#endif |
| + |
| +/// What the app does for its audio once it has launched: loads the |
| +/// noise suppressor's model in the background (DeepFilterPreload), so the |
| +/// first test or call does not wait for it. Nothing on iPhone, which runs |
| +/// RNNoise (made in well under a millisecond). |
| +@MainActor public enum AudioWarmUp { |
| + private static var done = false |
| + |
| + /// Once per launch; later calls (a window shown again) do nothing. |
| + public static func afterLaunch() { |
| + guard !done else { return } |
| + done = true |
| + #if canImport(DeepFilter) |
| + DeepFilterPreload.preload(after: 3) |
| + #endif |
| + } |
| +} |
diff --git a/native/apple/Sources/CallsKit/Call.swift b/native/apple/Sources/CallsKit/Call.swift| index 6484edb..a0fb899 100644 |
| --- a/native/apple/Sources/CallsKit/Call.swift |
| +++ b/native/apple/Sources/CallsKit/Call.swift |
| @@ -52,7 +52,7 @@ private final class Events: NSObject, BridgeListenerProtocol { |
| didSet { |
| guard screenMode != oldValue else { return } |
| ScreenShareMode.remembered = screenMode |
| - media.screenMode = screenMode |
| + built?.screenMode = screenMode |
| } |
| } |
| private var screenReports: [String: ScreenSendReport] = [:] |
| @@ -95,8 +95,9 @@ private final class Events: NSObject, BridgeListenerProtocol { |
| @Published public var reduceNoise: Bool = UserDefaults.standard.object(forKey: Call.reduceNoiseKey) as? Bool ?? true { |
| didSet { |
| UserDefaults.standard.set(reduceNoise, forKey: Call.reduceNoiseKey) |
| - media.audio.reduceNoise = reduceNoise |
| - audioProcessing = media.audio.report() |
| + guard let built else { return } |
| + built.audio.reduceNoise = reduceNoise |
| + audioProcessing = built.audio.report() |
| } |
| } |
| static let reduceNoiseKey = "reduceBackgroundNoise" |
| @@ -123,17 +124,28 @@ private final class Events: NSObject, BridgeListenerProtocol { |
| fileprivate var epoch = UUID() |
| private var bridge: BridgeClient? |
| private var events: Events? |
| - private lazy var media = MediaEngine(call: self) |
| + /// The call's media, made by the first join (MediaEngine.make, off the |
| + /// main thread) and kept for the next. |
| + private var built: MediaEngine? |
| + /// The media, made here and now only if something reaches for it |
| + /// before any join has — which nothing in the app does. |
| + private var media: MediaEngine { |
| + if let built { return built } |
| + AudioLog.info("call media made on the main thread, before a join") |
| + let made = MediaEngine(call: self) |
| + built = made |
| + return made |
| + } |
| /// The audio test's view of this call's microphone (AudioSelfTestModel): |
| /// the capture hook, nil outside a call, and whether WebRTC has opened it. |
| - var captureDenoiser: CaptureDenoiser? { bridge == nil ? nil : media.audio.denoiser } |
| - var captureStarted: Bool { bridge != nil && media.audio.capturing } |
| + var captureDenoiser: CaptureDenoiser? { bridge == nil ? nil : built?.audio.denoiser } |
| + var captureStarted: Bool { bridge != nil && built?.audio.capturing == true } |
| /// For "Copy audio diagnostics" in the panel's ⓘ and in Test audio |
| /// during a call: this call's microphone, devices, engine and hook. |
| func audioDiagnosticSections() -> [(String, [String])] { |
| var lines = [ |
| - "in a call: \(bridge != nil); muted \(muted); microphone track \(bridge != nil && media.localAudio != nil)", |
| + "in a call: \(bridge != nil); muted \(muted); microphone track \(bridge != nil && built?.localAudio != nil)", |
| "people here: \(self.state?.peers.count ?? 0)", |
| "microphone meter: \(levels.mic.silent ? "silent" : "step \(levels.mic.step)"), speaking \(levels.mic.speaking), sending from \(levels.mic.source.isEmpty ? "-" : levels.mic.source)", |
| ] |
| @@ -145,7 +157,7 @@ private final class Events: NSObject, BridgeListenerProtocol { |
| + ["output: \(audio.outputName)"] |
| #endif |
| var sections: [(String, [String])] = [("Call", lines), ("Devices", routeLines)] |
| - if bridge != nil { sections.append(("Audio engine and processing", media.audio.diagnostics())) } |
| + if bridge != nil, let built { sections.append(("Audio engine and processing", built.audio.diagnostics())) } |
| return sections |
| } |
| @@ -193,6 +205,13 @@ private final class Events: NSObject, BridgeListenerProtocol { |
| receivedChatCount = 0 |
| title = "Calls" |
| standing = "" |
| + // The factory and its audio device module, built on the audio queue |
| + // the first time: the call's window is up while it is made. |
| + if built == nil { |
| + let made = await MediaEngine.make(call: self) |
| + if built == nil { built = made } |
| + } |
| + guard epoch == joiningEpoch, bridge === client else { throw CancellationError() } |
| try await media.prepare(capture: capture) |
| guard epoch == joiningEpoch, bridge === client else { throw CancellationError() } |
| localTrack = media.localVideo |
| @@ -213,7 +232,7 @@ private final class Events: NSObject, BridgeListenerProtocol { |
| let old = bridge |
| bridge = nil |
| events = nil |
| - media.close() |
| + built?.close() |
| // This call's microphone and speaker choices end with it (see AudioRoutes). |
| #if os(macOS) |
| audio.reset() |
| @@ -255,12 +274,20 @@ private final class Events: NSObject, BridgeListenerProtocol { |
| } |
| public func mute() { |
| muted.toggle() |
| - media.localAudio?.isEnabled = !muted |
| + // On the audio queue: switching a track waits out an engine start, |
| + // 1,136 ms measured on the main thread with a Bluetooth switch |
| + // simulated — and the first thing someone does as they join is often |
| + // to mute. |
| + let track = media.localAudio |
| + let on = !muted |
| + AudioWork.post { track?.isEnabled = on } |
| reportMedia() |
| } |
| public func toggleCamera() { |
| cameraOff.toggle() |
| - media.localVideo?.isEnabled = !cameraOff |
| + let track = media.localVideo |
| + let on = !cameraOff |
| + AudioWork.post { track?.isEnabled = on } |
| reportMedia() |
| } |
| @@ -297,7 +324,7 @@ private final class Events: NSObject, BridgeListenerProtocol { |
| @Published public private(set) var cameras: [CameraDevice] = [] |
| public func refreshDevices() { |
| - let available = media.cameras |
| + let available = built?.cameras ?? [] |
| if cameras != available { cameras = available } |
| if let currentCamera, !available.contains(where: { $0.id == currentCamera }) { |
| self.currentCamera = nil |
diff --git a/native/apple/Sources/CallsKit/CallAudio.swift b/native/apple/Sources/CallsKit/CallAudio.swift| index 5c24b5f..f3d53df 100644 |
| --- a/native/apple/Sources/CallsKit/CallAudio.swift |
| +++ b/native/apple/Sources/CallsKit/CallAudio.swift |
| @@ -105,6 +105,16 @@ public struct ProcessingReport: Equatable, Sendable { |
| /// made. |
| @MainActor final class CallAudio { |
| let factory: RTCPeerConnectionFactory |
| + /// The factory's audio device module, fetched once, off the main |
| + /// thread. Fetching it, and every property and call on it, is a blocking |
| + /// hop to WebRTC's worker thread, which is held for the whole of an |
| + /// engine start, a Bluetooth switch included (1,544 ms measured on the |
| + /// main thread with the switch simulated). So it is only ever asked on |
| + /// the audio queue; the main thread reads `state`. |
| + let module: RTCAudioDeviceModule |
| + /// The module's state for the main thread to read: never by asking the |
| + /// module, whose every property waits out an engine start (ModuleState). |
| + let state: ModuleState |
| /// Nil for the iPhone tests' silent device, which takes the constructor |
| /// that has no processing module. |
| let processing: RTCDefaultAudioProcessingModule? |
| @@ -125,43 +135,57 @@ public struct ProcessingReport: Equatable, Sendable { |
| var enginePin: AudioEnginePin { devices.pin } |
| #endif |
| - /// `audioDevice` is the iPhone tests' silent device. It takes the old |
| - /// constructor, which has no audio device module to speak of. |
| - /// |
| - /// `input`, on Mac, replaces the microphone and the speaker both: the |
| - /// engine runs in manual rendering mode, touching no device at all. |
| - init( |
| - routes: AudioRoutes, audioDevice: RTCAudioDevice?, input: AudioInputSource? = nil, |
| - reduceNoise: Bool = true |
| - ) { |
| + /// What building a CallAudio takes, made off the main thread (`make`): |
| + /// the peer connection factory with its audio device module and |
| + /// processing module, and the capture hook. Making the factory alone |
| + /// took 100–135 ms on an M1 Max. |
| + struct Parts: @unchecked Sendable { |
| + let factory: RTCPeerConnectionFactory |
| + let module: RTCAudioDeviceModule |
| + let state: ModuleState |
| + let processing: RTCDefaultAudioProcessingModule? |
| + let denoiser: CaptureDenoiser |
| + #if os(macOS) |
| + let devices: EngineWatch |
| + #endif |
| + } |
| + |
| + /// Builds the parts. Nonisolated, and slow: `make` runs it on the audio |
| + /// queue. `routes` is only held (weakly) by the module's observer, which |
| + /// reaches it on the main queue. |
| + nonisolated static func parts( |
| + routes: AudioRoutes, defaults: UserDefaults, audioDevice: RTCAudioDevice?, input: AudioInputSource?, |
| + denoiser: CaptureDenoiser? |
| + ) -> Parts { |
| let encoder = RTCDefaultVideoEncoderFactory() |
| let decoder = RTCDefaultVideoDecoderFactory() |
| - self.reduceNoise = reduceNoise |
| - denoiser = CaptureDenoiser(suppressor: CallAudio.makeSuppressor()) |
| + let denoiser = denoiser ?? CaptureDenoiser(suppressor: CallAudio.makeSuppressor()) |
| #if os(iOS) |
| - plan = ProcessingPlan.plan(iPhone: true, reduceNoise: reduceNoise, rate: nil, voiceIsolation: false) |
| if let audioDevice { |
| - processing = nil |
| - factory = RTCPeerConnectionFactory( |
| + let factory = RTCPeerConnectionFactory( |
| encoderFactory: encoder, decoderFactory: decoder, audioDevice: audioDevice) |
| - return |
| + let module = factory.audioDeviceModule |
| + return Parts( |
| + factory: factory, module: module, state: ModuleState(factory: factory, module: module), processing: nil, |
| + denoiser: denoiser) |
| } |
| let processing = RTCDefaultAudioProcessingModule( |
| config: nil, capturePostProcessingDelegate: nil, renderPreProcessingDelegate: nil) |
| - self.processing = processing |
| // The platform default on iPhone is the voice-processing I/O unit it |
| // has always been. |
| - factory = RTCPeerConnectionFactory( |
| + let factory = RTCPeerConnectionFactory( |
| audioDeviceModuleType: .platformDefault, bypassVoiceProcessing: false, |
| encoderFactory: encoder, decoderFactory: decoder, audioProcessingModule: processing) |
| + let module = factory.audioDeviceModule |
| + return Parts( |
| + factory: factory, module: module, state: ModuleState(factory: factory, module: module), |
| + processing: processing, denoiser: denoiser) |
| #else |
| - plan = ProcessingPlan.plan(iPhone: false, reduceNoise: reduceNoise, rate: nil, voiceIsolation: false) |
| let processing = RTCDefaultAudioProcessingModule( |
| config: nil, capturePostProcessingDelegate: nil, renderPreProcessingDelegate: nil) |
| - self.processing = processing |
| // AVAudioEngine rather than the CoreAudio module: it is the one with a |
| // device list and a way to choose from it. |
| - factory = RTCPeerConnectionFactory( |
| + let factory = RTCPeerConnectionFactory( |
| audioDeviceModuleType: .audioEngine, bypassVoiceProcessing: false, |
| encoderFactory: encoder, decoderFactory: decoder, audioProcessingModule: processing) |
| let module = factory.audioDeviceModule |
| @@ -174,17 +198,75 @@ public struct ProcessingReport: Equatable, Sendable { |
| // module, and is what runs now. |
| _ = module.setPlatformVoiceProcessingAllowed(false) |
| let pin = AudioEnginePin() |
| - devices = EngineWatch( |
| + let devices = EngineWatch( |
| routes: routes, module: module, input: input, pin: pin, |
| channels: InputChannelMapper( |
| - defaults: routes.defaults, layout: pin.inputLayout, explicitInput: { pin.choices.input })) |
| + defaults: defaults, layout: pin.inputLayout, explicitInput: { pin.choices.input })) |
| module.observer = devices |
| if input != nil { _ = module.setManualRenderingMode(true) } |
| + return Parts( |
| + factory: factory, module: module, state: ModuleState(factory: factory, module: module), |
| + processing: processing, denoiser: denoiser, devices: devices) |
| + #endif |
| + } |
| + |
| + /// Builds a CallAudio with its slow half on the audio queue: the main |
| + /// thread only attaches the result. What a call and the audio test use. |
| + static func make( |
| + routes: AudioRoutes, audioDevice: RTCAudioDevice?, input: AudioInputSource? = nil, |
| + reduceNoise: Bool = true, denoiser: CaptureDenoiser? = nil |
| + ) async -> CallAudio { |
| + #if os(macOS) |
| + let defaults = routes.defaults |
| + #else |
| + let defaults = UserDefaults.standard |
| + #endif |
| + let parts = await AudioWork.run { |
| + CallAudio.parts(routes: routes, defaults: defaults, audioDevice: audioDevice, input: input, denoiser: denoiser) |
| + } |
| + return CallAudio(parts: parts, routes: routes, reduceNoise: reduceNoise) |
| + } |
| + |
| + /// `audioDevice` is the iPhone tests' silent device. It takes the old |
| + /// constructor, which has no audio device module to speak of. |
| + /// |
| + /// `input`, on Mac, replaces the microphone and the speaker both: the |
| + /// engine runs in manual rendering mode, touching no device at all. |
| + /// |
| + /// Builds on the calling thread; the app uses `make`. |
| + convenience init( |
| + routes: AudioRoutes, audioDevice: RTCAudioDevice?, input: AudioInputSource? = nil, |
| + reduceNoise: Bool = true |
| + ) { |
| + #if os(macOS) |
| + let defaults = routes.defaults |
| + #else |
| + let defaults = UserDefaults.standard |
| + #endif |
| + self.init( |
| + parts: CallAudio.parts(routes: routes, defaults: defaults, audioDevice: audioDevice, input: input, denoiser: nil), |
| + routes: routes, reduceNoise: reduceNoise) |
| + } |
| + |
| + /// Attaches built parts: cheap, and on the main actor. |
| + init(parts: Parts, routes: AudioRoutes, reduceNoise: Bool) { |
| + factory = parts.factory |
| + module = parts.module |
| + state = parts.state |
| + processing = parts.processing |
| + denoiser = parts.denoiser |
| + self.reduceNoise = reduceNoise |
| + #if os(iOS) |
| + plan = ProcessingPlan.plan(iPhone: true, reduceNoise: reduceNoise, rate: nil, voiceIsolation: false) |
| + #else |
| + plan = ProcessingPlan.plan(iPhone: false, reduceNoise: reduceNoise, rate: nil, voiceIsolation: false) |
| + devices = parts.devices |
| // Attached in manual rendering too, so the interop harness drives the |
| // same routing code as the app; no device is opened either way. |
| let watch = devices |
| - routes.attach(module, pin: pin, inputRate: { watch.lastInputRate }) |
| + routes.attach(module, state: state, pin: devices.pin, inputRate: { watch.lastInputRate }) |
| #endif |
| + guard let processing else { return } |
| // The listener first, then the delegate: attaching the delegate can |
| // initialise it at once, and that report used to go nowhere. |
| denoiser.rateChanged = { [weak self] rate in |
| @@ -200,10 +282,27 @@ public struct ProcessingReport: Equatable, Sendable { |
| // delegate is never called (seen in the spike, M150). The property is |
| // weak; `denoiser` is what keeps it alive. |
| processing.capturePostProcessingDelegate = denoiser |
| - CallAudio.upgrade(denoiser) |
| + upgraded = CallAudio.upgrade(denoiser) |
| denoiser.enabled = plan.suppressor |
| } |
| + /// Whether this CallAudio's suppressor is (or is becoming) DeepFilterNet, |
| + /// so a fresh spare is loaded for the next one when it is done. |
| + private var upgraded = false |
| + |
| + deinit { |
| + #if canImport(DeepFilter) |
| + if upgraded { DeepFilterPreload.finished() } |
| + #endif |
| + // The factory goes on the audio queue, after everything queued for |
| + // it: a stop or a close queued there must never find its module gone |
| + // (it did, in the harness: a crash in `stopRecording`). Its teardown, |
| + // which stops the engine, then happens there too, not on the main |
| + // thread. |
| + let held: (RTCPeerConnectionFactory, RTCAudioDeviceModule) = (factory, module) |
| + AudioWork.post { withExtendedLifetime(held) {} } |
| + } |
| + |
| /// Whether the microphone is capturing. The module opens it when the |
| /// first connection starts sending, not when the track is made. Either |
| /// the module says it is recording, or the capture hook has been handed |
| @@ -211,7 +310,7 @@ public struct ProcessingReport: Equatable, Sendable { |
| /// interop gate's manual rendering shows up). |
| var capturing: Bool { |
| #if os(macOS) |
| - return factory.audioDeviceModule.isRecording || denoiser.buffersSeen > buffersAtStart |
| + return state.read().isRecording || denoiser.buffersSeen > buffersAtStart |
| #else |
| // Not asked on iPhone: its module is the platform one, whose |
| // recording state has not been checked against a device here. |
| @@ -229,6 +328,11 @@ public struct ProcessingReport: Equatable, Sendable { |
| /// Set while this CallAudio asked the module to record by itself (the |
| /// audio test before a call); undone by `stopRecordingAlone`. |
| private var recordingAlone = false |
| + /// Whether the module really was started by this CallAudio's ask, as |
| + /// opposed to already recording. Only read and written on the audio |
| + /// queue, where the start and the stop run in order. |
| + private final class Started: @unchecked Sendable { var byUs = false } |
| + private var started = Started() |
| /// Opens the microphone outside a call, as a call's first sending stream |
| /// does: the module's `initRecording` then `startRecording`, which is |
| @@ -243,27 +347,42 @@ public struct ProcessingReport: Equatable, Sendable { |
| /// 2026-09-26). The interop harness did not see it: in manual rendering |
| /// the module runs its input whenever its output runs. |
| /// |
| + /// On the audio queue, never the main thread (0.3.9): `initRecording` |
| + /// waits on WebRTC's worker thread while the engine starts, and with a |
| + /// Bluetooth headset that includes its move to the hands-free profile, |
| + /// one to two seconds. In 0.3.8 the main thread waited with it. |
| + /// |
| /// - Parameter ask: false only for the harness's control run, which |
| /// shows that without it the module is never asked to record. |
| /// - Returns: 0, or the module's first error (-9000: no microphone |
| /// permission, as the module sees it). |
| @discardableResult |
| - func startRecordingAlone(ask: Bool = true) -> Int { |
| - let module = factory.audioDeviceModule |
| + func startRecordingAlone(ask: Bool = true) async -> Int { |
| guard ask else { |
| AudioLog.info("test: not asking the module to record (control run)") |
| return 0 |
| } |
| - guard !module.isRecording else { |
| + let module = self.module |
| + // Set before the wait, so a stop asked for during it is queued, and |
| + // runs after the start, and closes what it opened. |
| + recordingAlone = true |
| + let started = self.started |
| + let began = CaptureDenoiser.now() |
| + let (initialised, result, recording, already) = await AudioWork.run { () -> (Int, Int, Bool, Bool) in |
| + guard !module.isRecording else { return (0, 0, true, true) } |
| + let initialised = module.initRecording() |
| + let result = initialised == 0 ? module.startRecording() : 0 |
| + started.byUs = true |
| + return (initialised, result, module.isRecording, false) |
| + } |
| + if already { |
| AudioLog.info("test: the module is already recording") |
| - return 0 |
| + } else { |
| + AudioLog.info( |
| + "test: asked the module to record (init \(initialised), start \(result)) in \(Int((CaptureDenoiser.now() - began) * 1000)) ms off the main thread; recording \(recording)" |
| + ) |
| } |
| - let initialised = module.initRecording() |
| - let started = initialised == 0 ? module.startRecording() : 0 |
| - recordingAlone = true |
| - AudioLog.info( |
| - "test: asked the module to record (init \(initialised), start \(started)); recording \(module.isRecording)") |
| - return initialised != 0 ? initialised : started |
| + return initialised != 0 ? initialised : result |
| } |
| /// A module error, in words, for the audio test. |
| @@ -277,11 +396,24 @@ public struct ProcessingReport: Equatable, Sendable { |
| } |
| } |
| + /// Closes what `startRecordingAlone` opened, on the audio queue after |
| + /// it: never waited for here (the engine's stop can take as long as its |
| + /// start, a headset moving back to A2DP). |
| func stopRecordingAlone() { |
| guard recordingAlone else { return } |
| recordingAlone = false |
| - let result = factory.audioDeviceModule.stopRecording() |
| - AudioLog.info("test: stopped the module's recording (\(result))") |
| + let module = self.module |
| + let factory = self.factory |
| + let started = self.started |
| + self.started = Started() |
| + AudioWork.post { |
| + // The module is only as alive as its factory. |
| + withExtendedLifetime(factory) {} |
| + guard started.byUs else { return } |
| + started.byUs = false |
| + let result = module.stopRecording() |
| + AudioLog.info("test: stopped the module's recording (\(result))") |
| + } |
| } |
| /// Whether the module has been asked to enable its microphone since |
| @@ -298,17 +430,29 @@ public struct ProcessingReport: Equatable, Sendable { |
| } |
| /// The suppressor a call starts on. See `upgrade`. |
| - static func makeSuppressor() -> NoiseSuppressor { RNNoiseSuppressor() } |
| + nonisolated static func makeSuppressor() -> NoiseSuppressor { RNNoiseSuppressor() } |
| /// DeepFilterNet 3 on Mac, where the app carries its model: loaded off |
| /// the main thread, then swapped in. RNNoise otherwise, and always on |
| /// iPhone (DeepFilterNet's cost there has not been measured on a phone). |
| - private static func upgrade(_ denoiser: CaptureDenoiser) { |
| + /// |
| + /// The model preloaded after launch is taken when there is one |
| + /// (DeepFilterPreload), so the suppressor is DeepFilterNet from the |
| + /// first buffer. Returns whether it is, or will be, DeepFilterNet. |
| + private static func upgrade(_ denoiser: CaptureDenoiser) -> Bool { |
| #if canImport(DeepFilter) |
| - guard let model = DeepFilterSuppressor.model else { return } |
| + if denoiser.engineName == "DeepFilterNet 3" { return true } |
| + if let ready = DeepFilterPreload.take() { |
| + denoiser.replace(with: ready) |
| + return true |
| + } |
| + guard let model = DeepFilterSuppressor.model else { return false } |
| DispatchQueue.global(qos: .userInitiated).async { |
| if let deep = DeepFilterSuppressor(model: model) { denoiser.replace(with: deep) } |
| } |
| + return true |
| + #else |
| + return false |
| #endif |
| } |
| @@ -343,17 +487,37 @@ public struct ProcessingReport: Equatable, Sendable { |
| denoiser.enabled = reduceNoise |
| denoiser.standAside = voiceIsolation |
| guard let track, processing != nil else { return } |
| - let result = track.setAudioProcessingOptions(CallAudio.options(plan)) |
| - refused = result.isSuccess ? nil : result.message |
| - // Platform modes can be refused where Apple's unit cannot give them |
| - // (a simulator, a route without it). Then let WebRTC choose where each |
| - // enabled stage runs; what was off stays off, so the plan's rule — |
| - // never two suppressors, never gain before ours — still holds. |
| - if !result.isSuccess { |
| - let fallback = track.setAudioProcessingOptions(CallAudio.options(plan, automatic: true)) |
| - if !fallback.isSuccess { refused = fallback.message } |
| + // Set on the audio queue: setting a track's processing waits out an |
| + // engine start (1,061 ms measured on the main thread with a Bluetooth |
| + // switch simulated), and a rate change — which calls this — arrives |
| + // exactly then. Queued ahead of anything that sends the track, so the |
| + // options are still in place before it is ever sent. |
| + let plan = self.plan |
| + let factory = self.factory |
| + applied += 1 |
| + let generation = applied |
| + AudioWork.post { |
| + withExtendedLifetime(factory) {} |
| + var refused: String? |
| + let result = track.setAudioProcessingOptions(CallAudio.options(plan)) |
| + refused = result.isSuccess ? nil : result.message |
| + // Platform modes can be refused where Apple's unit cannot give them |
| + // (a simulator, a route without it). Then let WebRTC choose where each |
| + // enabled stage runs; what was off stays off, so the plan's rule — |
| + // never two suppressors, never gain before ours — still holds. |
| + if !result.isSuccess { |
| + let fallback = track.setAudioProcessingOptions(CallAudio.options(plan, automatic: true)) |
| + if !fallback.isSuccess { refused = fallback.message } |
| + } |
| + DispatchQueue.main.async { |
| + MainActor.assumeIsolated { [weak self] in |
| + guard let self, self.applied == generation else { return } |
| + self.refused = refused |
| + } |
| + } |
| } |
| } |
| + private var applied = 0 |
| static func options(_ plan: ProcessingPlan, automatic: Bool = false) -> RTCAudioProcessingOptions { |
| func component(_ place: ProcessingPlan.Where) -> RTCAudioProcessingComponentOptions { |
| @@ -375,10 +539,10 @@ public struct ProcessingReport: Equatable, Sendable { |
| /// For "Copy audio diagnostics": the module, the engine, the hook. |
| func diagnostics() -> [String] { |
| var lines: [String] = [] |
| - let module = factory.audioDeviceModule |
| + let module = state.read() |
| #if os(macOS) |
| lines.append( |
| - "module: manual rendering \(module.isManualRenderingMode), engine running \(module.isEngineRunning), playout initialised \(module.isPlayoutInitialized), playing \(module.isPlaying), recording initialised \(module.isRecordingInitialized), recording \(module.isRecording), microphone muted \(module.isMicrophoneMuted)" |
| + "module: manual rendering \(module.isManualRenderingMode), engine running \(module.isEngineRunning), playout initialised \(module.isPlayoutInitialized), playing \(module.isPlaying), recording initialised \(module.isRecordingInitialized), recording \(module.isRecording), microphone muted \(module.isMicrophoneMuted)\(module.stale ? " (as last read: the module is busy)" : "")" |
| ) |
| let choices = enginePin.choices |
| lines.append( |
| @@ -401,7 +565,11 @@ public struct ProcessingReport: Equatable, Sendable { |
| /// What is actually running, as WebRTC reports it — not what was asked. |
| func report() -> ProcessingReport { |
| - let state = factory.audioProcessingState |
| + guard let state = state.read().processing else { |
| + return ProcessingReport( |
| + suppression: denoiser.state, echoCancellation: "unknown", noiseSuppression: "unknown", |
| + gainControl: "unknown", refused: refused, started: capturing) |
| + } |
| func says(_ component: RTCAudioProcessingComponentState) -> String { |
| switch component.effective { |
| case .disabled: return "off" |
| @@ -428,7 +596,7 @@ public struct ProcessingReport: Equatable, Sendable { |
| /// for a real microphone — where the chosen input of a multi-channel one |
| /// is mapped (InputChannel.swift). Starting and stopping are watched so a |
| /// device the engine cannot start on is noticed rather than left silent. |
| - private final class EngineWatch: NSObject, RTCAudioDeviceModuleDelegate, @unchecked Sendable { |
| + final class EngineWatch: NSObject, RTCAudioDeviceModuleDelegate, @unchecked Sendable { |
| weak var routes: AudioRoutes? |
| weak var module: RTCAudioDeviceModule? |
| let input: AudioInputSource? |
| @@ -497,6 +665,14 @@ public struct ProcessingReport: Equatable, Sendable { |
| lock.lock() |
| requested = true |
| lock.unlock() |
| + // The harness's stand-in for a Bluetooth headset changing profile, |
| + // which holds up the engine's start here, on WebRTC's worker |
| + // thread. Zero in the app. |
| + let delay = AudioStartProbe.engineStartDelay |
| + if delay > 0, input == nil { |
| + AudioLog.info("harness: holding the engine's start \(Int(delay * 1000)) ms, as a Bluetooth switch does") |
| + Thread.sleep(forTimeInterval: delay) |
| + } |
| } |
| note("will enable: playout \(isPlayoutEnabled), recording \(isRecordingEnabled), voice processing \(isVoiceProcessingEnabled)") |
| guard input == nil, !engine.isInManualRenderingMode else { |
| @@ -571,16 +747,20 @@ public struct ProcessingReport: Equatable, Sendable { |
| // The module retries a start a few times and then gives up without |
| // telling its observer. Looked at once it has had time: an engine |
| // that is not running, with no stop since, did not start. |
| - DispatchQueue.main.asyncAfter(deadline: .now() + 2) { [weak self] in |
| - MainActor.assumeIsolated { |
| - guard let self, let module = self.module else { return } |
| - self.lock.lock() |
| - let now = (self.starts, self.stops) |
| - self.lock.unlock() |
| - guard now == seen, !module.isEngineRunning else { return } |
| - AudioLog.error("engine did not start on \(target.description)") |
| - self.note("did not start on \(target.description)") |
| - self.routes?.engineFailed(target, reason: "the audio engine would not start on it") |
| + // Asked on the audio queue: the module's answer waits out any start |
| + // in progress. |
| + AudioWork.queue.asyncAfter(deadline: .now() + 2) { [weak self] in |
| + guard let self, let module = self.module else { return } |
| + self.lock.lock() |
| + let now = (self.starts, self.stops) |
| + self.lock.unlock() |
| + guard now == seen, !module.isEngineRunning else { return } |
| + AudioLog.error("engine did not start on \(target.description)") |
| + self.note("did not start on \(target.description)") |
| + DispatchQueue.main.async { [weak self] in |
| + MainActor.assumeIsolated { |
| + self?.routes?.engineFailed(target, reason: "the audio engine would not start on it") |
| + } |
| } |
| } |
| return 0 |
| @@ -649,3 +829,90 @@ public struct ProcessingReport: Equatable, Sendable { |
| ) -> Int { 0 } |
| } |
| #endif |
| + |
| +/// The audio device module's state, for the main thread to read. |
| +/// |
| +/// Every property of LiveKit's module is a blocking hop to WebRTC's worker |
| +/// thread, and the worker holds on to it for the whole of an engine start |
| +/// or rebuild — a Bluetooth headset's profile switch included. Measured |
| +/// with that switch simulated at 1.5 s: `isRecording`, `isEngineRunning`, |
| +/// `isPlaying`, the device lists and the factory's processing state each |
| +/// held the main thread for 0.9 to 1.2 s. The meters read it ten times a |
| +/// second. |
| +/// |
| +/// So the main thread never asks the module. It asks this, which reads the |
| +/// module on a queue of its own and waits at most a few milliseconds for |
| +/// the answer; if the module is busy it gets the last answer, marked stale, |
| +/// and the fresh one is there for the next look. |
| +final class ModuleState: @unchecked Sendable { |
| + struct Snapshot: @unchecked Sendable { |
| + var isRecording = false |
| + var isPlaying = false |
| + var isEngineRunning = false |
| + var isManualRenderingMode = false |
| + var isPlayoutInitialized = false |
| + var isRecordingInitialized = false |
| + var isMicrophoneMuted = false |
| + var processing: RTCAudioProcessingState? |
| + /// The module did not answer in time: this is its last answer. |
| + var stale = false |
| + } |
| + |
| + private weak var factory: RTCPeerConnectionFactory? |
| + private weak var module: RTCAudioDeviceModule? |
| + private let queue = DispatchQueue(label: "net.amadan.oficina.calls.audio-state", qos: .userInitiated) |
| + private let lock = NSLock() |
| + private var latest = Snapshot() |
| + private var reading: DispatchGroup? |
| + /// How long a read on the main thread may wait for the module. |
| + static let patience: TimeInterval = 0.005 |
| + |
| + /// Reads once, here and now: made off the main thread with the parts. |
| + init(factory: RTCPeerConnectionFactory, module: RTCAudioDeviceModule) { |
| + self.factory = factory |
| + self.module = module |
| + latest = ModuleState.snapshot(factory: factory, module: module) |
| + } |
| + |
| + func read(patience: TimeInterval = ModuleState.patience) -> Snapshot { |
| + lock.lock() |
| + let group: DispatchGroup |
| + if let reading { |
| + group = reading |
| + } else { |
| + group = DispatchGroup() |
| + group.enter() |
| + reading = group |
| + queue.async { [self] in |
| + let fresh = factory.flatMap { f in module.map { ModuleState.snapshot(factory: f, module: $0) } } |
| + lock.lock() |
| + if let fresh { latest = fresh } |
| + reading = nil |
| + lock.unlock() |
| + group.leave() |
| + } |
| + } |
| + lock.unlock() |
| + let answered = group.wait(timeout: .now() + patience) == .success |
| + lock.lock() |
| + defer { lock.unlock() } |
| + var out = latest |
| + out.stale = !answered |
| + return out |
| + } |
| + |
| + private static func snapshot(factory: RTCPeerConnectionFactory, module: RTCAudioDeviceModule) -> Snapshot { |
| + var s = Snapshot() |
| + s.isRecording = module.isRecording |
| + s.isPlaying = module.isPlaying |
| + #if os(macOS) |
| + s.isEngineRunning = module.isEngineRunning |
| + s.isManualRenderingMode = module.isManualRenderingMode |
| + s.isPlayoutInitialized = module.isPlayoutInitialized |
| + s.isRecordingInitialized = module.isRecordingInitialized |
| + s.isMicrophoneMuted = module.isMicrophoneMuted |
| + #endif |
| + s.processing = factory.audioProcessingState |
| + return s |
| + } |
| +} |
diff --git a/native/apple/Sources/CallsKit/Media.swift b/native/apple/Sources/CallsKit/Media.swift| index 06b45c0..df3ea7d 100644 |
| --- a/native/apple/Sources/CallsKit/Media.swift |
| +++ b/native/apple/Sources/CallsKit/Media.swift |
| @@ -28,16 +28,35 @@ import LiveKitWebRTC |
| /// The audio device module and processing behind the factory. See CallAudio. |
| let audio: CallAudio |
| - init(call: Call) { |
| - self.call = call |
| + /// Builds on the calling thread: only for a call touched before it |
| + /// joined. A join uses `make`. |
| + convenience init(call: Call) { |
| RTCInitializeSSL() |
| #if os(iOS) |
| - audio = CallAudio(routes: call.audio, audioDevice: call.customAudioDevice) |
| + self.init(call: call, audio: CallAudio(routes: call.audio, audioDevice: call.customAudioDevice)) |
| #else |
| - audio = CallAudio(routes: call.audio, audioDevice: nil, input: call.audioInput) |
| + self.init(call: call, audio: CallAudio(routes: call.audio, audioDevice: nil, input: call.audioInput)) |
| #endif |
| + } |
| + |
| + init(call: Call, audio: CallAudio) { |
| + self.call = call |
| + self.audio = audio |
| factory = audio.factory |
| } |
| + |
| + /// The media for a call, with its factory and audio device module built |
| + /// on the audio queue (CallAudio.make): 100–200 ms the main thread no |
| + /// longer spends when someone joins. |
| + static func make(call: Call) async -> MediaEngine { |
| + await AudioWork.run { RTCInitializeSSL() } |
| + #if os(iOS) |
| + let audio = await CallAudio.make(routes: call.audio, audioDevice: call.customAudioDevice) |
| + #else |
| + let audio = await CallAudio.make(routes: call.audio, audioDevice: nil, input: call.audioInput) |
| + #endif |
| + return MediaEngine(call: call, audio: audio) |
| + } |
| func prepare(capture supplied: VideoCapture?) async throws { |
| close() |
| let source = factory.videoSource() |
| @@ -155,8 +174,13 @@ import LiveKitWebRTC |
| closePeers() |
| capture?.stop() |
| capture = nil |
| - localVideo?.isEnabled = false |
| - localAudio?.isEnabled = false |
| + // On the audio queue: a track's switch waits out an engine start. |
| + let video = localVideo |
| + let audio = localAudio |
| + AudioWork.post { |
| + video?.isEnabled = false |
| + audio?.isEnabled = false |
| + } |
| localVideo = nil |
| localAudio = nil |
| } |
| @@ -220,9 +244,8 @@ import LiveKitWebRTC |
| ] |
| for peer in peers.values { |
| guard let pc = peer.pc else { continue } |
| - let report: RTCStatisticsReport = await withCheckedContinuation { c in |
| - pc.statistics { c.resume(returning: $0) } |
| - } |
| + let ownScreen = await peer.ownScreenMid() |
| + let report = await PeerCalls.statistics(pc) |
| for stat in report.statistics.values { |
| let kind = (stat.values["kind"] as? String) ?? (stat.values["mediaType"] as? String) ?? "" |
| let mid = stat.values["mid"] as? String ?? "" |
| @@ -253,7 +276,7 @@ import LiveKitWebRTC |
| } |
| if stat.type == "outbound-rtp" { |
| if kind == "video" { |
| - let field = peer.isOwnScreen(mid: mid) ? "screenBytesSent" : "videoBytesSent" |
| + let field = !mid.isEmpty && mid == ownScreen ? "screenBytesSent" : "videoBytesSent" |
| values[field, default: 0] += |
| (stat.values["bytesSent"] as? NSNumber)?.uint64Value ?? 0 |
| } |
| @@ -277,15 +300,18 @@ import LiveKitWebRTC |
| for (id, peer) in peers { |
| guard let pc = peer.pc else { continue } |
| let (peak, buffers) = peer.tap.take() |
| - let state = pc.iceConnectionState |
| + // Read on the audio queue, as every call into the connection is |
| + // (PeerCalls): five times a second, and each one a blocking hop to |
| + // WebRTC's signalling thread. |
| + let (state, receivers) = await PeerCalls.read(pc) { pc in |
| + (pc.iceConnectionState, stats ? pc.receivers.filter { $0.track?.kind == "audio" } : []) |
| + } |
| var reading = PeerAudioReading( |
| peak: peak, buffers: buffers, connected: state == .connected || state == .completed) |
| if stats { |
| var bytes: UInt64 = 0 |
| - for receiver in pc.receivers where receiver.track?.kind == "audio" { |
| - let report: RTCStatisticsReport = await withCheckedContinuation { c in |
| - pc.statistics(for: receiver) { c.resume(returning: $0) } |
| - } |
| + for receiver in receivers { |
| + let report = await PeerCalls.statistics(pc, for: receiver) |
| for stat in report.statistics.values where stat.type == "inbound-rtp" { |
| bytes += (stat.values["bytesReceived"] as? NSNumber)?.uint64Value ?? 0 |
| } |
| @@ -311,10 +337,8 @@ import LiveKitWebRTC |
| func screenReports() async -> [String: ScreenSendReport] { |
| var reports: [String: ScreenSendReport] = [:] |
| for (id, peer) in peers { |
| - guard let pc = peer.pc, let mid = peer.ownScreenMid else { continue } |
| - let report: RTCStatisticsReport = await withCheckedContinuation { c in |
| - pc.statistics { c.resume(returning: $0) } |
| - } |
| + guard let pc = peer.pc, let mid = await peer.ownScreenMid() else { continue } |
| + let report = await PeerCalls.statistics(pc) |
| let flat = report.statistics.mapValues { (type: $0.type, values: $0.values as [String: Any]) } |
| if var read = ScreenSendReport.read(flat, mid: mid) { |
| read.timestamp = report.timestamp_us / 1_000_000 |
| @@ -394,24 +418,44 @@ struct PeerAudioReading: Equatable, Sendable { |
| } |
| return c |
| } |
| + /// Makes the connection on the audio queue, as the first thing this |
| + /// peer's work does: everything that arrives for it waits behind it. |
| + /// Making one waits on WebRTC's worker thread, which can be busy |
| + /// starting the audio engine for someone else (846 ms measured on the |
| + /// main thread with a Bluetooth switch simulated). |
| func open(ice: ICEConfig) { |
| guard let engine else { return } |
| lastICE = ice |
| - pc = engine.factory.peerConnection( |
| - with: configuration(ice), |
| - constraints: RTCMediaConstraints(mandatoryConstraints: nil, optionalConstraints: nil), |
| - delegate: self) |
| - if let audio = engine.localAudio { _ = pc?.add(audio, streamIds: ["calls"]) } |
| - if let video = engine.localVideo { |
| - sentVideo = pc?.add(video, streamIds: ["calls"]) != nil |
| + let factory = engine.factory |
| + let config = configuration(ice) |
| + let audio = engine.localAudio |
| + let video = engine.localVideo |
| + let delegate: RTCPeerConnectionDelegate = self |
| + enqueue { [weak self] in |
| + let (made, sent) = await AudioWork.run { () -> (RTCPeerConnection?, Bool) in |
| + guard |
| + let pc = factory.peerConnection( |
| + with: config, constraints: RTCMediaConstraints(mandatoryConstraints: nil, optionalConstraints: nil), |
| + delegate: delegate) |
| + else { return (nil, false) } |
| + if let audio { _ = pc.add(audio, streamIds: ["calls"]) } |
| + let sent = video.map { pc.add($0, streamIds: ["calls"]) != nil } ?? false |
| + return (pc, sent) |
| + } |
| + guard let self, !self.closed, let made else { |
| + if let made { PeerCalls.close(made) } |
| + return |
| + } |
| + self.pc = made |
| + self.sentVideo = sent |
| + if let limit = self.engine?.uplinkLimit { self.limitUplink(limit) } |
| + // A connection built while this app is already sharing gets the |
| + // screen with everything else: someone joining mid-share sees it. |
| + // Last, as in the browser, so the camera and microphone take the |
| + // m-lines the far side built for them and the screen a place of its |
| + // own after them. |
| + self.addScreen() |
| } |
| - if let limit = engine.uplinkLimit { limitUplink(limit) } |
| - // A connection built while this app is already sharing gets the |
| - // screen with everything else: someone joining mid-share sees it. |
| - // Last, as in the browser, so the camera and microphone take the |
| - // m-lines the far side built for them and the screen a place of its |
| - // own after them. |
| - addScreen() |
| } |
| /// Gives this connection the screen on a `sendonly` transceiver in a |
| @@ -425,31 +469,58 @@ struct PeerAudioReading: Equatable, Sendable { |
| /// up each sending a screen in the place the other built for something |
| /// else. The browser's mutualshare gate found that; the deferred ones |
| /// are added when the connection settles. |
| + /// |
| + /// Added on the audio queue, as every call into the connection is; a |
| + /// share that ended while it was being added is taken away again. |
| func addScreen() { |
| - guard let pc, !closed, screenTx == nil, let track = engine?.localScreen, |
| - pc.signalingState == .stable |
| + guard let pc, !closed, screenTx == nil, !addingScreen, let track = engine?.localScreen, let engine |
| else { return } |
| - let setup = RTCRtpTransceiverInit() |
| - setup.direction = .sendOnly |
| - setup.streamIds = ["screen"] |
| - screenTx = pc.addTransceiver(with: track, init: setup) |
| - if let tx = screenTx, let engine { |
| - try? tx.setCodecPreferences( |
| - ScreenSendPolicy.preferringH264(engine.factory.rtpSenderCapabilities(forKind: "video").codecs), |
| - error: ()) |
| + addingScreen = true |
| + let factory = engine.factory |
| + Task { @MainActor [weak self] in |
| + let added = await PeerCalls.read(pc) { pc -> RTCRtpTransceiver? in |
| + guard pc.signalingState == .stable else { return nil } |
| + let setup = RTCRtpTransceiverInit() |
| + setup.direction = .sendOnly |
| + setup.streamIds = ["screen"] |
| + guard let tx = pc.addTransceiver(with: track, init: setup) else { return nil } |
| + try? tx.setCodecPreferences( |
| + ScreenSendPolicy.preferringH264(factory.rtpSenderCapabilities(forKind: "video").codecs), error: ()) |
| + return tx |
| + } |
| + guard let self else { return } |
| + self.addingScreen = false |
| + guard let added else { return } |
| + self.screenTx = added |
| + self.screenMidCache = nil |
| + guard !self.closed, self.pc === pc, self.engine?.localScreen === track else { |
| + self.removeScreen() |
| + return |
| + } |
| + self.applySendPolicy() |
| } |
| - applySendPolicy() |
| } |
| + private var addingScreen = false |
| /// The screen at its mode's parameters and the camera capped beside |
| /// it, or the camera back at its defaults when there is no screen. |
| /// Sender parameters only: nothing here renegotiates. |
| + /// |
| + /// Applied on the audio queue: a sender's parameters are read and set on |
| + /// WebRTC's worker thread, and this runs each time a negotiation settles |
| + /// — just when the worker may be starting the audio engine. |
| func applySendPolicy() { |
| guard let pc, !closed, let engine else { return } |
| - if let screen = screenTx?.sender { screen.apply(engine.screenPlan) } |
| - let camera = engine.localVideo?.trackId |
| - if let sender = pc.senders.first(where: { $0.track != nil && $0.track?.trackId == camera }) { |
| - sender.apply(ScreenSendPolicy.camera(sharing: screenTx != nil)) |
| + let screen = screenTx?.sender |
| + let screenPlan = engine.screenPlan |
| + let cameraTrack = engine.localVideo |
| + let camera = ScreenSendPolicy.camera(sharing: screenTx != nil) |
| + AudioWork.post { |
| + if let screen { screen.apply(screenPlan) } |
| + let id = cameraTrack?.trackId |
| + if let sender = pc.senders.first(where: { $0.track != nil && $0.track?.trackId == id }) { |
| + sender.apply(camera) |
| + } |
| } |
| } |
| @@ -464,7 +535,8 @@ struct PeerAudioReading: Equatable, Sendable { |
| func removeScreen() { |
| guard let tx = screenTx else { return } |
| screenTx = nil |
| - tx.stopInternal() |
| + screenMidCache = nil |
| + AudioWork.post { tx.stopInternal() } |
| applySendPolicy() |
| renegotiate() |
| } |
| @@ -474,16 +546,16 @@ struct PeerAudioReading: Equatable, Sendable { |
| /// owed, and made the next time it settles. |
| private func renegotiate() { |
| guard let pc, !closed else { return } |
| - guard pc.signalingState == .stable else { |
| - owesOffer = true |
| - return |
| - } |
| Task { @MainActor [weak self] in |
| + guard await PeerCalls.read(pc, { $0.signalingState }) == .stable else { |
| + self?.owesOffer = true |
| + return |
| + } |
| guard let self else { return } |
| if self.polite { try? await Task.sleep(nanoseconds: 200_000_000) } |
| self.enqueue { [weak self] in |
| guard let self, let pc = self.pc else { return } |
| - if pc.signalingState != .stable { |
| + if await PeerCalls.read(pc, { $0.signalingState }) != .stable { |
| self.owesOffer = true |
| return |
| } |
| @@ -495,19 +567,37 @@ struct PeerAudioReading: Equatable, Sendable { |
| /// The connection is back at stable: the one moment a deferred screen |
| /// can be added, and an owed offer made. |
| private func settled() { |
| - guard let pc, !closed, pc.signalingState == .stable else { return } |
| - addScreen() |
| - applySendPolicy() |
| - if owesOffer { |
| - owesOffer = false |
| - renegotiate() |
| + guard let pc, !closed else { return } |
| + Task { @MainActor [weak self] in |
| + // Read on the audio queue, as every call into the connection is. |
| + let stable = await PeerCalls.read(pc) { $0.signalingState } == .stable |
| + guard let self, stable, !self.closed, self.pc === pc else { return } |
| + self.addScreen() |
| + self.applySendPolicy() |
| + if self.owesOffer { |
| + self.owesOffer = false |
| + self.renegotiate() |
| + } |
| } |
| } |
| func isScreen(mid: String) -> Bool { !mid.isEmpty && mid == screenMid } |
| - func isOwnScreen(mid: String) -> Bool { !mid.isEmpty && mid == screenTx?.mid } |
| - var ownScreenMid: String? { screenTx?.mid.isEmpty == false ? screenTx?.mid : nil } |
| + /// The mid our screen went out on, once negotiated: read on the audio |
| + /// queue (a transceiver's mid is a hop to WebRTC's signalling thread), |
| + /// and kept once known. |
| + func ownScreenMid() async -> String? { |
| + if let screenMidCache { return screenMidCache } |
| + guard let pc, let tx = screenTx else { return nil } |
| + let mid = await PeerCalls.read(pc) { _ in tx.mid } |
| + guard !mid.isEmpty, screenTx === tx else { return nil } |
| + screenMidCache = mid |
| + return mid |
| + } |
| + private var screenMidCache: String? |
| func limitUplink(_ bps: Int?) { |
| - _ = pc?.setBweMinBitrateBps(nil, currentBitrateBps: nil, maxBitrateBps: bps.map { NSNumber(value: $0) }) |
| + guard let pc else { return } |
| + AudioWork.post { |
| + _ = pc.setBweMinBitrateBps(nil, currentBitrateBps: nil, maxBitrateBps: bps.map { NSNumber(value: $0) }) |
| + } |
| } |
| /// The `share` message: on draws the captioned tile ahead of the media, |
| @@ -564,7 +654,9 @@ struct PeerAudioReading: Equatable, Sendable { |
| screenTx = nil |
| work?.cancel() |
| work = nil |
| - pc?.close() |
| + // Closing can stop the audio engine (the last connection), which |
| + // takes as long as starting it: on the audio queue. |
| + if let pc { PeerCalls.close(pc) } |
| pc = nil |
| candidates = [] |
| } |
| @@ -580,21 +672,18 @@ struct PeerAudioReading: Equatable, Sendable { |
| guard !closed, let body = json(payload) else { return } |
| engine?.call?.send(type: type, to: id, gen: generation, payload: body) |
| } |
| + // Descriptions and candidates are applied from the audio queue |
| + // (PeerCalls). Applying one waits for WebRTC's worker thread, which |
| + // starts the audio engine for the first stream in each direction — a |
| + // Bluetooth headset's profile switch included — and up to 0.3.8 the |
| + // main thread waited with it. |
| private func setLocal(_ desc: RTCSessionDescription) async throws { |
| guard let pc, !closed else { return } |
| - try await withCheckedThrowingContinuation { (c: CheckedContinuation<Void, Error>) in |
| - pc.setLocalDescription(desc) { error in |
| - if let error { c.resume(throwing: error) } else { c.resume() } |
| - } |
| - } |
| + try await PeerCalls.set(pc, local: true, desc) |
| } |
| private func setRemote(_ desc: RTCSessionDescription) async throws { |
| guard let pc, !closed else { return } |
| - try await withCheckedThrowingContinuation { (c: CheckedContinuation<Void, Error>) in |
| - pc.setRemoteDescription(desc) { error in |
| - if let error { c.resume(throwing: error) } else { c.resume() } |
| - } |
| - } |
| + try await PeerCalls.set(pc, local: false, desc) |
| } |
| private func makeDescription(answer: Bool, restart: Bool = false) async throws |
| -> RTCSessionDescription |
| @@ -602,23 +691,10 @@ struct PeerAudioReading: Equatable, Sendable { |
| guard let pc else { throw CancellationError() } |
| let constraints = RTCMediaConstraints( |
| mandatoryConstraints: restart ? ["IceRestart": "true"] : nil, optionalConstraints: nil) |
| - return try await withCheckedThrowingContinuation { c in |
| - let completion: @Sendable (RTCSessionDescription?, Error?) -> Void = { desc, error in |
| - if let desc { |
| - c.resume(returning: desc) |
| - } else { |
| - c.resume(throwing: error ?? CancellationError()) |
| - } |
| - } |
| - if answer { |
| - pc.answer(for: constraints, completionHandler: completion) |
| - } else { |
| - pc.offer(for: constraints, completionHandler: completion) |
| - } |
| - } |
| + return try await PeerCalls.describe(pc, answer: answer, constraints) |
| } |
| private func offer(restart: Bool = false) async throws { |
| - guard let pc, !closed, pc.signalingState == .stable else { return } |
| + guard let pc, !closed, await PeerCalls.read(pc, { $0.signalingState }) == .stable, !closed else { return } |
| let desc = try await makeDescription(answer: false, restart: restart) |
| guard !closed else { return } |
| try await setLocal(desc) |
| @@ -630,7 +706,7 @@ struct PeerAudioReading: Equatable, Sendable { |
| guard let self, let pc = self.pc, !self.closed else { return } |
| if message.type == "candidate", let candidate = message.payload?.candidate { |
| if self.ignoreOffer { return } |
| - if pc.remoteDescription == nil { |
| + if await PeerCalls.read(pc, { $0.remoteDescription == nil }) { |
| guard self.candidates.count < 256 else { throw CancellationError() } |
| self.candidates.append(candidate) |
| } else { |
| @@ -639,8 +715,10 @@ struct PeerAudioReading: Equatable, Sendable { |
| return |
| } |
| guard let desc = message.payload?.description else { return } |
| + let signaling = await PeerCalls.read(pc) { $0.signalingState } |
| + guard !self.closed else { return } |
| if message.type == "offer", desc.type == "offer" { |
| - let collision = pc.signalingState != .stable |
| + let collision = signaling != .stable |
| self.ignoreOffer = collision && !self.polite |
| if self.ignoreOffer { return } |
| if collision { try await self.setLocal(RTCSessionDescription(type: .rollback, sdp: "")) } |
| @@ -650,9 +728,7 @@ struct PeerAudioReading: Equatable, Sendable { |
| let answer = try await self.makeDescription(answer: true) |
| try await self.setLocal(answer) |
| self.send("answer", ["description": ["type": "answer", "sdp": answer.sdp]]) |
| - } else if message.type == "answer", desc.type == "answer", |
| - pc.signalingState == .haveLocalOffer |
| - { |
| + } else if message.type == "answer", desc.type == "answer", signaling == .haveLocalOffer { |
| try await self.setRemote(RTCSessionDescription(type: .answer, sdp: desc.sdp)) |
| self.ignoreOffer = false |
| try await self.drain() |
| @@ -664,9 +740,7 @@ struct PeerAudioReading: Equatable, Sendable { |
| let ice = RTCIceCandidate( |
| sdp: candidate.candidate, sdpMLineIndex: candidate.sdpMLineIndex ?? 0, |
| sdpMid: candidate.sdpMid) |
| - try await withCheckedThrowingContinuation { (c: CheckedContinuation<Void, Error>) in |
| - pc.add(ice) { error in if let error { c.resume(throwing: error) } else { c.resume() } } |
| - } |
| + try await PeerCalls.add(pc, ice) |
| } |
| private func drain() async throws { |
| let saved = candidates |
| @@ -678,7 +752,8 @@ struct PeerAudioReading: Equatable, Sendable { |
| lastICE = ice |
| enqueue { [weak self] in |
| guard let self, let pc = self.pc else { return } |
| - if pc.setConfiguration(self.configuration(ice)) { try await self.offer(restart: true) } |
| + let config = self.configuration(ice) |
| + if await PeerCalls.read(pc, { $0.setConfiguration(config) }) { try await self.offer(restart: true) } |
| } |
| } |
| nonisolated func peerConnectionShouldNegotiate(_ peerConnection: RTCPeerConnection) { |
diff --git a/native/apple/Sources/CallsMac/CallsMac.swift b/native/apple/Sources/CallsMac/CallsMac.swift| index cb85552..93e0348 100644 |
| --- a/native/apple/Sources/CallsMac/CallsMac.swift |
| +++ b/native/apple/Sources/CallsMac/CallsMac.swift |
| @@ -23,6 +23,7 @@ import LiveKitWebRTC |
| .onAppear { |
| CallsAppDelegate.model = model |
| model.restore() |
| + AudioWarmUp.afterLaunch() |
| } |
| .miniWindow(for: model) |
| .onChange(of: model.joining, initial: true) { updater.isBusy = model.joining } |
diff --git a/native/apple/Tests/CallsKitTests/AudioRoutesTests.swift b/native/apple/Tests/CallsKitTests/AudioRoutesTests.swift| index 3fb5adb..ae5dee0 100644 |
| --- a/native/apple/Tests/CallsKitTests/AudioRoutesTests.swift |
| +++ b/native/apple/Tests/CallsKitTests/AudioRoutesTests.swift |
| @@ -389,6 +389,36 @@ |
| /// The real CoreAudio on this machine: the inventory the app lists at |
| /// launch, with no engine and no permission needed. |
| + /// Which microphone the test says it is switching: a Bluetooth headset |
| + /// not yet at its hands-free rate, and nothing else. |
| + final class BluetoothSwitchTests: XCTestCase { |
| + private let inventory = AudioInventory( |
| + devices: [ |
| + SystemAudioDevice( |
| + uid: "airpods", name: "Paul’s AirPods Pro", inputChannels: 1, outputChannels: 2, sampleRate: 48_000, |
| + transport: "bluetooth"), |
| + SystemAudioDevice( |
| + uid: "airpods-hfp", name: "Headset", inputChannels: 1, outputChannels: 1, sampleRate: 24_000, |
| + transport: "bluetooth"), |
| + SystemAudioDevice( |
| + uid: "BuiltInMicrophoneDevice", name: "MacBook Air Microphone", inputChannels: 1, outputChannels: 0, |
| + sampleRate: 48_000, transport: "built-in"), |
| + ], defaultInput: "airpods", defaultOutput: "airpods") |
| + |
| + func testABluetoothHeadsetInItsListeningProfileIsSwitching() { |
| + XCTAssertEqual(AudioRoutes.bluetoothSwitching(uid: "airpods", inventory), "Paul’s AirPods Pro") |
| + } |
| + |
| + func testOneAlreadyAtItsHandsFreeRateIsNot() { |
| + XCTAssertNil(AudioRoutes.bluetoothSwitching(uid: "airpods-hfp", inventory)) |
| + } |
| + |
| + func testABuiltInMicrophoneOrNothingIsNot() { |
| + XCTAssertNil(AudioRoutes.bluetoothSwitching(uid: "BuiltInMicrophoneDevice", inventory)) |
| + XCTAssertNil(AudioRoutes.bluetoothSwitching(uid: nil, inventory)) |
| + } |
| + } |
| + |
| final class CoreAudioInventoryTests: XCTestCase { |
| func testTheInventoryIsReadyWithoutAnEngine() { |
| let inventory = CoreAudioDevices.inventory() |
| @@ -412,7 +442,7 @@ |
| /// default microphone and that speaker before the module wires it, the |
| /// engine runs there, and going back to the default rebuilds it. |
| final class EnginePinIntegrationTests: XCTestCase { |
| - @MainActor func testAChosenSpeakerRunsTheEngineOnItAndBack() throws { |
| + @MainActor func testAChosenSpeakerRunsTheEngineOnItAndBack() async throws { |
| let inventory = CoreAudioDevices.inventory() |
| guard let speaker = inventory.defaultOutput, let microphone = inventory.defaultInput else { |
| throw XCTSkip("this machine has no default input and output") |
| @@ -437,6 +467,8 @@ |
| XCTAssertEqual(audio.enginePin.lastApplied, expected) |
| try routes.use(output: "") |
| + // The rebuild runs on the audio queue (0.3.9), never the main thread. |
| + await AudioWork.drain() |
| XCTAssertEqual(audio.enginePin.lastApplied, .systemDefault, "rebuilt on the default") |
| XCTAssertTrue(module.isEngineRunning) |
| } |
diff --git a/native/apple/Tests/CallsKitTests/AudioSelfTestTests.swift b/native/apple/Tests/CallsKitTests/AudioSelfTestTests.swift| index 5cb76de..febc1b8 100644 |
| --- a/native/apple/Tests/CallsKitTests/AudioSelfTestTests.swift |
| +++ b/native/apple/Tests/CallsKitTests/AudioSelfTestTests.swift |
| @@ -202,12 +202,36 @@ private final class TestClock: @unchecked Sendable { |
| var allowed = true |
| var starts = 0 |
| var stops = 0 |
| + /// Set: `start` waits until it is released, as a start held up by a |
| + /// Bluetooth headset's switch does. |
| + var holdStart = false |
| + var failStart: Error? |
| + var switching: String? |
| + /// Whether the microphone would be open now: started and not stopped |
| + /// since. |
| + var open = false |
| + private var held: CheckedContinuation<Void, Never>? |
| init() { denoiser.audioProcessingInitialize(sampleRate: 48_000, channels: 1) } |
| func permitted() async -> Bool { allowed } |
| - func start() async throws { starts += 1 } |
| - func stop() { stops += 1 } |
| + func switchingMicrophone() async -> String? { switching } |
| + func start() async throws { |
| + starts += 1 |
| + if holdStart { await withCheckedContinuation { held = $0 } } |
| + if let failStart { throw failStart } |
| + open = true |
| + } |
| + func stop() { |
| + stops += 1 |
| + open = false |
| + } |
| + /// Lets a held start finish. |
| + func release() { |
| + held?.resume() |
| + held = nil |
| + } |
| + var isHeld: Bool { held != nil } |
| func levels() -> (input: CaptureLevel.Reading, output: CaptureLevel.Reading) { |
| (denoiser.inputLevel.take(), denoiser.level.take()) |
| } |
| @@ -433,4 +457,174 @@ private final class TestClock: @unchecked Sendable { |
| } |
| XCTAssertEqual(published, first, "the same level and verdict are not published again") |
| } |
| + |
| + // MARK: The start, off the main thread |
| + |
| + /// Waits (yielding the main actor) until `condition` holds. |
| + private func until(_ condition: () -> Bool) async { |
| + for _ in 0..<200 where !condition() { await Task.yield() } |
| + } |
| + |
| + /// The window opens at once and says it is starting; the verdict and |
| + /// meter take over only once the microphone has started. |
| + func testStartingIsSaidUntilTheMicrophoneHasStarted() async { |
| + let (test, capture, _, clock) = make { $0.holdStart = true } |
| + let opening = Task { await test.open() } |
| + await until { capture.isHeld } |
| + XCTAssertEqual(test.startPhase, .starting(switching: nil)) |
| + XCTAssertEqual(test.startingSentence, "Starting microphone…") |
| + XCTAssertEqual(test.micSentence, "Starting microphone…") |
| + XCTAssertNil(test.micAdvice) |
| + capture.release() |
| + await opening.value |
| + XCTAssertEqual(test.startPhase, .running) |
| + XCTAssertNil(test.startingSentence) |
| + capture.feed(8_000) |
| + clock.now += 0.1 |
| + test.tick() |
| + XCTAssertEqual(test.micVerdict, .working) |
| + test.close() |
| + } |
| + |
| + /// A Bluetooth headset is named while it switches. |
| + func testABluetoothSwitchIsSaid() async { |
| + let (test, capture, _, _) = make { |
| + $0.holdStart = true |
| + $0.switching = "Paul’s AirPods Pro" |
| + } |
| + let opening = Task { await test.open() } |
| + await until { capture.isHeld } |
| + XCTAssertEqual(test.startPhase, .starting(switching: "Paul’s AirPods Pro")) |
| + XCTAssertEqual(test.startingSentence, "Switching Paul’s AirPods Pro to microphone mode…") |
| + capture.release() |
| + await opening.value |
| + XCTAssertEqual(test.startPhase, .running) |
| + test.close() |
| + } |
| + |
| + /// Closed while the microphone starts: the start, when it finishes, is |
| + /// stopped at once, and nothing is left open or judged. |
| + func testClosingMidStartLeavesNothingOpen() async { |
| + let (test, capture, _, _) = make { $0.holdStart = true } |
| + let opening = Task { await test.open() } |
| + await until { capture.isHeld } |
| + test.close() |
| + XCTAssertEqual(test.startPhase, .cancelled) |
| + capture.release() |
| + await opening.value |
| + XCTAssertFalse(capture.open, "the microphone the late start opened was closed") |
| + XCTAssertEqual(capture.starts, 1) |
| + XCTAssertGreaterThanOrEqual(capture.stops, 1) |
| + XCTAssertEqual(test.startPhase, .cancelled) |
| + XCTAssertNil(test.micProblem, "a cancelled start is not reported as a failure") |
| + } |
| + |
| + /// Closed while permission is being asked: nothing starts at all. |
| + func testClosingWhileAskingStartsNothing() async { |
| + let (test, capture, _, _) = make() |
| + test.close() |
| + await test.open() |
| + XCTAssertEqual(capture.starts, 0) |
| + XCTAssertEqual(test.startPhase, .cancelled) |
| + } |
| + |
| + func testAStartThatFailsSaysWhy() async { |
| + let (test, capture, _, _) = make { |
| + $0.failStart = NSError(domain: "t", code: 1, userInfo: [NSLocalizedDescriptionKey: "no device"]) |
| + } |
| + await test.open() |
| + XCTAssertEqual(test.startPhase, .failed) |
| + XCTAssertEqual(test.micProblem, "The microphone would not start: no device") |
| + XCTAssertFalse(capture.open) |
| + test.close() |
| + } |
| + |
| + func testOpeningTwiceStartsOnce() async { |
| + let (test, capture, _, _) = make() |
| + await test.open() |
| + await test.open() |
| + XCTAssertEqual(capture.starts, 1) |
| + test.close() |
| + test.close() |
| + XCTAssertEqual(capture.stops, 1) |
| + } |
| +} |
| + |
| +/// The start's rules alone. |
| +final class MicStartTests: XCTestCase { |
| + func testTheHappyPath() { |
| + var start = MicStart() |
| + XCTAssertTrue(start.open()) |
| + XCTAssertEqual(start.phase, .asking) |
| + XCTAssertEqual(start.sentence, "Asking to use the microphone…") |
| + XCTAssertTrue(start.permission(true)) |
| + XCTAssertEqual(start.sentence, "Starting microphone…") |
| + start.microphone(bluetoothSwitching: "AirPods") |
| + XCTAssertEqual(start.sentence, "Switching AirPods to microphone mode…") |
| + XCTAssertEqual(start.finished(ok: true), .keep) |
| + XCTAssertEqual(start.phase, .running) |
| + XCTAssertNil(start.sentence) |
| + XCTAssertTrue(start.close(), "a running microphone is stopped on close") |
| + XCTAssertEqual(start.phase, .cancelled) |
| + } |
| + |
| + func testOpenOnlyOnce() { |
| + var start = MicStart() |
| + XCTAssertTrue(start.open()) |
| + XCTAssertFalse(start.open()) |
| + start.close() |
| + XCTAssertFalse(start.open(), "a closed test does not open again") |
| + } |
| + |
| + func testNoPermission() { |
| + var start = MicStart() |
| + _ = start.open() |
| + XCTAssertFalse(start.permission(false)) |
| + XCTAssertEqual(start.phase, .failed) |
| + XCTAssertFalse(start.close(), "nothing to stop") |
| + } |
| + |
| + func testPermissionAfterClose() { |
| + var start = MicStart() |
| + _ = start.open() |
| + XCTAssertFalse(start.close(), "nothing started yet") |
| + XCTAssertFalse(start.permission(true), "the answer comes back to a closed window") |
| + XCTAssertEqual(start.phase, .cancelled) |
| + } |
| + |
| + func testALateStartIsStopped() { |
| + var start = MicStart() |
| + _ = start.open() |
| + _ = start.permission(true) |
| + XCTAssertTrue(start.close(), "a start in flight may open the microphone") |
| + XCTAssertEqual(start.finished(ok: true), .stop) |
| + XCTAssertEqual(start.phase, .cancelled) |
| + } |
| + |
| + func testALateFailureIsQuiet() { |
| + var start = MicStart() |
| + _ = start.open() |
| + _ = start.permission(true) |
| + start.close() |
| + XCTAssertEqual(start.finished(ok: false), .failed) |
| + XCTAssertEqual(start.phase, .cancelled) |
| + } |
| + |
| + func testRestartAfterPlayback() { |
| + var start = MicStart() |
| + _ = start.open() |
| + _ = start.permission(true) |
| + _ = start.finished(ok: true) |
| + XCTAssertTrue(start.restart()) |
| + XCTAssertTrue(start.isStarting) |
| + XCTAssertEqual(start.finished(ok: true), .keep) |
| + start.close() |
| + XCTAssertFalse(start.restart(), "not after close") |
| + } |
| + |
| + func testTheBluetoothNameOnlyWhileStarting() { |
| + var start = MicStart() |
| + start.microphone(bluetoothSwitching: "AirPods") |
| + XCTAssertEqual(start.phase, .idle) |
| + } |
| } |
diff --git a/native/ios/project.yml b/native/ios/project.yml| index a74c77d..36fa266 100644 |
| --- a/native/ios/project.yml |
| +++ b/native/ios/project.yml |
| @@ -10,7 +10,7 @@ settings: |
| base: |
| SWIFT_VERSION: '5.0' |
| CODE_SIGN_STYLE: Automatic |
| - MARKETING_VERSION: '0.3.8' |
| + MARKETING_VERSION: '0.3.9' |
| CURRENT_PROJECT_VERSION: '1' |
| # The App Group the app and its broadcast extension share: the socket |
| # the iPhone's screen crosses lives in its container. One setting |