| 1 | import Foundation |
| 2 | import LiveKitWebRTC |
| 3 | import RNNoise |
| 4 | import os |
| 5 | |
| 6 | #if canImport(DeepFilter) |
| 7 | import DeepFilter |
| 8 | #endif |
| 9 | |
| 10 | // Noise suppression for the microphone, run inside WebRTC's capture |
| 11 | // processing. |
| 12 | // |
| 13 | // WebRTC's audio processing runs high-pass filter → echo canceller → |
| 14 | // noise suppressor → gain control, then hands each 10 ms frame to a |
| 15 | // "capture post-processing" delegate before encoding. Its own noise |
| 16 | // suppressor is the classic statistical one: good on steady hiss, poor on |
| 17 | // wind, typing and a room's echo, which are what people actually hear. |
| 18 | // So WebRTC's is switched off, and a neural suppressor runs in the |
| 19 | // delegate instead, after the echo canceller has taken out the far side. |
| 20 | // |
| 21 | // The delegate is skipped by WebRTC while the microphone is muted (its |
| 22 | // output is unused), and runs on WebRTC's audio thread: nothing here may |
| 23 | // block, allocate per frame, or wait for the main actor. |
| 24 | |
| 25 | /// A mono, frame-at-a-time neural noise suppressor. |
| 26 | /// |
| 27 | /// `process` works in place on one frame of `frameLength` samples in |
| 28 | /// WebRTC's float scale — ±32768 for full scale, int16's range as floats — |
| 29 | /// and returns how sure it is that the frame is speech, 0 to 1. Each |
| 30 | /// implementation owns its own conversion to whatever scale its model |
| 31 | /// wants. |
| 32 | public protocol NoiseSuppressor: AnyObject { |
| 33 | /// Shown in the audio readout. |
| 34 | var name: String { get } |
| 35 | /// Samples per frame at 48 kHz, the only rate either model runs at. |
| 36 | var frameLength: Int { get } |
| 37 | func process(_ frame: UnsafeMutablePointer<Float>) -> Float |
| 38 | } |
| 39 | |
| 40 | /// How far either suppressor may take noise down, in dB. Unlimited, a |
| 41 | /// room goes to digital silence between words, which a listener hears as |
| 42 | /// the line dropping, and a model can hollow out a voice with the noise |
| 43 | /// around it. 30 dB takes a room to near silence and leaves speech its own |
| 44 | /// texture. |
| 45 | public let noiseAttenuationLimitDb: Float = 30 |
| 46 | |
| 47 | /// xiph's RNNoise: a small recurrent network, BSD-3, 480-sample frames at |
| 48 | /// 48 kHz, and — unusually — in WebRTC's own ±32768 scale already, so the |
| 49 | /// samples go in and come out untouched by any conversion. |
| 50 | /// |
| 51 | /// RNNoise has no attenuation limit of its own, so one is made the usual |
| 52 | /// way: the untouched input is mixed back in at the limit. |
| 53 | public final class RNNoiseSuppressor: NoiseSuppressor { |
| 54 | public let name = "RNNoise" |
| 55 | public let frameLength: Int |
| 56 | private let state: OpaquePointer |
| 57 | private let scratch: UnsafeMutablePointer<Float> |
| 58 | private let floor = pow(10, -noiseAttenuationLimitDb / 20) |
| 59 | |
| 60 | public init() { |
| 61 | state = rnnoise_create(nil) |
| 62 | frameLength = Int(rnnoise_get_frame_size()) |
| 63 | scratch = .allocate(capacity: frameLength) |
| 64 | } |
| 65 | deinit { |
| 66 | rnnoise_destroy(state) |
| 67 | scratch.deallocate() |
| 68 | } |
| 69 | public func process(_ frame: UnsafeMutablePointer<Float>) -> Float { |
| 70 | scratch.update(from: frame, count: frameLength) |
| 71 | let speech = rnnoise_process_frame(state, frame, scratch) |
| 72 | for i in 0..<frameLength { frame[i] += floor * scratch[i] } |
| 73 | return speech |
| 74 | } |
| 75 | } |
| 76 | |
| 77 | #if canImport(DeepFilter) |
| 78 | /// DeepFilterNet 3 (Rikorose/DeepFilterNet's libDF, MIT or Apache-2.0): |
| 79 | /// a larger model than RNNoise, markedly better on non-stationary noise |
| 80 | /// (typing, wind, voices in the next room). 480-sample hop at 48 kHz, |
| 81 | /// mono, in ±1 floats — so samples are scaled by 1/32768 on the way in |
| 82 | /// and back on the way out. It adds about 30 ms of latency (two frames |
| 83 | /// of lookahead and the overlap of its transform). |
| 84 | public final class DeepFilterSuppressor: NoiseSuppressor { |
| 85 | public let name = "DeepFilterNet 3" |
| 86 | public let frameLength: Int |
| 87 | private let state: OpaquePointer |
| 88 | private let input: UnsafeMutablePointer<Float> |
| 89 | private let output: UnsafeMutablePointer<Float> |
| 90 | |
| 91 | /// The model, where the app carries one: `CALLS_DEEPFILTER_MODEL` |
| 92 | /// first (tests and the interop harness), then the bundle. |
| 93 | public static var model: URL? { |
| 94 | if let path = ProcessInfo.processInfo.environment["CALLS_DEEPFILTER_MODEL"] { |
| 95 | return URL(fileURLWithPath: path) |
| 96 | } |
| 97 | return Bundle.main.url(forResource: "DeepFilterNet3_onnx", withExtension: "tar.gz") |
| 98 | } |
| 99 | |
| 100 | /// Nil when there is no model file. libDF aborts the process on a |
| 101 | /// model it cannot read, so the file is checked here first. |
| 102 | public init?(model: URL, attenuationLimit: Float = noiseAttenuationLimitDb) { |
| 103 | guard FileManager.default.isReadableFile(atPath: model.path) else { return nil } |
| 104 | guard let state = df_create(model.path, attenuationLimit, nil) else { return nil } |
| 105 | self.state = state |
| 106 | frameLength = Int(df_get_frame_length(state)) |
| 107 | input = .allocate(capacity: frameLength) |
| 108 | output = .allocate(capacity: frameLength) |
| 109 | } |
| 110 | deinit { |
| 111 | df_free(state) |
| 112 | input.deallocate() |
| 113 | output.deallocate() |
| 114 | } |
| 115 | |
| 116 | /// Returns the model's local SNR estimate mapped to 0...1: at or below |
| 117 | /// 5 dB reads as noise, at or above 15 dB as speech. |
| 118 | public func process(_ frame: UnsafeMutablePointer<Float>) -> Float { |
| 119 | for i in 0..<frameLength { input[i] = frame[i] / 32768 } |
| 120 | let snr = df_process_frame(state, input, output) |
| 121 | for i in 0..<frameLength { frame[i] = output[i] * 32768 } |
| 122 | return min(max((snr - 5) / 10, 0), 1) |
| 123 | } |
| 124 | } |
| 125 | #endif |
| 126 | |
| 127 | /// A gentle gain after the suppressor, instead of WebRTC's gain control. |
| 128 | /// |
| 129 | /// WebRTC's gain control runs *before* the capture post-processing hook, |
| 130 | /// so left on it would raise the room along with a quiet voice and then |
| 131 | /// hand the suppressor louder noise to remove — the pumping a denoised |
| 132 | /// call is known for. It is switched off while a suppressor runs, and this |
| 133 | /// takes its place, after the noise is gone. |
| 134 | /// |
| 135 | /// Conservative on purpose: it only ever lifts (0 to +12 dB), it only |
| 136 | /// learns from frames the suppressor calls speech, it rises slowly |
| 137 | /// (about 1 dB a second of speech) and falls fast, and a per-frame peak |
| 138 | /// limit keeps it from ever clipping. A voice already at a sensible level |
| 139 | /// is left alone. |
| 140 | public struct SpeechGain { |
| 141 | /// Speech level aimed for, dBFS RMS. -26 is the usual target for |
| 142 | /// conversational speech (ITU-T P.56 practice). |
| 143 | public static let targetDb: Float = -26 |
| 144 | public static let maxGainDb: Float = 12 |
| 145 | public static let voiced: Float = 0.6 |
| 146 | public private(set) var gainDb: Float = 0 |
| 147 | |
| 148 | public init() {} |
| 149 | |
| 150 | public mutating func apply(_ frame: UnsafeMutablePointer<Float>, count: Int, speech: Float) { |
| 151 | var energy: Float = 0 |
| 152 | var peak: Float = 0 |
| 153 | for i in 0..<count { |
| 154 | energy += frame[i] * frame[i] |
| 155 | peak = max(peak, abs(frame[i])) |
| 156 | } |
| 157 | let rms = (energy / Float(max(count, 1))).squareRoot() / 32768 |
| 158 | if speech >= SpeechGain.voiced, rms > 0.001 { // above -60 dBFS |
| 159 | let levelDb = 20 * log10(rms) |
| 160 | let wanted = min(max(SpeechGain.targetDb - levelDb, 0), SpeechGain.maxGainDb) |
| 161 | // Up by 0.01 dB a frame (1 dB/s), down by up to 1 dB a frame. |
| 162 | gainDb += min(max(wanted - gainDb, -1), 0.01) |
| 163 | } |
| 164 | guard gainDb > 0.01 else { return } |
| 165 | var gain = pow(10, gainDb / 20) |
| 166 | let limit: Float = 0.9 * 32768 |
| 167 | if peak * gain > limit { gain = max(1, limit / peak) } |
| 168 | for i in 0..<count { frame[i] *= gain } |
| 169 | } |
| 170 | } |
| 171 | |
| 172 | /// What the suppressor is doing, for the audio readout. |
| 173 | public enum SuppressionState: Equatable, Sendable { |
| 174 | /// Toggle off, or nothing has asked yet. |
| 175 | case off |
| 176 | /// Processing frames; `cpu` is the share of real time spent in it on |
| 177 | /// the audio thread, 0 to 1 — wall clock, so preemption counts too. |
| 178 | case running(engine: String, cpu: Double) |
| 179 | /// Initialised and waiting — muted (WebRTC skips processing then), or |
| 180 | /// no audio has flowed yet. |
| 181 | case idle(engine: String) |
| 182 | /// The capture runs at a rate the models do not (a Bluetooth headset's |
| 183 | /// 16 kHz, say): frames pass through untouched. |
| 184 | case unsupportedRate(Int) |
| 185 | /// Apple's Voice Isolation is on; two suppressors in a row would only |
| 186 | /// take more of the voice with them. |
| 187 | case standingAside |
| 188 | } |
| 189 | |
| 190 | /// The frame adapter between WebRTC's capture hook and a suppressor. |
| 191 | /// |
| 192 | /// WebRTC hands over 10 ms at a time, 480 samples at 48 kHz, which is one |
| 193 | /// frame of either model; a buffer that is a whole number of frames is |
| 194 | /// processed frame by frame, and anything else passes through untouched. |
| 195 | /// So does every rate but 48 kHz, and every channel count but one. |
| 196 | /// |
| 197 | /// The suppressor is made once and kept across WebRTC's |
| 198 | /// initialise/release cycles, which happen whenever the processing is |
| 199 | /// reconfigured: a model can take a moment to load, and the audio thread |
| 200 | /// is no place to do it. `release` stops processing; it does not free. |
| 201 | public final class CaptureDenoiser: NSObject, RTCAudioCustomProcessingDelegate, @unchecked Sendable { |
| 202 | public static let supportedRate = 48_000 |
| 203 | |
| 204 | private struct State { |
| 205 | var enabled = true |
| 206 | var standAside = false |
| 207 | var rate = 0 |
| 208 | var channels = 0 |
| 209 | var initialised = false |
| 210 | var frames: UInt64 = 0 |
| 211 | /// Every buffer WebRTC has handed over, processed or not: proof that |
| 212 | /// the microphone is capturing. |
| 213 | var buffers: UInt64 = 0 |
| 214 | /// Buffers since the last initialise; the first is logged. |
| 215 | var buffersSinceInit: UInt64 = 0 |
| 216 | var busySeconds: Double = 0 |
| 217 | var lastFrame: Double = -.infinity |
| 218 | var cpu: Double = 0 |
| 219 | } |
| 220 | private let lock = OSAllocatedUnfairLock(initialState: State()) |
| 221 | /// Read under the lock once per buffer, so it can be replaced mid-call. |
| 222 | private var current: NoiseSuppressor |
| 223 | private var gain = SpeechGain() |
| 224 | private let clock: () -> Double |
| 225 | /// Called (on the audio thread) when WebRTC reinitialises at a rate the |
| 226 | /// suppressor can or cannot take, so the processing options can follow. |
| 227 | public var rateChanged: (@Sendable (Int) -> Void)? |
| 228 | |
| 229 | public init(suppressor: NoiseSuppressor, clock: @escaping () -> Double = CaptureDenoiser.now) { |
| 230 | current = suppressor |
| 231 | self.clock = clock |
| 232 | } |
| 233 | |
| 234 | /// Swaps the model, between buffers. A call starts on RNNoise and moves |
| 235 | /// to DeepFilterNet once its model has loaded off the main thread |
| 236 | /// (a third of a second on an M1 Max; the audio thread is no place for |
| 237 | /// it, and neither is joining). |
| 238 | public func replace(with suppressor: NoiseSuppressor) { |
| 239 | lock.withLock { _ in current = suppressor } |
| 240 | } |
| 241 | |
| 242 | private var suppressor: NoiseSuppressor { lock.withLock { _ in current } } |
| 243 | |
| 244 | public static func now() -> Double { ProcessInfo.processInfo.systemUptime } |
| 245 | |
| 246 | public var engineName: String { suppressor.name } |
| 247 | |
| 248 | /// The toggle. Off passes frames through untouched. |
| 249 | public var enabled: Bool { |
| 250 | get { lock.withLock { $0.enabled } } |
| 251 | set { lock.withLock { $0.enabled = newValue } } |
| 252 | } |
| 253 | /// Set while Apple's Voice Isolation is doing the job. |
| 254 | public var standAside: Bool { |
| 255 | get { lock.withLock { $0.standAside } } |
| 256 | set { lock.withLock { $0.standAside = newValue } } |
| 257 | } |
| 258 | public var framesProcessed: UInt64 { lock.withLock { $0.frames } } |
| 259 | |
| 260 | public var state: SuppressionState { |
| 261 | let now = clock() |
| 262 | return lock.withLock { s in |
| 263 | if !s.enabled { return .off } |
| 264 | if s.standAside { return .standingAside } |
| 265 | if s.initialised, s.rate != CaptureDenoiser.supportedRate { return .unsupportedRate(s.rate) } |
| 266 | if s.initialised, now - s.lastFrame < 1 { return .running(engine: current.name, cpu: s.cpu) } |
| 267 | return .idle(engine: current.name) |
| 268 | } |
| 269 | } |
| 270 | |
| 271 | // MARK: RTCAudioCustomProcessingDelegate |
| 272 | |
| 273 | public func audioProcessingInitialize(sampleRate sampleRateHz: Int, channels: Int) { |
| 274 | lock.withLock { s in |
| 275 | s.rate = sampleRateHz |
| 276 | s.channels = channels |
| 277 | s.initialised = true |
| 278 | s.buffersSinceInit = 0 |
| 279 | } |
| 280 | AudioLog.info( |
| 281 | "capture hook initialised at \(sampleRateHz) Hz, \(channels) ch; noise reduction \(sampleRateHz == CaptureDenoiser.supportedRate ? "can run" : "passes it through (meters still read it)")") |
| 282 | // Every time, not only on a change. WebRTC initialises its processing |
| 283 | // at 16 kHz when the module is made, long before any microphone opens, |
| 284 | // and that first call can arrive before anyone is listening; a |
| 285 | // Bluetooth headset that really does capture at 16 kHz then looked |
| 286 | // like "no change", and the plan never heard about it — leaving |
| 287 | // WebRTC's own suppressor off while the readout said it was standing |
| 288 | // in. The listener decides what is a change. |
| 289 | rateChanged?(sampleRateHz) |
| 290 | } |
| 291 | |
| 292 | public var buffersSeen: UInt64 { lock.withLock { $0.buffers } } |
| 293 | |
| 294 | /// Counts one buffer; true for the first since the last initialise. |
| 295 | /// Apart from WebRTC's buffer type so the tests can drive it. |
| 296 | @discardableResult |
| 297 | public func countBuffer() -> Bool { |
| 298 | lock.withLock { s in |
| 299 | s.buffers += 1 |
| 300 | s.buffersSinceInit += 1 |
| 301 | return s.buffersSinceInit == 1 |
| 302 | } |
| 303 | } |
| 304 | |
| 305 | /// The rate and channels WebRTC last initialised the hook with (0 |
| 306 | /// before it has): for the diagnostics. |
| 307 | public var format: (rate: Int, channels: Int, initialised: Bool) { |
| 308 | lock.withLock { s in (s.rate, s.channels, s.initialised) } |
| 309 | } |
| 310 | |
| 311 | public func audioProcessingProcess(audioBuffer: RTCAudioBuffer) { |
| 312 | if countBuffer() { |
| 313 | AudioLog.info( |
| 314 | "capture hook: first buffer since initialise, \(audioBuffer.frames) frames x \(audioBuffer.channels) ch") |
| 315 | } |
| 316 | guard audioBuffer.channels >= 1 else { return } |
| 317 | handle(audioBuffer.rawBuffer(forChannel: 0), frames: audioBuffer.frames, channels: audioBuffer.channels) |
| 318 | } |
| 319 | |
| 320 | /// Everything the hook does with one buffer, apart from WebRTC's buffer |
| 321 | /// type so the tests drive this exact path: the meter before, the |
| 322 | /// suppressor (mono at 48 kHz only), then the meter after and the sink — |
| 323 | /// at every rate and channel count, so a Bluetooth headset's 16 or 24 |
| 324 | /// kHz, which the suppressor passes through, is metered and recorded |
| 325 | /// all the same. |
| 326 | public func handle(_ samples: UnsafeMutablePointer<Float>, frames: Int, channels: Int) { |
| 327 | inputLevel.record(samples, frames: frames, at: clock()) |
| 328 | if channels == 1 { process(samples, frames: frames) } |
| 329 | observe(samples, frames: frames) |
| 330 | } |
| 331 | |
| 332 | /// The same buffers measured on the way *in*, before the suppressor: |
| 333 | /// what the microphone delivers after WebRTC's echo canceller and |
| 334 | /// high-pass filter. "Is the microphone picking anything up" is asked |
| 335 | /// of this one, because a quiet room the suppressor has taken down by |
| 336 | /// 30 dB can look dead afterwards and is not. |
| 337 | public let inputLevel = CaptureLevel() |
| 338 | |
| 339 | /// What leaves the hook, measured: the level of every buffer after the |
| 340 | /// suppressor and gain have had it (or passed it through), which is the |
| 341 | /// audio that is encoded and sent. Read by the microphone meter. |
| 342 | public let level = CaptureLevel() |
| 343 | |
| 344 | /// Something that wants the processed audio itself — the audio test's |
| 345 | /// recording. Called on the audio thread with WebRTC's ±32768 floats and |
| 346 | /// the rate WebRTC is processing at; it must copy and return, never |
| 347 | /// block. Nil almost always. |
| 348 | public var sink: (@Sendable (UnsafePointer<Float>, Int, Int) -> Void)? { |
| 349 | get { lock.withLock { _ in sinkBox } } |
| 350 | set { lock.withLock { _ in sinkBox = newValue } } |
| 351 | } |
| 352 | private var sinkBox: (@Sendable (UnsafePointer<Float>, Int, Int) -> Void)? |
| 353 | |
| 354 | /// The measuring half of the hook, apart from WebRTC's buffer type so |
| 355 | /// the tests can drive it: after `process`, whatever the channel count |
| 356 | /// or rate, so a stream the suppressor passes through is metered too. |
| 357 | public func observe(_ samples: UnsafePointer<Float>, frames: Int) { |
| 358 | guard frames > 0 else { return } |
| 359 | level.record(samples, frames: frames, at: clock()) |
| 360 | let (sink, rate) = lock.withLock { s in (sinkBox, s.rate) } |
| 361 | sink?(samples, frames, rate) |
| 362 | } |
| 363 | |
| 364 | public func audioProcessingRelease() { |
| 365 | lock.withLock { $0.initialised = false } |
| 366 | } |
| 367 | |
| 368 | // MARK: The adapter itself, apart from WebRTC's buffer type |
| 369 | |
| 370 | /// Processes one buffer in place. Public for the tests, which cannot |
| 371 | /// make an RTCAudioBuffer. |
| 372 | public func process(_ samples: UnsafeMutablePointer<Float>, frames: Int) { |
| 373 | let (run, suppressor) = lock.withLock { s in |
| 374 | ( |
| 375 | s.enabled && !s.standAside && s.initialised && s.channels == 1 |
| 376 | && s.rate == CaptureDenoiser.supportedRate, current |
| 377 | ) |
| 378 | } |
| 379 | let length = suppressor.frameLength |
| 380 | guard run, frames > 0, frames % length == 0 else { return } |
| 381 | let started = clock() |
| 382 | var offset = 0 |
| 383 | while offset < frames { |
| 384 | let frame = samples + offset |
| 385 | let speech = suppressor.process(frame) |
| 386 | gain.apply(frame, count: length, speech: speech) |
| 387 | offset += length |
| 388 | } |
| 389 | let finished = clock() |
| 390 | let audio = Double(frames) / Double(CaptureDenoiser.supportedRate) |
| 391 | lock.withLock { s in |
| 392 | s.frames += UInt64(frames / length) |
| 393 | s.lastFrame = finished |
| 394 | // A smoothed share of one core: time spent against audio covered. |
| 395 | let share = (finished - started) / audio |
| 396 | s.cpu = s.frames <= UInt64(frames / length) ? share : s.cpu * 0.98 + share * 0.02 |
| 397 | } |
| 398 | } |
| 399 | } |
| 400 | |
| 401 | /// The level of the processed microphone, kept for a meter to read. |
| 402 | /// |
| 403 | /// Written on the audio thread once per buffer (10 ms), read on the main |
| 404 | /// actor a few times a second. What a read gets is the loudest buffer |
| 405 | /// since the last read — a peak hold — so a meter sampled at 10 Hz does |
| 406 | /// not miss a syllable that fell between two samples. Levels are 0...1 of |
| 407 | /// full scale (WebRTC's ±32768). |
| 408 | public final class CaptureLevel: @unchecked Sendable { |
| 409 | public struct Reading: Equatable, Sendable { |
| 410 | /// Loudest buffer RMS since the last read, 0...1. |
| 411 | public var rms: Float |
| 412 | /// Loudest single sample since the last read, 0...1. |
| 413 | public var peak: Float |
| 414 | /// Buffers measured since the last read. Zero means nothing flowed: |
| 415 | /// muted (WebRTC skips the hook then) or not capturing. |
| 416 | public var buffers: Int |
| 417 | /// When the last buffer was measured, on the clock the adapter was |
| 418 | /// given (uptime seconds by default). |
| 419 | public var at: Double |
| 420 | } |
| 421 | private let lock = OSAllocatedUnfairLock(initialState: Reading(rms: 0, peak: 0, buffers: 0, at: -.infinity)) |
| 422 | |
| 423 | public init() {} |
| 424 | |
| 425 | public func record(_ samples: UnsafePointer<Float>, frames: Int, at time: Double) { |
| 426 | var energy: Float = 0 |
| 427 | var peak: Float = 0 |
| 428 | for i in 0..<frames { |
| 429 | let v = samples[i] |
| 430 | energy += v * v |
| 431 | peak = max(peak, abs(v)) |
| 432 | } |
| 433 | let rms = (energy / Float(frames)).squareRoot() / 32768 |
| 434 | let top = min(peak / 32768, 1) |
| 435 | lock.withLock { r in |
| 436 | r.rms = max(r.rms, min(rms, 1)) |
| 437 | r.peak = max(r.peak, top) |
| 438 | r.buffers += 1 |
| 439 | r.at = time |
| 440 | } |
| 441 | } |
| 442 | |
| 443 | /// The reading since the last call, and a fresh start for the next. |
| 444 | public func take() -> Reading { |
| 445 | lock.withLock { r in |
| 446 | let out = r |
| 447 | r.rms = 0 |
| 448 | r.peak = 0 |
| 449 | r.buffers = 0 |
| 450 | return out |
| 451 | } |
| 452 | } |
| 453 | } |
| 454 | |
amadan