| 1 | import Combine |
| 2 | import Foundation |
| 3 | #if os(iOS) |
| 4 | import AVFoundation |
| 5 | #endif |
| 6 | // Go serializes this API with a mutex; Objective-C bindings carry no Sendable annotation. |
| 7 | @preconcurrency import CallsBridge |
| 8 | import LiveKitWebRTC |
| 9 | |
| 10 | // The Go callback only enqueues. The main actor owns the interface and media; |
| 11 | // an epoch prevents callbacks from a closed room reaching a later call. |
| 12 | private final class Events: NSObject, BridgeListenerProtocol { |
| 13 | weak var call: Call? |
| 14 | let epoch: UUID |
| 15 | init(call: Call, epoch: UUID) { |
| 16 | self.call = call |
| 17 | self.epoch = epoch |
| 18 | } |
| 19 | func onEvent(_ event: String?) { |
| 20 | guard let event, let data = event.data(using: .utf8) else { return } |
| 21 | guard let decoded = try? JSONDecoder().decode(WireEvent.self, from: data) else { |
| 22 | DispatchQueue.main.async { [weak self] in self?.call?.invalidEvents += 1 } |
| 23 | return |
| 24 | } |
| 25 | DispatchQueue.main.async { [weak self] in |
| 26 | guard let self, let call = self.call, call.epoch == self.epoch else { return } |
| 27 | call.receive(decoded) |
| 28 | } |
| 29 | } |
| 30 | } |
| 31 | |
| 32 | @MainActor public final class Call: ObservableObject { |
| 33 | @Published public private(set) var state: RoomState? { |
| 34 | didSet { arrangement.roster(state) } |
| 35 | } |
| 36 | /// This viewer's layout, pin and order of people (0.4.0). |
| 37 | public let arrangement = CallArrangement() |
| 38 | @Published public private(set) var title = "Calls" |
| 39 | /// Whether `title` is one someone chose, shown over the call, rather |
| 40 | /// than one Calls wrote, which is not (CallTitle). |
| 41 | @Published public private(set) var named = false |
| 42 | @Published public private(set) var standing = "" |
| 43 | @Published public private(set) var status = "Ready to join" |
| 44 | @Published public private(set) var remoteTracks: [String: RTCVideoTrack] = [:] |
| 45 | /// §5.4: the peers showing a screen here, oldest first, and the screens |
| 46 | /// that have arrived. Two things and not one, because the `share` |
| 47 | /// message usually beats the media by a renegotiation, and the tile |
| 48 | /// carries its caption before there is anything to show in it. |
| 49 | @Published public private(set) var screens: [String] = [] |
| 50 | @Published public private(set) var remoteScreens: [String: RTCVideoTrack] = [:] |
| 51 | /// Whether this app is sending a screen, and what the last attempt to |
| 52 | /// had to say — which is usually nothing. |
| 53 | @Published public private(set) var sharingScreen = false |
| 54 | @Published public private(set) var screenNote = "" |
| 55 | private var screenPending = false |
| 56 | /// How this app's own screen is going out, while it is sharing: the |
| 57 | /// slowest connection's, read once a second. See ScreenSendReport. |
| 58 | @Published public private(set) var screenReport: ScreenSendReport? |
| 59 | /// "Optimise for": what a shared screen gives up first. Remembered, and |
| 60 | /// applied at once to a share in progress (see ScreenSendPolicy). |
| 61 | @Published public var screenMode: ScreenShareMode = .remembered { |
| 62 | didSet { |
| 63 | guard screenMode != oldValue else { return } |
| 64 | ScreenShareMode.remembered = screenMode |
| 65 | built?.screenMode = screenMode |
| 66 | } |
| 67 | } |
| 68 | private var screenReports: [String: ScreenSendReport] = [:] |
| 69 | private var screenWatch: Task<Void, Never>? |
| 70 | /// The sharer's watchdog (ScreenWatchdog) and what it is doing, for the |
| 71 | /// panel's screen line. |
| 72 | private var screenWatchdog = ScreenWatchdog() |
| 73 | @Published public private(set) var screenWatchStatus = "" |
| 74 | /// The panel's second screen line: encoded and sent rates, keyframes, |
| 75 | /// and the watchdog (ScreenSendReport.detail). |
| 76 | @Published public private(set) var screenDetail = "" |
| 77 | /// The quiet notice while someone's view of this app's screen is being |
| 78 | /// put right: "Vicky's view of your screen stalled — recovering". |
| 79 | @Published public private(set) var screenRecovery = "" |
| 80 | /// The receiver's watchdog, and the list last reported as |
| 81 | /// `screenStalled` (spec §15.21). |
| 82 | private var screenReceiveWatch = ScreenReceiveWatch() |
| 83 | private var reportedStalled: [String] = [] |
| 84 | /// Each arriving screen's statistics at the last look, for the stall |
| 85 | /// log's "what moved" and for "Copy call diagnostics". |
| 86 | private var screenReceived: [String: ScreenReceiveReport] = [:] |
| 87 | /// Where each arriving screen stood when it last moved, so a stall can |
| 88 | /// be described against it. |
| 89 | private var screenReceivedMoving: [String: ScreenReceiveReport] = [:] |
| 90 | private var screenReattaches = 0 |
| 91 | /// "Reconnecting…", said only for a problem that lasts (spec §15.23): |
| 92 | /// the whole call's, and the people whose tiles say it on their own. |
| 93 | private var reconnectWatch = ReconnectWatch() |
| 94 | @Published public private(set) var reconnecting = false |
| 95 | @Published public private(set) var reconnectingPeers: Set<String> = [] |
| 96 | /// "<name> is connecting and can't hear you yet" (spec §15.24): the |
| 97 | /// watch, and the people it is said for now — on their tiles, and as a |
| 98 | /// dot in the mini window when they are not the one it shows. |
| 99 | private var connectingWatch = ConnectingWatch() |
| 100 | @Published public private(set) var connectingPeers: Set<String> = [] |
| 101 | /// This app's own half: whose audio path is established (HearingRule), |
| 102 | /// per connection generation; whose receiver reports have shown this |
| 103 | /// app's audio reaching them; and the list last reported as `hearing`. |
| 104 | private var heardFrom: [String: (epoch: Int64, heard: Bool)] = [:] |
| 105 | private var ackedBy: [String: Int64] = [:] |
| 106 | private var reportedHearing: [String] = [] |
| 107 | /// Quality reports (spec §15.26): the notices tallied over the window, |
| 108 | /// the running totals the last report was taken from, the loop that |
| 109 | /// sends one every `qualityInterval`, and whether the server has said to |
| 110 | /// stop (its kill switch) or the final one has gone. |
| 111 | private var qualityTally = QualityEventTally() |
| 112 | private var qualityBuilder = QualityReportBuilder() |
| 113 | private var qualityWatch: Task<Void, Never>? |
| 114 | private var qualityStopped = false |
| 115 | private var qualityFinalSent = false |
| 116 | /// Thirty seconds, as the ruling has it; the gate shortens it |
| 117 | /// (CALLS_QUALITY_INTERVAL, read by the interop harness). |
| 118 | @_spi(Fixtures) public static var qualityInterval: Double = 30 |
| 119 | /// "Report a problem" (spec §15.26): the sheet, while it is open. |
| 120 | @Published public var problemReport: ProblemReportModel? |
| 121 | /// Until when the last call can still be reported: ten minutes after it |
| 122 | /// ended, which is as long as the server keeps its report token. The |
| 123 | /// closed bridge client is kept that long — its transport and token |
| 124 | /// outlive the room — with the call's diagnostics as they were at the end. |
| 125 | @Published public private(set) var reportableUntil: Date? |
| 126 | private var reportClient: BridgeClient? |
| 127 | private var reportDiagnostics = "" |
| 128 | static let reportWindow: TimeInterval = 600 |
| 129 | /// A fixture call answers a report itself, for the UI test of the sheet. |
| 130 | private var fixtureReports = false |
| 131 | /// The phase the notices are drawn for: the core's, except that its |
| 132 | /// `reconnecting` is only said once ReconnectWatch says so, and is said |
| 133 | /// for a second after it is over. |
| 134 | public var noticePhase: String? { |
| 135 | if reconnecting { return "reconnecting" } |
| 136 | if state?.phase == "reconnecting" { return "admitted" } |
| 137 | return state?.phase |
| 138 | } |
| 139 | private var screenWatchFrozenUntil = 0.0 |
| 140 | @Published public private(set) var localTrack: RTCVideoTrack? |
| 141 | @Published public private(set) var muted = false |
| 142 | @Published public private(set) var cameraOff = false |
| 143 | @Published public private(set) var floor = false |
| 144 | @Published public private(set) var chat: [String] = [] |
| 145 | @Published public private(set) var receivedChatCount = 0 |
| 146 | @Published public private(set) var invitationURL: URL? |
| 147 | @Published public private(set) var invitationError = "" |
| 148 | @Published public private(set) var loadingInvitation = false |
| 149 | /// A brief message over the stage, such as "Invite link copied". |
| 150 | @Published public internal(set) var toast: String? |
| 151 | var toastSerial = 0 |
| 152 | /// What the last device change had to say, and usually nothing. |
| 153 | @Published public private(set) var deviceNote = "" |
| 154 | /// The audio-health readout, once a call has been up long enough for |
| 155 | /// one. Nil means no sample yet, which is not the same as healthy. |
| 156 | @Published public private(set) var audioHealth: AudioHealth? |
| 157 | /// The camera in use. Published rather than read through the capture on |
| 158 | /// demand, because a picker bound to a plain computed property never |
| 159 | /// learns that a switch finished. |
| 160 | @Published public private(set) var currentCamera: String? |
| 161 | /// What the camera is doing (see CameraStatus), and a fallback worth |
| 162 | /// saying. Nil status for a capture that does not say, and outside a |
| 163 | /// call. |
| 164 | @Published public private(set) var cameraStatus: CameraStatus? |
| 165 | @Published public private(set) var cameraNote = "" |
| 166 | /// The working camera in use when it is not the remembered one, which |
| 167 | /// the panel offers to remember. |
| 168 | @Published public private(set) var cameraToKeep: CameraDevice? |
| 169 | /// A real camera, so the panel shows its row even with nothing in it. |
| 170 | @Published public private(set) var offersCameras = false |
| 171 | /// Audio routing belongs to this call. |
| 172 | public let audio = AudioRoutes() |
| 173 | /// "Reduce background noise": the neural suppressor in place of WebRTC's |
| 174 | /// own. On by default, and remembered. |
| 175 | @Published public var reduceNoise: Bool = UserDefaults.standard.object(forKey: Call.reduceNoiseKey) as? Bool ?? true { |
| 176 | didSet { |
| 177 | UserDefaults.standard.set(reduceNoise, forKey: Call.reduceNoiseKey) |
| 178 | guard let built else { return } |
| 179 | built.audio.reduceNoise = reduceNoise |
| 180 | audioProcessing = built.audio.report() |
| 181 | } |
| 182 | } |
| 183 | static let reduceNoiseKey = "reduceBackgroundNoise" |
| 184 | /// Which processing is running on the microphone, for the readout. |
| 185 | @Published public private(set) var audioProcessing: ProcessingReport? |
| 186 | /// The meters (spec §15.20): your microphone and everyone's sound, in |
| 187 | /// objects of their own so a meter moving does not redraw the call. |
| 188 | public let levels = AudioLevels() |
| 189 | private var levelWatch: Task<Void, Never>? |
| 190 | /// What was last reported as `noSoundFrom`, and the queue that reports |
| 191 | /// it: the bridge's send is a request, and not for the main thread. |
| 192 | private var reportedNoSound: [String] = [] |
| 193 | private let noSoundQueue = DispatchQueue(label: "calls.no-sound-from") |
| 194 | /// Received bytes per person at the last statistics read, and whether |
| 195 | /// they had risen: "arriving" for the no-sound rule. |
| 196 | private var audioBytes: [String: UInt64] = [:] |
| 197 | private var audioArriving: [String: Bool] = [:] |
| 198 | /// Some connection is up, and so carrying the microphone to someone |
| 199 | /// (every connection is given the track when it is made). Read from the |
| 200 | /// peer sampling, five times a second; the microphone warning waits on |
| 201 | /// it (MicSample.sending). |
| 202 | private var sendingAudio = false |
| 203 | /// The loudest remote sound since the gate last asked (sampleStats). |
| 204 | private var remotePeakSinceSample: Float = 0 |
| 205 | private var micPeakSinceSample: Float = 0 |
| 206 | private var cameraWork: Task<Void, Never>? |
| 207 | private var cameraSwitches = 0 |
| 208 | private let knocks = KnockWatch() |
| 209 | fileprivate var epoch = UUID() |
| 210 | private var bridge: BridgeClient? |
| 211 | private var events: Events? |
| 212 | /// The call's media, made by the first join (MediaEngine.make, off the |
| 213 | /// main thread) and kept for the next. |
| 214 | private var built: MediaEngine? |
| 215 | /// The media, made here and now only if something reaches for it |
| 216 | /// before any join has — which nothing in the app does. |
| 217 | private var media: MediaEngine { |
| 218 | if let built { return built } |
| 219 | AudioLog.info("call media made on the main thread, before a join") |
| 220 | let made = MediaEngine(call: self) |
| 221 | built = made |
| 222 | return made |
| 223 | } |
| 224 | /// The audio test's view of this call's microphone (AudioSelfTestModel): |
| 225 | /// the capture hook, nil outside a call, and whether WebRTC has opened it. |
| 226 | var captureDenoiser: CaptureDenoiser? { bridge == nil ? nil : built?.audio.denoiser } |
| 227 | var captureStarted: Bool { bridge != nil && built?.audio.capturing == true } |
| 228 | |
| 229 | /// For "Copy audio diagnostics" in the panel's ⓘ and in Test audio |
| 230 | /// during a call: this call's microphone, devices, engine and hook. |
| 231 | func audioDiagnosticSections() -> [(String, [String])] { |
| 232 | var lines = [ |
| 233 | "in a call: \(bridge != nil); muted \(muted); microphone track \(bridge != nil && built?.localAudio != nil)", |
| 234 | "people here: \(self.state?.peers.count ?? 0)", |
| 235 | "microphone meter: \(levels.mic.silent ? "silent" : "step \(levels.mic.step)"), speaking \(levels.mic.speaking), sending from \(levels.mic.source.isEmpty ? "-" : levels.mic.source)", |
| 236 | ] |
| 237 | if let health = audioHealth { lines.append("audio health: \(health.summary)") } |
| 238 | #if os(macOS) |
| 239 | let routeLines = audio.diagnostics() |
| 240 | #else |
| 241 | let routeLines = audio.inputs.map { "input: \($0.name) [\($0.id)]\($0.id == audio.currentInput ? " (current)" : "")" } |
| 242 | + ["output: \(audio.outputName)"] |
| 243 | #endif |
| 244 | var sections: [(String, [String])] = [("Call", lines), ("Devices", routeLines)] |
| 245 | if bridge != nil, let built { sections.append(("Audio engine and processing", built.audio.diagnostics())) } |
| 246 | return sections |
| 247 | } |
| 248 | |
| 249 | public func audioDiagnosticsText() -> String { |
| 250 | diagnosticsText { self.name(of: $0) } |
| 251 | } |
| 252 | |
| 253 | /// "Copy call diagnostics" as "Report a problem" sends it (spec §15.26): |
| 254 | /// everyone in it named "peer abc123", the first six characters of their |
| 255 | /// peer id, rather than by name. The server turns those into its tags. |
| 256 | /// Device names stay, as the spec says. |
| 257 | public func anonymousDiagnosticsText() -> String { |
| 258 | diagnosticsText(label: Self.anonymousLabel) |
| 259 | } |
| 260 | |
| 261 | nonisolated static func anonymousLabel(_ peer: String) -> String { "peer " + peer.prefix(6) } |
| 262 | |
| 263 | private func diagnosticsText(label: (String) -> String) -> String { |
| 264 | AudioDiagnosticsReport.make( |
| 265 | context: "a call's details", |
| 266 | sections: audioDiagnosticSections() + peopleDiagnosticSections(label: label) |
| 267 | + screenDiagnosticSections(label: label)) |
| 268 | } |
| 269 | |
| 270 | /// Everyone else in the call, as this app sees them: what they report |
| 271 | /// about their own media and hearing, and what this app is saying |
| 272 | /// about them. |
| 273 | private func peopleDiagnosticSections(label: (String) -> String) -> [(String, [String])] { |
| 274 | guard let state else { return [] } |
| 275 | var lines: [String] = [] |
| 276 | for person in state.peers where person.peer != state.peer { |
| 277 | var facts = ["connection \(person.gen ?? 0)"] |
| 278 | if let media = person.media { |
| 279 | facts.append(media.muted ? "muted" : "unmuted") |
| 280 | facts.append(media.cameraOff ? "camera off" : "camera on") |
| 281 | if let hearing = media.hearing { facts.append(hearing.contains(state.peer) ? "hears you" : "does not hear you yet") } |
| 282 | if media.noSoundFrom?.contains(state.peer) == true { facts.append("hears no sound from you") } |
| 283 | if media.audioBlocked == true { facts.append("their browser blocked sound") } |
| 284 | } else { |
| 285 | facts.append("no media report") |
| 286 | } |
| 287 | if levels.noSoundFrom.contains(person.peer) { facts.append("no sound from them here") } |
| 288 | if reconnectingPeers.contains(person.peer) { facts.append("reconnecting") } |
| 289 | if connectingPeers.contains(person.peer) { facts.append("connecting, can't hear you yet") } |
| 290 | lines.append("\(label(person.peer)): " + facts.joined(separator: ", ")) |
| 291 | } |
| 292 | return lines.isEmpty ? [] : [("People", lines)] |
| 293 | } |
| 294 | |
| 295 | /// The screen's part of "Copy call diagnostics": how it is leaving, what |
| 296 | /// the watchdog has done, and whose screens have stalled here. |
| 297 | private func screenDiagnosticSections(label name: (String) -> String) -> [(String, [String])] { |
| 298 | var lines: [String] = [] |
| 299 | if sharingScreen { |
| 300 | lines.append("Sharing (\(screenMode.rawValue))") |
| 301 | for (peer, report) in screenReports.sorted(by: { $0.key < $1.key }) { |
| 302 | lines.append("to \(name(peer)): \(report.summary)") |
| 303 | lines.append(" frames captured \(report.framesCaptured), encoded \(report.framesEncoded), sent \(report.framesSent), keyframes \(report.keyFramesEncoded), asked for \(report.keyframeRequests), limitation \(report.limitation.isEmpty ? "none" : report.limitation)") |
| 304 | } |
| 305 | if let capture = media.screenCaptureCounts { |
| 306 | lines.append("capture: \(capture.heartbeats ?? 0) reports, \(capture.delivered) frames delivered") |
| 307 | } |
| 308 | lines.append("watchdog: \(screenWatchdog.status)") |
| 309 | } else { |
| 310 | lines.append("Not sharing") |
| 311 | } |
| 312 | if sharingScreen { |
| 313 | for peer in (state?.peers ?? []).map(\.peer) where screenWatchdog.escalation(of: peer) != .none { |
| 314 | lines.append("recovering \(name(peer))'s view: \(screenWatchdog.escalation(of: peer))") |
| 315 | } |
| 316 | lines.append( |
| 317 | "recoveries: \(screenWatchdog.keyframes) keyframes, \(screenWatchdog.encoderRecoveries) fresh encoders, \(screenWatchdog.reAdds) re-adds, \(screenWatchdog.rebuilds) rebuilds, \(screenWatchdog.reshares) shares again, \(screenWatchdog.failures) given up") |
| 318 | } |
| 319 | // The receiver's half: every screen arriving here, as its statistics |
| 320 | // have it, whether or not it has stalled. |
| 321 | for (peer, report) in screenReceived.sorted(by: { $0.key < $1.key }) { |
| 322 | lines.append("from \(name(peer)): \(report.summary)") |
| 323 | } |
| 324 | if !reportedStalled.isEmpty { |
| 325 | lines.append("stalled here: " + reportedStalled.map { name($0) }.joined(separator: ", ")) |
| 326 | } |
| 327 | if screenReattaches > 0 { lines.append("pictures attached again here: \(screenReattaches)") } |
| 328 | let log = AudioDiagnosticsStore.screen.lines(last: AudioDiagnosticsStore.lineLimit) |
| 329 | let connection = AudioDiagnosticsStore.connection.lines(last: 50) |
| 330 | return [ |
| 331 | ("Screen", lines), ("Screen log (last \(AudioDiagnosticsStore.lineLimit) lines)", log.isEmpty ? ["(nothing logged)"] : log), |
| 332 | ("Connection log (last 50 lines)", connection.isEmpty ? ["(nothing logged)"] : connection), |
| 333 | ] |
| 334 | } |
| 335 | private var ice: ICEConfig? |
| 336 | private var pendingMedia: [Message] = [] |
| 337 | fileprivate var invalidEvents: UInt64 = 0 |
| 338 | let playAudio: Bool |
| 339 | |
| 340 | #if os(iOS) |
| 341 | let customAudioDevice: RTCAudioDevice? |
| 342 | public init(audioDevice: RTCAudioDevice? = nil, playAudio: Bool = true) { |
| 343 | customAudioDevice = audioDevice |
| 344 | self.playAudio = playAudio |
| 345 | } |
| 346 | #else |
| 347 | let audioInput: AudioInputSource? |
| 348 | /// `audioInput` stands in for the microphone (the interop harness). |
| 349 | public init(playAudio: Bool = true, audioInput: AudioInputSource? = nil) { |
| 350 | self.playAudio = playAudio |
| 351 | self.audioInput = audioInput |
| 352 | } |
| 353 | #endif |
| 354 | public func join(origin: String, room: String, cookies: String, capture: VideoCapture? = nil) |
| 355 | async throws |
| 356 | { |
| 357 | let old = bridge |
| 358 | bridge = nil |
| 359 | close() |
| 360 | let joiningEpoch = epoch |
| 361 | if let old { await Task.detached { old.leave() }.value } |
| 362 | guard epoch == joiningEpoch else { throw CancellationError() } |
| 363 | let listener = Events(call: self, epoch: epoch) |
| 364 | var error: NSError? |
| 365 | guard let client = BridgeNewClient(origin, room, cookies, listener, &error) else { |
| 366 | throw error ?? NSError(domain: "Calls", code: 1) |
| 367 | } |
| 368 | events = listener |
| 369 | bridge = client |
| 370 | arrangement.begin(meeting: origin + "/" + room) |
| 371 | muted = false |
| 372 | cameraOff = false |
| 373 | chat = [] |
| 374 | receivedChatCount = 0 |
| 375 | title = "Calls" |
| 376 | standing = "" |
| 377 | // The factory and its audio device module, built on the audio queue |
| 378 | // the first time: the call's window is up while it is made. |
| 379 | if built == nil { |
| 380 | let made = await MediaEngine.make(call: self) |
| 381 | if built == nil { built = made } |
| 382 | } |
| 383 | guard epoch == joiningEpoch, bridge === client else { throw CancellationError() } |
| 384 | try await media.prepare(capture: capture) |
| 385 | guard epoch == joiningEpoch, bridge === client else { throw CancellationError() } |
| 386 | localTrack = media.localVideo |
| 387 | cameraChanged() |
| 388 | reportMedia() |
| 389 | media.audio.reduceNoise = reduceNoise |
| 390 | audioProcessing = media.audio.report() |
| 391 | watchLevels() |
| 392 | startQualityWatch() |
| 393 | KnockNotice.requestPermission() |
| 394 | // Asking Apple at join, once there is a session to register the |
| 395 | // answer against. Apple replies on its own schedule, at the app |
| 396 | // delegate, and may never reply at all. |
| 397 | RemoteKnocks.register() |
| 398 | try client.start() |
| 399 | } |
| 400 | public func close() { |
| 401 | epoch = UUID() |
| 402 | let old = bridge |
| 403 | bridge = nil |
| 404 | events = nil |
| 405 | built?.close() |
| 406 | // This call's microphone and speaker choices end with it (see AudioRoutes). |
| 407 | #if os(macOS) |
| 408 | audio.reset() |
| 409 | #endif |
| 410 | cameras = [] |
| 411 | remoteTracks = [:] |
| 412 | screens = [] |
| 413 | remoteScreens = [:] |
| 414 | sharingScreen = false |
| 415 | screenNote = "" |
| 416 | endScreenWatch() |
| 417 | localTrack = nil |
| 418 | ice = nil |
| 419 | state = nil |
| 420 | pendingMedia = [] |
| 421 | invitationURL = nil |
| 422 | invitationError = "" |
| 423 | loadingInvitation = false |
| 424 | deviceNote = "" |
| 425 | audioHealth = nil |
| 426 | audioProcessing = nil |
| 427 | endLevelWatch() |
| 428 | qualityWatch?.cancel() |
| 429 | qualityWatch = nil |
| 430 | currentCamera = nil |
| 431 | cameraStatus = nil |
| 432 | cameraNote = "" |
| 433 | cameraToKeep = nil |
| 434 | knocks.reset() |
| 435 | cameraSwitches += 1 |
| 436 | cameraWork?.cancel() |
| 437 | cameraWork = nil |
| 438 | if let old { DispatchQueue.global().async { old.close() } } |
| 439 | } |
| 440 | public func leave() async { |
| 441 | // The final quality report reads the connections' statistics, so it is |
| 442 | // built before they close, and sent beside the leave rather than after |
| 443 | // it: quitting gives the two together three seconds. |
| 444 | let final = await finalQualityReport() |
| 445 | let old = bridge |
| 446 | if let old { keepForReports(old) } |
| 447 | bridge = nil |
| 448 | close() |
| 449 | status = "You left the meeting" |
| 450 | if let old { |
| 451 | await Task.detached { |
| 452 | await withTaskGroup(of: Void.self) { group in |
| 453 | if let final { group.addTask { _ = old.sendQuality(final) } } |
| 454 | group.addTask { old.leave() } |
| 455 | } |
| 456 | }.value |
| 457 | } |
| 458 | } |
| 459 | public func mute() { |
| 460 | muted.toggle() |
| 461 | // On the audio queue: switching a track waits out an engine start, |
| 462 | // 1,136 ms measured on the main thread with a Bluetooth switch |
| 463 | // simulated — and the first thing someone does as they join is often |
| 464 | // to mute. |
| 465 | let track = media.localAudio |
| 466 | let on = !muted |
| 467 | AudioWork.post { track?.isEnabled = on } |
| 468 | reportMedia() |
| 469 | } |
| 470 | public func toggleCamera() { |
| 471 | cameraOff.toggle() |
| 472 | let track = media.localVideo |
| 473 | let on = !cameraOff |
| 474 | AudioWork.post { track?.isEnabled = on } |
| 475 | reportMedia() |
| 476 | } |
| 477 | |
| 478 | // --------------------------------------------------------------- |
| 479 | // Media state (§15.19) |
| 480 | // --------------------------------------------------------------- |
| 481 | |
| 482 | /// Tells the core what this app is sending. The core reports it once |
| 483 | /// admitted, again whenever a roster says the server has not got it — |
| 484 | /// a resume, a restart — and at once on a change, so this is called on |
| 485 | /// each toggle and once the tracks exist, and nowhere else. No track at |
| 486 | /// all is muted, or off, as far as anyone listening can tell. |
| 487 | private func reportMedia() { |
| 488 | try? bridge?.setMedia(muted || media.localAudio == nil, cameraOff: cameraOff || media.localVideo == nil) |
| 489 | } |
| 490 | |
| 491 | /// A remote peer's reported state, or nil when it has not reported. |
| 492 | public func peerMedia(_ peer: String) -> PeerMedia? { Self.peerMedia(peer, in: state) } |
| 493 | /// Whether a remote peer has said it is muted. Unknown is not muted. |
| 494 | public func isMuted(_ peer: String) -> Bool { peerMedia(peer)?.muted ?? false } |
| 495 | /// Whether a remote peer has said its camera is off. Unknown is not off. |
| 496 | public func isCameraOff(_ peer: String) -> Bool { peerMedia(peer)?.cameraOff ?? false } |
| 497 | nonisolated static func peerMedia(_ peer: String, in state: RoomState?) -> PeerMedia? { |
| 498 | state?.peers.first { $0.peer == peer }?.media |
| 499 | } |
| 500 | |
| 501 | // --------------------------------------------------------------- |
| 502 | // Devices |
| 503 | // --------------------------------------------------------------- |
| 504 | |
| 505 | /// The cameras available, and the one in use. Read through the running |
| 506 | /// capture rather than from AVFoundation directly, so what the picker |
| 507 | /// offers is what the call could actually switch to. |
| 508 | @Published public private(set) var cameras: [CameraDevice] = [] |
| 509 | |
| 510 | public func refreshDevices() { |
| 511 | let available = built?.cameras ?? [] |
| 512 | if cameras != available { cameras = available } |
| 513 | if let currentCamera, !available.contains(where: { $0.id == currentCamera }) { |
| 514 | self.currentCamera = nil |
| 515 | } |
| 516 | audio.refresh() |
| 517 | } |
| 518 | |
| 519 | /// The capture said something changed on its own: a camera came or |
| 520 | /// went, a fallback, the first frame. Everything camera-shaped is read |
| 521 | /// back from it. |
| 522 | func cameraChanged() { |
| 523 | offersCameras = media.offersCameras |
| 524 | refreshDevices() |
| 525 | let now = media.currentCamera |
| 526 | if currentCamera != now { currentCamera = now } |
| 527 | let status = media.cameraStatus |
| 528 | if cameraStatus != status { cameraStatus = status } |
| 529 | let note = media.cameraNote |
| 530 | if cameraNote != note { cameraNote = note } |
| 531 | let keep = media.cameraToKeep |
| 532 | if cameraToKeep != keep { cameraToKeep = keep } |
| 533 | } |
| 534 | |
| 535 | /// What your own tile says in place of your picture, or nil to show it. |
| 536 | /// Never an open-ended "Waiting for video": it says what it is waiting |
| 537 | /// for, or why there is nothing. |
| 538 | public var selfPlaceholder: String? { |
| 539 | if state?.phase == "ended" { return "Not in the call" } |
| 540 | if localTrack == nil { return "Starting camera…" } |
| 541 | return cameraStatus?.tileText |
| 542 | } |
| 543 | |
| 544 | /// The panel's "Try again", for a call with no working camera. |
| 545 | public func retryCamera() async { |
| 546 | await media.retryCamera() |
| 547 | cameraChanged() |
| 548 | } |
| 549 | |
| 550 | /// Switches camera mid-call. Serialised: two picks are two capture |
| 551 | /// restarts on one device, and run at once the survivor is whichever |
| 552 | /// session happened to open last, which is not the one the person |
| 553 | /// chose. The counter decides who wins; the queue decides that only |
| 554 | /// one runs. |
| 555 | public func useCamera(_ id: String) async { |
| 556 | cameraSwitches += 1 |
| 557 | let mine = cameraSwitches |
| 558 | let previous = cameraWork |
| 559 | let work = Task { @MainActor [weak self] in |
| 560 | await previous?.value |
| 561 | guard let self, mine == self.cameraSwitches, !Task.isCancelled else { return } |
| 562 | do { |
| 563 | try await self.media.use(camera: id) |
| 564 | guard mine == self.cameraSwitches, !Task.isCancelled else { return } |
| 565 | self.deviceNote = "" |
| 566 | } catch is CancellationError { |
| 567 | } catch { |
| 568 | guard mine == self.cameraSwitches, !Task.isCancelled else { return } |
| 569 | self.deviceNote = error.localizedDescription |
| 570 | } |
| 571 | // Read back rather than assume: a refused switch left the capture |
| 572 | // wherever it actually is, and the picker has to follow the camera |
| 573 | // and not the intention. |
| 574 | guard mine == self.cameraSwitches, !Task.isCancelled else { return } |
| 575 | self.cameraChanged() |
| 576 | } |
| 577 | cameraWork = work |
| 578 | await work.value |
| 579 | } |
| 580 | |
| 581 | /// Records something a device change had to say. The panel has no |
| 582 | /// place of its own to put an error, and a route that would not change |
| 583 | /// belongs beside the control that would not change it. |
| 584 | public func noteDevice(_ text: String) { deviceNote = text } |
| 585 | |
| 586 | /// Polls the audio-health counters while a call is up. |
| 587 | /// |
| 588 | /// This exists because of a call on 2026-09-17 that came back as an |
| 589 | /// unintelligible growl, and because nothing in this app recorded |
| 590 | /// anything about it — no log, no counters, no artefact. A reproduction |
| 591 | /// would have produced nothing either. (Until the move to LiveKit's |
| 592 | /// WebRTC build it was also the only instrument on macOS, which had no |
| 593 | /// way to switch away from a suspect device.) |
| 594 | /// |
| 595 | /// Five seconds because this is a readout a person glances at when |
| 596 | /// something sounds wrong, not a time series. |
| 597 | public func watchAudio() async { |
| 598 | while !Task.isCancelled { |
| 599 | try? await Task.sleep(nanoseconds: 5_000_000_000) |
| 600 | guard !Task.isCancelled, bridge != nil else { return } |
| 601 | refreshDevices() |
| 602 | audioHealth = AudioHealth(await media.stats()) |
| 603 | media.audio.refresh() |
| 604 | audioProcessing = media.audio.report() |
| 605 | } |
| 606 | } |
| 607 | // --------------------------------------------------------------- |
| 608 | // Levels (§15.20) |
| 609 | // --------------------------------------------------------------- |
| 610 | |
| 611 | /// What a tile shows about its sound: yours, or a remote person's. |
| 612 | public var selfAudio: TileAudio { .you(levels.mic) } |
| 613 | public func tileAudio(_ peer: String) -> TileAudio { |
| 614 | .peer(levels.peer(peer), name: name(of: peer)) |
| 615 | } |
| 616 | |
| 617 | /// Reads the meters ten times a second while the call is up: your |
| 618 | /// microphone every tick, everyone else's every other (5 Hz), and the |
| 619 | /// received byte counts once a second. What is published only changes |
| 620 | /// when a meter step or a line does (see AudioLevels). |
| 621 | private func watchLevels() { |
| 622 | levelWatch?.cancel() |
| 623 | levels.reset() |
| 624 | resetConnecting() |
| 625 | reportedNoSound = [] |
| 626 | audioBytes = [:] |
| 627 | audioArriving = [:] |
| 628 | levelWatch = Task { @MainActor [weak self] in |
| 629 | var tick = 0 |
| 630 | while !Task.isCancelled { |
| 631 | try? await Task.sleep(nanoseconds: 100_000_000) |
| 632 | guard let self, !Task.isCancelled, self.bridge != nil else { return } |
| 633 | tick += 1 |
| 634 | let now = ProcessInfo.processInfo.systemUptime |
| 635 | self.sampleMic(now: now) |
| 636 | if tick % 2 == 0 { await self.samplePeers(stats: tick % 10 == 0, now: now) } |
| 637 | if tick % 10 == 5 { await self.watchRemoteScreens(now: now) } |
| 638 | } |
| 639 | } |
| 640 | } |
| 641 | |
| 642 | private func endLevelWatch() { |
| 643 | levelWatch?.cancel() |
| 644 | levelWatch = nil |
| 645 | screenReceiveWatch = ScreenReceiveWatch() |
| 646 | reportedStalled = [] |
| 647 | screenReceived = [:] |
| 648 | screenReceivedMoving = [:] |
| 649 | levels.reset() |
| 650 | resetConnecting() |
| 651 | reportedNoSound = [] |
| 652 | audioBytes = [:] |
| 653 | audioArriving = [:] |
| 654 | sendingAudio = false |
| 655 | } |
| 656 | |
| 657 | private func resetConnecting() { |
| 658 | connectingWatch = ConnectingWatch() |
| 659 | if !connectingPeers.isEmpty { connectingPeers = [] } |
| 660 | heardFrom = [:] |
| 661 | ackedBy = [:] |
| 662 | // The core starts every report with `hearing: []` (spec §15.24). |
| 663 | reportedHearing = [] |
| 664 | } |
| 665 | |
| 666 | private func sampleMic(now: Double) { |
| 667 | let denoiser = media.audio.denoiser |
| 668 | let sent = denoiser.level.take() |
| 669 | let heard = denoiser.inputLevel.take() |
| 670 | micPeakSinceSample = max(micPeakSinceSample, sent.peak) |
| 671 | #if os(macOS) |
| 672 | let source = audio.inputSource |
| 673 | let channels = audio.inputChannels.count > 1 |
| 674 | let capturing = media.audio.capturing |
| 675 | #else |
| 676 | let source = audio.inputs.first { $0.id == audio.currentInput }?.name ?? "" |
| 677 | let channels = false |
| 678 | let capturing = media.audio.hookFlowing |
| 679 | #endif |
| 680 | levels.updateMic( |
| 681 | MicSample( |
| 682 | muted: muted || media.localAudio == nil, capturing: capturing, |
| 683 | inputPeak: heard.peak, inputBuffers: heard.buffers, sentPeak: sent.peak, sending: sendingAudio), |
| 684 | now: now, source: source, channels: channels) |
| 685 | AudioDiagnosticsStore.shared.record( |
| 686 | .init( |
| 687 | at: now, input: heard.peak, inputBuffers: heard.buffers, output: sent.peak, outputBuffers: sent.buffers, |
| 688 | source: muted ? "call (muted)" : capturing ? "call" : "call (mic not open)")) |
| 689 | } |
| 690 | |
| 691 | private func samplePeers(stats: Bool, now: Double) async { |
| 692 | // §15.24: the people whose audio path is not established yet are read |
| 693 | // closely, five times a second, until it is; the people in their first |
| 694 | // moments who do not say who they hear are asked about this app's audio. |
| 695 | let mine = state?.peer ?? "" |
| 696 | let joining = connectingWatch.joining |
| 697 | var detail = Set<String>() |
| 698 | var acks = Set<String>() |
| 699 | for person in state?.peers ?? [] where person.peer != mine { |
| 700 | let epoch = person.gen ?? 0 |
| 701 | if heardFrom[person.peer].map({ $0.epoch != epoch || !$0.heard }) ?? true { detail.insert(person.peer) } |
| 702 | if joining.contains(person.peer), HearingRule.hearsYou(person.media, me: mine) == nil, |
| 703 | ackedBy[person.peer] != epoch |
| 704 | { |
| 705 | acks.insert(person.peer) |
| 706 | } |
| 707 | } |
| 708 | let readings = await media.peerAudio(stats: stats, detail: detail, acks: acks) |
| 709 | guard levelWatch != nil, let state else { return } |
| 710 | sendingAudio = readings.values.contains { $0.connected } |
| 711 | let me = state.peer |
| 712 | var present = Set<String>() |
| 713 | var people: [String: ReconnectWatch.Peer] = [:] |
| 714 | var arrivals: [String: ConnectingWatch.Peer] = [:] |
| 715 | for person in state.peers where person.peer != me { |
| 716 | guard let reading = readings[person.peer] else { continue } |
| 717 | present.insert(person.peer) |
| 718 | if stats, let bytes = reading.bytes { |
| 719 | audioArriving[person.peer] = bytes > (audioBytes[person.peer] ?? 0) |
| 720 | audioBytes[person.peer] = bytes |
| 721 | } |
| 722 | let epoch = person.gen ?? 0 |
| 723 | var was = heardFrom[person.peer] ?? (epoch, false) |
| 724 | if was.epoch != epoch { was = (epoch, false) } |
| 725 | let heard = HearingRule.heard( |
| 726 | was: was.heard, |
| 727 | .init( |
| 728 | connected: reading.connected, bad: reading.gone, hasReceiver: reading.hasReceiver, |
| 729 | receiverLive: reading.receiverLive, packets: reading.packets ?? 0, decoded: reading.buffers, |
| 730 | senderMuted: person.media?.muted == true)) |
| 731 | heardFrom[person.peer] = (epoch, heard) |
| 732 | if reading.acked == true { ackedBy[person.peer] = epoch } |
| 733 | arrivals[person.peer] = ConnectingWatch.Peer( |
| 734 | epoch: epoch, hearsMe: HearingRule.hearsYou(person.media, me: me), reported: person.media != nil, |
| 735 | connected: reading.connected, acked: ackedBy[person.peer] == epoch) |
| 736 | } |
| 737 | for id in Array(heardFrom.keys) where !present.contains(id) { heardFrom.removeValue(forKey: id) } |
| 738 | for id in Array(ackedBy.keys) where !present.contains(id) { ackedBy.removeValue(forKey: id) } |
| 739 | for id in connectingWatch.observe(at: now, people: arrivals) { |
| 740 | CallLog.info("\(id.prefix(6)) can hear us") |
| 741 | AudioAnnouncer.say(ConnectingWords.canHear(name(of: id))) |
| 742 | } |
| 743 | let nowJoining = connectingWatch.joining |
| 744 | if connectingPeers != connectingWatch.shown { connectingPeers = connectingWatch.shown } |
| 745 | for person in state.peers where person.peer != me { |
| 746 | guard let reading = readings[person.peer] else { continue } |
| 747 | // Someone in their first moments has not lost anything yet: §15.24 |
| 748 | // speaks for them until it gives up. |
| 749 | people[person.peer] = |
| 750 | nowJoining.contains(person.peer) |
| 751 | ? ReconnectWatch.Peer() |
| 752 | : ReconnectWatch.Peer( |
| 753 | connectionBad: reading.connectionBad, |
| 754 | audioMissing: audioArriving[person.peer] == false && person.media?.muted != true) |
| 755 | let said = ConnectingWatch.tile( |
| 756 | name: person.name, media: person.media, me: me, muted: muted, |
| 757 | joining: nowJoining.contains(person.peer), shown: connectingPeers.contains(person.peer), |
| 758 | reconnecting: reconnectingPeers.contains(person.peer)) |
| 759 | remotePeakSinceSample = max(remotePeakSinceSample, reading.peak) |
| 760 | levels.updatePeer( |
| 761 | person.peer, name: person.name, |
| 762 | sample: PeerAudioSample( |
| 763 | reported: person.media, connected: reading.connected, |
| 764 | arriving: audioArriving[person.peer] ?? false, level: reading.peak), |
| 765 | notices: said.notices, connecting: said.connecting, |
| 766 | now: now) |
| 767 | } |
| 768 | watchReconnect(now: now, people: people) |
| 769 | observeQuality(now: now, state: state) |
| 770 | levels.settle(present: present) |
| 771 | for id in Array(audioBytes.keys) where !present.contains(id) { |
| 772 | audioBytes.removeValue(forKey: id) |
| 773 | audioArriving.removeValue(forKey: id) |
| 774 | } |
| 775 | reportNoSound(levels.noSoundFrom) |
| 776 | reportHearing(heardFrom.filter { $0.value.heard }.keys.sorted(), present: present) |
| 777 | } |
| 778 | |
| 779 | /// §15.24: whose audio this app is receiving and playing. Sent only on a |
| 780 | /// change, in order, off the main thread, as `noSoundFrom` is. |
| 781 | private func reportHearing(_ peers: [String], present: Set<String>) { |
| 782 | guard peers != reportedHearing, let client = bridge else { return } |
| 783 | let moved = HearingRule.moved(from: reportedHearing, to: peers, present: present) |
| 784 | reportedHearing = peers |
| 785 | guard moved else { return } |
| 786 | let body = json(peers) ?? "[]" |
| 787 | noSoundQueue.async { try? client.setHearing(body) } |
| 788 | } |
| 789 | |
| 790 | /// One look by ReconnectWatch (spec §15.23), and the words it says. |
| 791 | private func watchReconnect(now: Double, people: [String: ReconnectWatch.Peer]) { |
| 792 | reconnectWatch.observe(at: now, streamDown: state?.phase == "reconnecting", people: people) |
| 793 | if reconnectingPeers != reconnectWatch.peers { |
| 794 | for id in reconnectWatch.peers.subtracting(reconnectingPeers) { CallLog.info("reconnecting to \(id.prefix(6)): shown") } |
| 795 | reconnectingPeers = reconnectWatch.peers |
| 796 | } |
| 797 | guard reconnecting != reconnectWatch.call else { return } |
| 798 | reconnecting = reconnectWatch.call |
| 799 | CallLog.info(reconnecting ? "reconnecting: shown (a problem that lasted)" : "reconnecting: cleared") |
| 800 | switch state?.phase { |
| 801 | case "reconnecting": status = reconnecting ? ReconnectWords.call : "In the meeting" |
| 802 | case "admitted": status = reconnecting ? ReconnectWords.call : "In the meeting" |
| 803 | default: break |
| 804 | } |
| 805 | } |
| 806 | |
| 807 | /// §15.20's receiving half: who is reaching this app as silence. The |
| 808 | /// core keeps only roster peers and reports on a change; this sends only |
| 809 | /// on a change too, in order, off the main thread. |
| 810 | private func reportNoSound(_ peers: [String]) { |
| 811 | guard peers != reportedNoSound, let client = bridge else { return } |
| 812 | reportedNoSound = peers |
| 813 | let body = json(peers) ?? "[]" |
| 814 | noSoundQueue.async { try? client.setNoSoundFrom(body) } |
| 815 | } |
| 816 | |
| 817 | public func retryJoin() { send(type: "join", to: "", gen: 0, payload: "") } |
| 818 | public func loadInvitation() async { |
| 819 | guard let client = bridge, !loadingInvitation else { return } |
| 820 | let currentEpoch = epoch |
| 821 | invitationURL = nil |
| 822 | invitationError = "" |
| 823 | loadingInvitation = true |
| 824 | defer { if epoch == currentEpoch { loadingInvitation = false } } |
| 825 | do { |
| 826 | let link = try await Task.detached { |
| 827 | var error: NSError? |
| 828 | let value = client.shareLink(&error) |
| 829 | if let error { throw error } |
| 830 | return value |
| 831 | }.value |
| 832 | guard epoch == currentEpoch else { return } |
| 833 | invitationURL = URL(string: link) |
| 834 | } catch { |
| 835 | guard epoch == currentEpoch else { return } |
| 836 | invitationError = "Could not get the invitation. Try again." |
| 837 | } |
| 838 | } |
| 839 | public func admit(_ peer: String) { |
| 840 | send(type: "admit", to: "", gen: 0, payload: json(["peer": peer]) ?? "") |
| 841 | } |
| 842 | @discardableResult public func sendChat(_ text: String) -> Bool { |
| 843 | guard !text.isEmpty else { return false } |
| 844 | guard let payload = json(["text": text]), payload.utf8.count <= 2059 else { |
| 845 | status = "Keep the message within 2 KB." |
| 846 | return false |
| 847 | } |
| 848 | guard let bridge else { return false } |
| 849 | do { |
| 850 | try bridge.send("chat", to: "", generation: 0, payload: payload) |
| 851 | return true |
| 852 | } catch { |
| 853 | status = "Could not send the message. Try again." |
| 854 | return false |
| 855 | } |
| 856 | } |
| 857 | func send(type: String, to: String, gen: Int64, payload: String) { |
| 858 | do { try bridge?.send(type, to: to, generation: gen, payload: payload) } catch { |
| 859 | status = "Could not send to the meeting. Try again." |
| 860 | } |
| 861 | } |
| 862 | fileprivate func receive(_ event: WireEvent) { |
| 863 | switch event.type { |
| 864 | case "sent", "send-failed": |
| 865 | guard let message = event.signal, message.type == "chat", |
| 866 | let text = message.payload?.text |
| 867 | else { return } |
| 868 | appendChat("\(event.type == "sent" ? "You" : "Not sent"): \(text)") |
| 869 | case "info": |
| 870 | if let info = event.info { |
| 871 | title = info.title |
| 872 | named = info.named ?? false |
| 873 | standing = info.standing |
| 874 | floor = info.floor |
| 875 | } |
| 876 | case "state": |
| 877 | guard let state = event.state else { return } |
| 878 | self.state = state |
| 879 | // Only a member is sent a waiting list at all (spec §4.3), so this |
| 880 | // needs no check of its own for who is allowed to be told. |
| 881 | KnockNotice.announce( |
| 882 | knocks.arrivals(waiting: state.waiting, visible: KnockNotice.appIsFrontmost)) |
| 883 | switch state.phase { |
| 884 | case "admitted": status = reconnecting ? ReconnectWords.call : "In the meeting" |
| 885 | case "waiting": status = "Waiting to be let in" |
| 886 | case "full": status = "The meeting is full. Try joining again shortly." |
| 887 | // Said only once it has lasted (ReconnectWatch, spec §15.23): a |
| 888 | // stream renewal is a reconnect of half a second nobody should see. |
| 889 | case "reconnecting": if reconnecting { status = ReconnectWords.call } |
| 890 | case "ended": |
| 891 | status = |
| 892 | [ |
| 893 | "left": "You left the meeting", "ended": "This meeting has ended", |
| 894 | "replaced": "This meeting is open elsewhere", |
| 895 | "removed": "You were removed from this meeting", |
| 896 | "standing": "You no longer have access", "expired": "Your invitation has expired", |
| 897 | "signin": "Sign in again to join", "unavailable": "This meeting is unavailable", |
| 898 | ][state.reason ?? ""] ?? "The call has ended" |
| 899 | endedForReports() |
| 900 | media.close() |
| 901 | remoteTracks = [:] |
| 902 | screens = [] |
| 903 | remoteScreens = [:] |
| 904 | sharingScreen = false |
| 905 | endScreenWatch() |
| 906 | endLevelWatch() |
| 907 | localTrack = nil |
| 908 | cameraChanged() |
| 909 | default: status = "Connecting…" |
| 910 | } |
| 911 | if state.peer.isEmpty { |
| 912 | media.closePeers() |
| 913 | remoteTracks = [:] |
| 914 | screens = [] |
| 915 | remoteScreens = [:] |
| 916 | ice = nil |
| 917 | } |
| 918 | if let ice { media.reconcile(state: state, ice: ice) } |
| 919 | case "ice": |
| 920 | guard let config = event.ice else { return } |
| 921 | let renewing = ice != nil |
| 922 | ice = config |
| 923 | floor = config.iceServers.isEmpty |
| 924 | if let state { media.reconcile(state: state, ice: config) } |
| 925 | if renewing { media.updateICE(config) } |
| 926 | let pending = pendingMedia |
| 927 | pendingMedia = [] |
| 928 | for message in pending { media.receive(message) } |
| 929 | case "ice-unavailable": status = "Could not refresh the call connection. Retrying…" |
| 930 | case "message": |
| 931 | guard let message = event.message else { return } |
| 932 | if message.type == "chat", let text = message.payload?.text { |
| 933 | appendChat("\(message.name ?? "Participant"): \(text)") |
| 934 | receivedChatCount += 1 |
| 935 | } |
| 936 | if message.type == "share" { |
| 937 | media.receiveShare(message) |
| 938 | return |
| 939 | } |
| 940 | if ice == nil, ["offer", "answer", "candidate"].contains(message.type) { |
| 941 | if pendingMedia.count < 256 { pendingMedia.append(message) } |
| 942 | } else { |
| 943 | media.receive(message) |
| 944 | } |
| 945 | default: break |
| 946 | } |
| 947 | } |
| 948 | private func appendChat(_ line: String) { |
| 949 | chat.append(line) |
| 950 | if chat.count > 200 { chat.removeFirst(chat.count - 200) } |
| 951 | } |
| 952 | func setRemote(_ track: RTCVideoTrack, peer: String) { remoteTracks[peer] = track } |
| 953 | func removeRemote(_ peer: String) { |
| 954 | remoteTracks.removeValue(forKey: peer) |
| 955 | dropScreen(peer) |
| 956 | } |
| 957 | |
| 958 | // --------------------------------------------------------------- |
| 959 | // Screens (§5.4) |
| 960 | // --------------------------------------------------------------- |
| 961 | |
| 962 | /// The name the roster gives a peer, which is the only place a name |
| 963 | /// comes from: the hub put it there, and nothing a peer sends can. |
| 964 | public func name(of peer: String) -> String { |
| 965 | state?.peers.first { $0.peer == peer }?.name ?? "Participant" |
| 966 | } |
| 967 | |
| 968 | /// Shares a screen (§5.4): the capture asks the person what to share, |
| 969 | /// every connection gets it on a transceiver of its own, and then the |
| 970 | /// room is told — the label, not the media, so peers can caption the |
| 971 | /// tile before the renegotiation lands. |
| 972 | /// |
| 973 | /// A refusal at the picker says one sentence, the browser's, and does |
| 974 | /// not diagnose which kind of refusal it was. |
| 975 | public func shareScreen(_ capture: ScreenCapture) async { |
| 976 | // The picker stays open for as long as a person takes to answer it, |
| 977 | // and the button is live the whole time. Without this a second press |
| 978 | // starts a second capture that nothing would ever stop. |
| 979 | guard bridge != nil, !sharingScreen, !screenPending else { return } |
| 980 | screenPending = true |
| 981 | defer { screenPending = false } |
| 982 | let joined = epoch |
| 983 | screenNote = "" |
| 984 | media.screenMode = screenMode |
| 985 | do { |
| 986 | try await media.startScreen(capture) { [weak self] in |
| 987 | // The capture ended on its own: the system's stop control, or |
| 988 | // the shared window closing. The same ending as the button. |
| 989 | guard let self, self.epoch == joined else { return } |
| 990 | self.stopSharingScreen() |
| 991 | } |
| 992 | } catch is CancellationError { |
| 993 | return |
| 994 | } catch { |
| 995 | guard epoch == joined else { return } |
| 996 | screenNote = "Screen sharing was not allowed." |
| 997 | return |
| 998 | } |
| 999 | guard epoch == joined, bridge != nil else { |
| 1000 | media.stopScreen() |
| 1001 | return |
| 1002 | } |
| 1003 | sharingScreen = true |
| 1004 | screenWatchdog.reset() |
| 1005 | ScreenLog.info("sharing started (\(screenMode.rawValue))") |
| 1006 | send(type: "share", to: "", gen: 0, payload: json(["on": true]) ?? "") |
| 1007 | watchScreen() |
| 1008 | } |
| 1009 | |
| 1010 | /// Reads the screen's outbound statistics once a second while sharing, |
| 1011 | /// for the device panel's line and the gate. |
| 1012 | private func watchScreen() { |
| 1013 | screenWatch?.cancel() |
| 1014 | screenWatch = Task { @MainActor [weak self] in |
| 1015 | while !Task.isCancelled { |
| 1016 | try? await Task.sleep(nanoseconds: 1_000_000_000) |
| 1017 | guard let self, !Task.isCancelled, self.sharingScreen else { return } |
| 1018 | let fresh = await self.media.screenReports() |
| 1019 | guard !Task.isCancelled, self.sharingScreen else { return } |
| 1020 | var measured: [String: ScreenSendReport] = [:] |
| 1021 | for (peer, report) in fresh { |
| 1022 | measured[peer] = report.measured(since: self.screenReports[peer]) |
| 1023 | } |
| 1024 | let previous = self.screenReports |
| 1025 | self.screenReports = measured |
| 1026 | let worst = ScreenSendReport.worst(Array(measured.values)) |
| 1027 | self.screenReport = worst |
| 1028 | // Motion mode scales from the source's own size, which only the |
| 1029 | // statistics know (a window can be any shape, and re-picked). |
| 1030 | if let worst { self.media.screenSource(width: worst.sourceWidth, height: worst.sourceHeight) } |
| 1031 | await self.runScreenWatchdog(measured, previous: previous) |
| 1032 | } |
| 1033 | } |
| 1034 | } |
| 1035 | |
| 1036 | /// Off only for the gate's measurement of what happens without it. |
| 1037 | @_spi(Fixtures) public static var screenWatchdogEnabled = true |
| 1038 | |
| 1039 | /// One look by the sharer's watchdog, and whatever it asks for. |
| 1040 | private func runScreenWatchdog( |
| 1041 | _ reports: [String: ScreenSendReport], previous previousReports: [String: ScreenSendReport] |
| 1042 | ) async { |
| 1043 | guard sharingScreen, let capture = media.screenCaptureCounts else { return } |
| 1044 | let me = state?.peer ?? "" |
| 1045 | let stalledBy = Set( |
| 1046 | (state?.peers ?? []).filter { $0.peer != me && ($0.media?.screenStalled ?? []).contains(me) }.map(\.peer)) |
| 1047 | let sending = reports.mapValues { |
| 1048 | ScreenSendCounts( |
| 1049 | encoded: $0.framesEncoded, sent: $0.framesSent, keyframes: $0.keyFramesEncoded, |
| 1050 | bandwidthLimited: $0.limitation == "bandwidth") |
| 1051 | } |
| 1052 | let failuresBefore = screenWatchdog.failures |
| 1053 | var actions = screenWatchdog.observe( |
| 1054 | at: ProcessInfo.processInfo.systemUptime, capture: capture, sending: sending, stalledBy: stalledBy) |
| 1055 | if !Self.screenWatchdogEnabled { actions = [] } |
| 1056 | countScreenSteps(actions, failed: screenWatchdog.failures - failuresBefore, stalledBy: stalledBy) |
| 1057 | for action in actions { |
| 1058 | switch action { |
| 1059 | case .restartCapture: |
| 1060 | ScreenLog.info("restarting the capture") |
| 1061 | let ok = await media.restartScreenCapture() |
| 1062 | if !ok { ScreenLog.error("the capture could not be restarted") } |
| 1063 | case .keyframe(let peer): |
| 1064 | ScreenLog.info("keyframe for \(ScreenWatchdog.short(peer))") |
| 1065 | media.forceScreenKeyframe(peer) |
| 1066 | case .recreateEncoder(let peer): |
| 1067 | ScreenLog.info("fresh encoder for \(ScreenWatchdog.short(peer))") |
| 1068 | media.recreateScreenEncoder(peer) |
| 1069 | case .reAdd(let peer): |
| 1070 | ScreenLog.info("screen re-added for \(ScreenWatchdog.short(peer))") |
| 1071 | media.reAddScreen(peer) |
| 1072 | sayShareAgain() |
| 1073 | case .rebuild(let peer): |
| 1074 | ScreenLog.info("screen rebuilt for \(ScreenWatchdog.short(peer)): new transceiver, same size and mode") |
| 1075 | media.reAddScreen(peer) |
| 1076 | sayShareAgain() |
| 1077 | case .reshare: |
| 1078 | ScreenLog.info("sharing again for everyone, from the same capture") |
| 1079 | await media.reshareScreen() |
| 1080 | sayShareAgain() |
| 1081 | } |
| 1082 | } |
| 1083 | screenWatchStatus = screenWatchdog.status |
| 1084 | if let (peer, worst) = reports.min(by: { $0.value.framesPerSecond < $1.value.framesPerSecond }) { |
| 1085 | screenDetail = worst.detail(since: previousReports[peer], watchdog: screenWatchdog.status) |
| 1086 | } |
| 1087 | var recovering: [String] = [] |
| 1088 | var restarting: [String] = [] |
| 1089 | var failed: [String] = [] |
| 1090 | for peer in stalledBy.sorted(by: { name(of: $0) < name(of: $1) }) { |
| 1091 | switch screenWatchdog.escalation(of: peer) { |
| 1092 | case .rebuilt, .reshared: restarting.append(name(of: peer)) |
| 1093 | case .failed: failed.append(name(of: peer)) |
| 1094 | case .none, .keyframes: recovering.append(name(of: peer)) |
| 1095 | } |
| 1096 | } |
| 1097 | screenRecovery = ScreenWords.notice(recovering: recovering, restarting: restarting, failed: failed) |
| 1098 | } |
| 1099 | |
| 1100 | /// A new transceiver's track arrives unlabelled; say it again. |
| 1101 | private func sayShareAgain() { |
| 1102 | Task { @MainActor [weak self] in |
| 1103 | try? await Task.sleep(nanoseconds: 2_000_000_000) |
| 1104 | guard let self, self.sharingScreen else { return } |
| 1105 | self.send(type: "share", to: "", gen: 0, payload: json(["on": true]) ?? "") |
| 1106 | } |
| 1107 | } |
| 1108 | private func endScreenWatch() { |
| 1109 | screenWatch?.cancel() |
| 1110 | screenWatch = nil |
| 1111 | screenReports = [:] |
| 1112 | screenReport = nil |
| 1113 | screenWatchdog.reset() |
| 1114 | screenWatchStatus = "" |
| 1115 | screenRecovery = "" |
| 1116 | screenDetail = "" |
| 1117 | } |
| 1118 | |
| 1119 | /// The receiver's watchdog, once a second from the level watch: whose |
| 1120 | /// screens have stopped moving here, reported as `screenStalled` when |
| 1121 | /// that changes (spec §15.21), each stall logged with what the |
| 1122 | /// statistics say is still arriving, and the picture attached again |
| 1123 | /// four seconds in (spec §15.22). |
| 1124 | private func watchRemoteScreens(now: Double) async { |
| 1125 | var stalled: [String] = [] |
| 1126 | if !screens.isEmpty { |
| 1127 | let reports = await media.remoteScreenReports() |
| 1128 | var frames = reports.mapValues(\.framesDecoded) |
| 1129 | // The gate's frozen picture: the counts held where they were. |
| 1130 | if now < screenWatchFrozenUntil { frames = frames.mapValues { _ in 0 } } |
| 1131 | let sharing = Set(screens) |
| 1132 | let before = screenReceived |
| 1133 | screenReceived = reports.filter { sharing.contains($0.key) } |
| 1134 | stalled = screenReceiveWatch.observe(at: now, decoded: frames.filter { sharing.contains($0.key) }) |
| 1135 | for (peer, report) in screenReceived where !stalled.contains(peer) { |
| 1136 | if before[peer].map({ $0.framesDecoded != report.framesDecoded || $0.mid != report.mid }) ?? true { |
| 1137 | screenReceivedMoving[peer] = report |
| 1138 | } |
| 1139 | } |
| 1140 | for peer in stalled where !reportedStalled.contains(peer) { |
| 1141 | let report = screenReceived[peer] ?? ScreenReceiveReport() |
| 1142 | let moving = screenReceivedMoving[peer] |
| 1143 | ScreenLog.info( |
| 1144 | "screen from \(ScreenWatchdog.short(peer)) stalled here: reporting it; since it last moved \(report.moved(since: moving)) — \(report.diagnosis(since: moving))") |
| 1145 | ScreenLog.info("screen from \(ScreenWatchdog.short(peer)): \(report.summary)") |
| 1146 | } |
| 1147 | for peer in screenReceiveWatch.reattach { |
| 1148 | let report = screenReceived[peer] ?? ScreenReceiveReport() |
| 1149 | ScreenLog.info( |
| 1150 | "screen from \(ScreenWatchdog.short(peer)) still stalled: attaching its picture again; \(report.moved(since: screenReceivedMoving[peer]))") |
| 1151 | reattachScreen(peer) |
| 1152 | } |
| 1153 | } else { |
| 1154 | screenReceiveWatch = ScreenReceiveWatch() |
| 1155 | screenReceived = [:] |
| 1156 | screenReceivedMoving = [:] |
| 1157 | } |
| 1158 | guard stalled != reportedStalled, let client = bridge else { return } |
| 1159 | for peer in reportedStalled where !stalled.contains(peer) { |
| 1160 | let report = screenReceived[peer] |
| 1161 | ScreenLog.info( |
| 1162 | "screen from \(ScreenWatchdog.short(peer)) moving here again\(report.map { ": " + $0.summary } ?? "")") |
| 1163 | } |
| 1164 | reportedStalled = stalled |
| 1165 | let body = json(stalled) ?? "[]" |
| 1166 | noSoundQueue.async { try? client.setScreenStalled(body) } |
| 1167 | } |
| 1168 | |
| 1169 | /// A stalled screen's picture dropped and attached again: a new renderer |
| 1170 | /// on the same track, for a picture stopped on this side. |
| 1171 | private func reattachScreen(_ peer: String) { |
| 1172 | guard let track = remoteScreens[peer] else { return } |
| 1173 | screenReattaches += 1 |
| 1174 | qualityTally.count(.reattach, for: peer) |
| 1175 | remoteScreens.removeValue(forKey: peer) |
| 1176 | Task { @MainActor [weak self] in |
| 1177 | try? await Task.sleep(nanoseconds: 150_000_000) |
| 1178 | guard let self, self.screens.contains(peer), self.remoteScreens[peer] == nil else { return } |
| 1179 | self.remoteScreens[peer] = track |
| 1180 | } |
| 1181 | } |
| 1182 | |
| 1183 | /// Stops sharing, from every way out: the button, the system's own stop |
| 1184 | /// control, and the capture ending under us. |
| 1185 | public func stopSharingScreen() { |
| 1186 | guard sharingScreen else { return } |
| 1187 | sharingScreen = false |
| 1188 | ScreenLog.info("sharing stopped") |
| 1189 | endScreenWatch() |
| 1190 | // What was never built has no screen to stop. |
| 1191 | built?.stopScreen() |
| 1192 | send(type: "share", to: "", gen: 0, payload: json(["on": false]) ?? "") |
| 1193 | } |
| 1194 | |
| 1195 | /// A peer has said it is sharing, or its screen has arrived. Either one |
| 1196 | /// is enough to draw the tile. |
| 1197 | func announceScreen(_ peer: String) { |
| 1198 | if !screens.contains(peer) { screens.append(peer) } |
| 1199 | } |
| 1200 | func setScreen(_ track: RTCVideoTrack, peer: String) { |
| 1201 | announceScreen(peer) |
| 1202 | remoteScreens[peer] = track |
| 1203 | } |
| 1204 | /// Both of §5.4's endings meet here — the `share {on:false}` message |
| 1205 | /// and the receiver going away with its stopped transceiver — and |
| 1206 | /// whichever comes first wins. The second finds nothing to do. |
| 1207 | func dropScreen(_ peer: String) { |
| 1208 | screens.removeAll { $0 == peer } |
| 1209 | remoteScreens.removeValue(forKey: peer) |
| 1210 | } |
| 1211 | func mediaFailed() { status = "A participant’s media could not connect." } |
| 1212 | /// Caps the call's uplink estimate at `bps` (nil lifts it): the gate's |
| 1213 | /// stand-in for a thin connection, which it has no other way to get. |
| 1214 | public func limitUplink(_ bps: Int?) { media.limitUplink(bps) } |
| 1215 | /// For the gate: this app's receiver watch sees every screen frozen for |
| 1216 | /// `seconds`, so its stall report and the sharer's answer can be driven |
| 1217 | /// end to end. |
| 1218 | @_spi(Fixtures) public func freezeScreenWatchForTest(seconds: Double) { |
| 1219 | screenWatchFrozenUntil = ProcessInfo.processInfo.systemUptime + seconds |
| 1220 | } |
| 1221 | /// For the gate's encoder stall (see MediaEngine). |
| 1222 | @_spi(Fixtures) public func stallScreenEncoderForTest() { media.stallScreenEncoderForTest() } |
| 1223 | /// For the gate: the watchdog's share again (spec §15.22), on demand. |
| 1224 | @_spi(Fixtures) public func reshareScreenForTest() async { |
| 1225 | guard sharingScreen else { return } |
| 1226 | ScreenLog.info("sharing again for everyone, from the same capture (asked for by the gate)") |
| 1227 | await media.reshareScreen() |
| 1228 | sayShareAgain() |
| 1229 | } |
| 1230 | public func sampleStats() async -> [String: UInt64] { |
| 1231 | var values = await media.stats() |
| 1232 | values["hasICE"] = ice == nil ? 0 : 1 |
| 1233 | values["remoteTracks"] = UInt64(remoteTracks.count) |
| 1234 | values["remoteScreens"] = UInt64(remoteScreens.count) |
| 1235 | values["screens"] = UInt64(screens.count) |
| 1236 | values["invalidEvents"] = invalidEvents |
| 1237 | // §15.24: people still in their first moments, people whose tiles say |
| 1238 | // they cannot hear you yet, and how many this app reports it hears. |
| 1239 | values["connectingJoining"] = UInt64(connectingWatch.joining.count) |
| 1240 | values["connectingShown"] = UInt64(connectingPeers.count) |
| 1241 | values["hearing"] = UInt64(reportedHearing.count) |
| 1242 | // This app's own screen, as it leaves: the slowest connection's. |
| 1243 | if let screen = screenReport { |
| 1244 | values["screenSendWidth"] = UInt64(screen.width) |
| 1245 | values["screenSendHeight"] = UInt64(screen.height) |
| 1246 | values["screenSendFpsTenths"] = UInt64((screen.framesPerSecond * 10).rounded()) |
| 1247 | values["screenCapturedFpsTenths"] = UInt64((screen.capturedPerSecond * 10).rounded()) |
| 1248 | values["screenFramesSent"] = screen.framesSent |
| 1249 | values["screenFramesEncoded"] = screen.framesEncoded |
| 1250 | values["screenFramesCaptured"] = screen.framesCaptured |
| 1251 | values["screenLimitation"] = screen.limitationCode |
| 1252 | values["screenLimitedBandwidthMs"] = UInt64(((screen.limitationSeconds["bandwidth"] ?? 0) * 1000).rounded()) |
| 1253 | values["screenLimitedCpuMs"] = UInt64(((screen.limitationSeconds["cpu"] ?? 0) * 1000).rounded()) |
| 1254 | values["screenTargetBitrate"] = screen.targetBitrate |
| 1255 | values["screenSentBitrate"] = screen.sentBitrate |
| 1256 | values["screenEncodeMsTenths"] = UInt64((screen.encodeMsPerFrame * 10).rounded()) |
| 1257 | values["screenHardwareEncoder"] = screen.hardwareEncoder ? 1 : 0 |
| 1258 | values["screenCodecH264"] = screen.codec.lowercased().hasSuffix("h264") ? 1 : 0 |
| 1259 | values["screenPath"] = screen.pathCode |
| 1260 | values["screenRttMs"] = UInt64((screen.roundTripTime * 1000).rounded()) |
| 1261 | values["screenPacketsLost"] = UInt64(max(0, screen.packetsLost)) |
| 1262 | } |
| 1263 | // The screen watchdogs (ScreenWatchdog): what the sharer's has done, |
| 1264 | // how many stall reports it has heard, and whose screens this app has |
| 1265 | // reported stalled. |
| 1266 | values["screenWatchdogCaptureRestarts"] = UInt64(screenWatchdog.captureRestarts) |
| 1267 | values["screenWatchdogKeyframes"] = UInt64(screenWatchdog.keyframes) |
| 1268 | values["screenWatchdogEncoderRecoveries"] = UInt64(screenWatchdog.encoderRecoveries) |
| 1269 | values["screenWatchdogReAdds"] = UInt64(screenWatchdog.reAdds) |
| 1270 | values["screenStallReports"] = UInt64(screenWatchdog.stallReports) |
| 1271 | values["screenStalledHere"] = UInt64(reportedStalled.count) |
| 1272 | values["screenWatchdogRebuilds"] = UInt64(screenWatchdog.rebuilds) |
| 1273 | values["screenWatchdogReshares"] = UInt64(screenWatchdog.reshares) |
| 1274 | values["screenWatchdogFailures"] = UInt64(screenWatchdog.failures) |
| 1275 | values["screenReattaches"] = UInt64(screenReattaches) |
| 1276 | values["iceRestarts"] = media.iceRestarts |
| 1277 | values["screenKeyframes"] = screenReport?.keyFramesEncoded ?? 0 |
| 1278 | if let capture = media.screenCaptureCounts { |
| 1279 | values["screenCaptureHeartbeats"] = capture.heartbeats ?? 0 |
| 1280 | values["screenCaptureDelivered"] = capture.delivered |
| 1281 | } |
| 1282 | values["rosterPeers"] = UInt64(state?.peers.count ?? 0) |
| 1283 | values["noiseFramesProcessed"] = media.audio.denoiser.framesProcessed |
| 1284 | values["noiseSuppressor"] = media.audio.denoiser.engineName == "RNNoise" ? 1 : 2 |
| 1285 | if case .running(_, let cpu) = media.audio.denoiser.state { |
| 1286 | values["noiseCpuPermille"] = UInt64((cpu * 1000).rounded()) |
| 1287 | } |
| 1288 | // What WebRTC reports running, not what was asked of it. |
| 1289 | let running = media.audio.report() |
| 1290 | values["webrtcEchoCancellation"] = running.echoCancellation == "off" ? 0 : 1 |
| 1291 | values["webrtcNoiseSuppression"] = running.noiseSuppression == "off" ? 0 : 1 |
| 1292 | values["webrtcGainControl"] = running.gainControl.hasPrefix("Calls") || running.gainControl == "off" ? 0 : 1 |
| 1293 | // Whether the readout would say the microphone has started. |
| 1294 | values["audioReadoutStarted"] = running.started ? 1 : 0 |
| 1295 | // §15.20's meters, as the tiles see them: the loudest remote sound and |
| 1296 | // the loudest sent sound since the last sample, in thousandths of full |
| 1297 | // scale; who is in "No sound from"; and whether your own tile says |
| 1298 | // the microphone is picking nothing up. |
| 1299 | values["remoteAudioPeakPermille"] = UInt64((remotePeakSinceSample * 1000).rounded()) |
| 1300 | values["micPeakPermille"] = UInt64((micPeakSinceSample * 1000).rounded()) |
| 1301 | remotePeakSinceSample = 0 |
| 1302 | micPeakSinceSample = 0 |
| 1303 | values["noSoundFrom"] = UInt64(levels.noSoundFrom.count) |
| 1304 | values["micSilent"] = levels.mic.silent ? 1 : 0 |
| 1305 | // The speaking glow behind your own name (0.3.7): lit while what is |
| 1306 | // sent is speech that has kept up. |
| 1307 | values["micGlow"] = levels.mic.glow.glow.lit ? 1 : 0 |
| 1308 | return values |
| 1309 | } |
| 1310 | } |
| 1311 | |
| 1312 | // --------------------------------------------------------------- |
| 1313 | // Quality and problem reports (§15.26) |
| 1314 | // --------------------------------------------------------------- |
| 1315 | |
| 1316 | extension Call { |
| 1317 | /// Sends a quality report every `qualityInterval` while admitted: the |
| 1318 | /// window starts at admission, and each report covers the time since |
| 1319 | /// the last. Read and built on the main actor, as the statistics reads |
| 1320 | /// for the meters are; sent off it, because the bridge's send is a |
| 1321 | /// request. |
| 1322 | fileprivate func startQualityWatch() { |
| 1323 | qualityWatch?.cancel() |
| 1324 | qualityTally = QualityEventTally() |
| 1325 | qualityBuilder = QualityReportBuilder() |
| 1326 | qualityStopped = false |
| 1327 | qualityFinalSent = false |
| 1328 | qualityWatch = Task { @MainActor [weak self] in |
| 1329 | while !Task.isCancelled { |
| 1330 | try? await Task.sleep(nanoseconds: 1_000_000_000) |
| 1331 | guard let self, !Task.isCancelled, let client = self.bridge else { return } |
| 1332 | let now = ProcessInfo.processInfo.systemUptime |
| 1333 | guard self.qualityBuilder.started else { |
| 1334 | if self.admittedForReports { |
| 1335 | self.qualityBuilder.start(at: now) |
| 1336 | self.qualityTally = QualityEventTally() |
| 1337 | } |
| 1338 | continue |
| 1339 | } |
| 1340 | guard self.qualityBuilder.elapsed(at: now) >= Self.qualityInterval else { continue } |
| 1341 | guard self.admittedForReports, !self.qualityStopped, QualityReporting.enabled() else { |
| 1342 | // Off, stopped, or between participations: the window moves on |
| 1343 | // with nothing sent, so a report never covers time it did not see. |
| 1344 | self.qualityBuilder.start(at: now) |
| 1345 | self.qualityTally = QualityEventTally() |
| 1346 | continue |
| 1347 | } |
| 1348 | let samples = await self.built?.qualitySamples() ?? [:] |
| 1349 | guard !Task.isCancelled, self.bridge === client else { return } |
| 1350 | guard let body = self.qualityReport(at: ProcessInfo.processInfo.systemUptime, samples: samples, final: false) |
| 1351 | else { continue } |
| 1352 | Task.detached { [weak self] in |
| 1353 | let answer = client.sendQuality(body) |
| 1354 | if answer == "stop" { |
| 1355 | await MainActor.run { [weak self] in |
| 1356 | self?.qualityStopped = true |
| 1357 | CallLog.info("quality reports: the server asked for no more this call") |
| 1358 | } |
| 1359 | } |
| 1360 | } |
| 1361 | } |
| 1362 | } |
| 1363 | } |
| 1364 | |
| 1365 | private var admittedForReports: Bool { |
| 1366 | guard let phase = state?.phase else { return false } |
| 1367 | return phase == "admitted" || phase == "reconnecting" |
| 1368 | } |
| 1369 | |
| 1370 | private func qualityReport(at now: Double, samples: [String: QualityPeerSample], final: Bool) -> String? { |
| 1371 | qualityBuilder.build( |
| 1372 | at: now, samples: samples, events: &qualityTally, client: QualityClientInfo.current(), |
| 1373 | devices: qualityDevices(), final: final |
| 1374 | ).encoded() |
| 1375 | } |
| 1376 | |
| 1377 | /// The report sent at leave, built while the connections are still up; |
| 1378 | /// nil when there is none to send. |
| 1379 | fileprivate func finalQualityReport() async -> String? { |
| 1380 | guard let client = bridge, qualityBuilder.started, !qualityStopped, !qualityFinalSent, |
| 1381 | QualityReporting.enabled() |
| 1382 | else { return nil } |
| 1383 | let samples = await built?.qualitySamples() ?? [:] |
| 1384 | guard bridge === client, !qualityFinalSent else { return nil } |
| 1385 | qualityFinalSent = true |
| 1386 | return qualityReport(at: ProcessInfo.processInfo.systemUptime, samples: samples, final: true) |
| 1387 | } |
| 1388 | |
| 1389 | /// The call ended under this app (ended, removed, open elsewhere): the |
| 1390 | /// final report goes now, with the notices but without statistics — the |
| 1391 | /// connections are closing as it is built — and the call stays |
| 1392 | /// reportable from here. |
| 1393 | fileprivate func endedForReports() { |
| 1394 | guard let client = bridge else { return } |
| 1395 | keepForReports(client) |
| 1396 | guard qualityBuilder.started, !qualityStopped, !qualityFinalSent, QualityReporting.enabled() else { return } |
| 1397 | qualityFinalSent = true |
| 1398 | guard let body = qualityReport(at: ProcessInfo.processInfo.systemUptime, samples: [:], final: true) else { return } |
| 1399 | Task.detached { _ = client.sendQuality(body) } |
| 1400 | } |
| 1401 | |
| 1402 | /// The kinds of microphone and speaker in use. Never their names. |
| 1403 | private func qualityDevices() -> QualityDevices? { |
| 1404 | #if os(macOS) |
| 1405 | let kinds = audio.deviceKinds() |
| 1406 | return QualityDevices(input: kinds.input, output: kinds.output) |
| 1407 | #else |
| 1408 | let route = AVAudioSession.sharedInstance().currentRoute |
| 1409 | return QualityDevices( |
| 1410 | input: DeviceKind.ios(port: route.inputs.first?.portType.rawValue), |
| 1411 | output: DeviceKind.ios(port: route.outputs.first?.portType.rawValue)) |
| 1412 | #endif |
| 1413 | } |
| 1414 | |
| 1415 | /// One look by the level watch, tallied for the window's events. |
| 1416 | fileprivate func observeQuality(now: Double, state: RoomState) { |
| 1417 | guard qualityBuilder.started else { return } |
| 1418 | let me = state.peer |
| 1419 | var fromUs = Set<String>() |
| 1420 | var blocked = Set<String>() |
| 1421 | for person in state.peers where person.peer != me { |
| 1422 | if person.media?.noSoundFrom?.contains(me) == true { fromUs.insert(person.peer) } |
| 1423 | if person.media?.audioBlocked == true { blocked.insert(person.peer) } |
| 1424 | } |
| 1425 | qualityTally.observe( |
| 1426 | at: now, call: reconnecting, reconnecting: reconnectingPeers, cantHear: connectingPeers, |
| 1427 | noSound: Set(levels.noSoundFrom), noSoundFromUs: fromUs, audioBlocked: blocked, |
| 1428 | screenStalled: Set(reportedStalled)) |
| 1429 | } |
| 1430 | |
| 1431 | /// The sharer's watchdog's steps, toward the peers they were for. |
| 1432 | fileprivate func countScreenSteps(_ actions: [ScreenRecovery], failed: Int, stalledBy: Set<String>) { |
| 1433 | let me = state?.peer ?? "" |
| 1434 | for action in actions { |
| 1435 | switch action { |
| 1436 | case .keyframe(let peer): qualityTally.count(.keyframe, for: peer) |
| 1437 | case .reAdd(let peer), .rebuild(let peer): qualityTally.count(.rebuild, for: peer) |
| 1438 | case .reshare: |
| 1439 | for person in state?.peers ?? [] where person.peer != me { qualityTally.count(.reshare, for: person.peer) } |
| 1440 | case .restartCapture, .recreateEncoder: break |
| 1441 | } |
| 1442 | } |
| 1443 | guard failed > 0 else { return } |
| 1444 | for peer in stalledBy where screenWatchdog.escalation(of: peer) == .failed { |
| 1445 | qualityTally.count(.failure, for: peer) |
| 1446 | } |
| 1447 | } |
| 1448 | |
| 1449 | /// Keeps a call's closed client, and its diagnostics as they are now, |
| 1450 | /// for "Report a problem" during the ten minutes after it. |
| 1451 | fileprivate func keepForReports(_ client: BridgeClient) { |
| 1452 | guard reportClient !== client else { return } |
| 1453 | reportClient = client |
| 1454 | reportDiagnostics = anonymousDiagnosticsText() |
| 1455 | let until = Date().addingTimeInterval(Self.reportWindow) |
| 1456 | reportableUntil = until |
| 1457 | Task { @MainActor [weak self] in |
| 1458 | try? await Task.sleep(nanoseconds: UInt64(Self.reportWindow * 1_000_000_000)) |
| 1459 | guard let self, self.reportableUntil == until else { return } |
| 1460 | self.reportableUntil = nil |
| 1461 | self.reportClient = nil |
| 1462 | self.reportDiagnostics = "" |
| 1463 | } |
| 1464 | } |
| 1465 | |
| 1466 | /// Whether "Report a problem" has anything to report: a call in |
| 1467 | /// progress, or one that ended less than ten minutes ago. |
| 1468 | public var canReportProblem: Bool { |
| 1469 | bridge != nil || fixtureReports || (reportableUntil.map { $0 > Date() } ?? false) |
| 1470 | } |
| 1471 | |
| 1472 | /// Opens the sheet. In a call it reports this call, its diagnostics read |
| 1473 | /// now; after one, the call that ended, as it was when it ended. |
| 1474 | public func reportProblem() { |
| 1475 | let info = QualityClientInfo.current() |
| 1476 | let deliver: @Sendable (String) async -> ProblemOutcome |
| 1477 | if let client = bridge ?? (canReportProblem ? reportClient : nil) { |
| 1478 | let diagnostics = bridge != nil ? anonymousDiagnosticsText() : reportDiagnostics |
| 1479 | deliver = { note in |
| 1480 | let body = ProblemReportBody(note: note, diagnostics: diagnostics, client: info).encoded() |
| 1481 | return await Task.detached { ProblemOutcome(bridgeAnswer: client.sendProblem(body)) }.value |
| 1482 | } |
| 1483 | } else if fixtureReports { |
| 1484 | // The UI test's call has no server: it is answered here, after a |
| 1485 | // moment, with a reference of the server's shape. |
| 1486 | deliver = { _ in |
| 1487 | try? await Task.sleep(nanoseconds: 300_000_000) |
| 1488 | return .sent(ref: "K7Q2M9PD") |
| 1489 | } |
| 1490 | } else { |
| 1491 | return |
| 1492 | } |
| 1493 | problemReport = ProblemReportModel(deliver: deliver) |
| 1494 | } |
| 1495 | } |
| 1496 | |
| 1497 | // --------------------------------------------------------------- |
| 1498 | // Fixtures |
| 1499 | // --------------------------------------------------------------- |
| 1500 | |
| 1501 | extension Call { |
| 1502 | /// Puts the call into a described state without a server, for the |
| 1503 | /// offline snapshots of the call views and the iPhone's UI test of its |
| 1504 | /// controls. `state` is a roster in the wire's own JSON. Never used by |
| 1505 | /// a call that has joined anything. |
| 1506 | @_spi(Fixtures) public func showFixture( |
| 1507 | title: String, standing: String = "member", status: String = "In the meeting", |
| 1508 | state json: String, screens: [String] = [], sharingScreen: Bool = false, |
| 1509 | muted: Bool = false, floor: Bool = false, named: Bool = false |
| 1510 | ) { |
| 1511 | self.title = title |
| 1512 | self.named = named |
| 1513 | self.standing = standing |
| 1514 | self.state = try? JSONDecoder().decode(RoomState.self, from: Data(json.utf8)) |
| 1515 | self.status = status |
| 1516 | // A fixture's reconnecting is one that has lasted (spec §15.23). |
| 1517 | self.reconnecting = self.state?.phase == "reconnecting" |
| 1518 | self.screens = screens |
| 1519 | self.sharingScreen = sharingScreen |
| 1520 | self.muted = muted |
| 1521 | self.floor = floor |
| 1522 | self.fixtureReports = true |
| 1523 | } |
| 1524 | |
| 1525 | /// Your own microphone, camera and screen, set over a fixture for the |
| 1526 | /// snapshots of the controls in each state. |
| 1527 | @_spi(Fixtures) public func showFixtureSelf(muted: Bool, cameraOff: Bool, sharingScreen: Bool) { |
| 1528 | self.muted = muted |
| 1529 | self.cameraOff = cameraOff |
| 1530 | self.sharingScreen = sharingScreen |
| 1531 | } |
| 1532 | |
| 1533 | /// Spec §15.24 for the snapshots: these people have just joined and the |
| 1534 | /// grace has passed, so each tile says what it would — "connecting", or |
| 1535 | /// a report of theirs that replaces it. |
| 1536 | @_spi(Fixtures) public func showFixtureConnecting(_ peers: [String]) { |
| 1537 | let me = state?.peer ?? "" |
| 1538 | var shown: Set<String> = [] |
| 1539 | for person in state?.peers ?? [] where peers.contains(person.peer) { |
| 1540 | let said = ConnectingWatch.tile( |
| 1541 | name: person.name, media: person.media, me: me, muted: muted, joining: true, shown: true, |
| 1542 | reconnecting: false) |
| 1543 | if said.connecting != nil { shown.insert(person.peer) } |
| 1544 | levels.peer(person.peer).show( |
| 1545 | state: person.media?.muted == true ? .muted : .meter(0), speaking: false, notices: said.notices, |
| 1546 | connecting: said.connecting) |
| 1547 | } |
| 1548 | connectingPeers = shown |
| 1549 | } |
| 1550 | |
| 1551 | /// Everyone's glow, and yours, held at one level, for the snapshots. |
| 1552 | @_spi(Fixtures) public func showFixtureSpeaking(level: Float) { |
| 1553 | levels.mic.glow.showFixture(level: level) |
| 1554 | // Nobody muted glows: muted, the speaking gate is shut. |
| 1555 | if muted { levels.mic.glow.reset() } |
| 1556 | for peer in state?.peers ?? [] where peer.peer != state?.peer && !isMuted(peer.peer) { |
| 1557 | levels.peer(peer.peer).glow.showFixture(level: level) |
| 1558 | } |
| 1559 | levels.othersGlow.showFixture(level: level) |
| 1560 | levels.anyoneSpeaking.set(levels.othersGlow.glow.lit) |
| 1561 | } |
| 1562 | } |
| 1563 | |
amadan