| 1 | #if os(macOS) |
| 2 | import CallsBroadcastKit |
| 3 | import CoreMedia |
| 4 | import Foundation |
| 5 | import ScreenCaptureKit |
| 6 | import LiveKitWebRTC |
| 7 | import os |
| 8 | |
| 9 | /// A display or a window, chosen with the system's own picker and |
| 10 | /// delivered into the call's screen source. |
| 11 | /// |
| 12 | /// The picker is SCContentSharingPicker (macOS 14, the app's floor), and |
| 13 | /// that choice does more than save drawing one: content a person picks |
| 14 | /// there is content they have consented to share, so the app needs no |
| 15 | /// Screen Recording permission of its own and never sends anyone to |
| 16 | /// System Settings to grant one. There is nothing in Info.plist for it |
| 17 | /// either — macOS has no usage string for screen capture. The same holds |
| 18 | /// in the App Sandbox, which is why this is also the path a sandboxed |
| 19 | /// build can keep. |
| 20 | /// |
| 21 | /// The capture ends on its own when the person stops it from the menu |
| 22 | /// bar's sharing indicator, or the shared window closes; either one |
| 23 | /// reaches `ended`, which the call treats exactly like its own button. |
| 24 | /// |
| 25 | /// It can also be started again on the same content (`restart`), which |
| 26 | /// is what the screen watchdog asks for when the capture stops reporting |
| 27 | /// anything at all (see ScreenWatchdog), and what a capture error other |
| 28 | /// than the person stopping it gets before the share is given up. |
| 29 | @MainActor public final class MacScreenCapture: NSObject, ScreenCapture { |
| 30 | private var stream: SCStream? |
| 31 | private var frames: ScreenFrames? |
| 32 | private var filter: SCContentFilter? |
| 33 | private var source: RTCVideoSource? |
| 34 | private var ended: (@MainActor () -> Void)? |
| 35 | private var choosing: CheckedContinuation<SCContentFilter, Error>? |
| 36 | private let queue = DispatchQueue(label: "calls.screen", qos: .userInteractive) |
| 37 | // Counts from the captures this one has replaced, so the totals the |
| 38 | // watchdog reads run on across a restart. |
| 39 | private var earlier = ScreenCaptureCounts(heartbeats: 0, delivered: 0) |
| 40 | // Restarts after a capture error, for `didStopWithError`'s budget. |
| 41 | private var errorRestarts: [Date] = [] |
| 42 | private var restarting = false |
| 43 | |
| 44 | public override init() {} |
| 45 | |
| 46 | public var counts: ScreenCaptureCounts { |
| 47 | let now = frames?.counts ?? ScreenCaptureCounts(heartbeats: 0, delivered: 0) |
| 48 | return ScreenCaptureCounts( |
| 49 | heartbeats: (earlier.heartbeats ?? 0) + (now.heartbeats ?? 0), |
| 50 | delivered: earlier.delivered + now.delivered) |
| 51 | } |
| 52 | |
| 53 | public func start(source: RTCVideoSource, ended: @escaping @MainActor () -> Void) async throws { |
| 54 | let picker = SCContentSharingPicker.shared |
| 55 | var configuration = SCContentSharingPickerConfiguration() |
| 56 | configuration.allowedPickerModes = [.singleDisplay, .singleWindow] |
| 57 | // The call's own window is the one thing nobody in it needs to see: |
| 58 | // shared, it is a hall of mirrors of their own faces. |
| 59 | if let own = Bundle.main.bundleIdentifier { |
| 60 | configuration.excludedBundleIDs = [own] |
| 61 | } |
| 62 | picker.defaultConfiguration = configuration |
| 63 | picker.maximumStreamCount = 1 |
| 64 | picker.add(self) |
| 65 | picker.isActive = true |
| 66 | let filter: SCContentFilter |
| 67 | do { |
| 68 | let picked = try await withCheckedThrowingContinuation { continuation in |
| 69 | choosing = continuation |
| 70 | picker.present() |
| 71 | } |
| 72 | filter = await Self.withoutCalls(picked) |
| 73 | } catch { |
| 74 | release() |
| 75 | throw error |
| 76 | } |
| 77 | do { |
| 78 | try await begin(filter: filter, source: source) |
| 79 | } catch { |
| 80 | release() |
| 81 | throw error |
| 82 | } |
| 83 | self.ended = ended |
| 84 | } |
| 85 | |
| 86 | /// A given display, with no picker: the interop gate's real capture, |
| 87 | /// which is the only way to put ScreenCaptureKit and the hardware |
| 88 | /// encoder under test together. Needs Screen Recording permission for |
| 89 | /// the process, which the picker path never does. |
| 90 | @_spi(Fixtures) public func start( |
| 91 | display: SCDisplay, source: RTCVideoSource, ended: @escaping @MainActor () -> Void |
| 92 | ) async throws { |
| 93 | try await begin(filter: SCContentFilter(display: display, excludingWindows: []), source: source) |
| 94 | self.ended = ended |
| 95 | } |
| 96 | |
| 97 | private func begin(filter: SCContentFilter, source: RTCVideoSource) async throws { |
| 98 | let frames = ScreenFrames(source: source, queue: queue) |
| 99 | let stream = SCStream(filter: filter, configuration: Self.configuration(for: filter), delegate: self) |
| 100 | do { |
| 101 | try stream.addStreamOutput(frames, type: .screen, sampleHandlerQueue: queue) |
| 102 | try await stream.startCapture() |
| 103 | } catch { |
| 104 | frames.stop() |
| 105 | throw error |
| 106 | } |
| 107 | self.stream = stream |
| 108 | self.frames = frames |
| 109 | self.filter = filter |
| 110 | self.source = source |
| 111 | let configured = Self.configuration(for: filter) |
| 112 | frameSize = (configured.width, configured.height) |
| 113 | } |
| 114 | public private(set) var frameSize: (width: Int, height: Int)? |
| 115 | |
| 116 | /// The same content into the same source, on a capture of its own: |
| 117 | /// a fresh stream with a fresh pool of surfaces. What stopping and |
| 118 | /// sharing again did by hand, without the picker and without the |
| 119 | /// call hearing about it — the transceivers and tracks stay as they |
| 120 | /// are, and only the frames behind them are new. |
| 121 | public func restart() async -> Bool { |
| 122 | guard !restarting, let filter, let source, let old = stream else { return false } |
| 123 | restarting = true |
| 124 | defer { restarting = false } |
| 125 | earlier = counts |
| 126 | stream = nil |
| 127 | frames?.stop() |
| 128 | frames = nil |
| 129 | try? await old.stopCapture() |
| 130 | guard filter === self.filter, self.source === source else { return false } |
| 131 | do { |
| 132 | try await begin(filter: filter, source: source) |
| 133 | ScreenLog.info("capture restarted") |
| 134 | return true |
| 135 | } catch { |
| 136 | ScreenLog.error("capture restart failed: \(Self.describe(error))") |
| 137 | return false |
| 138 | } |
| 139 | } |
| 140 | |
| 141 | public func stop() { |
| 142 | ended = nil |
| 143 | let stream = self.stream |
| 144 | self.stream = nil |
| 145 | frames?.stop() |
| 146 | frames = nil |
| 147 | filter = nil |
| 148 | source = nil |
| 149 | release() |
| 150 | stream?.stopCapture { _ in } |
| 151 | } |
| 152 | |
| 153 | private func release() { |
| 154 | let picker = SCContentSharingPicker.shared |
| 155 | picker.remove(self) |
| 156 | picker.isActive = false |
| 157 | choosing?.resume(throwing: CancellationError()) |
| 158 | choosing = nil |
| 159 | } |
| 160 | |
| 161 | /// The capture ending under the call, rather than by it. |
| 162 | private func finished() { |
| 163 | let ended = self.ended |
| 164 | stop() |
| 165 | ended?() |
| 166 | } |
| 167 | |
| 168 | /// The picked content with Calls taken out of it. |
| 169 | /// |
| 170 | /// A whole display, as the picker hands it back, includes this app's |
| 171 | /// own windows: the call window, and the small floating one it shows |
| 172 | /// while it is in the background. Shared, they are a hall of mirrors |
| 173 | /// of everyone's faces — and the floating one is always on top of |
| 174 | /// whatever the person meant to show. So a display is rebuilt as the |
| 175 | /// same display excluding this application, which also covers windows |
| 176 | /// that open after sharing starts. A single window needs nothing: this |
| 177 | /// app's are left out of the picker, so the window is someone else's. |
| 178 | /// |
| 179 | /// The display comes from the filter itself on macOS 15.2 and later, |
| 180 | /// and before that from matching its frame. The application comes from |
| 181 | /// the current process's own shareable content, which is the one query |
| 182 | /// that asks for no Screen Recording permission. On a macOS too old |
| 183 | /// for that (before 14.4) the display is shared as picked; the |
| 184 | /// floating window's own `sharingType` is then the only thing keeping |
| 185 | /// it out, and the call window is in the share. |
| 186 | static func withoutCalls(_ filter: SCContentFilter) async -> SCContentFilter { |
| 187 | guard filter.style == .display, #available(macOS 14.4, *), |
| 188 | let content = try? await SCShareableContent.currentProcess, |
| 189 | let calls = content.applications.first(where: { $0.processID == getpid() }) |
| 190 | else { return filter } |
| 191 | var display: SCDisplay? |
| 192 | if #available(macOS 15.2, *) { display = filter.includedDisplays.first } |
| 193 | if display == nil { display = content.displays.first { $0.frame == filter.contentRect } } |
| 194 | guard let display else { return filter } |
| 195 | return SCContentFilter(display: display, excludingApplications: [calls], exceptingWindows: []) |
| 196 | } |
| 197 | |
| 198 | /// Full resolution up to 1920 on the long side, at up to 30 frames a |
| 199 | /// second. Larger is sharper than any receiver will be shown it and |
| 200 | /// costs the encoder for nothing. What gives way under pressure is the |
| 201 | /// "Optimise for" mode's (ScreenSendPolicy). |
| 202 | /// |
| 203 | /// Six surfaces in the queue. The queue depth is not a queue of frames |
| 204 | /// waiting to be sent — a late frame is dropped on arrival (see |
| 205 | /// ScreenFrames) — it is the pool of surfaces the capture draws into, |
| 206 | /// and every surface the encoder is still holding, or this capture is |
| 207 | /// keeping for its repeat, is one the capture cannot use. When they |
| 208 | /// are all held the capture stops outright: no frames and no idle |
| 209 | /// reports either, until one comes back. Measured on 2026-09-28 at a |
| 210 | /// depth of three, holding three surfaces stopped a display capture |
| 211 | /// after three frames, and holding two did not. 0.4.0 went from five |
| 212 | /// to three and began holding one for the repeat, which left the |
| 213 | /// hardware encoder two frames in flight before the screen froze |
| 214 | /// (Paul, 2026-09-28: "after a while, my screen froze on her end"). |
| 215 | /// The depth above. Settable only for the interop gate, which measures |
| 216 | /// the depth 0.4.1 shipped with against this one. |
| 217 | @_spi(Fixtures) public static var queueDepth = 6 |
| 218 | |
| 219 | static func configuration(for filter: SCContentFilter) -> SCStreamConfiguration { |
| 220 | let configuration = SCStreamConfiguration() |
| 221 | let rect = filter.contentRect |
| 222 | let scale = CGFloat(filter.pointPixelScale) |
| 223 | let size = fitted(width: rect.width * scale, height: rect.height * scale, longest: 1920) |
| 224 | configuration.width = size.width |
| 225 | configuration.height = size.height |
| 226 | configuration.minimumFrameInterval = CMTime(value: 1, timescale: 30) |
| 227 | configuration.pixelFormat = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange |
| 228 | configuration.showsCursor = true |
| 229 | configuration.queueDepth = queueDepth |
| 230 | return configuration |
| 231 | } |
| 232 | } |
| 233 | |
| 234 | extension MacScreenCapture: SCContentSharingPickerObserver { |
| 235 | nonisolated public func contentSharingPicker( |
| 236 | _ picker: SCContentSharingPicker, didCancelFor stream: SCStream? |
| 237 | ) { |
| 238 | Task { @MainActor [weak self] in |
| 239 | guard let self else { return } |
| 240 | if stream == nil || stream === self.stream { |
| 241 | if let choosing = self.choosing { |
| 242 | self.choosing = nil |
| 243 | choosing.resume(throwing: ScreenShareRefused()) |
| 244 | } |
| 245 | } |
| 246 | } |
| 247 | } |
| 248 | |
| 249 | nonisolated public func contentSharingPicker( |
| 250 | _ picker: SCContentSharingPicker, didUpdateWith filter: SCContentFilter, for stream: SCStream? |
| 251 | ) { |
| 252 | Task { @MainActor [weak self] in |
| 253 | guard let self else { return } |
| 254 | if let choosing = self.choosing { |
| 255 | self.choosing = nil |
| 256 | choosing.resume(returning: filter) |
| 257 | return |
| 258 | } |
| 259 | // Chosen again from the menu bar mid-share: the same stream, now |
| 260 | // on something else. The call does not need to hear about it. |
| 261 | guard let current = self.stream, stream == nil || stream === current else { return } |
| 262 | let next = await Self.withoutCalls(filter) |
| 263 | guard current === self.stream else { return } |
| 264 | self.filter = next |
| 265 | current.updateContentFilter(next) { _ in } |
| 266 | current.updateConfiguration(Self.configuration(for: next)) { _ in } |
| 267 | } |
| 268 | } |
| 269 | |
| 270 | nonisolated public func contentSharingPickerStartDidFailWithError(_ error: Error) { |
| 271 | Task { @MainActor [weak self] in |
| 272 | guard let self, let choosing = self.choosing else { return } |
| 273 | self.choosing = nil |
| 274 | choosing.resume(throwing: error) |
| 275 | } |
| 276 | } |
| 277 | } |
| 278 | |
| 279 | extension MacScreenCapture: SCStreamDelegate { |
| 280 | /// The person stopping the share from the menu bar, or the shared |
| 281 | /// window or display going away, ends it: that is what they asked for, |
| 282 | /// or all there is. Anything else — the capture service interrupted, |
| 283 | /// the system stopping the stream across a sleep or a display change — |
| 284 | /// is started again on the same content, up to three times a minute, |
| 285 | /// before the share is given up. Up to 0.4.1 every error ended the |
| 286 | /// share, and a person whose capture had merely hiccuped was left to |
| 287 | /// notice and share again. |
| 288 | nonisolated public func stream(_ stream: SCStream, didStopWithError error: Error) { |
| 289 | Task { @MainActor [weak self] in |
| 290 | guard let self, stream === self.stream else { return } |
| 291 | ScreenLog.info("capture stopped: \(Self.describe(error))") |
| 292 | let now = Date() |
| 293 | self.errorRestarts.removeAll { now.timeIntervalSince($0) > 60 } |
| 294 | guard Self.recoverable(error), self.errorRestarts.count < 3 else { |
| 295 | self.finished() |
| 296 | return |
| 297 | } |
| 298 | self.errorRestarts.append(now) |
| 299 | if await !self.restart() { self.finished() } |
| 300 | } |
| 301 | } |
| 302 | |
| 303 | /// Whether a capture error is worth starting again after. |
| 304 | nonisolated static func recoverable(_ error: Error) -> Bool { |
| 305 | let error = error as NSError |
| 306 | guard error.domain == SCStreamErrorDomain else { return true } |
| 307 | switch SCStreamError.Code(rawValue: error.code) { |
| 308 | case .userStopped?, .noCaptureSource?, .noWindowList?, .noDisplayList?, .userDeclined?, |
| 309 | .missingEntitlements?: |
| 310 | return false |
| 311 | default: |
| 312 | return true |
| 313 | } |
| 314 | } |
| 315 | |
| 316 | nonisolated static func describe(_ error: Error) -> String { |
| 317 | let error = error as NSError |
| 318 | return "\(error.domain) \(error.code)" |
| 319 | } |
| 320 | } |
| 321 | |
| 322 | struct ScreenShareRefused: Error {} |
| 323 | |
| 324 | /// Hands each complete frame to the source. Called on the capture's own |
| 325 | /// queue, never the main actor: a screen is thirty frames a second and |
| 326 | /// the main actor has an interface to draw. |
| 327 | /// |
| 328 | /// Two things beyond passing frames on: |
| 329 | /// |
| 330 | /// - A frame that reaches here more than a tenth of a second after it |
| 331 | /// was captured is dropped: the queue is behind, and a newer frame is |
| 332 | /// already waiting. Sending it would only add its lateness to the |
| 333 | /// picture everyone is watching. |
| 334 | /// - An idle screen sends nothing (ScreenCaptureKit reports only |
| 335 | /// changes), and the encoder only makes a keyframe when a frame comes |
| 336 | /// to encode. So a receiver that lost a packet on a still screen, or |
| 337 | /// joined during one, could wait on a broken or empty picture until |
| 338 | /// something moved. The last frame is sent again once a second while |
| 339 | /// nothing changes, which gives the encoder its chance. |
| 340 | private final class ScreenFrames: NSObject, SCStreamOutput, @unchecked Sendable { |
| 341 | private let source: RTCVideoSource |
| 342 | private let capturer: RTCVideoCapturer |
| 343 | private let queue: DispatchQueue |
| 344 | private var timer: DispatchSourceTimer? |
| 345 | // Only touched on `queue`. Holding the newest frame pins one of the |
| 346 | // capture's surfaces, but it is the one the encoder was just given |
| 347 | // and is usually still holding anyway. |
| 348 | private var last: CVPixelBuffer? |
| 349 | private var lastSent: UInt64 = 0 |
| 350 | // Read on the main actor by the watchdog; written on `queue`. |
| 351 | private let tally = OSAllocatedUnfairLock(initialState: ScreenCaptureCounts(heartbeats: 0, delivered: 0)) |
| 352 | var counts: ScreenCaptureCounts { tally.withLock { $0 } } |
| 353 | |
| 354 | static let stale = 0.1 |
| 355 | static let repeatAfter: UInt64 = 1_000_000_000 |
| 356 | |
| 357 | init(source: RTCVideoSource, queue: DispatchQueue) { |
| 358 | self.source = source |
| 359 | self.queue = queue |
| 360 | capturer = RTCVideoCapturer(delegate: source) |
| 361 | super.init() |
| 362 | let timer = DispatchSource.makeTimerSource(queue: queue) |
| 363 | timer.schedule(deadline: .now() + .milliseconds(500), repeating: .milliseconds(500), leeway: .milliseconds(50)) |
| 364 | timer.setEventHandler { [weak self] in self?.repeatIfIdle() } |
| 365 | timer.resume() |
| 366 | self.timer = timer |
| 367 | } |
| 368 | |
| 369 | func stop() { |
| 370 | timer?.cancel() |
| 371 | timer = nil |
| 372 | queue.async { self.last = nil } |
| 373 | } |
| 374 | |
| 375 | func stream( |
| 376 | _ stream: SCStream, didOutputSampleBuffer sample: CMSampleBuffer, of type: SCStreamOutputType |
| 377 | ) { |
| 378 | guard type == .screen, sample.isValid else { return } |
| 379 | // Anything at all, idle reports included, is the capture alive. |
| 380 | tally.withLock { $0.heartbeats = ($0.heartbeats ?? 0) + 1 } |
| 381 | guard let pixels = sample.imageBuffer else { return } |
| 382 | // Only complete frames carry pixels. An idle screen sends status |
| 383 | // frames and nothing else; the repeat below covers it. |
| 384 | guard |
| 385 | let attachments = CMSampleBufferGetSampleAttachmentsArray(sample, createIfNecessary: false) |
| 386 | as? [[SCStreamFrameInfo: Any]], |
| 387 | let raw = attachments.first?[.status] as? Int, |
| 388 | SCFrameStatus(rawValue: raw) == .complete |
| 389 | else { return } |
| 390 | let captured = CMTimeGetSeconds(sample.presentationTimeStamp) |
| 391 | let now = CMTimeGetSeconds(CMClockGetTime(CMClockGetHostTimeClock())) |
| 392 | last = pixels |
| 393 | if now - captured > Self.stale { return } |
| 394 | deliver(pixels, at: captured) |
| 395 | } |
| 396 | |
| 397 | private func repeatIfIdle() { |
| 398 | guard let last else { return } |
| 399 | let now = CMTimeGetSeconds(CMClockGetTime(CMClockGetHostTimeClock())) |
| 400 | let ns = UInt64(max(0, now) * 1_000_000_000) |
| 401 | guard ns >= lastSent, ns - lastSent >= Self.repeatAfter else { return } |
| 402 | deliver(last, at: now) |
| 403 | } |
| 404 | |
| 405 | private func deliver(_ pixels: CVPixelBuffer, at seconds: Double) { |
| 406 | let ns = UInt64(max(0, seconds) * 1_000_000_000) |
| 407 | // Never backwards: a repeat stamped just after a real frame would |
| 408 | // otherwise be followed by an older one. |
| 409 | guard ns > lastSent else { return } |
| 410 | lastSent = ns |
| 411 | tally.withLock { $0.delivered += 1 } |
| 412 | source.capturer( |
| 413 | capturer, |
| 414 | didCapture: RTCVideoFrame( |
| 415 | buffer: RTCCVPixelBuffer(pixelBuffer: pixels), rotation: ._0, timeStampNs: Int64(ns))) |
| 416 | } |
| 417 | } |
| 418 | #endif |
| 419 | |
amadan