| 1 | import CoreVideo |
| 2 | import Foundation |
| 3 | import CallsKit |
| 4 | import LiveKitWebRTC |
| 5 | |
| 6 | /// Two synthetic cameras behind the app's own camera management |
| 7 | /// (`ManagedCamera`), so the gate drives the same choice, switching and |
| 8 | /// recovery code the apps run — only the frames are made up. |
| 9 | /// |
| 10 | /// The cameras differ only in frame size, so that a switch is something |
| 11 | /// the far side can *see* rather than something this side merely claims. |
| 12 | /// There is no camera in CI; what the gate can still hold is the plumbing |
| 13 | /// — that a switch is serialised, that the track and its m-line survive |
| 14 | /// it, that frames keep arriving across it, and (`stall`) that a camera |
| 15 | /// which stops sending is noticed and restarted. |
| 16 | @MainActor final class SyntheticCameras: CameraBackend { |
| 17 | private var delegate: RTCVideoCapturerDelegate? |
| 18 | private var syntheticCapturer: RTCVideoCapturer? |
| 19 | private var task: Task<Void, Never>? |
| 20 | private var chosen = SyntheticCameras.first |
| 21 | /// Set by the gate: frames stop until the camera is next opened, as an |
| 22 | /// ATEM's do when its input changes format. |
| 23 | var stalled = false |
| 24 | nonisolated static let first = "synthetic-320" |
| 25 | nonisolated static let second = "synthetic-480" |
| 26 | private var width: Int { chosen == SyntheticCameras.second ? 480 : 320 } |
| 27 | private var height: Int { chosen == SyntheticCameras.second ? 360 : 240 } |
| 28 | var devicesChanged: (@MainActor () -> Void)? |
| 29 | var trouble: (@MainActor () -> Void)? |
| 30 | |
| 31 | func requestAccess() async throws {} |
| 32 | |
| 33 | var devices: [CameraDevice] { |
| 34 | [ |
| 35 | CameraDevice(id: SyntheticCameras.first, name: "Synthetic 320x240"), |
| 36 | CameraDevice(id: SyntheticCameras.second, name: "Synthetic 480x360"), |
| 37 | ] |
| 38 | } |
| 39 | func formatCount(_ id: String) -> Int { 1 } |
| 40 | |
| 41 | func open(_ id: String, format: Int, delegate: RTCVideoCapturerDelegate) async throws { |
| 42 | guard devices.contains(where: { $0.id == id }) else { |
| 43 | throw NSError( |
| 44 | domain: "Calls", code: 4, |
| 45 | userInfo: [NSLocalizedDescriptionKey: "That camera is no longer available."]) |
| 46 | } |
| 47 | chosen = id |
| 48 | stalled = false |
| 49 | self.delegate = delegate |
| 50 | syntheticCapturer = RTCVideoCapturer(delegate: delegate) |
| 51 | task?.cancel() |
| 52 | task = Task { @MainActor [weak self] in |
| 53 | while !Task.isCancelled { |
| 54 | self?.frame() |
| 55 | try? await Task.sleep(nanoseconds: 66_666_667) |
| 56 | } |
| 57 | } |
| 58 | } |
| 59 | func close() async { |
| 60 | task?.cancel() |
| 61 | task = nil |
| 62 | } |
| 63 | private func frame() { |
| 64 | guard !stalled, let delegate, let syntheticCapturer else { return } |
| 65 | var pixel: CVPixelBuffer? |
| 66 | CVPixelBufferCreate( |
| 67 | kCFAllocatorDefault, width, height, kCVPixelFormatType_32BGRA, |
| 68 | [kCVPixelBufferIOSurfacePropertiesKey: [:]] as CFDictionary, &pixel) |
| 69 | guard let pixel else { return } |
| 70 | CVPixelBufferLockBaseAddress(pixel, []) |
| 71 | if let base = CVPixelBufferGetBaseAddress(pixel) { |
| 72 | memset( |
| 73 | base, Int32(Int(Date().timeIntervalSince1970 * 30) % 200 + 30), |
| 74 | CVPixelBufferGetDataSize(pixel)) |
| 75 | } |
| 76 | CVPixelBufferUnlockBaseAddress(pixel, []) |
| 77 | delegate.capturer( |
| 78 | syntheticCapturer, |
| 79 | didCapture: RTCVideoFrame( |
| 80 | buffer: RTCCVPixelBuffer(pixelBuffer: pixel), rotation: ._0, |
| 81 | timeStampNs: Int64(ProcessInfo.processInfo.systemUptime * 1_000_000_000))) |
| 82 | } |
| 83 | } |
| 84 | |
| 85 | /// A screen with nothing on it but frames, for the gates: there is no |
| 86 | /// display to share in CI and no person to answer a picker. What it can |
| 87 | /// still hold is everything after the picker — the source, the sendonly |
| 88 | /// transceiver, the `share` message, and both of §5.4's endings, the |
| 89 | /// button's and the capture's own (`end`, which is the system's stop |
| 90 | /// control as far as the call can tell). |
| 91 | @MainActor final class SyntheticScreen: ScreenCapture { |
| 92 | private var source: RTCVideoSource? |
| 93 | private var capturer: RTCVideoCapturer? |
| 94 | private var task: Task<Void, Never>? |
| 95 | private var ended: (@MainActor () -> Void)? |
| 96 | /// A 1920×1080 page of text scrolling at 30 frames a second, for the |
| 97 | /// motion gate: what someone scrolling a document looks like to the |
| 98 | /// encoder, which is where a screencast source falls behind. |
| 99 | private let page: ScrollingPage? |
| 100 | |
| 101 | init(motion: Bool = false) { page = motion ? ScrollingPage() : nil } |
| 102 | |
| 103 | func start(source: RTCVideoSource, ended: @escaping @MainActor () -> Void) async throws { |
| 104 | self.source = source |
| 105 | self.ended = ended |
| 106 | let capturer = RTCVideoCapturer(delegate: source) |
| 107 | self.capturer = capturer |
| 108 | if let page { |
| 109 | // Off the main actor and on a timer of its own, as a real capture |
| 110 | // is: the gate measures WebRTC, not this process's main thread. |
| 111 | page.start(source: source, capturer: capturer) |
| 112 | return |
| 113 | } |
| 114 | task = Task { @MainActor [weak self] in |
| 115 | while !Task.isCancelled { |
| 116 | self?.frame() |
| 117 | try? await Task.sleep(nanoseconds: 100_000_000) |
| 118 | } |
| 119 | } |
| 120 | } |
| 121 | /// Frames the moving page has produced, for the gate. |
| 122 | var produced: Int { page?.produced ?? 0 } |
| 123 | /// A capture that has stalled: no frames, and nothing else either, until |
| 124 | /// the watchdog restarts it (`screen-stall`). |
| 125 | var stalled = false |
| 126 | private var made: UInt64 = 0 |
| 127 | private(set) var restarts = 0 |
| 128 | var counts: ScreenCaptureCounts { |
| 129 | let n = page.map { UInt64($0.produced) } ?? made |
| 130 | return ScreenCaptureCounts(heartbeats: n, delivered: n) |
| 131 | } |
| 132 | var frameSize: (width: Int, height: Int)? { page != nil ? (ScrollingPage.width, ScrollingPage.height) : (640, 360) } |
| 133 | func restart() async -> Bool { |
| 134 | guard source != nil else { return false } |
| 135 | restarts += 1 |
| 136 | stalled = false |
| 137 | page?.stalled = false |
| 138 | return true |
| 139 | } |
| 140 | func stall() { |
| 141 | stalled = true |
| 142 | page?.stalled = true |
| 143 | } |
| 144 | func stop() { |
| 145 | page?.stop() |
| 146 | task?.cancel() |
| 147 | task = nil |
| 148 | source = nil |
| 149 | capturer = nil |
| 150 | ended = nil |
| 151 | } |
| 152 | /// The capture ending on its own, as a shared window closing would. |
| 153 | func end() { |
| 154 | let ended = self.ended |
| 155 | stop() |
| 156 | ended?() |
| 157 | } |
| 158 | private func frame() { |
| 159 | guard let source, let capturer, !stalled else { return } |
| 160 | made += 1 |
| 161 | // A 16:9 screen, wider than either synthetic camera, so the far side |
| 162 | // can tell by size alone which one it is decoding. |
| 163 | var pixel: CVPixelBuffer? |
| 164 | CVPixelBufferCreate( |
| 165 | kCFAllocatorDefault, 640, 360, kCVPixelFormatType_32BGRA, |
| 166 | [kCVPixelBufferIOSurfacePropertiesKey: [:]] as CFDictionary, &pixel) |
| 167 | guard let pixel else { return } |
| 168 | CVPixelBufferLockBaseAddress(pixel, []) |
| 169 | if let base = CVPixelBufferGetBaseAddress(pixel) { |
| 170 | memset( |
| 171 | base, Int32(Int(Date().timeIntervalSince1970 * 10) % 100 + 120), |
| 172 | CVPixelBufferGetDataSize(pixel)) |
| 173 | } |
| 174 | CVPixelBufferUnlockBaseAddress(pixel, []) |
| 175 | source.capturer( |
| 176 | capturer, |
| 177 | didCapture: RTCVideoFrame( |
| 178 | buffer: RTCCVPixelBuffer(pixelBuffer: pixel), rotation: ._0, |
| 179 | timeStampNs: Int64(ProcessInfo.processInfo.systemUptime * 1_000_000_000))) |
| 180 | } |
| 181 | } |
| 182 | |
| 183 | /// Lines of dark "glyphs" on white, twice a screen tall, shown through a |
| 184 | /// 1920×1080 window that moves down 12 rows a frame and wraps: a fast |
| 185 | /// scroll through a dense document, at 30 frames a second, with a window |
| 186 | /// dragged across it. Built once; |
| 187 | /// each frame is row copies into a pooled buffer, on a queue of its own. |
| 188 | final class ScrollingPage: @unchecked Sendable { |
| 189 | static let width = 1920 |
| 190 | /// 1080 unless `CALLS_SYNTHETIC_SCREEN_HEIGHT` says otherwise: 1246 is a |
| 191 | /// 16:10 Retina display as the Mac's capture fits it to 1920 wide, the |
| 192 | /// size that meets H.264 level 3.1's block limit (spec §15.21, §15.22). |
| 193 | static let height = Int(ProcessInfo.processInfo.environment["CALLS_SYNTHETIC_SCREEN_HEIGHT"] ?? "") ?? 1080 |
| 194 | private let rows = ScrollingPage.height * 2 |
| 195 | private let texture: [UInt8] |
| 196 | private var offset = 0 |
| 197 | private var frame = 0 |
| 198 | private let queue = DispatchQueue(label: "calls.synthetic-screen", qos: .userInteractive) |
| 199 | private var timer: DispatchSourceTimer? |
| 200 | private var pool: CVPixelBufferPool? |
| 201 | private let lock = NSLock() |
| 202 | private var count = 0 |
| 203 | var produced: Int { lock.withLock { count } } |
| 204 | private var stalledFlag = false |
| 205 | /// Set by the gate's `screen-stall`: no frames until cleared. |
| 206 | var stalled: Bool { |
| 207 | get { lock.withLock { stalledFlag } } |
| 208 | set { lock.withLock { stalledFlag = newValue } } |
| 209 | } |
| 210 | |
| 211 | init() { |
| 212 | let stride = Self.width * 4 |
| 213 | var texture = [UInt8](repeating: 0xF4, count: stride * rows) |
| 214 | var seed: UInt64 = 0x9E37_79B9_7F4A_7C15 |
| 215 | func random(_ n: Int) -> Int { |
| 216 | seed = seed &* 6_364_136_223_846_793_005 &+ 1_442_695_040_888_963_407 |
| 217 | return Int((seed >> 33) % UInt64(n)) |
| 218 | } |
| 219 | let rows = self.rows |
| 220 | texture.withUnsafeMutableBytes { raw in |
| 221 | guard let base = raw.baseAddress else { return } |
| 222 | var line = 8 |
| 223 | while line + 16 < rows { |
| 224 | var x = 40 + random(40) |
| 225 | let end = Self.width - 40 - random(600) |
| 226 | while x < end { |
| 227 | let glyph = 5 + random(9) |
| 228 | let tall = 9 + random(6) |
| 229 | let shade = Int32(20 + random(60)) |
| 230 | for y in (line + 14 - tall)..<(line + 14) { |
| 231 | memset(base + y * stride + x * 4, shade, min(glyph, end - x) * 4) |
| 232 | } |
| 233 | x += glyph + 2 + (random(6) == 0 ? 8 : 0) |
| 234 | } |
| 235 | line += 22 |
| 236 | } |
| 237 | } |
| 238 | self.texture = texture |
| 239 | CVPixelBufferPoolCreate( |
| 240 | kCFAllocatorDefault, nil, |
| 241 | [ |
| 242 | kCVPixelBufferWidthKey: Self.width, kCVPixelBufferHeightKey: Self.height, |
| 243 | kCVPixelBufferPixelFormatTypeKey: kCVPixelFormatType_32BGRA, |
| 244 | kCVPixelBufferIOSurfacePropertiesKey: [:], |
| 245 | ] as CFDictionary, &pool) |
| 246 | } |
| 247 | |
| 248 | func start(source: RTCVideoSource, capturer: RTCVideoCapturer) { |
| 249 | // Strict: a lenient timer on a background process is coalesced to a |
| 250 | // third of this rate, which measured the timer and not WebRTC. |
| 251 | let timer = DispatchSource.makeTimerSource(flags: .strict, queue: queue) |
| 252 | timer.schedule(deadline: .now(), repeating: .nanoseconds(33_333_333), leeway: .milliseconds(2)) |
| 253 | timer.setEventHandler { [weak self] in |
| 254 | guard let self, !self.stalled, let pixel = self.next() else { return } |
| 255 | source.capturer( |
| 256 | capturer, |
| 257 | didCapture: RTCVideoFrame( |
| 258 | buffer: RTCCVPixelBuffer(pixelBuffer: pixel), rotation: ._0, |
| 259 | timeStampNs: Int64(DispatchTime.now().uptimeNanoseconds))) |
| 260 | self.lock.withLock { self.count += 1 } |
| 261 | } |
| 262 | self.timer = timer |
| 263 | timer.resume() |
| 264 | } |
| 265 | |
| 266 | func stop() { |
| 267 | timer?.cancel() |
| 268 | timer = nil |
| 269 | } |
| 270 | |
| 271 | private func bounce(_ travelled: Int, _ span: Int) -> Int { |
| 272 | let lap = travelled % (2 * span) |
| 273 | return lap < span ? lap : 2 * span - lap |
| 274 | } |
| 275 | |
| 276 | private func next() -> CVPixelBuffer? { |
| 277 | guard let pool else { return nil } |
| 278 | var pixel: CVPixelBuffer? |
| 279 | CVPixelBufferPoolCreatePixelBuffer(kCFAllocatorDefault, pool, &pixel) |
| 280 | guard let pixel else { return nil } |
| 281 | CVPixelBufferLockBaseAddress(pixel, []) |
| 282 | defer { CVPixelBufferUnlockBaseAddress(pixel, []) } |
| 283 | guard let base = CVPixelBufferGetBaseAddress(pixel) else { return nil } |
| 284 | let destination = CVPixelBufferGetBytesPerRow(pixel) |
| 285 | let stride = Self.width * 4 |
| 286 | let first = offset |
| 287 | let rows = self.rows |
| 288 | frame += 1 |
| 289 | // A window being dragged across the page — 800×450, moving 10 rows |
| 290 | // and 14 columns a frame and bouncing off the edges — whose content |
| 291 | // changes every half second, as a page loading or a slide advancing |
| 292 | // would. Together with the scroll beneath it, what someone moving |
| 293 | // things around a busy screen sends the encoder. |
| 294 | let span = (x: Self.width - 800, y: Self.height - 450) |
| 295 | let wx = bounce(frame * 14, span.x) |
| 296 | let wy = bounce(frame * 10, span.y) |
| 297 | let scene = frame / 15 |
| 298 | let jump = (scene &* 7919) % (rows - 450) |
| 299 | let shift = ((scene &* 104_729) % 900) * 4 |
| 300 | texture.withUnsafeBytes { raw in |
| 301 | guard let source = raw.baseAddress else { return } |
| 302 | for y in 0..<Self.height { |
| 303 | memcpy(base + y * destination, source + ((first + y) % rows) * stride, stride) |
| 304 | if y >= wy, y < wy + 450 { |
| 305 | memcpy(base + y * destination + wx * 4, source + (jump + y - wy) * stride + shift, 800 * 4) |
| 306 | } |
| 307 | } |
| 308 | } |
| 309 | offset = (offset + 12) % rows |
| 310 | return pixel |
| 311 | } |
| 312 | } |
| 313 | |
amadan