| 1 | import CoreGraphics |
| 2 | import Foundation |
| 3 | import LiveKitWebRTC |
| 4 | import SwiftUI |
| 5 | |
| 6 | /// What surrounds a shared screen on the stage (2026-10-05, Paul: "Should |
| 7 | /// the video view stay fullscreen even here?"). A screen is fitted whole, |
| 8 | /// never cropped, so a window of another shape leaves bars beside or |
| 9 | /// above it. Rather than black, they show the same picture enlarged, soft |
| 10 | /// and dimmed, as FaceTime does. |
| 11 | /// |
| 12 | /// Cheap by construction: twice a second, WebRTC scales one frame down to |
| 13 | /// `width` × `height` (libyuv, from whatever buffer the decoder gave), it |
| 14 | /// is converted and blurred at that size, and the picture is drawn |
| 15 | /// enlarged, its smooth upscaling doing the rest of the blur. Nothing runs |
| 16 | /// at the frame rate and nothing is blurred at the window's size. |
| 17 | public struct ScreenBackdrop: View { |
| 18 | let track: RTCVideoTrack |
| 19 | @StateObject private var sampler = BackdropSampler() |
| 20 | |
| 21 | public init(track: RTCVideoTrack) { self.track = track } |
| 22 | |
| 23 | public var body: some View { |
| 24 | ZStack { |
| 25 | CallsAppearance.stageSurface |
| 26 | if let image = sampler.image { |
| 27 | Image(decorative: image, scale: 1) |
| 28 | .resizable() |
| 29 | .interpolation(.high) |
| 30 | .aspectRatio(contentMode: .fill) |
| 31 | // Past the edges a little: the blur's own edge is darker. |
| 32 | .scaleEffect(1.15) |
| 33 | .opacity(BackdropSampler.dimmed) |
| 34 | .transition(.opacity) |
| 35 | } |
| 36 | } |
| 37 | .clipped() |
| 38 | .animation(.easeInOut(duration: 0.4), value: sampler.image != nil) |
| 39 | .onAppear { sampler.attach(track) } |
| 40 | .onChange(of: ObjectIdentifier(track)) { sampler.attach(track) } |
| 41 | .onDisappear { sampler.detach() } |
| 42 | .accessibilityHidden(true) |
| 43 | } |
| 44 | } |
| 45 | |
| 46 | /// Takes frames from a track at `interval` and keeps the latest as a small |
| 47 | /// blurred picture. |
| 48 | @MainActor final class BackdropSampler: ObservableObject { |
| 49 | /// The picture's size: 16:9, enough for the colours and their places. |
| 50 | nonisolated static let width = 64 |
| 51 | nonisolated static let height = 36 |
| 52 | /// How often a frame is taken. |
| 53 | nonisolated static let interval: TimeInterval = 0.5 |
| 54 | /// How much of the picture shows over the dark surface behind it. |
| 55 | nonisolated static let dimmed: Double = 0.45 |
| 56 | |
| 57 | @Published private(set) var image: CGImage? |
| 58 | private var track: RTCVideoTrack? |
| 59 | private var renderer: Renderer? |
| 60 | |
| 61 | func attach(_ next: RTCVideoTrack) { |
| 62 | guard track !== next else { return } |
| 63 | detach() |
| 64 | let renderer = Renderer { [weak self] image in |
| 65 | Task { @MainActor in self?.image = image } |
| 66 | } |
| 67 | next.add(renderer) |
| 68 | track = next |
| 69 | self.renderer = renderer |
| 70 | } |
| 71 | |
| 72 | func detach() { |
| 73 | if let renderer { track?.remove(renderer) } |
| 74 | track = nil |
| 75 | renderer = nil |
| 76 | image = nil |
| 77 | } |
| 78 | |
| 79 | /// WebRTC's side: called on its decoding thread for every frame, it |
| 80 | /// keeps one in `interval` and lets the rest go at once. |
| 81 | final class Renderer: NSObject, RTCVideoRenderer, @unchecked Sendable { |
| 82 | private let lock = NSLock() |
| 83 | private var last: TimeInterval = 0 |
| 84 | private let deliver: (CGImage) -> Void |
| 85 | init(deliver: @escaping (CGImage) -> Void) { self.deliver = deliver } |
| 86 | |
| 87 | func setSize(_ size: CGSize) {} |
| 88 | |
| 89 | func renderFrame(_ frame: RTCVideoFrame?) { |
| 90 | guard let frame else { return } |
| 91 | let now = ProcessInfo.processInfo.systemUptime |
| 92 | lock.lock() |
| 93 | let due = now - last >= BackdropSampler.interval |
| 94 | if due { last = now } |
| 95 | lock.unlock() |
| 96 | guard due, let image = BackdropSampler.picture(of: frame) else { return } |
| 97 | deliver(image) |
| 98 | } |
| 99 | } |
| 100 | |
| 101 | /// One frame as the backdrop's picture: scaled down by WebRTC, then |
| 102 | /// converted and blurred here. |
| 103 | nonisolated static func picture(of frame: RTCVideoFrame) -> CGImage? { |
| 104 | let source = frame.buffer |
| 105 | guard source.width > 0, source.height > 0 else { return nil } |
| 106 | let small = |
| 107 | source.cropAndScale?( |
| 108 | with: 0, offsetY: 0, cropWidth: source.width, cropHeight: source.height, |
| 109 | scaleWidth: Int32(width), scaleHeight: Int32(height)) ?? source |
| 110 | let planes = small.toI420() |
| 111 | var rgba = rgbaFromI420( |
| 112 | width: Int(planes.width), height: Int(planes.height), |
| 113 | y: planes.dataY, strideY: Int(planes.strideY), |
| 114 | u: planes.dataU, strideU: Int(planes.strideU), |
| 115 | v: planes.dataV, strideV: Int(planes.strideV), |
| 116 | outWidth: width, outHeight: height) |
| 117 | boxBlur(&rgba, width: width, height: height, radius: 3, passes: 2) |
| 118 | return image(rgba, width: width, height: height) |
| 119 | } |
| 120 | |
| 121 | /// I420 to RGBA (BT.601, video range) at `outWidth` × `outHeight`, |
| 122 | /// sampling the nearest source pixel when the sizes differ. Integer |
| 123 | /// arithmetic (coefficients × 256) over raw buffers. |
| 124 | nonisolated static func rgbaFromI420( |
| 125 | width: Int, height: Int, y: UnsafePointer<UInt8>, strideY: Int, |
| 126 | u: UnsafePointer<UInt8>, strideU: Int, v: UnsafePointer<UInt8>, strideV: Int, |
| 127 | outWidth: Int, outHeight: Int |
| 128 | ) -> [UInt8] { |
| 129 | var out = [UInt8](repeating: 255, count: outWidth * outHeight * 4) |
| 130 | guard width > 0, height > 0 else { return out } |
| 131 | out.withUnsafeMutableBufferPointer { buffer in |
| 132 | let o = buffer.baseAddress! |
| 133 | var i = 0 |
| 134 | for row in 0..<outHeight { |
| 135 | let sy = min(height - 1, row * height / outHeight) |
| 136 | let yRow = y + sy * strideY |
| 137 | let uRow = u + (sy >> 1) * strideU |
| 138 | let vRow = v + (sy >> 1) * strideV |
| 139 | for col in 0..<outWidth { |
| 140 | let sx = min(width - 1, col * width / outWidth) |
| 141 | let c = 298 * (Int(yRow[sx]) - 16) + 128 |
| 142 | let d = Int(uRow[sx >> 1]) - 128 |
| 143 | let e = Int(vRow[sx >> 1]) - 128 |
| 144 | o[i] = clamp8((c + 409 * e) >> 8) |
| 145 | o[i + 1] = clamp8((c - 100 * d - 208 * e) >> 8) |
| 146 | o[i + 2] = clamp8((c + 516 * d) >> 8) |
| 147 | i += 4 |
| 148 | } |
| 149 | } |
| 150 | } |
| 151 | return out |
| 152 | } |
| 153 | |
| 154 | @inline(__always) nonisolated private static func clamp8(_ value: Int) -> UInt8 { |
| 155 | UInt8(truncatingIfNeeded: value < 0 ? 0 : value > 255 ? 255 : value) |
| 156 | } |
| 157 | |
| 158 | /// A separable box blur, `passes` times (two is close to Gaussian), |
| 159 | /// with running sums, so each pass is linear in the pixels. |
| 160 | nonisolated static func boxBlur( |
| 161 | _ pixels: inout [UInt8], width: Int, height: Int, radius: Int, passes: Int |
| 162 | ) { |
| 163 | guard radius > 0, width > 0, height > 0 else { return } |
| 164 | var scratch = pixels |
| 165 | pixels.withUnsafeMutableBufferPointer { a in |
| 166 | scratch.withUnsafeMutableBufferPointer { b in |
| 167 | for _ in 0..<passes { |
| 168 | blurLines(a.baseAddress!, b.baseAddress!, count: height, length: width, step: 4, lineStep: width * 4, radius: radius) |
| 169 | blurLines(b.baseAddress!, a.baseAddress!, count: width, length: height, step: width * 4, lineStep: 4, radius: radius) |
| 170 | } |
| 171 | } |
| 172 | } |
| 173 | } |
| 174 | |
| 175 | nonisolated private static func blurLines( |
| 176 | _ source: UnsafeMutablePointer<UInt8>, _ out: UnsafeMutablePointer<UInt8>, count: Int, |
| 177 | length: Int, step: Int, lineStep: Int, radius: Int |
| 178 | ) { |
| 179 | for line in 0..<count { |
| 180 | let base = line * lineStep |
| 181 | for channel in 0..<3 { |
| 182 | var sum = 0 |
| 183 | var n = 0 |
| 184 | // The window [at - radius, at + radius], clipped to the line. |
| 185 | for k in 0...min(length - 1, radius) { |
| 186 | sum += Int(source[base + k * step + channel]) |
| 187 | n += 1 |
| 188 | } |
| 189 | for at in 0..<length { |
| 190 | out[base + at * step + channel] = UInt8(truncatingIfNeeded: sum / n) |
| 191 | let leaving = at - radius |
| 192 | let entering = at + radius + 1 |
| 193 | if leaving >= 0 { |
| 194 | sum -= Int(source[base + leaving * step + channel]) |
| 195 | n -= 1 |
| 196 | } |
| 197 | if entering < length { |
| 198 | sum += Int(source[base + entering * step + channel]) |
| 199 | n += 1 |
| 200 | } |
| 201 | } |
| 202 | } |
| 203 | for at in 0..<length { out[base + at * step + 3] = 255 } |
| 204 | } |
| 205 | } |
| 206 | |
| 207 | nonisolated static func image(_ rgba: [UInt8], width: Int, height: Int) -> CGImage? { |
| 208 | guard let provider = CGDataProvider(data: Data(rgba) as CFData) else { return nil } |
| 209 | return CGImage( |
| 210 | width: width, height: height, bitsPerComponent: 8, bitsPerPixel: 32, bytesPerRow: width * 4, |
| 211 | space: CGColorSpaceCreateDeviceRGB(), |
| 212 | bitmapInfo: CGBitmapInfo(rawValue: CGImageAlphaInfo.noneSkipLast.rawValue), |
| 213 | provider: provider, decode: nil, shouldInterpolate: true, intent: .defaultIntent) |
| 214 | } |
| 215 | } |
| 216 | |
| 217 | /// Where a picture goes in its tile, top-down: a screen fitted whole and |
| 218 | /// centred; a camera filling the tile, cropped evenly from the sides but, |
| 219 | /// from top and bottom, more from the bottom (`headroom`), since a face |
| 220 | /// is above the middle of the picture and a crop through the forehead |
| 221 | /// reads worse than one through the chest. The web's stage and tiles do |
| 222 | /// the same with object-position (calls.css). |
| 223 | public enum VideoFrame { |
| 224 | /// The share of a vertical crop taken from the top: 0.5 would be even. |
| 225 | public static let headroom: CGFloat = 0.35 |
| 226 | |
| 227 | public static func place(video: CGSize, in bounds: CGSize, fit: Bool) -> CGRect { |
| 228 | guard video.width > 0, video.height > 0 else { return CGRect(origin: .zero, size: bounds) } |
| 229 | let across = bounds.width / video.width |
| 230 | let down = bounds.height / video.height |
| 231 | let scale = fit ? min(across, down) : max(across, down) |
| 232 | let size = CGSize(width: video.width * scale, height: video.height * scale) |
| 233 | let spare = bounds.height - size.height |
| 234 | // Spare room (a fitted screen) is shared evenly; a crop (negative) |
| 235 | // takes headroom of it from the top. |
| 236 | let top = spare >= 0 || fit ? spare / 2 : spare * headroom |
| 237 | return CGRect(x: (bounds.width - size.width) / 2, y: top, width: size.width, height: size.height) |
| 238 | } |
| 239 | } |
| 240 | |
amadan