| 1 | import Accelerate |
| 2 | import CoreVideo |
| 3 | import Foundation |
| 4 | |
| 5 | /// The extension's half: connects to the app, and sends it each frame it |
| 6 | /// is given, smaller. |
| 7 | /// |
| 8 | /// Everything here is shaped by the extension's memory ceiling. One frame |
| 9 | /// buffer, allocated once and reused; the frame is scaled straight into |
| 10 | /// it from ReplayKit's own pixel buffer, which is released as soon as |
| 11 | /// `send` returns; and at most one frame is ever in flight. A frame that |
| 12 | /// arrives while the last is still being written, or sooner than the rate |
| 13 | /// allows, is dropped rather than queued — a queue is exactly how an |
| 14 | /// extension runs out of memory while the app is busy. |
| 15 | public final class BroadcastSender: @unchecked Sendable { |
| 16 | private let path: String |
| 17 | private let longest: CGFloat |
| 18 | private let interval: UInt64 |
| 19 | private let queue = DispatchQueue(label: "calls.broadcast.send") |
| 20 | private let lock = NSLock() |
| 21 | // Guarded by lock. |
| 22 | private var fd: Int32 = -1 |
| 23 | private var busy = false |
| 24 | private var last: UInt64 = 0 |
| 25 | private var finished = false |
| 26 | // Only touched on queue, or before connect. |
| 27 | private var buffer: UnsafeMutableRawPointer? |
| 28 | private var capacity = 0 |
| 29 | |
| 30 | /// Called once, on the sending queue, when the app has gone: it closed |
| 31 | /// the socket because the person stopped sharing there, or the call |
| 32 | /// ended. Not called for `close()`. |
| 33 | public var onClosed: (@Sendable () -> Void)? |
| 34 | |
| 35 | /// `longest` is the long side frames are sent at, at most; 1280 keeps a |
| 36 | /// phone's text legible on a laptop and the frame near a megabyte. |
| 37 | public init(path: String, longest: Int = 1280, framesPerSecond: Int = 15) { |
| 38 | self.path = path |
| 39 | self.longest = CGFloat(longest) |
| 40 | interval = 1_000_000_000 / UInt64(max(1, framesPerSecond)) |
| 41 | } |
| 42 | |
| 43 | deinit { buffer?.deallocate() } |
| 44 | |
| 45 | /// Connects to the app, trying for up to `timeout`: the app may still be |
| 46 | /// settling into the call as the broadcast's countdown ends. |
| 47 | public func connect(timeout: TimeInterval) -> Bool { |
| 48 | let until = Date().addingTimeInterval(timeout) |
| 49 | repeat { |
| 50 | if let fd = try? BroadcastSocket.connect(path: path) { |
| 51 | lock.lock() |
| 52 | self.fd = fd |
| 53 | lock.unlock() |
| 54 | return true |
| 55 | } |
| 56 | Thread.sleep(forTimeInterval: 0.1) |
| 57 | } while Date() < until |
| 58 | return false |
| 59 | } |
| 60 | |
| 61 | /// Offers one frame. Returns at once; the write happens on the sending |
| 62 | /// queue. `pixels` must be 4:2:0 bi-planar, which is what ReplayKit |
| 63 | /// delivers; anything else is dropped. |
| 64 | public func send(_ pixels: CVPixelBuffer, orientation: UInt16, timestampNs: UInt64) { |
| 65 | let format = CVPixelBufferGetPixelFormatType(pixels) |
| 66 | guard format == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange |
| 67 | || format == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange |
| 68 | else { return } |
| 69 | lock.lock() |
| 70 | let now = DispatchTime.now().uptimeNanoseconds |
| 71 | guard fd >= 0, !busy, !finished, now >= last + interval else { |
| 72 | lock.unlock() |
| 73 | return |
| 74 | } |
| 75 | busy = true |
| 76 | last = now |
| 77 | lock.unlock() |
| 78 | // Scaled here, on ReplayKit's thread, so the pixel buffer is not held |
| 79 | // past this call; only the copy crosses to the sending queue. |
| 80 | guard let length = pack(pixels, orientation: orientation, timestampNs: timestampNs) else { |
| 81 | lock.lock() |
| 82 | busy = false |
| 83 | lock.unlock() |
| 84 | return |
| 85 | } |
| 86 | queue.async { [self] in |
| 87 | let ok = writeAll(length) |
| 88 | lock.lock() |
| 89 | busy = false |
| 90 | let first = !ok && !finished |
| 91 | if !ok { finished = true } |
| 92 | lock.unlock() |
| 93 | if first { onClosed?() } |
| 94 | } |
| 95 | } |
| 96 | |
| 97 | public func close() { |
| 98 | lock.lock() |
| 99 | finished = true |
| 100 | let fd = self.fd |
| 101 | self.fd = -1 |
| 102 | lock.unlock() |
| 103 | if fd >= 0 { Darwin.close(fd) } |
| 104 | } |
| 105 | |
| 106 | /// Scales `pixels` into the frame buffer behind a header, and answers the |
| 107 | /// number of bytes to write. |
| 108 | private func pack(_ pixels: CVPixelBuffer, orientation: UInt16, timestampNs: UInt64) -> Int? { |
| 109 | let size = fitted( |
| 110 | width: CGFloat(CVPixelBufferGetWidth(pixels)), height: CGFloat(CVPixelBufferGetHeight(pixels)), |
| 111 | longest: longest) |
| 112 | let header = BroadcastFrameHeader( |
| 113 | width: size.width, height: size.height, orientation: orientation, timestampNs: timestampNs, |
| 114 | videoRange: CVPixelBufferGetPixelFormatType(pixels) |
| 115 | == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange) |
| 116 | let length = BroadcastFrameHeader.size + header.payloadLength |
| 117 | if capacity < length { |
| 118 | buffer?.deallocate() |
| 119 | buffer = UnsafeMutableRawPointer.allocate(byteCount: length, alignment: 16) |
| 120 | capacity = length |
| 121 | } |
| 122 | guard let buffer else { return nil } |
| 123 | header.write(to: buffer) |
| 124 | CVPixelBufferLockBaseAddress(pixels, .readOnly) |
| 125 | defer { CVPixelBufferUnlockBaseAddress(pixels, .readOnly) } |
| 126 | guard let luma = CVPixelBufferGetBaseAddressOfPlane(pixels, 0), |
| 127 | let chroma = CVPixelBufferGetBaseAddressOfPlane(pixels, 1) |
| 128 | else { return nil } |
| 129 | var sourceLuma = vImage_Buffer( |
| 130 | data: luma, height: vImagePixelCount(CVPixelBufferGetHeightOfPlane(pixels, 0)), |
| 131 | width: vImagePixelCount(CVPixelBufferGetWidthOfPlane(pixels, 0)), |
| 132 | rowBytes: CVPixelBufferGetBytesPerRowOfPlane(pixels, 0)) |
| 133 | var sourceChroma = vImage_Buffer( |
| 134 | data: chroma, height: vImagePixelCount(CVPixelBufferGetHeightOfPlane(pixels, 1)), |
| 135 | width: vImagePixelCount(CVPixelBufferGetWidthOfPlane(pixels, 1)), |
| 136 | rowBytes: CVPixelBufferGetBytesPerRowOfPlane(pixels, 1)) |
| 137 | let picture = buffer.advanced(by: BroadcastFrameHeader.size) |
| 138 | var targetLuma = vImage_Buffer( |
| 139 | data: picture, height: vImagePixelCount(size.height), width: vImagePixelCount(size.width), |
| 140 | rowBytes: size.width) |
| 141 | var targetChroma = vImage_Buffer( |
| 142 | data: picture.advanced(by: size.width * size.height), |
| 143 | height: vImagePixelCount(size.height / 2), width: vImagePixelCount(size.width / 2), |
| 144 | rowBytes: size.width) |
| 145 | let flags = vImage_Flags(kvImageNoFlags) |
| 146 | guard vImageScale_Planar8(&sourceLuma, &targetLuma, nil, flags) == kvImageNoError, |
| 147 | vImageScale_CbCr8(&sourceChroma, &targetChroma, nil, flags) == kvImageNoError |
| 148 | else { return nil } |
| 149 | return length |
| 150 | } |
| 151 | |
| 152 | private func writeAll(_ length: Int) -> Bool { |
| 153 | lock.lock() |
| 154 | let fd = self.fd |
| 155 | lock.unlock() |
| 156 | guard fd >= 0, let buffer else { return false } |
| 157 | var sent = 0 |
| 158 | while sent < length { |
| 159 | let n = Darwin.write(fd, buffer.advanced(by: sent), length - sent) |
| 160 | if n > 0 { |
| 161 | sent += n |
| 162 | } else if n < 0 && errno == EINTR { |
| 163 | continue |
| 164 | } else { |
| 165 | return false |
| 166 | } |
| 167 | } |
| 168 | return true |
| 169 | } |
| 170 | } |
| 171 | |
amadan