| 1 | import Foundation |
| 2 | import LiveKitWebRTC |
| 3 | |
| 4 | /// What a shared screen is for, which decides what WebRTC gives up first |
| 5 | /// when the uplink or the encoder cannot carry everything. |
| 6 | /// |
| 7 | /// A screencast source on its own keeps every pixel and drops frames, which |
| 8 | /// is right for a document and exactly wrong for someone scrolling, |
| 9 | /// dragging or playing video: the far side sees a slideshow of a screen |
| 10 | /// that is moving ("the screenshare isn't keeping up", 2026-09-24). |
| 11 | public enum ScreenShareMode: String, CaseIterable, Identifiable, Sendable { |
| 12 | /// Keeps it reasonably sharp while tracking motion. The default. |
| 13 | case balanced |
| 14 | /// Keeps full resolution and lets the frame rate fall: slides, code, text. |
| 15 | case detail |
| 16 | /// Keeps the frame rate and lets resolution fall, to 1280 at most. |
| 17 | case motion |
| 18 | |
| 19 | public var id: String { rawValue } |
| 20 | |
| 21 | public var title: String { |
| 22 | switch self { |
| 23 | case .balanced: return "Balanced" |
| 24 | case .detail: return "Text and detail" |
| 25 | case .motion: return "Motion and video" |
| 26 | } |
| 27 | } |
| 28 | |
| 29 | public var explanation: String { |
| 30 | switch self { |
| 31 | case .balanced: return "Keeps up with scrolling and moving windows, at full sharpness when the connection allows." |
| 32 | case .detail: return "Keeps small text sharp. Movement may look jerky." |
| 33 | case .motion: return "Keeps movement smooth. Text may look softer." |
| 34 | } |
| 35 | } |
| 36 | |
| 37 | static let key = "screenShareMode" |
| 38 | |
| 39 | /// The remembered choice, balanced until someone picks another. |
| 40 | public static var remembered: ScreenShareMode { |
| 41 | get { UserDefaults.standard.string(forKey: key).flatMap(Self.init(rawValue:)) ?? .balanced } |
| 42 | set { UserDefaults.standard.set(newValue.rawValue, forKey: key) } |
| 43 | } |
| 44 | } |
| 45 | |
| 46 | /// One sender's encoding, as the policy wants it. Nil means the encoder's |
| 47 | /// own default, and applying nil puts that default back. |
| 48 | struct EncodingPlan: Equatable { |
| 49 | enum Degradation: Equatable { case maintainResolution, maintainFramerate, balanced } |
| 50 | |
| 51 | var maxBitrate: Int? |
| 52 | var maxFramerate: Int? |
| 53 | var scaleResolutionDownBy: Double? |
| 54 | /// WebRTC's bitrate priority: 1 is everyone's default, 4 is "high". |
| 55 | var bitratePriority = 1.0 |
| 56 | var highPriority = false |
| 57 | var degradation: Degradation? |
| 58 | } |
| 59 | |
| 60 | /// The sender parameters for a screen and for the camera beside it. |
| 61 | /// |
| 62 | /// Measured on 2026-09-24 with the native/browser motion gate (a 1080p |
| 63 | /// page scrolling at 30 fps with a window dragged across it, H.264 on the |
| 64 | /// Mac's hardware encoder, received in Chromium), frames a second arriving |
| 65 | /// with the uplink open / capped at 2.5 / 1.5 / 1.0 Mbit/s: |
| 66 | /// |
| 67 | /// before (screencast, no parameters) 26 / 22 / 2 / 1 at 1920×1080 |
| 68 | /// detail (maintain resolution, 1920) 27 / 21 / 13 / 2 at 1920×1080 |
| 69 | /// balanced (maintain frame rate, 1920) 30 / 30 / 19 / 25 at 960–320 wide |
| 70 | /// motion (not screencast, 1280) 30 / 30 / 30 / 23 at 640–320 wide |
| 71 | /// |
| 72 | /// WebRTC's own "balanced" degradation gave 7 / 2 / 2 / 1.5 on the same |
| 73 | /// source, which is why the balanced mode does not use it. The same page |
| 74 | /// on software VP8 managed 2 fps even on the open link (80 ms a frame to |
| 75 | /// encode), which is why the screen asks for H.264 first. |
| 76 | /// |
| 77 | /// Sender parameters and nothing else, so none of it touches SDP: a mode |
| 78 | /// changed mid-share, the camera capped and restored, all without a |
| 79 | /// renegotiation for perfect negotiation to wobble over (see room.js's |
| 80 | /// tuneAudioSender for the same reasoning on the browser side). |
| 81 | public enum ScreenSendPolicy { |
| 82 | /// A screen longer than this on its long side is scaled down to it in |
| 83 | /// motion mode: 720p's worth of pixels is what 30 fps fits in. |
| 84 | static let motionLongestSide = 1280.0 |
| 85 | /// And never sent larger than this in any mode (the Mac's capture is |
| 86 | /// already fitted to it; this holds for anything else that shares). |
| 87 | static let longestSide = 1920.0 |
| 88 | |
| 89 | /// The camera's ceiling while a screen is going out: the screen is what |
| 90 | /// everyone is looking at, and the camera is a thumbnail beside it. |
| 91 | static let sharingCameraBitrate = 500_000 |
| 92 | static let sharingCameraHeight = 360.0 |
| 93 | |
| 94 | /// Whether the source is marked as a screencast. Motion is not: a |
| 95 | /// screencast is encoded as screen content, which is tuned for |
| 96 | /// sharpness, and in the measurement above the plain source held 30 fps |
| 97 | /// on a thinner link than the screencast did. |
| 98 | static func screencast(_ mode: ScreenShareMode) -> Bool { mode != .motion } |
| 99 | |
| 100 | /// The most 16×16 blocks a frame from a source that is *not* marked as |
| 101 | /// a screencast may have: H.264 level 3.1's 3,600, which is 1280×720. |
| 102 | /// |
| 103 | /// Chromium answers every H.264 offer at level 3.1, and the Mac's |
| 104 | /// hardware encoder, as this build of WebRTC sets it up, holds a camera- |
| 105 | /// like stream to the level it negotiated: a larger frame is refused by |
| 106 | /// VideoToolbox (kVTParameterErr, -12902) and nothing is encoded at all. |
| 107 | /// A screencast source is set up another way and is not held to it — |
| 108 | /// measured 2026-09-28 at 1920×1246, steady. So this is the motion |
| 109 | /// mode's cap, and the cap for any mode on a share that *started* in |
| 110 | /// motion mode (its source cannot become a screencast later). |
| 111 | /// |
| 112 | /// It is Paul's freeze of 2026-09-28 ("after a while, my screen froze on |
| 113 | /// her end"): a 16:10 display in motion mode is 1280×830 at its 1280 cap, |
| 114 | /// 4,160 blocks. It went out at 960×620 while the link was still |
| 115 | /// ramping, and the moment WebRTC let it up to full size, the encoder |
| 116 | /// stopped — for eight or nine seconds at a time, until WebRTC's quality |
| 117 | /// scaler stepped it down again, over and over; and for good if the mode |
| 118 | /// was then switched to Text and detail, which does not let the size |
| 119 | /// fall. A 16:9 page at 1280×720 is exactly 3,600, which is why the |
| 120 | /// motion gate's synthetic 1080p page never met it. |
| 121 | static let realtimeMacroblocks = 3600 |
| 122 | /// Off only for the gate, which measures 0.4.1's sizing against it. |
| 123 | @_spi(Fixtures) public static var levelCapped = true |
| 124 | |
| 125 | /// The screen's encoding, for a source of the given size (zero while |
| 126 | /// it is not known yet, which leaves the scale alone). `screencast` is |
| 127 | /// whether the source was made as one (see `realtimeMacroblocks`). |
| 128 | static func screen( |
| 129 | _ mode: ScreenShareMode, width: Int = 0, height: Int = 0, screencast: Bool? = nil |
| 130 | ) -> EncodingPlan { |
| 131 | let longest = Double(max(width, height)) |
| 132 | let capped = levelCapped && !(screencast ?? Self.screencast(mode)) && width > 0 && height > 0 |
| 133 | func fit(_ side: Double) -> Double? { |
| 134 | var scale = longest > side ? longest / side : 1 |
| 135 | if capped { |
| 136 | // Down until the frame's blocks fit the level, as WebRTC will |
| 137 | // size it: each side divided by the scale, rounded down. |
| 138 | func blocks(_ s: Double) -> Int { |
| 139 | ((Int(Double(width) / s) + 15) / 16) * ((Int(Double(height) / s) + 15) / 16) |
| 140 | } |
| 141 | while blocks(scale) > realtimeMacroblocks { scale *= 1.01 } |
| 142 | } |
| 143 | return scale > 1 ? scale : nil |
| 144 | } |
| 145 | switch mode { |
| 146 | case .detail: |
| 147 | return EncodingPlan( |
| 148 | maxBitrate: 3_500_000, maxFramerate: 30, scaleResolutionDownBy: fit(longestSide), |
| 149 | bitratePriority: 4, highPriority: true, degradation: .maintainResolution) |
| 150 | case .balanced: |
| 151 | // Full size while the link and the encoder can carry it, and a |
| 152 | // smaller picture rather than a slideshow when they cannot. |
| 153 | return EncodingPlan( |
| 154 | maxBitrate: 3_000_000, maxFramerate: 30, scaleResolutionDownBy: fit(longestSide), |
| 155 | bitratePriority: 4, highPriority: true, degradation: .maintainFramerate) |
| 156 | case .motion: |
| 157 | return EncodingPlan( |
| 158 | maxBitrate: 2_500_000, maxFramerate: 30, scaleResolutionDownBy: fit(motionLongestSide), |
| 159 | bitratePriority: 4, highPriority: true, degradation: .maintainFramerate) |
| 160 | } |
| 161 | } |
| 162 | |
| 163 | /// The sender's codecs with H.264 first, the rest after it in their own |
| 164 | /// order: Apple's hardware encoder where the far side can take H.264, |
| 165 | /// and everything else still on offer where it cannot, so a browser |
| 166 | /// without H.264 negotiates what it always did. Held on the screen's |
| 167 | /// transceiver so a renegotiation the far side offers keeps it. |
| 168 | static func preferringH264(_ codecs: [RTCRtpCodecCapability]) -> [RTCRtpCodecCapability] { |
| 169 | codecs.filter { $0.name.uppercased() == "H264" } + codecs.filter { $0.name.uppercased() != "H264" } |
| 170 | } |
| 171 | |
| 172 | /// The camera's encoding: capped while this app shares a screen, and |
| 173 | /// the encoder's own defaults again once it stops. `height` is the |
| 174 | /// camera's own frame height where known; the native camera is |
| 175 | /// captured nearest 640 wide, so 480 lines. |
| 176 | static func camera(sharing: Bool, height: Int = 480) -> EncodingPlan { |
| 177 | guard sharing else { return EncodingPlan() } |
| 178 | let scale = Double(height) > sharingCameraHeight ? Double(height) / sharingCameraHeight : nil |
| 179 | return EncodingPlan(maxBitrate: sharingCameraBitrate, scaleResolutionDownBy: scale) |
| 180 | } |
| 181 | } |
| 182 | |
| 183 | extension RTCRtpSender { |
| 184 | /// Applies a plan to this sender's first encoding, and reports whether |
| 185 | /// anything changed. Reads the parameters fresh each time: WebRTC |
| 186 | /// refuses a set that did not come from the latest get. |
| 187 | @discardableResult func apply(_ plan: EncodingPlan) -> Bool { |
| 188 | let parameters = self.parameters |
| 189 | guard let encoding = parameters.encodings.first else { return false } |
| 190 | let before = describe(parameters) |
| 191 | encoding.maxBitrateBps = plan.maxBitrate.map { NSNumber(value: $0) } |
| 192 | encoding.maxFramerate = plan.maxFramerate.map { NSNumber(value: $0) } |
| 193 | encoding.scaleResolutionDownBy = NSNumber(value: plan.scaleResolutionDownBy ?? 1) |
| 194 | encoding.bitratePriority = plan.bitratePriority |
| 195 | encoding.networkPriority = plan.highPriority ? .high : .low |
| 196 | switch plan.degradation { |
| 197 | case .maintainResolution?: |
| 198 | parameters.degradationPreference = NSNumber(value: RTCDegradationPreference.maintainResolution.rawValue) |
| 199 | case .maintainFramerate?: |
| 200 | parameters.degradationPreference = NSNumber(value: RTCDegradationPreference.maintainFramerate.rawValue) |
| 201 | case .balanced?: |
| 202 | parameters.degradationPreference = NSNumber(value: RTCDegradationPreference.balanced.rawValue) |
| 203 | case nil: |
| 204 | parameters.degradationPreference = nil |
| 205 | } |
| 206 | guard describe(parameters) != before else { return false } |
| 207 | self.parameters = parameters |
| 208 | return true |
| 209 | } |
| 210 | |
| 211 | private func describe(_ parameters: RTCRtpParameters) -> String { |
| 212 | guard let e = parameters.encodings.first else { return "" } |
| 213 | return [ |
| 214 | e.maxBitrateBps?.stringValue, e.maxFramerate?.stringValue, e.scaleResolutionDownBy?.stringValue, |
| 215 | String(e.bitratePriority), String(e.networkPriority.rawValue), |
| 216 | parameters.degradationPreference?.stringValue, |
| 217 | ].map { $0 ?? "-" }.joined(separator: ",") |
| 218 | } |
| 219 | } |
| 220 | |
amadan