amadan

oficina / meet Public

Merge branch 'fix/camera-recovery' into main

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude 64389111d96757ba7149dbe9bdd16c795a571938 parent e99c8daparent 042d45b
28 files changed, +2170 −288
  • callscore/room.go +15 −0
  • callscore/room_test.go +100 −0
  • callscore/transport.go +28 −0
  • internal/calls/home_native_test.go +94 −0
  • native/README.md +65 −12
  • native/apple/Sources/CallsInterop/Interop.swift +25 −1
  • native/apple/Sources/CallsInterop/SyntheticVideo.swift +36 −25
  • native/apple/Sources/CallsKit/Audio.swift +125 −62
  • native/apple/Sources/CallsKit/Call.swift +43 −3
  • native/apple/Sources/CallsKit/CallAudio.swift +36 −7
  • native/apple/Sources/CallsKit/CallsModel.swift +9 −0
  • native/apple/Sources/CallsKit/CameraRecovery.swift +424 −0
  • native/apple/Sources/CallsKit/Capture.swift +508 −93
  • native/apple/Sources/CallsKit/DevicePanel.swift +75 −8
  • native/apple/Sources/CallsKit/GuestJoinView.swift +0 −2
  • native/apple/Sources/CallsKit/MacCallView.swift +14 −3
  • native/apple/Sources/CallsKit/MacVideoTile.swift +8 −4
  • native/apple/Sources/CallsKit/Media.swift +27 −3
  • native/apple/Sources/CallsKit/MiniWindow.swift +2 −1
  • native/apple/Sources/CallsKit/NoiseSuppression.swift +15 −4
  • native/apple/Sources/CallsMac/CallsMac.swift +26 −1
  • native/apple/Tests/CallsKitTests/AudioHealthTests.swift +47 −2
  • native/apple/Tests/CallsKitTests/AudioRoutesTests.swift +72 −43
  • native/apple/Tests/CallsKitTests/CameraRecoveryTests.swift +329 −0
  • native/apple/Tests/CallsKitTests/NoiseSuppressionTests.swift +9 −4
  • native/ios/Sources/CallsPhone.swift +19 −9
  • native/ios/project.yml +1 −1
  • test/browser/native.mjs +18 −0

CI passed — run details

diff --git a/callscore/room.go b/callscore/room.go
index a02c121..06ec76b 100644
--- a/callscore/room.go
+++ b/callscore/room.go
@@ -157,6 +157,15 @@ func (r *Room) streamLoop() {
}
delay := 500 * time.Millisecond
+ // A fresh open refused with 409 means this session already holds a
+ // participation in the room. For an app that is its own ghost: the
+ // previous run of the app (quit, crashed, relaunched by an update)
+ // whose participation is still inside its thirty-second resume
+ // window. The session is the app's alone, so it takes its own place
+ // back, once, rather than ending the call as "open elsewhere" before
+ // it began. A second instance that really is elsewhere is told
+ // lost{replaced} on its own stream, and does not take it back.
+ tookOver := false
for r.ctx.Err() == nil {
r.mu.Lock()
peer := r.state.Peer
@@ -189,6 +198,12 @@ func (r *Room) streamLoop() {
r.terminal(reason)
return
case 409:
+ if peer == "" && !tookOver {
+ tookOver = true
+ if r.t.takeover(r.ctx) == nil {
+ continue
+ }
+ }
r.terminal("replaced")
return
case 401, 403, 302, 303, 307, 308:
diff --git a/callscore/room_test.go b/callscore/room_test.go
index cfd5689..c0d9aa5 100644
--- a/callscore/room_test.go
+++ b/callscore/room_test.go
@@ -302,3 +302,103 @@ func TestTargetQueueRetriesSameSequenceBeforeNextMessage(t *testing.T) {
}
}
}
+
+// A fresh open refused with 409 takes the session's own participation back
+// once, and a refused takeover — or a second 409 — is "replaced" as before.
+func TestFreshOpenConflictTakesOverOnce(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ takeover int
+ location string
+ conflicts int
+ want string
+ }{
+ {"taken back", 303, "/m/room", 1, "admitted"},
+ {"takeover refused", 404, "", 1, "replaced"},
+ {"sign-in redirect is a refusal", 303, "/signin", 1, "replaced"},
+ {"only once", 303, "/m/room", 2, "replaced"},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ var mu sync.Mutex
+ opens, takeovers := 0, 0
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+ switch req.URL.Path {
+ case "/m/room/ice":
+ fmt.Fprint(w, `{"iceServers":[],"expiresAt":0,"renewAt":0}`)
+ case "/m/room/signal":
+ w.WriteHeader(200)
+ case "/m/room/takeover":
+ mu.Lock()
+ takeovers++
+ mu.Unlock()
+ if req.Method != http.MethodPost || req.Header.Get("Origin") == "" {
+ t.Error("takeover must be a POST carrying Origin")
+ }
+ if tc.location != "" {
+ w.Header().Set("Location", tc.location)
+ }
+ w.WriteHeader(tc.takeover)
+ case "/m/room/stream":
+ mu.Lock()
+ opens++
+ n := opens
+ mu.Unlock()
+ if n <= tc.conflicts {
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(409)
+ fmt.Fprint(w, `{"peer":"ghost"}`)
+ return
+ }
+ w.Header().Set("Content-Type", "text/event-stream")
+ fmt.Fprint(w, "data: {\"type\":\"hello\",\"peer\":\"self\",\"boot\":\"boot\"}\n\n")
+ fmt.Fprint(w, "data: {\"type\":\"roster\",\"roster\":{\"peers\":[{\"peer\":\"self\",\"kind\":\"member\"}]}}\n\n")
+ w.(http.Flusher).Flush()
+ <-req.Context().Done()
+ default:
+ w.WriteHeader(404)
+ }
+ }))
+ defer server.Close()
+ states := make(chan Snapshot, 64)
+ r, err := New(server.URL, "room", server.Client(), func(e Event) {
+ if e.State != nil {
+ states <- *e.State
+ }
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err = r.Start(); err != nil {
+ t.Fatal(err)
+ }
+ defer r.Close()
+ timer := time.NewTimer(5 * time.Second)
+ defer timer.Stop()
+ for {
+ select {
+ case s := <-states:
+ got := ""
+ if s.Phase == "admitted" {
+ got = "admitted"
+ } else if s.Phase == "ended" {
+ got = s.Reason
+ }
+ if got == "" {
+ continue
+ }
+ if got != tc.want {
+ t.Fatalf("got %q, want %q", got, tc.want)
+ }
+ mu.Lock()
+ defer mu.Unlock()
+ if takeovers != 1 {
+ t.Fatalf("takeovers = %d, want exactly 1", takeovers)
+ }
+ return
+ case <-timer.C:
+ t.Fatal("no outcome")
+ }
+ }
+ })
+ }
+}
diff --git a/callscore/transport.go b/callscore/transport.go
index 2c4ebd2..082de13 100644
--- a/callscore/transport.go
+++ b/callscore/transport.go
@@ -245,6 +245,34 @@ func (t *transport) info(ctx context.Context) (Info, error) {
return info, nil
}
+// takeover ends this session's own live participation in the room — the
+// web's "Use this tab" (POST /m/{sqid}/takeover), which answers 303 back
+// to the room when it has done so. Any other answer, a redirect to
+// sign-in among them, is a refusal.
+func (t *transport) takeover(ctx context.Context) error {
+ ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+ defer cancel()
+ room := "/m/" + t.room
+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, t.origin+room+"/takeover", nil)
+ if err != nil {
+ return errors.New("cannot build Calls request")
+ }
+ req.Header.Set("Origin", t.origin)
+ res, err := t.http.Do(req)
+ if err != nil {
+ return errors.New("cannot reach Calls")
+ }
+ defer res.Body.Close()
+ _, _ = io.Copy(io.Discard, io.LimitReader(res.Body, maxResponse))
+ if res.StatusCode != http.StatusSeeOther {
+ return &HTTPError{Status: res.StatusCode}
+ }
+ if u, err := url.Parse(res.Header.Get("Location")); err != nil || u.Path != room {
+ return &HTTPError{Status: res.StatusCode}
+ }
+ return nil
+}
+
func (t *transport) shareLink(ctx context.Context) (string, error) {
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
diff --git a/internal/calls/home_native_test.go b/internal/calls/home_native_test.go
index 452283d..b6e5dcd 100644
--- a/internal/calls/home_native_test.go
+++ b/internal/calls/home_native_test.go
@@ -97,6 +97,100 @@ func TestNativeCoreUsesHomeSessionAndResumesCall(t *testing.T) {
}
}
+// Paul's "This meeting is open elsewhere" (Calls 0.3.1): an app that quit
+// or crashed mid-call leaves its participation inside the thirty-second
+// resume window, and the next run of the app, on the same kept session,
+// was refused its fresh stream with 409 — which the core read as
+// "replaced" before the call had begun. The core now takes its own place
+// back through the web's takeover, on the real Home-mode router with its
+// CSRF middleware, and joins.
+func TestNativeCoreTakesBackItsOwnGhostParticipation(t *testing.T) {
+ f := newHomeRouterFixture(t)
+ f.link(t)
+ started := f.do("POST", "/meetings/start", "")
+ match := regexp.MustCompile(`href="/m/([A-Za-z0-9]+)/join"`).FindStringSubmatch(started.Body.String())
+ if len(match) != 2 {
+ t.Fatal("missing new meeting")
+ }
+ room := match[1]
+ server := httptest.NewServer(f.handler)
+ defer server.Close()
+ target, _ := url.Parse(server.URL)
+ cookies := []*http.Cookie{}
+ for _, c := range f.cookies {
+ cookies = append(cookies, c)
+ }
+ client := &http.Client{Transport: nativeRoundTripper(func(req *http.Request) (*http.Response, error) {
+ copy := req.Clone(req.Context())
+ u := *req.URL
+ copy.URL = &u
+ copy.Host = req.URL.Host
+ copy.URL.Host = target.Host
+ copy.URL.Scheme = target.Scheme
+ for _, c := range cookies {
+ copy.AddCookie(c)
+ }
+ return http.DefaultTransport.RoundTrip(copy)
+ })}
+ admitted := func(native *callscore.Room, events chan callscore.Event) string {
+ t.Helper()
+ deadline := time.NewTimer(5 * time.Second)
+ defer deadline.Stop()
+ for {
+ select {
+ case e := <-events:
+ if e.State == nil {
+ continue
+ }
+ if e.State.Phase == "ended" {
+ t.Fatalf("call ended before it began: %q", e.State.Reason)
+ }
+ if e.State.Phase == "admitted" && e.State.Peer != "" {
+ return string(e.State.Peer)
+ }
+ case <-deadline.C:
+ t.Fatal("native client was never admitted")
+ }
+ }
+ }
+ firstEvents := make(chan callscore.Event, 128)
+ first, err := callscore.New(homeTestOrigin, room, client, func(e callscore.Event) {
+ select {
+ case firstEvents <- e:
+ default:
+ }
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := first.Start(); err != nil {
+ t.Fatal(err)
+ }
+ ghost := admitted(first, firstEvents)
+ // The app goes away without leaving: no leave signal, the stream
+ // simply drops, as it does when the process ends.
+ first.Close()
+
+ events := make(chan callscore.Event, 128)
+ second, err := callscore.New(homeTestOrigin, room, client, func(e callscore.Event) { events <- e })
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := second.Start(); err != nil {
+ t.Fatal(err)
+ }
+ defer second.Close()
+ peer := admitted(second, events)
+ if peer == ghost {
+ t.Fatal("the new run should have a participation of its own")
+ }
+ for _, p := range second.Snapshot().Peers {
+ if string(p.Peer) == ghost {
+ t.Fatal("the ghost participation is still in the room")
+ }
+ }
+}
+
func TestNativeCoreCannotUseAgentBearerAsHumanSession(t *testing.T) {
f := newHomeRouterFixture(t)
f.link(t)
diff --git a/native/README.md b/native/README.md
index 862ace6..6e2ed13 100644
--- a/native/README.md
+++ b/native/README.md
@@ -110,6 +110,46 @@ connection already holds does not change and nothing renegotiates — the
native counterpart of the browser's `replaceTrack`. Switches are serialised;
the last pick wins.
+The camera is remembered and watched (`CameraDirector`, `ManagedCamera`, held
+by `CameraRecoveryTests`), because of an ATEM Mini that dropped off USB mid-call
+in 0.3.1 and left the self tile on "Waiting for video":
+
+- **Remembered.** A camera chosen in the panel is kept in UserDefaults by its
+ `uniqueID`, with its name for while it is missing. A call starts on it if it
+ is connected, otherwise on the built-in camera (the front one on iPhone),
+ otherwise on whatever there is, and the panel says so ("ATEM Mini isn't
+ connected; using FaceTime HD Camera").
+- **Followed.** Cameras connecting and disconnecting are observed —
+ AVFoundation's connect and disconnect notifications and, on Mac, the
+ discovery session's device list by KVO — with a look every three seconds as
+ well while there is no working camera. The camera in use disconnecting falls
+ back to another at once ("ATEM Mini disconnected; using FaceTime HD Camera");
+ the remembered camera coming back, or appearing later, is switched back to.
+- **Watched.** Frames are counted on their way to the video source. No first
+ frame within three seconds of opening, or none for three seconds after they
+ had been arriving, restarts capture on the same device; still nothing tries
+ the device's next format, then another camera. With nothing to fall back to,
+ the camera is named as having no signal and kept, and a frame arriving later
+ is noticed. A camera that fell silent is returned to only when it reconnects.
+- **Formats.** Nearest 640 wide, then nearest 30 frames a second without going
+ over, at a rate the format really supports: 30 where a range covers it, or
+ the format's own rate (an ATEM's 1080p formats each run at exactly one of
+ 23.98, 24, 25, 29.97, 30, 50, 59.94 or 60), set on the device exactly when it
+ is not a whole number. An open that has not finished in six seconds counts as
+ refused.
+- **Restarted** after a capture session runtime error, an interruption ending,
+ or the Mac waking, at most once every five seconds.
+- **Said.** Your own tile says what the camera is doing instead of an endless
+ "Waiting for video": "Starting camera…", "No signal from ATEM Mini — check
+ its input", "Camera disconnected", "No camera found". The panel's Camera row
+ is always there in a call — "No camera found" with **Try again** when there
+ is none, and the picker with a plain-words line when the camera is not
+ working.
+
+The native/browser gate drives the same code with synthetic cameras, and stalls
+one mid-call: the watchdog has to restart it and the browser has to decode the
+picture again on the same connection.
+
The microphone is a picker on both platforms, and on Mac so is the speaker.
On iPhone it is an audio session route. On Mac it is a device on the call's own
WebRTC audio device module — LiveKit's AVAudioEngine module, which lists input
@@ -122,17 +162,24 @@ audio device API at all.
The Mac's choices follow these rules (`AudioRoutes`, held by
`AudioRoutesTests`):
-- **System default** is the first entry of each picker, and where every call
- starts. It follows macOS: a new default device, or the device changing its
- sample rate, is followed by the module itself, which listens for both and
- rebuilds its engine.
-- An explicit choice keeps its device until you change it or the device
- disconnects; a change of system default does not move it.
-- A chosen device that disconnects falls back to the system default, and the
- panel says so.
-- Leaving the call clears the choices; the next call starts on the system
- default.
-- Nothing changes Sound settings.
+- **Remembered** across calls and launches, in UserDefaults: the device's
+ CoreAudio UID, and its name for while it is missing. Calls 0.3.1 forgot them
+ at the end of every call. The panel says "Calls remembers these for next
+ time."
+- **System default** is the first entry of each picker, where someone who has
+ never chosen starts, and a choice like any other (remembered when picked). It
+ follows macOS: a new default device, or the device changing its sample rate,
+ is followed by the module itself, which listens for both and rebuilds its
+ engine.
+- An explicit choice keeps its device; a change of system default does not
+ move it.
+- A chosen device that is not connected — at join or mid-call — puts the call on
+ the system default for now, and the panel says so ("Scarlett 18i8 isn't
+ connected; using MacBook Pro Microphone"). The choice is kept, and the call
+ goes back to the device when it reappears; CoreAudio's device-list property
+ is listened to for that as well as the module's own report. No poller.
+- Nothing changes Sound settings. Mute is not remembered: every call starts
+ unmuted.
The app keeps the choice itself rather than asking the module, because
LiveKit's engine module (M150) cannot say which device it is on —
@@ -194,7 +241,13 @@ echo canceller has removed the far side. `ProcessingPlan` holds the rules:
The audio readout says which suppressor is running and at what share of a
core, and what WebRTC reports as running for echo cancellation, noise
suppression and gain — what it reports, not what was asked. WebRTC skips the
-hook while the microphone is muted, and the readout says so.
+hook while the microphone is muted, and the readout says so. WebRTC opens the
+microphone only once a connection is sending, so a person alone in a call has
+none running yet; until then the processing module reports its construction
+default (16 kHz, everything off), and on Mac the readout says "The microphone
+has not started yet" instead of repeating that. Every initialisation of the
+capture hook is reported to the plan, a repeat of the same rate included, so a
+microphone really capturing at 16 kHz brings WebRTC's own suppressor back.
**On Mac the suppressor is DeepFilterNet 3**; on iPhone it is RNNoise.
diff --git a/native/apple/Sources/CallsInterop/Interop.swift b/native/apple/Sources/CallsInterop/Interop.swift
index fc6d4c7..ddeadee 100644
--- a/native/apple/Sources/CallsInterop/Interop.swift
+++ b/native/apple/Sources/CallsInterop/Interop.swift
@@ -14,6 +14,11 @@ struct Input: Decodable {
}
@main struct Interop {
@MainActor static var screen: SyntheticScreen?
+ @MainActor static let cameras = SyntheticCameras()
+ // A defaults domain of its own, so the gate's camera choices are never
+ // remembered into anyone's real preferences.
+ @MainActor static let camera = ManagedCamera(
+ backend: cameras, defaults: UserDefaults(suiteName: "calls-interop-\(UUID().uuidString)")!)
@MainActor static func main() async {
if CommandLine.arguments.count == 3, CommandLine.arguments[1] == "--lobby-snapshots" {
do { try await LobbySnapshots.save(to: CommandLine.arguments[2]) } catch { exit(1) }
@@ -27,7 +32,7 @@ struct Input: Decodable {
let preview = input.chatSnapshot.map { ChatPreview(call: call, path: $0) }
do {
try await call.join(
- origin: input.origin, room: input.room, cookies: input.cookies, capture: SyntheticVideo())
+ origin: input.origin, room: input.room, cookies: input.cookies, capture: Interop.camera)
} catch {
print("FAIL native start")
exit(1)
@@ -73,6 +78,13 @@ struct Input: Decodable {
print("{\"audioSwitched\":\(succeeded)}")
fflush(stdout)
}
+ // The camera stops sending, and the watchdog has to notice and
+ // restart it: frames come back on the next open.
+ if command == "camera-stall" {
+ await MainActor.run { Interop.cameras.stalled = true }
+ print("{\"cameraStalled\":true}")
+ fflush(stdout)
+ }
if command.hasPrefix("camera ") {
let id = String(command.dropFirst("camera ".count))
await call.useCamera(id)
@@ -128,6 +140,18 @@ struct Input: Decodable {
// place a screen says whose it is.
"screenCaptions": call.screens.map { screenCaption(call.name(of: $0)) },
"sharingScreen": call.sharingScreen,
+ "cameraOpens": Interop.camera.opens,
+ "cameraStatus": {
+ switch call.cameraStatus {
+ case .live?: return "live"
+ case .starting?: return "starting"
+ case .noSignal?: return "no-signal"
+ case .disconnected?: return "disconnected"
+ case .failed?: return "failed"
+ case .noCamera?: return "none"
+ case nil: return ""
+ }
+ }(),
]), let text = String(data: data, encoding: .utf8) {
print(text)
fflush(stdout)
diff --git a/native/apple/Sources/CallsInterop/SyntheticVideo.swift b/native/apple/Sources/CallsInterop/SyntheticVideo.swift
index 7697662..186a6b4 100644
--- a/native/apple/Sources/CallsInterop/SyntheticVideo.swift
+++ b/native/apple/Sources/CallsInterop/SyntheticVideo.swift
@@ -3,39 +3,52 @@ import Foundation
import CallsKit
import LiveKitWebRTC
-@MainActor final class SyntheticVideo: VideoCapture {
- private var source: RTCVideoSource?
+/// Two synthetic cameras behind the app's own camera management
+/// (`ManagedCamera`), so the gate drives the same choice, switching and
+/// recovery code the apps run — only the frames are made up.
+///
+/// The cameras differ only in frame size, so that a switch is something
+/// the far side can *see* rather than something this side merely claims.
+/// There is no camera in CI; what the gate can still hold is the plumbing
+/// — that a switch is serialised, that the track and its m-line survive
+/// it, that frames keep arriving across it, and (`stall`) that a camera
+/// which stops sending is noticed and restarted.
+@MainActor final class SyntheticCameras: CameraBackend {
+ private var delegate: RTCVideoCapturerDelegate?
private var syntheticCapturer: RTCVideoCapturer?
private var task: Task<Void, Never>?
- // Two cameras, differing only in frame size, so that a switch is
- // something the far side can *see* rather than something this side
- // merely claims. There is no camera in CI; what the gate can still hold
- // is the plumbing — that a switch is serialised, that the track and its
- // m-line survive it, and that frames keep arriving across it.
- private var chosen = SyntheticVideo.first
+ private var chosen = SyntheticCameras.first
+ /// Set by the gate: frames stop until the camera is next opened, as an
+ /// ATEM's do when its input changes format.
+ var stalled = false
nonisolated static let first = "synthetic-320"
nonisolated static let second = "synthetic-480"
- private var width: Int { chosen == SyntheticVideo.second ? 480 : 320 }
- private var height: Int { chosen == SyntheticVideo.second ? 360 : 240 }
+ private var width: Int { chosen == SyntheticCameras.second ? 480 : 320 }
+ private var height: Int { chosen == SyntheticCameras.second ? 360 : 240 }
+ var devicesChanged: (@MainActor () -> Void)?
+ var trouble: (@MainActor () -> Void)?
+
+ func requestAccess() async throws {}
- nonisolated var cameras: [CameraDevice] {
+ var devices: [CameraDevice] {
[
- CameraDevice(id: SyntheticVideo.first, name: "Synthetic 320x240"),
- CameraDevice(id: SyntheticVideo.second, name: "Synthetic 480x360"),
+ CameraDevice(id: SyntheticCameras.first, name: "Synthetic 320x240"),
+ CameraDevice(id: SyntheticCameras.second, name: "Synthetic 480x360"),
]
}
- var currentCamera: String? { chosen }
- func use(camera id: String) async throws {
- guard cameras.contains(where: { $0.id == id }) else {
+ func formatCount(_ id: String) -> Int { 1 }
+
+ func open(_ id: String, format: Int, delegate: RTCVideoCapturerDelegate) async throws {
+ guard devices.contains(where: { $0.id == id }) else {
throw NSError(
domain: "Calls", code: 4,
userInfo: [NSLocalizedDescriptionKey: "That camera is no longer available."])
}
chosen = id
- }
- func start(source: RTCVideoSource) async throws {
- self.source = source
- syntheticCapturer = RTCVideoCapturer(delegate: source)
+ stalled = false
+ self.delegate = delegate
+ syntheticCapturer = RTCVideoCapturer(delegate: delegate)
+ task?.cancel()
task = Task { @MainActor [weak self] in
while !Task.isCancelled {
self?.frame()
@@ -43,14 +56,12 @@ import LiveKitWebRTC
}
}
}
- func stop() {
+ func close() async {
task?.cancel()
task = nil
- source = nil
- syntheticCapturer = nil
}
private func frame() {
- guard let source, let syntheticCapturer else { return }
+ guard !stalled, let delegate, let syntheticCapturer else { return }
var pixel: CVPixelBuffer?
CVPixelBufferCreate(
kCFAllocatorDefault, width, height, kCVPixelFormatType_32BGRA,
@@ -63,7 +74,7 @@ import LiveKitWebRTC
CVPixelBufferGetDataSize(pixel))
}
CVPixelBufferUnlockBaseAddress(pixel, [])
- source.capturer(
+ delegate.capturer(
syntheticCapturer,
didCapture: RTCVideoFrame(
buffer: RTCCVPixelBuffer(pixelBuffer: pixel), rotation: ._0,
diff --git a/native/apple/Sources/CallsKit/Audio.swift b/native/apple/Sources/CallsKit/Audio.swift
index 87431b4..ca07f56 100644
--- a/native/apple/Sources/CallsKit/Audio.swift
+++ b/native/apple/Sources/CallsKit/Audio.swift
@@ -1,5 +1,8 @@
import AVFoundation
import Foundation
+#if os(macOS)
+ import CoreAudio
+#endif
import LiveKitWebRTC
@@ -24,20 +27,26 @@ public struct AudioDevice: Identifiable, Hashable, Sendable {
/// do at all (its macOS slice had no audio device API). Until a call has
/// made that module, the Mac lists are empty.
///
-/// On Mac the choices belong to this call, and the rules are:
+/// On Mac the person's choices are remembered — across calls and across
+/// launches — and the rules are:
///
-/// - **System default** is an entry of its own and is where every call
-/// starts. It follows macOS — a new default device, or a device changing
-/// its sample rate, is the module's to follow, and it does (it listens
-/// for both and rebuilds its engine).
-/// - An explicit choice stays until the person changes it or the device
-/// disconnects. A change of system default does not move it.
-/// - A chosen device that disconnects falls back to the system default,
-/// and `routeNote` says so.
-/// - Leaving the call clears the choices, so the next call starts on the
-/// system default again.
+/// - **System default** is an entry of its own, where someone who has
+/// never chosen starts, and a choice like any other: picked, it is what
+/// is remembered. It follows macOS — a new default device, or a device
+/// changing its sample rate, is the module's to follow, and it does (it
+/// listens for both and rebuilds its engine).
+/// - A chosen device is kept by its CoreAudio UID, with its name beside it
+/// for while it is missing. A change of system default does not move it.
+/// - A chosen device that is not connected — at join, or disconnecting
+/// mid-call — puts the call on the system default for now, and
+/// `routeNote` says so ("Scarlett 18i8 isn't connected; using MacBook Pro
+/// Microphone"). The choice is kept, and when the device comes back the
+/// call goes back to it on its own.
/// - Nothing here changes the Mac's Sound settings.
///
+/// Calls 0.3.1 forgot the choices at the end of every call, so a Scarlett
+/// chosen yesterday was the MacBook's microphone today (Paul, 2026-09-24).
+///
/// The choice is kept here rather than read back from the module, because
/// the engine module in this build cannot say which device it is on:
/// `AudioEngineDevice` does not implement `GetRecordingDevice` or
@@ -61,18 +70,28 @@ public struct AudioDevice: Identifiable, Hashable, Sendable {
public let selectable = true
#if os(macOS)
- /// Said when a chosen device has gone and the call has fallen back to
- /// the system default. Cleared by the next choice and by leaving.
+ /// Said while a remembered device is missing and the call is on the
+ /// system default instead. Clears itself when the device is back.
@Published public private(set) var routeNote = ""
private var control: AudioModuleControl?
- /// The person's choices for this call. Nil follows the system default.
- private var chosenInput: String?
- private var chosenOutput: String?
+ private let defaults: UserDefaults
+ /// The device each side of the module was last moved to. Nil is where
+ /// the module starts: following the system default.
+ private var appliedInput: String?
+ private var appliedOutput: String?
+ private var listener: AudioObjectPropertyListenerBlock?
#endif
- public init() {
- refresh()
- }
+ #if os(macOS)
+ public init(defaults: UserDefaults = .standard) {
+ self.defaults = defaults
+ refresh()
+ }
+ #else
+ public init() {
+ refresh()
+ }
+ #endif
public func refresh() {
#if os(iOS)
@@ -93,34 +112,46 @@ public struct AudioDevice: Identifiable, Hashable, Sendable {
}
let ins = control.devices(true)
let outs = control.devices(false)
- var lost: [String] = []
- if let id = chosenInput, !ins.contains(where: { $0.id == id }) {
- chosenInput = nil
- lost.append("microphone")
- // The module has already let go of a device that left its list;
- // asking for the default again does not depend on that.
- if let fallback = ins.first(where: \.isDefault) { _ = control.select(fallback.id, true) }
- }
- if let id = chosenOutput, !outs.contains(where: { $0.id == id }) {
- chosenOutput = nil
- lost.append("speaker")
- if let fallback = outs.first(where: \.isDefault) { _ = control.select(fallback.id, false) }
- }
- if !lost.isEmpty {
- routeNote =
- "The \(lost.joined(separator: " and ")) you chose disconnected. This call is using the system default."
- }
+ let input = follow(AudioPreference.load(input: true, defaults), ins, input: true, applied: &appliedInput)
+ let output = follow(AudioPreference.load(input: false, defaults), outs, input: false, applied: &appliedOutput)
+ update(\.routeNote, [input.note, output.note].compactMap { $0 }.joined(separator: " "))
update(\.inputs, ins.map(AudioRoutes.device))
update(\.outputs, outs.map(AudioRoutes.device))
- update(\.currentInput, chosenInput ?? ins.first(where: \.isDefault)?.id)
- update(\.currentOutput, chosenOutput ?? outs.first(where: \.isDefault)?.id)
+ update(\.currentInput, input.current)
+ update(\.currentOutput, output.current)
update(\.outputName, outputs.first { $0.id == currentOutput }?.name ?? "")
#endif
}
#if os(macOS)
- /// The panel refreshes every second while it is open; publishing only
- /// what changed keeps an open menu from being rebuilt under the pointer.
+ /// One side of the module, brought into line with the remembered
+ /// choice: the chosen device when it is listed, the system default
+ /// when it is not (with a note), and the module moved only when that
+ /// is somewhere other than where it already is.
+ private func follow(
+ _ preference: AudioPreference?, _ list: [ModuleDevice], input: Bool, applied: inout String?
+ ) -> (current: String?, note: String?) {
+ guard let control else { return (nil, nil) }
+ let fallback = list.first(where: \.isDefault)
+ var target = fallback?.id
+ var note: String?
+ if let preference, !preference.id.isEmpty {
+ if list.contains(where: { $0.id == preference.id }) {
+ target = preference.id
+ } else {
+ let what = preference.name.isEmpty ? "The \(input ? "microphone" : "speaker") you chose" : preference.name
+ let using = fallback.map { $0.name } ?? "the system default"
+ note = "\(what) isn’t connected; using \(using)."
+ }
+ }
+ if let target, (applied ?? fallback?.id) != target, control.select(target, input) {
+ applied = target
+ }
+ return (target, note)
+ }
+
+ /// The panel refreshes while it is open; publishing only what changed
+ /// keeps an open menu from being rebuilt under the pointer.
private func update<Value: Equatable>(
_ key: ReferenceWritableKeyPath<AudioRoutes, Value>, _ value: Value
) {
@@ -129,13 +160,35 @@ public struct AudioDevice: Identifiable, Hashable, Sendable {
func attach(_ module: RTCAudioDeviceModule) {
attach(AudioModuleControl(module))
+ listenForDevices()
}
func attach(_ control: AudioModuleControl) {
self.control = control
+ appliedInput = nil
+ appliedOutput = nil
refresh()
}
+ /// CoreAudio's own device list, listened to directly: the module
+ /// reports list changes too, but a remembered device coming back is
+ /// the moment the call should move to it, and that is worth hearing
+ /// twice rather than not at all. No poller.
+ private func listenForDevices() {
+ guard listener == nil else { return }
+ var address = AudioObjectPropertyAddress(
+ mSelector: kAudioHardwarePropertyDevices, mScope: kAudioObjectPropertyScopeGlobal,
+ mElement: kAudioObjectPropertyElementMain)
+ let block: AudioObjectPropertyListenerBlock = { [weak self] _, _ in
+ DispatchQueue.main.async { MainActor.assumeIsolated { self?.refresh() } }
+ }
+ if AudioObjectAddPropertyListenerBlock(
+ AudioObjectID(kAudioObjectSystemObject), &address, DispatchQueue.main, block) == noErr
+ {
+ listener = block
+ }
+ }
+
/// The module lists the system default as a device of its own, which
/// follows whatever macOS is set to. Said as such, so choosing it is
/// understood as "whatever the Mac uses" rather than one microphone.
@@ -145,9 +198,9 @@ public struct AudioDevice: Identifiable, Hashable, Sendable {
selectable: true)
}
- /// Chooses the microphone on Mac, mid-call if one is up. The engine is
- /// reconfigured underneath the running track; nothing renegotiates.
- /// An empty id is the system default.
+ /// Chooses the microphone on Mac, mid-call if one is up, and remembers
+ /// it. The engine is reconfigured underneath the running track;
+ /// nothing renegotiates. An empty id is the system default.
public func use(input id: String) throws {
try choose(id, input: true, what: "microphone")
}
@@ -156,34 +209,22 @@ public struct AudioDevice: Identifiable, Hashable, Sendable {
try choose(id, input: false, what: "speaker")
}
- /// The system default is recorded as no choice at all, so that it is
- /// followed rather than held.
+ /// The system default is remembered as a choice of its own (an empty
+ /// UID), so that it is followed rather than held.
private func choose(_ id: String, input: Bool, what: String) throws {
guard let control else { throw AudioRoutes.gone(what) }
let list = control.devices(input)
let device = id.isEmpty ? list.first(where: \.isDefault) : list.first { $0.id == id }
guard let device, control.select(device.id, input) else { throw AudioRoutes.gone(what) }
- if input { chosenInput = device.isDefault ? nil : device.id } else {
- chosenOutput = device.isDefault ? nil : device.id
- }
- routeNote = ""
+ if input { appliedInput = device.id } else { appliedOutput = device.id }
+ AudioPreference(id: device.isDefault ? "" : device.id, name: device.isDefault ? "" : device.name)
+ .save(input: input, defaults)
refresh()
}
- /// Leaving the call: the choices were this call's, so the module goes
- /// back to following macOS for the next one. With the call's audio
- /// stopped this only records the device; nothing is opened.
+ /// Leaving the call. The choices are remembered, so there is nothing
+ /// to undo; what is left is to look again at what is connected.
func reset() {
- chosenInput = nil
- chosenOutput = nil
- routeNote = ""
- if let control {
- for input in [true, false] {
- if let fallback = control.devices(input).first(where: \.isDefault) {
- _ = control.select(fallback.id, input)
- }
- }
- }
refresh()
}
@@ -252,6 +293,28 @@ public struct AudioDevice: Identifiable, Hashable, Sendable {
}
#if os(macOS)
+ /// A remembered microphone or speaker: the CoreAudio UID the module
+ /// lists it by, and its name for while it is missing. An empty UID is
+ /// the system default, chosen.
+ struct AudioPreference: Equatable, Sendable {
+ var id: String
+ var name: String
+
+ private static func keys(_ input: Bool) -> (String, String) {
+ input ? ("audioInputUID", "audioInputName") : ("audioOutputUID", "audioOutputName")
+ }
+ static func load(input: Bool, _ defaults: UserDefaults) -> AudioPreference? {
+ let (idKey, nameKey) = keys(input)
+ guard let id = defaults.string(forKey: idKey) else { return nil }
+ return AudioPreference(id: id, name: defaults.string(forKey: nameKey) ?? "")
+ }
+ func save(input: Bool, _ defaults: UserDefaults) {
+ let (idKey, nameKey) = Self.keys(input)
+ defaults.set(id, forKey: idKey)
+ defaults.set(name, forKey: nameKey)
+ }
+ }
+
/// A device as the audio device module lists it.
struct ModuleDevice: Equatable, Sendable {
let id: String
diff --git a/native/apple/Sources/CallsKit/Call.swift b/native/apple/Sources/CallsKit/Call.swift
index 40dff4d..dbfe124 100644
--- a/native/apple/Sources/CallsKit/Call.swift
+++ b/native/apple/Sources/CallsKit/Call.swift
@@ -61,6 +61,13 @@ private final class Events: NSObject, BridgeListenerProtocol {
/// demand, because a picker bound to a plain computed property never
/// learns that a switch finished.
@Published public private(set) var currentCamera: String?
+ /// What the camera is doing (see CameraStatus), and a fallback worth
+ /// saying. Nil status for a capture that does not say, and outside a
+ /// call.
+ @Published public private(set) var cameraStatus: CameraStatus?
+ @Published public private(set) var cameraNote = ""
+ /// A real camera, so the panel shows its row even with nothing in it.
+ @Published public private(set) var offersCameras = false
/// Audio routing belongs to this call.
public let audio = AudioRoutes()
/// "Reduce background noise": the neural suppressor in place of WebRTC's
@@ -126,9 +133,8 @@ private final class Events: NSObject, BridgeListenerProtocol {
try await media.prepare(capture: capture)
guard epoch == joiningEpoch, bridge === client else { throw CancellationError() }
localTrack = media.localVideo
- currentCamera = media.currentCamera
+ cameraChanged()
reportMedia()
- refreshDevices()
media.audio.reduceNoise = reduceNoise
audioProcessing = media.audio.report()
KnockNotice.requestPermission()
@@ -165,6 +171,8 @@ private final class Events: NSObject, BridgeListenerProtocol {
audioHealth = nil
audioProcessing = nil
currentCamera = nil
+ cameraStatus = nil
+ cameraNote = ""
knocks.reset()
cameraSwitches += 1
cameraWork?.cancel()
@@ -230,6 +238,35 @@ private final class Events: NSObject, BridgeListenerProtocol {
audio.refresh()
}
+ /// The capture said something changed on its own: a camera came or
+ /// went, a fallback, the first frame. Everything camera-shaped is read
+ /// back from it.
+ func cameraChanged() {
+ offersCameras = media.offersCameras
+ refreshDevices()
+ let now = media.currentCamera
+ if currentCamera != now { currentCamera = now }
+ let status = media.cameraStatus
+ if cameraStatus != status { cameraStatus = status }
+ let note = media.cameraNote
+ if cameraNote != note { cameraNote = note }
+ }
+
+ /// What your own tile says in place of your picture, or nil to show it.
+ /// Never an open-ended "Waiting for video": it says what it is waiting
+ /// for, or why there is nothing.
+ public var selfPlaceholder: String? {
+ if state?.phase == "ended" { return "Not in the call" }
+ if localTrack == nil { return "Starting camera…" }
+ return cameraStatus?.tileText
+ }
+
+ /// The panel's "Try again", for a call with no working camera.
+ public func retryCamera() async {
+ await media.retryCamera()
+ cameraChanged()
+ }
+
/// Switches camera mid-call. Serialised: two picks are two capture
/// restarts on one device, and run at once the survivor is whichever
/// session happened to open last, which is not the one the person
@@ -255,7 +292,7 @@ private final class Events: NSObject, BridgeListenerProtocol {
// wherever it actually is, and the picker has to follow the camera
// and not the intention.
guard mine == self.cameraSwitches, !Task.isCancelled else { return }
- self.currentCamera = self.media.currentCamera
+ self.cameraChanged()
}
cameraWork = work
await work.value
@@ -372,6 +409,7 @@ private final class Events: NSObject, BridgeListenerProtocol {
remoteScreens = [:]
sharingScreen = false
localTrack = nil
+ cameraChanged()
default: status = "Connecting…"
}
if state.peer.isEmpty {
@@ -513,6 +551,8 @@ private final class Events: NSObject, BridgeListenerProtocol {
values["webrtcEchoCancellation"] = running.echoCancellation == "off" ? 0 : 1
values["webrtcNoiseSuppression"] = running.noiseSuppression == "off" ? 0 : 1
values["webrtcGainControl"] = running.gainControl.hasPrefix("Calls") || running.gainControl == "off" ? 0 : 1
+ // Whether the readout would say the microphone has started.
+ values["audioReadoutStarted"] = running.started ? 1 : 0
return values
}
}
diff --git a/native/apple/Sources/CallsKit/CallAudio.swift b/native/apple/Sources/CallsKit/CallAudio.swift
index 0229345..5489cf0 100644
--- a/native/apple/Sources/CallsKit/CallAudio.swift
+++ b/native/apple/Sources/CallsKit/CallAudio.swift
@@ -65,9 +65,18 @@ public struct ProcessingReport: Equatable, Sendable {
public var gainControl: String
/// Set when WebRTC refused the options asked of it.
public var refused: String?
+ /// False until the microphone is actually capturing. WebRTC opens it
+ /// only once there is someone to send to, so a person alone in a call
+ /// has no microphone running yet, and what the processing module says
+ /// before then is its construction-time default (16 kHz, everything
+ /// off) rather than anything about their microphone.
+ public var started = true
/// One or two plain sentences.
public var summary: String {
+ guard started else {
+ return "The microphone has not started yet. It starts when someone else is in the call, and this will say what is running on it then."
+ }
var lines: [String] = []
switch suppression {
case .off: lines.append("Background noise reduction is off.")
@@ -105,6 +114,7 @@ public struct ProcessingReport: Equatable, Sendable {
private var voiceIsolation = false
private var refused: String?
private(set) var plan: ProcessingPlan
+ private var buffersAtStart: UInt64 = 0
var reduceNoise: Bool {
didSet { if reduceNoise != oldValue { apply() } }
}
@@ -168,12 +178,8 @@ public struct ProcessingReport: Equatable, Sendable {
// same routing code as the app; no device is opened either way.
routes.attach(module)
#endif
- // Set after the factory exists: handed to the module's initialiser, the
- // delegate is never called (seen in the spike, M150). The property is
- // weak; `denoiser` is what keeps it alive.
- processing.capturePostProcessingDelegate = denoiser
- CallAudio.upgrade(denoiser)
- denoiser.enabled = plan.suppressor
+ // The listener first, then the delegate: attaching the delegate can
+ // initialise it at once, and that report used to go nowhere.
denoiser.rateChanged = { [weak self] rate in
DispatchQueue.main.async {
MainActor.assumeIsolated {
@@ -183,6 +189,27 @@ public struct ProcessingReport: Equatable, Sendable {
}
}
}
+ // Set after the factory exists: handed to the module's initialiser, the
+ // delegate is never called (seen in the spike, M150). The property is
+ // weak; `denoiser` is what keeps it alive.
+ processing.capturePostProcessingDelegate = denoiser
+ CallAudio.upgrade(denoiser)
+ denoiser.enabled = plan.suppressor
+ }
+
+ /// Whether the microphone is capturing. The module opens it when the
+ /// first connection starts sending, not when the track is made. Either
+ /// the module says it is recording, or the capture hook has been handed
+ /// audio since this call's track was made (which is also how the
+ /// interop gate's manual rendering shows up).
+ var capturing: Bool {
+ #if os(macOS)
+ return factory.audioDeviceModule.isRecording || denoiser.buffersSeen > buffersAtStart
+ #else
+ // Not asked on iPhone: its module is the platform one, whose
+ // recording state has not been checked against a device here.
+ return true
+ #endif
}
/// The suppressor a call starts on. See `upgrade`.
@@ -205,6 +232,7 @@ public struct ProcessingReport: Equatable, Sendable {
func makeTrack() -> RTCAudioTrack {
let track = factory.audioTrack(withTrackId: "microphone")
self.track = track
+ buffersAtStart = denoiser.buffersSeen
apply()
return track
}
@@ -275,7 +303,8 @@ public struct ProcessingReport: Equatable, Sendable {
if plan.suppressor, gain == "off" { gain = "Calls, after noise reduction" }
return ProcessingReport(
suppression: denoiser.state, echoCancellation: says(state.echoCancellation),
- noiseSuppression: says(state.noiseSuppression), gainControl: gain, refused: refused)
+ noiseSuppression: says(state.noiseSuppression), gainControl: gain, refused: refused,
+ started: capturing)
}
}
diff --git a/native/apple/Sources/CallsKit/CallsModel.swift b/native/apple/Sources/CallsKit/CallsModel.swift
index b0a441f..8485039 100644
--- a/native/apple/Sources/CallsKit/CallsModel.swift
+++ b/native/apple/Sources/CallsKit/CallsModel.swift
@@ -632,6 +632,15 @@ import Foundation
}
}
}
+ /// Leaves the call and returns when the leave has been sent (or given
+ /// up on): for quitting, where the process is about to end.
+ public func leaveNow() async {
+ callEpoch = UUID()
+ starting = false
+ cancelSignIn()
+ await call.leave()
+ joining = false
+ }
public func leave() {
callEpoch = UUID()
starting = false
diff --git a/native/apple/Sources/CallsKit/CameraRecovery.swift b/native/apple/Sources/CallsKit/CameraRecovery.swift
new file mode 100644
index 0000000..10b57c0
--- /dev/null
+++ b/native/apple/Sources/CallsKit/CameraRecovery.swift
@@ -0,0 +1,424 @@
+import Foundation
+
+/// What the camera is doing, said the way the self tile and the device
+/// panel say it.
+///
+/// This exists because of a report against Calls 0.3.1: an ATEM Mini that
+/// had dropped off USB left the self tile on "Waiting for video" and the
+/// device panel with no Camera row at all, and nothing on screen said what
+/// had happened or offered a way back. Every state here has words of its
+/// own, and none of them is "waiting" without saying for what.
+public enum CameraStatus: Equatable, Sendable {
+ /// Opening a camera, and no frame from it yet.
+ case starting(String)
+ /// Frames are arriving from the named camera.
+ case live(String)
+ /// The camera opened but sends nothing — an ATEM with no HDMI input, or
+ /// one that is changing format.
+ case noSignal(String)
+ /// The camera in use went away and there is no other to use.
+ case disconnected(String)
+ /// There is no camera at all.
+ case noCamera
+ /// The camera would not open, and there is no other to use.
+ case failed(String)
+
+ /// What the self tile says in place of video. Nil while frames arrive.
+ public var tileText: String? {
+ switch self {
+ case .starting: return "Starting camera…"
+ case .live: return nil
+ case .noSignal(let name): return "No signal from \(name) — check its input"
+ case .disconnected: return "Camera disconnected"
+ case .noCamera: return "No camera found"
+ case .failed(let name): return "\(name) would not start"
+ }
+ }
+
+ /// The plain-words line under the camera picker. Nil when all is well.
+ public var panelText: String? {
+ switch self {
+ case .live: return nil
+ case .starting(let name): return "Starting \(name)…"
+ case .noSignal(let name):
+ return "\(name) is connected but sending no picture. Check its input, or choose another camera."
+ case .disconnected(let name): return "\(name) disconnected. Calls will use it again when it is back."
+ case .noCamera: return "No camera found"
+ case .failed(let name): return "\(name) would not start. Try again, or choose another camera."
+ }
+ }
+
+ var live: Bool {
+ if case .live = self { return true }
+ return false
+ }
+ var disconnected: Bool {
+ if case .disconnected = self { return true }
+ return false
+ }
+}
+
+/// The person's camera, remembered between calls and launches.
+///
+/// Kept by `uniqueID`, which survives unplugging and replugging, with the
+/// name beside it so the panel can say which camera it is waiting for
+/// while that camera is not connected.
+public struct CameraPreference: Equatable, Sendable {
+ public var id: String
+ public var name: String
+ public init(id: String, name: String) {
+ self.id = id
+ self.name = name
+ }
+
+ static let idKey = "preferredCameraID"
+ static let nameKey = "preferredCameraName"
+
+ public static func load(_ defaults: UserDefaults = .standard) -> CameraPreference? {
+ guard let id = defaults.string(forKey: idKey), !id.isEmpty else { return nil }
+ return CameraPreference(id: id, name: defaults.string(forKey: nameKey) ?? "")
+ }
+
+ public func save(_ defaults: UserDefaults = .standard) {
+ defaults.set(id, forKey: Self.idKey)
+ defaults.set(name, forKey: Self.nameKey)
+ }
+}
+
+/// The first-frame and stall watchdog, as a pure state machine over a
+/// frame counter and a clock.
+///
+/// A camera that opens is not a camera that works: the ATEM Mini opens
+/// happily with nothing plugged into it and delivers no frame, ever. So
+/// the counter the capturer feeds is watched, and a camera with no first
+/// frame within `firstFrame`, or no frame for `stall` after it had been
+/// sending, is escalated one step at a time:
+///
+/// 1. restart capture on the same device and format,
+/// 2. try the device's next format,
+/// 3. fall back to another camera.
+///
+/// Frames arriving again at any point put it back at the bottom of the
+/// ladder. Each step waits for the capture to be opened again before
+/// the clock restarts, so a slow open is never counted as a stall.
+struct CameraWatchdog: Equatable {
+ enum Step: Equatable { case restart, nextFormat, fallback }
+
+ var firstFrame: TimeInterval = 3
+ var stall: TimeInterval = 3
+
+ /// How far up the ladder this device has gone since its last frame.
+ private(set) var level = 0
+ /// Whether a frame has arrived since the capture was last opened.
+ private(set) var flowing = false
+ /// When the clock started: the open, or the last new frame. Nil while
+ /// nothing is being watched (between a step and the next open).
+ private var since: TimeInterval?
+ private var count: UInt64 = 0
+ /// Top of the ladder with nowhere to go: no more steps, but a frame
+ /// arriving still counts as the camera working again.
+ private(set) var exhausted = false
+
+ /// The capture has (re)opened. The clock starts for a first frame.
+ mutating func opened(at now: TimeInterval, frames: UInt64) {
+ since = now
+ count = frames
+ flowing = false
+ exhausted = false
+ }
+
+ /// Nothing left to try: stop escalating, keep listening for frames.
+ mutating func giveUp() {
+ since = nil
+ exhausted = true
+ }
+
+ /// A different camera, or the person's own choice: the ladder starts
+ /// again from the bottom.
+ mutating func reset() {
+ level = 0
+ since = nil
+ flowing = false
+ exhausted = false
+ }
+
+ /// Stops watching until the next open.
+ mutating func pause() { since = nil }
+
+ /// Frames are not expected just now (an iPhone app in the background):
+ /// the clock is held where it is, without climbing the ladder.
+ mutating func hold(at now: TimeInterval, frames: UInt64) {
+ guard since != nil else { return }
+ since = now
+ count = frames
+ }
+
+ /// Called on every tick with the frame counter. Answers the next step
+ /// when one is due, and stops watching until `opened` is called again.
+ mutating func observe(frames: UInt64, at now: TimeInterval) -> Step? {
+ if frames != count {
+ count = frames
+ if since != nil || exhausted {
+ since = now
+ flowing = true
+ level = 0
+ exhausted = false
+ }
+ return nil
+ }
+ guard let started = since else { return nil }
+ guard now - started >= (flowing ? stall : firstFrame) else { return nil }
+ since = nil
+ let step: Step = level == 0 ? .restart : level == 1 ? .nextFormat : .fallback
+ level = min(level + 1, 2)
+ return step
+ }
+}
+
+/// What the camera capture should do next.
+enum CameraCommand: Equatable {
+ /// Open this device with the format at this rank (0 is the best).
+ case open(String, format: Int)
+ /// Stop capturing: there is nothing to capture from.
+ case stop
+}
+
+/// The rules that decide which camera a call is on, and what it says
+/// about it — apart from AVFoundation, so they can be held by tests.
+///
+/// - At join the remembered camera is used if it is connected, otherwise
+/// the built-in one, otherwise whatever there is.
+/// - Choosing a camera remembers it.
+/// - The camera in use disconnecting falls back to another at once, and
+/// says so ("ATEM Mini disconnected; using FaceTime HD Camera").
+/// - The remembered camera reconnecting, or appearing later, is switched
+/// back to, without renegotiation (the capturer feeds the same source).
+/// - A camera that sends nothing is restarted, then tried in its next
+/// format, then fallen back from; with nothing to fall back to, it is
+/// named as having no signal and kept, in case its signal returns.
+/// - Session errors and waking from sleep restart capture, at most once
+/// every `restartSpacing` seconds.
+@MainActor final class CameraDirector {
+ private(set) var devices: [CameraDevice] = []
+ private(set) var active: String?
+ private(set) var format = 0
+ private(set) var status: CameraStatus = .noCamera
+ /// A fallback the person should know about. Cleared by their own
+ /// choice and by the remembered camera coming back.
+ private(set) var note = ""
+ private(set) var watchdog = CameraWatchdog()
+ /// Cameras fallen back from for sending nothing, while they stay
+ /// connected. Not returned to automatically until they reconnect.
+ private var silent: Set<String> = []
+ /// Cameras that would not open, this round.
+ private var refused: Set<String> = []
+ private var lastRestart: TimeInterval = -.infinity
+ var restartSpacing: TimeInterval = 5
+
+ private let defaults: UserDefaults
+ /// How many formats a device has to try, ranked. Supplied by the
+ /// capture; the tests give numbers.
+ private let formats: (String) -> Int
+
+ init(defaults: UserDefaults = .standard, formats: @escaping (String) -> Int) {
+ self.defaults = defaults
+ self.formats = formats
+ }
+
+ var preference: CameraPreference? { CameraPreference.load(defaults) }
+
+ private func name(_ id: String?) -> String {
+ guard let id else { return "The camera" }
+ if let device = devices.first(where: { $0.id == id }) { return device.name }
+ if let preference, preference.id == id, !preference.name.isEmpty { return preference.name }
+ return "The camera"
+ }
+
+ /// The camera a call should be on, given what is connected: the
+ /// remembered one, then the built-in one, then the first. Pure, and
+ /// what `start` and every fallback ask.
+ nonisolated static func choose(
+ _ devices: [CameraDevice], preferred: String?, excluding: Set<String> = []
+ ) -> CameraDevice? {
+ let usable = devices.filter { !excluding.contains($0.id) }
+ if let preferred, let device = usable.first(where: { $0.id == preferred }) { return device }
+ return usable.first(where: \.builtIn) ?? usable.first
+ }
+
+ /// A call starts: pick the camera and open it.
+ func start(devices: [CameraDevice]) -> CameraCommand? {
+ self.devices = devices
+ silent = []
+ refused = []
+ note = ""
+ watchdog.reset()
+ return pick(missingNote: true)
+ }
+
+ /// Chooses from scratch, excluding what has failed this round.
+ private func pick(missingNote: Bool) -> CameraCommand? {
+ let preferred = preference
+ guard let device = Self.choose(devices, preferred: preferred?.id, excluding: silent.union(refused))
+ else {
+ active = nil
+ status = devices.isEmpty ? .noCamera : status
+ if devices.isEmpty { note = "" }
+ return .stop
+ }
+ if missingNote, let preferred, preferred.id != device.id,
+ !devices.contains(where: { $0.id == preferred.id })
+ {
+ let missing = preferred.name.isEmpty ? "Your camera" : preferred.name
+ note = "\(missing) isn’t connected; using \(device.name)"
+ }
+ return open(device.id)
+ }
+
+ private func open(_ id: String, format: Int = 0) -> CameraCommand {
+ if active != id { watchdog.reset() }
+ active = id
+ self.format = format
+ status = .starting(name(id))
+ watchdog.pause()
+ return .open(id, format: format)
+ }
+
+ /// The capture opened `id`; the watchdog starts timing a first frame.
+ func opened(_ id: String, at now: TimeInterval, frames: UInt64) {
+ guard id == active else { return }
+ refused.remove(id)
+ watchdog.opened(at: now, frames: frames)
+ }
+
+ /// The capture could not open `id`. Another camera is tried; with none
+ /// left, the panel says so and offers to try again.
+ func openFailed(_ id: String) -> CameraCommand? {
+ guard id == active else { return nil }
+ let failedName = name(id)
+ refused.insert(id)
+ if let other = Self.choose(devices, preferred: preference?.id, excluding: silent.union(refused)) {
+ note = "\(failedName) would not start; using \(other.name)"
+ return open(other.id)
+ }
+ status = .failed(failedName)
+ watchdog.pause()
+ return nil
+ }
+
+ /// A tick while frames are not expected — the iPhone app in the
+ /// background, where the system stops every camera.
+ func hold(frames: UInt64, at now: TimeInterval) {
+ watchdog.hold(at: now, frames: frames)
+ }
+
+ /// A tick: the frame counter, read.
+ func tick(frames: UInt64, at now: TimeInterval) -> CameraCommand? {
+ guard let id = active else { return nil }
+ let step = watchdog.observe(frames: frames, at: now)
+ if watchdog.flowing, !status.live {
+ status = .live(name(id))
+ silent.remove(id)
+ }
+ guard let step else { return nil }
+ switch step {
+ case .restart:
+ return open(id, format: format)
+ case .nextFormat:
+ if format + 1 < formats(id) { return open(id, format: format + 1) }
+ return fallBack(from: id)
+ case .fallback:
+ return fallBack(from: id)
+ }
+ }
+
+ private func fallBack(from id: String) -> CameraCommand? {
+ let silentName = name(id)
+ if let other = Self.choose(
+ devices, preferred: preference?.id, excluding: silent.union(refused).union([id]))
+ {
+ silent.insert(id)
+ note = "No signal from \(silentName); using \(other.name)"
+ return open(other.id)
+ }
+ // Nothing to fall back to. Stay on it, say so, and keep counting: an
+ // ATEM whose input comes back starts sending on the same session.
+ status = .noSignal(silentName)
+ watchdog.giveUp()
+ return nil
+ }
+
+ /// The connected cameras changed.
+ func devicesChanged(_ now: [CameraDevice]) -> CameraCommand? {
+ let before = Set(devices.map(\.id))
+ let previous = devices
+ devices = now
+ let present = Set(now.map(\.id))
+ // A camera that left is forgiven: when it comes back it is a new
+ // session, and may be sending again.
+ silent.formIntersection(present)
+ refused.formIntersection(present)
+ let preferred = preference
+ if let id = active, !present.contains(id) {
+ let gone = previous.first(where: { $0.id == id })?.name ?? name(id)
+ if let other = Self.choose(now, preferred: preferred?.id, excluding: silent.union(refused)) {
+ note = "\(gone) disconnected; using \(other.name)"
+ return open(other.id)
+ }
+ active = nil
+ note = ""
+ status = .disconnected(gone)
+ watchdog.reset()
+ return .stop
+ }
+ // The remembered camera is back (or has just appeared): go back to it.
+ if let preferred, active != preferred.id, present.contains(preferred.id),
+ !before.contains(preferred.id) || active == nil
+ {
+ note = ""
+ return open(preferred.id)
+ }
+ // Nothing running and something to run on.
+ if active == nil, !now.isEmpty {
+ if case .disconnected = status { note = "" }
+ return pick(missingNote: true)
+ }
+ // "Camera disconnected" stays said until something comes back.
+ if active == nil, now.isEmpty, !status.disconnected { status = .noCamera }
+ return nil
+ }
+
+ /// The person chose a camera: it is remembered and used.
+ func choose(_ id: String) -> CameraCommand? {
+ guard let device = devices.first(where: { $0.id == id }) else { return nil }
+ CameraPreference(id: device.id, name: device.name).save(defaults)
+ silent.remove(id)
+ refused.remove(id)
+ note = ""
+ watchdog.reset()
+ return open(id)
+ }
+
+ /// "Try again": everything that failed is forgiven and the choice is
+ /// made from scratch.
+ func retry() -> CameraCommand? {
+ silent = []
+ refused = []
+ note = ""
+ watchdog.reset()
+ return pick(missingNote: true)
+ }
+
+ /// A capture session error or interruption, or the Mac waking: restart
+ /// on the same device, no more often than `restartSpacing`.
+ func restart(at now: TimeInterval) -> CameraCommand? {
+ guard let id = active, now - lastRestart >= restartSpacing else { return nil }
+ lastRestart = now
+ return open(id, format: format)
+ }
+
+ /// The call is over.
+ func stop() {
+ active = nil
+ watchdog.reset()
+ }
+}
diff --git a/native/apple/Sources/CallsKit/Capture.swift b/native/apple/Sources/CallsKit/Capture.swift
index 55bbc5b..a39a622 100644
--- a/native/apple/Sources/CallsKit/Capture.swift
+++ b/native/apple/Sources/CallsKit/Capture.swift
@@ -1,6 +1,13 @@
import AVFoundation
import Foundation
import LiveKitWebRTC
+import os
+
+#if os(macOS)
+ import AppKit
+#else
+ import UIKit
+#endif
/// One camera the person can choose, named the way a menu wants it.
///
@@ -10,9 +17,13 @@ import LiveKitWebRTC
public struct CameraDevice: Identifiable, Hashable, Sendable {
public let id: String
public let name: String
- public init(id: String, name: String) {
+ /// The Mac's own camera (the iPhone's front one): where a call goes
+ /// when the remembered camera is not connected.
+ public let builtIn: Bool
+ public init(id: String, name: String, builtIn: Bool = false) {
self.id = id
self.name = name
+ self.builtIn = builtIn
}
}
@@ -29,12 +40,33 @@ public struct CameraDevice: Identifiable, Hashable, Sendable {
var cameras: [CameraDevice] { get }
var currentCamera: String? { get }
func use(camera id: String) async throws
+
+ /// Whether the call should always show a Camera row, even with no
+ /// camera to list: true for real cameras, where "No camera found" is
+ /// something a person needs to be told.
+ var offersCameras: Bool { get }
+ /// What the camera is doing, for the self tile and the panel. Nil for a
+ /// capture that does not say.
+ var cameraStatus: CameraStatus? { get }
+ /// A fallback worth saying ("ATEM Mini disconnected; using FaceTime HD
+ /// Camera"), and usually nothing.
+ var cameraNote: String { get }
+ /// Called on the main actor whenever any of the above changes on its
+ /// own — a camera connecting, a fallback, the first frame.
+ func observe(_ changed: @escaping @MainActor () -> Void)
+ /// The panel's "Try again".
+ func retryCamera() async
}
extension VideoCapture {
public var cameras: [CameraDevice] { [] }
public var currentCamera: String? { nil }
public func use(camera id: String) async throws {}
+ public var offersCameras: Bool { false }
+ public var cameraStatus: CameraStatus? { nil }
+ public var cameraNote: String { "" }
+ public func observe(_ changed: @escaping @MainActor () -> Void) {}
+ public func retryCamera() async {}
}
/// A screen, as §5.4 sends one: frames into a source of its own, which
@@ -51,131 +83,514 @@ extension VideoCapture {
func stop()
}
-@MainActor final class CameraCapture: VideoCapture {
- private var capturer: RTCCameraVideoCapturer?
- private var source: RTCVideoSource?
- private var epoch = UUID()
- private(set) var currentCamera: String?
+// ---------------------------------------------------------------------
+// The managed camera: device choice, recovery and the watchdog
+// ---------------------------------------------------------------------
+
+/// Where frames actually come from, under the recovery rules: AVFoundation
+/// in the apps, synthetic frames in the interop gate. Everything about
+/// *which* camera and *when to try again* lives in `ManagedCamera` and
+/// `CameraDirector`; a backend only lists, opens and closes.
+@MainActor public protocol CameraBackend: AnyObject {
+ /// Asks for camera (and microphone) permission. Throws when refused.
+ func requestAccess() async throws
+ /// The cameras connected now.
+ var devices: [CameraDevice] { get }
+ /// How many formats `open` can be asked for on this device, ranked.
+ func formatCount(_ id: String) -> Int
+ /// Opens the device in its ranked format, delivering frames to
+ /// `delegate`. Throws when it will not open.
+ func open(_ id: String, format: Int, delegate: RTCVideoCapturerDelegate) async throws
+ func close() async
+ /// Set by the managed camera: the device list changed.
+ var devicesChanged: (@MainActor () -> Void)? { get set }
+ /// Set by the managed camera: the session reported an error or an
+ /// interruption, or the machine woke. Capture should be restarted.
+ var trouble: (@MainActor () -> Void)? { get set }
+}
+
+/// Counts frames on their way to the video source, which is all the
+/// watchdog needs to know whether a camera is working. The capturer holds
+/// its delegate weakly, so the managed camera holds this.
+final class FrameCounter: NSObject, RTCVideoCapturerDelegate, @unchecked Sendable {
+ private let source: RTCVideoSource
+ private let lock = OSAllocatedUnfairLock(initialState: UInt64(0))
+ init(source: RTCVideoSource) { self.source = source }
+ var count: UInt64 { lock.withLock { $0 } }
+ func capturer(_ capturer: RTCVideoCapturer, didCapture frame: RTCVideoFrame) {
+ lock.withLock { $0 += 1 }
+ source.capturer(capturer, didCapture: frame)
+ }
+}
+
+/// A camera under the recovery rules (see `CameraDirector`): remembered
+/// between calls, followed as devices come and go, and watched for frames.
+///
+/// Every open and close runs one at a time, and an open the director has
+/// since moved on from is skipped, so the last decision is the one that
+/// holds.
+@MainActor public final class ManagedCamera: VideoCapture {
+ private let backend: CameraBackend
+ private let director: CameraDirector
+ private var counter: FrameCounter?
+ private var work: Task<Void, Never>?
+ private var ticker: Task<Void, Never>?
+ private var running = false
+ private var changed: (@MainActor () -> Void)?
+ private(set) public var currentCamera: String?
+ /// Opens performed, for the gate: a stall that recovered is one more.
+ public private(set) var opens = 0
+ private let clock: () -> TimeInterval
+ var tickInterval: Duration = .milliseconds(500)
+
+ public init(
+ backend: CameraBackend, defaults: UserDefaults = .standard,
+ clock: @escaping () -> TimeInterval = { ProcessInfo.processInfo.systemUptime }
+ ) {
+ self.backend = backend
+ self.clock = clock
+ director = CameraDirector(defaults: defaults, formats: { [weak backend] in backend?.formatCount($0) ?? 1 })
+ backend.devicesChanged = { [weak self] in self?.devicesChanged() }
+ backend.trouble = { [weak self] in self?.trouble() }
+ }
+
+ public var cameras: [CameraDevice] { backend.devices }
+ public var offersCameras: Bool { true }
+ public var cameraStatus: CameraStatus? { running ? director.status : nil }
+ public var cameraNote: String { running ? director.note : "" }
+ public func observe(_ changed: @escaping @MainActor () -> Void) { self.changed = changed }
+
+ public func start(source: RTCVideoSource) async throws {
+ try await backend.requestAccess()
+ counter = FrameCounter(source: source)
+ running = true
+ ticker = Task { @MainActor [weak self] in
+ var slow = 0
+ while !Task.isCancelled {
+ guard let self else { return }
+ try? await Task.sleep(for: self.tickInterval)
+ guard !Task.isCancelled else { return }
+ self.tick()
+ // Belt and braces for the device notifications: while there is no
+ // working camera, look again every few seconds.
+ slow += 1
+ if slow % 6 == 0, !(self.director.status.live) { self.devicesChanged() }
+ }
+ }
+ // Returns once the first open has been tried, so a call starts with
+ // its camera (or its reason for having none) already known.
+ await perform(director.start(devices: backend.devices))
+ }
+
+ public func stop() {
+ running = false
+ ticker?.cancel()
+ ticker = nil
+ director.stop()
+ currentCamera = nil
+ let backend = backend
+ let previous = work
+ work = Task { @MainActor in
+ await previous?.value
+ await backend.close()
+ }
+ counter = nil
+ }
+
+ public func use(camera id: String) async throws {
+ guard running else { return }
+ guard let command = director.choose(id) else {
+ throw NSError(
+ domain: "Calls", code: 4,
+ userInfo: [NSLocalizedDescriptionKey: "That camera is no longer available."])
+ }
+ await perform(command)
+ }
+
+ public func retryCamera() async {
+ guard running else { return }
+ await perform(director.retry())
+ }
+
+ private func tick() {
+ guard running, let counter else { return }
+ #if os(iOS)
+ // iOS stops every camera while the app is in the background; that
+ // is not a stall to recover from.
+ if UIApplication.shared.applicationState != .active {
+ director.hold(frames: counter.count, at: clock())
+ return
+ }
+ #endif
+ if let command = director.tick(frames: counter.count, at: clock()) {
+ Task { await self.perform(command) }
+ }
+ notify()
+ }
+
+ private func devicesChanged() {
+ guard running else {
+ changed?()
+ return
+ }
+ if let command = director.devicesChanged(backend.devices) {
+ Task { await self.perform(command) }
+ }
+ notify()
+ }
+
+ private func trouble() {
+ guard running else { return }
+ if let command = director.restart(at: clock()) {
+ Task { await self.perform(command) }
+ }
+ }
+
+ private var published: (CameraStatus, String, String?)?
+ private func notify() {
+ let now = (director.status, director.note, currentCamera)
+ if let published, published == now { return }
+ published = now
+ changed?()
+ }
+
+ /// Runs a command after whatever is running, and returns when it has.
+ private func perform(_ command: CameraCommand?) async {
+ guard let command else {
+ notify()
+ return
+ }
+ let previous = work
+ let task = Task { @MainActor [weak self] in
+ await previous?.value
+ await self?.execute(command)
+ }
+ work = task
+ await task.value
+ }
+
+ private func execute(_ command: CameraCommand) async {
+ guard running else { return }
+ switch command {
+ case .stop:
+ await backend.close()
+ currentCamera = nil
+ notify()
+ case .open(let id, let format):
+ // The director has moved on (a later pick, a fallback): skip.
+ guard director.active == id, director.format == format, let counter else { return }
+ await backend.close()
+ currentCamera = nil
+ notify()
+ guard running, director.active == id else { return }
+ do {
+ try await backend.open(id, format: format, delegate: counter)
+ opens += 1
+ guard running, director.active == id else { return }
+ currentCamera = id
+ director.opened(id, at: clock(), frames: counter.count)
+ } catch {
+ guard running, director.active == id else { return }
+ if let next = director.openFailed(id) {
+ notify()
+ await execute(next)
+ return
+ }
+ }
+ notify()
+ }
+ }
+}
- private static var devices: [AVCaptureDevice] {
+// ---------------------------------------------------------------------
+// AVFoundation
+// ---------------------------------------------------------------------
+
+/// The cameras connected to this device, followed as they come and go:
+/// AVFoundation's connect and disconnect notifications, and on Mac the
+/// discovery session's device list by KVO. One for the process, so the
+/// panel can list cameras whether or not a capture is running.
+@MainActor public final class CameraWatch {
+ public static let shared = CameraWatch()
+ public private(set) var cameras: [CameraDevice] = []
+ private var listeners: [UUID: @MainActor () -> Void] = [:]
+ private var tokens: [NSObjectProtocol] = []
+ #if os(macOS)
+ private let session = AVCaptureDevice.DiscoverySession(
+ deviceTypes: [.builtInWideAngleCamera, .external, .continuityCamera],
+ mediaType: .video, position: .unspecified)
+ private var kvo: NSKeyValueObservation?
+ #endif
+
+ private init() {
+ for name in [AVCaptureDevice.wasConnectedNotification, AVCaptureDevice.wasDisconnectedNotification] {
+ tokens.append(
+ NotificationCenter.default.addObserver(forName: name, object: nil, queue: .main) { _ in
+ MainActor.assumeIsolated { CameraWatch.shared.refresh() }
+ })
+ }
#if os(macOS)
- AVCaptureDevice.DiscoverySession(
- deviceTypes: [.builtInWideAngleCamera, .external, .continuityCamera],
- mediaType: .video, position: .unspecified
- ).devices
+ kvo = session.observe(\.devices, options: [.new]) { _, _ in
+ DispatchQueue.main.async { MainActor.assumeIsolated { CameraWatch.shared.refresh() } }
+ }
+ #endif
+ cameras = Self.list(devices)
+ }
+
+ var devices: [AVCaptureDevice] {
+ #if os(macOS)
+ session.devices
#else
RTCCameraVideoCapturer.captureDevices()
#endif
}
- var cameras: [CameraDevice] {
- Self.devices.map {
- CameraDevice(id: $0.uniqueID, name: $0.localizedName)
+ func device(_ id: String) -> AVCaptureDevice? { devices.first { $0.uniqueID == id } }
+
+ static func list(_ devices: [AVCaptureDevice]) -> [CameraDevice] {
+ devices.filter(\.isConnected).map {
+ #if os(macOS)
+ let builtIn = $0.deviceType == .builtInWideAngleCamera
+ #else
+ let builtIn = $0.position == .front
+ #endif
+ return CameraDevice(id: $0.uniqueID, name: $0.localizedName, builtIn: builtIn)
+ }
+ }
+
+ public func refresh() {
+ let now = Self.list(devices)
+ guard now != cameras else { return }
+ cameras = now
+ for listener in listeners.values { listener() }
+ }
+
+ func listen(_ listener: @escaping @MainActor () -> Void) -> UUID {
+ let id = UUID()
+ listeners[id] = listener
+ return id
+ }
+ func unlisten(_ id: UUID) { listeners.removeValue(forKey: id) }
+}
+
+/// The formats a camera is opened in, best first, and the frame rate to
+/// ask for in each — held apart from AVFoundation so the rule can be
+/// tested with numbers.
+///
+/// Nearest 640 wide first (a call does not need more, and the encoder
+/// scales down anyway), then nearest 30 frames a second without going
+/// over, then the device's own order. The rate asked for is always one
+/// the format supports: 30 where a range covers it, otherwise the
+/// fastest rate at or under 30, otherwise the slowest the format offers.
+/// An ATEM Mini offers 1080p at exactly 23.98, 24, 25, 29.97, 30, 50,
+/// 59.94 and 60 — ranges of one rate each — and asking one of those for a
+/// rate outside its range is an exception AVFoundation raises and WebRTC
+/// swallows, leaving the camera at whatever rate it was on.
+public struct CameraFormatChoice: Equatable, Sendable {
+ public struct Candidate: Equatable, Sendable {
+ public var width: Int
+ public var height: Int
+ /// Each supported range as (min, max) frames a second.
+ public var ranges: [ClosedRange<Double>]
+ public init(width: Int, height: Int, ranges: [ClosedRange<Double>]) {
+ self.width = width
+ self.height = height
+ self.ranges = ranges
+ }
+ }
+
+ /// The rate to run a format at, and whether it is a whole number WebRTC
+ /// can be given as-is.
+ public static func rate(for ranges: [ClosedRange<Double>], target: Double = 30) -> Double? {
+ guard !ranges.isEmpty else { return nil }
+ if ranges.contains(where: { $0.contains(target) }) { return target }
+ let under = ranges.map(\.upperBound).filter { $0 <= target }
+ if let best = under.max() { return best }
+ return ranges.map(\.lowerBound).min()
+ }
+
+ /// The candidates' indexes, best first, one per distinct size and rate.
+ public static func rank(_ candidates: [Candidate], width target: Int = 640) -> [Int] {
+ var seen = Set<String>()
+ return candidates.indices.sorted { a, b in
+ let ca = candidates[a], cb = candidates[b]
+ let wa = abs(ca.width - target), wb = abs(cb.width - target)
+ if wa != wb { return wa < wb }
+ let ra = 30 - (rate(for: ca.ranges) ?? 0), rb = 30 - (rate(for: cb.ranges) ?? 0)
+ if abs(ra) != abs(rb) { return (ra >= 0 ? ra : 1000 - ra) < (rb >= 0 ? rb : 1000 - rb) }
+ return a < b
+ }.filter {
+ let c = candidates[$0]
+ let key = "\(c.width)x\(c.height)@\(rate(for: c.ranges) ?? 0)"
+ return seen.insert(key).inserted
}
}
+}
+
+/// AVFoundation's camera, through WebRTC's capturer.
+@MainActor final class AVCameraBackend: CameraBackend {
+ private var capturer: RTCCameraVideoCapturer?
+ private var observed: [NSObjectProtocol] = []
+ private var listener: UUID?
+ #if os(macOS)
+ private var woke: NSObjectProtocol?
+ #endif
+ var devicesChanged: (@MainActor () -> Void)?
+ var trouble: (@MainActor () -> Void)?
+ /// How long an open may take before it counts as refused. A camera
+ /// that re-enumerates mid-open can leave the completion uncalled.
+ static let openTimeout: TimeInterval = 6
+
+ init() {
+ let watch = CameraWatch.shared
+ listener = watch.listen { [weak self] in self?.devicesChanged?() }
+ #if os(macOS)
+ woke = (
+ NSWorkspace.shared.notificationCenter.addObserver(
+ forName: NSWorkspace.didWakeNotification, object: nil, queue: .main
+ ) { [weak self] _ in
+ MainActor.assumeIsolated {
+ CameraWatch.shared.refresh()
+ self?.trouble?()
+ }
+ })
+ #endif
+ }
- func start(source: RTCVideoSource) async throws {
- let preparing = epoch
+ func requestAccess() async throws {
for kind in [AVMediaType.video, .audio] {
let granted = await AVCaptureDevice.requestAccess(for: kind)
- guard preparing == epoch else { throw CancellationError() }
if !granted {
throw NSError(
domain: "Calls", code: 2,
- userInfo: [
- NSLocalizedDescriptionKey:
- "Allow camera and microphone access in Settings to join."
- ])
+ userInfo: [NSLocalizedDescriptionKey: "Allow camera and microphone access in Settings to join."])
}
}
- let devices = Self.devices
- #if os(iOS)
- let preferredDevice = devices.first { $0.position == .front }
- #else
- let preferredDevice = devices.first
- #endif
- guard let device = preferredDevice ?? devices.first else {
+ }
+
+ var devices: [CameraDevice] {
+ CameraWatch.shared.refresh()
+ return CameraWatch.shared.cameras
+ }
+
+ private func formats(_ device: AVCaptureDevice) -> [AVCaptureDevice.Format] {
+ let all = RTCCameraVideoCapturer.supportedFormats(for: device)
+ let candidates = all.map { format -> CameraFormatChoice.Candidate in
+ let size = CMVideoFormatDescriptionGetDimensions(format.formatDescription)
+ return CameraFormatChoice.Candidate(
+ width: Int(size.width), height: Int(size.height),
+ ranges: format.videoSupportedFrameRateRanges.map { $0.minFrameRate...$0.maxFrameRate })
+ }
+ return CameraFormatChoice.rank(candidates).map { all[$0] }
+ }
+
+ func formatCount(_ id: String) -> Int {
+ guard let device = CameraWatch.shared.device(id) else { return 0 }
+ return formats(device).count
+ }
+
+ func open(_ id: String, format index: Int, delegate: RTCVideoCapturerDelegate) async throws {
+ guard let device = CameraWatch.shared.device(id) else {
throw NSError(
- domain: "Calls", code: 3, userInfo: [NSLocalizedDescriptionKey: "No camera is available."])
+ domain: "Calls", code: 4, userInfo: [NSLocalizedDescriptionKey: "That camera is no longer available."])
}
- self.source = source
- let capture = RTCCameraVideoCapturer(delegate: source)
- capturer = capture
- try await run(capture, device: device, preparing: preparing)
- }
-
- /// Switches the camera without touching the track.
- ///
- /// The capturer keeps delivering into the same RTCVideoSource, so the
- /// RTCVideoTrack every peer connection is already sending does not
- /// change and no renegotiation happens — the native counterpart of the
- /// browser's replaceTrack, and for the same reason: an m-line that
- /// exists and keeps existing is the one thing perfect negotiation never
- /// has to hear about.
- ///
- /// If opening the new camera fails, try to restore the previous capture.
- func use(camera id: String) async throws {
- guard let capture = capturer, source != nil else { return }
- guard let device = Self.devices.first(where: { $0.uniqueID == id })
- else {
+ let ranked = formats(device)
+ guard !ranked.isEmpty else {
throw NSError(
- domain: "Calls", code: 4,
- userInfo: [NSLocalizedDescriptionKey: "That camera is no longer available."])
+ domain: "Calls", code: 3, userInfo: [NSLocalizedDescriptionKey: "No camera is available."])
}
- epoch = UUID()
- let preparing = epoch
- let previous = currentCamera
- await capture.stopCapture()
- currentCamera = nil
- guard preparing == epoch else { throw CancellationError() }
+ let format = ranked[min(index, ranked.count - 1)]
+ let ranges = format.videoSupportedFrameRateRanges
+ let rate = CameraFormatChoice.rate(for: ranges.map { $0.minFrameRate...$0.maxFrameRate }) ?? 30
+ let capture = capturer ?? makeCapturer(delegate: delegate)
+ // WebRTC takes a whole number of frames a second. For a rate that is
+ // not one (29.97), it is given the nearest, and the exact duration
+ // from the format's own range is set on the device afterwards.
+ let fps = max(1, Int(rate.rounded()))
+ let once = OSAllocatedUnfairLock(initialState: false)
do {
- try await run(capture, device: device, preparing: preparing)
+ try await start(capture, device: device, format: format, fps: fps, once: once)
} catch {
- if preparing == epoch, let previous,
- let old = Self.devices.first(where: { $0.uniqueID == previous })
- {
- try? await run(capture, device: old, preparing: preparing)
- }
+ // A failed or abandoned open may still finish later on the
+ // capturer's own queue and start delivering frames from this
+ // device. The capturer is cut loose — no delegate, stopped, not
+ // reused — so those frames reach nobody, and the next open gets a
+ // fresh one.
+ capture.delegate = nil
+ capture.stopCapture(completionHandler: {})
+ capturer = nil
throw error
}
+ if Double(fps) != rate,
+ let range = ranges.first(where: { $0.minFrameRate <= rate && rate <= $0.maxFrameRate })
+ {
+ let duration = rate == range.maxFrameRate ? range.minFrameDuration : range.maxFrameDuration
+ if (try? device.lockForConfiguration()) != nil {
+ device.activeVideoMinFrameDuration = duration
+ device.activeVideoMaxFrameDuration = duration
+ device.unlockForConfiguration()
+ }
+ }
}
- /// The half start and use share: pick the format nearest 640 wide, cap
- /// the frame rate at 30, and start. Held together so a camera chosen
- /// mid-call is opened exactly the way the one chosen at join was.
- private func run(_ capture: RTCCameraVideoCapturer, device: AVCaptureDevice, preparing: UUID)
- async throws
- {
- guard
- let format = RTCCameraVideoCapturer.supportedFormats(for: device).min(by: {
- abs(Int(CMVideoFormatDescriptionGetDimensions($0.formatDescription).width) - 640)
- < abs(Int(CMVideoFormatDescriptionGetDimensions($1.formatDescription).width) - 640)
- })
- else {
- throw NSError(
- domain: "Calls", code: 3, userInfo: [NSLocalizedDescriptionKey: "No camera is available."])
- }
+ private func start(
+ _ capture: RTCCameraVideoCapturer, device: AVCaptureDevice, format: AVCaptureDevice.Format,
+ fps: Int, once: OSAllocatedUnfairLock<Bool>
+ ) async throws {
try await withCheckedThrowingContinuation { (continuation: CheckedContinuation<Void, Error>) in
- capture.startCapture(
- with: device, format: format,
- fps: max(
- 1, min(30, Int(format.videoSupportedFrameRateRanges.map { $0.maxFrameRate }.max() ?? 30)))
- ) { error in
+ let finish: @Sendable (Error?) -> Void = { error in
+ let first = once.withLock { done -> Bool in
+ defer { done = true }
+ return !done
+ }
+ guard first else { return }
if let error { continuation.resume(throwing: error) } else { continuation.resume() }
}
+ capture.startCapture(with: device, format: format, fps: fps) { finish($0) }
+ DispatchQueue.main.asyncAfter(deadline: .now() + AVCameraBackend.openTimeout) {
+ finish(
+ NSError(
+ domain: "Calls", code: 6, userInfo: [NSLocalizedDescriptionKey: "The camera did not start."]))
+ }
}
- if preparing != epoch {
- await capture.stopCapture()
- throw CancellationError()
+ }
+
+ private func makeCapturer(delegate: RTCVideoCapturerDelegate) -> RTCCameraVideoCapturer {
+ let capture = RTCCameraVideoCapturer(delegate: delegate)
+ capturer = capture
+ let session = capture.captureSession
+ // A runtime error, or an interruption ending: WebRTC's capturer
+ // retries a runtime error once itself; these restart it again, rate
+ // limited by the director.
+ for name in [
+ AVCaptureSession.runtimeErrorNotification, AVCaptureSession.interruptionEndedNotification,
+ ] {
+ observed.append(
+ NotificationCenter.default.addObserver(forName: name, object: session, queue: .main) {
+ [weak self] _ in
+ MainActor.assumeIsolated { self?.trouble?() }
+ })
}
- currentCamera = device.uniqueID
+ return capture
}
- func stop() {
- epoch = UUID()
- capturer?.stopCapture()
- capturer = nil
- source = nil
- currentCamera = nil
+ func close() async {
+ guard let capturer else { return }
+ await capturer.stopCapture()
}
+
+ deinit {
+ for token in observed { NotificationCenter.default.removeObserver(token) }
+ #if os(macOS)
+ if let woke { NSWorkspace.shared.notificationCenter.removeObserver(woke) }
+ #endif
+ if let listener {
+ Task { @MainActor in CameraWatch.shared.unlisten(listener) }
+ }
+ }
+}
+
+/// The platform camera: AVFoundation under the recovery rules.
+@MainActor final class CameraCapture {
+ static func make() -> ManagedCamera { ManagedCamera(backend: AVCameraBackend()) }
}
diff --git a/native/apple/Sources/CallsKit/DevicePanel.swift b/native/apple/Sources/CallsKit/DevicePanel.swift
index 121eae2..07a29bc 100644
--- a/native/apple/Sources/CallsKit/DevicePanel.swift
+++ b/native/apple/Sources/CallsKit/DevicePanel.swift
@@ -28,13 +28,7 @@ import SwiftUI
public var body: some View {
VStack(alignment: .leading, spacing: 16) {
- // A picker even for one camera: a Continuity or external camera can
- // arrive while the panel is open, and the list refreshes under it.
- if !call.cameras.isEmpty {
- picker(
- "Camera", selection: $camera,
- options: call.cameras.map { ($0.id, $0.name) }, identifier: "cameraPicker")
- }
+ cameraRow
#if os(iOS)
if audio.inputs.count > 1 {
@@ -92,6 +86,45 @@ import SwiftUI
}
}
+ /// Always there in a call with a real camera — a camera that has gone
+ /// is exactly when a person needs this row, and 0.3.1 hid it then. A
+ /// picker even for one camera: a Continuity or external camera can
+ /// arrive while the panel is open, and the list follows it.
+ @ViewBuilder private var cameraRow: some View {
+ let state = CameraRowState(
+ cameras: call.cameras.count, offers: call.offersCameras, status: call.cameraStatus)
+ if state != .hidden {
+ VStack(alignment: .leading, spacing: 6) {
+ if state == .noCamera {
+ row("Camera", "No camera found")
+ } else {
+ picker(
+ "Camera", selection: $camera,
+ options: call.cameras.map { ($0.id, $0.name) }, identifier: "cameraPicker")
+ }
+ if case .picker(let line, _) = state, let line {
+ Text(line).font(.callout).foregroundStyle(.secondary)
+ .fixedSize(horizontal: false, vertical: true)
+ .accessibilityIdentifier("cameraStatus")
+ }
+ if !call.cameraNote.isEmpty {
+ Label(call.cameraNote, systemImage: "info.circle")
+ .font(.callout).foregroundStyle(.secondary)
+ .fixedSize(horizontal: false, vertical: true)
+ .accessibilityIdentifier("cameraNote")
+ }
+ if state.offersRetry {
+ Button("Try again", systemImage: "arrow.clockwise") {
+ Task { await call.retryCamera() }
+ }
+ .buttonStyle(.bordered)
+ .accessibilityIdentifier("cameraRetry")
+ .accessibilityHint("Looks for cameras again and restarts the camera.")
+ }
+ }
+ }
+ }
+
#if os(iOS)
private var microphonePicker: some View {
VStack(alignment: .leading, spacing: 6) {
@@ -172,7 +205,7 @@ import SwiftUI
options: audio.outputs.map { ($0.id, $0.name) }, identifier: "speakerPicker")
}
if !audio.inputs.isEmpty || !audio.outputs.isEmpty {
- Text("Changes apply to this call only.")
+ Text("Calls remembers these for next time.")
.font(.footnote).foregroundStyle(.secondary)
}
if !audio.routeNote.isEmpty {
@@ -243,3 +276,37 @@ import SwiftUI
#endif
}
}
+
+/// What the panel's Camera row shows, as a rule the tests can hold.
+enum CameraRowState: Equatable {
+ /// A capture with nothing to choose (the interop gate's synthetic one).
+ case hidden
+ /// A real camera was expected and none is connected.
+ case noCamera
+ /// The picker, with a plain-words line when the camera is not working,
+ /// and "Try again" when trying again could help.
+ case picker(line: String?, retry: Bool)
+
+ init(cameras: Int, offers: Bool, status: CameraStatus?) {
+ if cameras == 0 {
+ self = offers ? .noCamera : .hidden
+ return
+ }
+ switch status {
+ case .noCamera?, .live?, nil:
+ self = .picker(line: nil, retry: false)
+ case .starting?:
+ self = .picker(line: status?.panelText, retry: false)
+ case .noSignal?, .disconnected?, .failed?:
+ self = .picker(line: status?.panelText, retry: true)
+ }
+ }
+
+ var offersRetry: Bool {
+ switch self {
+ case .hidden: return false
+ case .noCamera: return true
+ case .picker(_, let retry): return retry
+ }
+ }
+}
diff --git a/native/apple/Sources/CallsKit/GuestJoinView.swift b/native/apple/Sources/CallsKit/GuestJoinView.swift
index b269b38..8e8ea0c 100644
--- a/native/apple/Sources/CallsKit/GuestJoinView.swift
+++ b/native/apple/Sources/CallsKit/GuestJoinView.swift
@@ -49,8 +49,6 @@ import SwiftUI
.foregroundStyle(.red).fixedSize(horizontal: false, vertical: true)
.accessibilityLabel("Error: \(problem)")
}
- } header: {
- Text("Invite link")
} footer: {
Text(
"Paste the link someone sent you. Your browser opens to confirm who you are for this meeting; nothing is saved on this device, and your Calls server stays \(model.serverHost)."
diff --git a/native/apple/Sources/CallsKit/MacCallView.swift b/native/apple/Sources/CallsKit/MacCallView.swift
index cfb36d5..7725a84 100644
--- a/native/apple/Sources/CallsKit/MacCallView.swift
+++ b/native/apple/Sources/CallsKit/MacCallView.swift
@@ -4,6 +4,7 @@
public struct MacCallView: View {
@ObservedObject private var call: Call
private let leave: () -> Void
+ private let rejoin: (() -> Void)?
@State private var grid: Bool
@State private var showingChat: Bool
@State private var chosen: StageItem?
@@ -12,10 +13,12 @@
@StateObject private var chatWindow = MacChatWindow()
public init(
- call: Call, grid: Bool = false, showingChat: Bool = false, leave: @escaping () -> Void
+ call: Call, grid: Bool = false, showingChat: Bool = false, leave: @escaping () -> Void,
+ rejoin: (() -> Void)? = nil
) {
self.call = call
self.leave = leave
+ self.rejoin = rejoin
_grid = State(initialValue: grid)
_showingChat = State(initialValue: showingChat)
}
@@ -101,6 +104,12 @@
}
}
if call.state?.phase == "full" { Button("Try joining again") { call.retryJoin() } }
+ // An ended call used to stay on screen as a call with no way back
+ // into it but Leave and the lobby.
+ if call.state?.phase == "ended", let rejoin {
+ Button("Join again", action: rejoin).buttonStyle(.borderedProminent)
+ .accessibilityIdentifier("joinAgain")
+ }
controls
}
.onAppear {
@@ -147,7 +156,8 @@
.aspectRatio(16 / 9, contentMode: .fit)
}
MacVideoTile(
- track: call.localTrack, label: "You", mirrored: true, cameraOff: call.cameraOff
+ track: call.localTrack, label: "You", mirrored: true, cameraOff: call.cameraOff,
+ placeholder: call.selfPlaceholder
)
.aspectRatio(16 / 9, contentMode: .fit)
ForEach(peers) { peer in
@@ -192,7 +202,8 @@
.frame(width: size.width, height: size.height)
}
MacVideoTile(
- track: call.localTrack, label: "You", mirrored: true, cameraOff: call.cameraOff
+ track: call.localTrack, label: "You", mirrored: true, cameraOff: call.cameraOff,
+ placeholder: call.selfPlaceholder
)
.frame(width: size.width, height: size.height)
}
diff --git a/native/apple/Sources/CallsKit/MacVideoTile.swift b/native/apple/Sources/CallsKit/MacVideoTile.swift
index a0619b6..9ba4ab0 100644
--- a/native/apple/Sources/CallsKit/MacVideoTile.swift
+++ b/native/apple/Sources/CallsKit/MacVideoTile.swift
@@ -14,11 +14,14 @@
/// the edge of a screen is where its menu bar and its scroll bars are,
/// and a face survives losing its margins where a document does not.
var screen = false
+ /// Your own tile: what the camera is doing, in place of a picture that
+ /// is not arriving (see `Call.selfPlaceholder`). Nil shows the video.
+ var placeholder: String? = nil
var body: some View {
ZStack {
Color.black
- if let track, !cameraOff {
+ if let track, !cameraOff, placeholder == nil {
MacVideoView(track: track, fit: screen).scaleEffect(x: mirrored ? -1 : 1, y: 1)
} else {
VStack(spacing: 12) {
@@ -27,7 +30,7 @@
? "video.slash" : screen ? "rectangle.on.rectangle" : "person.crop.circle"
)
.font(.largeTitle)
- Text(waiting).font(.callout)
+ Text(waiting).font(.callout).multilineTextAlignment(.center).padding(.horizontal, 12)
}.foregroundStyle(.white.opacity(0.8))
}
}
@@ -40,10 +43,11 @@
.clipShape(RoundedRectangle(cornerRadius: 16))
.accessibilityElement(children: .ignore)
.accessibilityLabel(tileAccessibilityLabel(label, muted: muted))
- .accessibilityValue(cameraOff ? "Camera off" : track == nil ? waiting : "Video on")
+ .accessibilityValue(cameraOff ? "Camera off" : track == nil || placeholder != nil ? waiting : "Video on")
}
private var waiting: String {
- cameraOff ? "Camera off" : screen ? "Waiting for the screen" : "Waiting for video"
+ cameraOff
+ ? "Camera off" : placeholder ?? (screen ? "Waiting for the screen" : "Waiting for video")
}
}
diff --git a/native/apple/Sources/CallsKit/Media.swift b/native/apple/Sources/CallsKit/Media.swift
index fa1f692..bb0360b 100644
--- a/native/apple/Sources/CallsKit/Media.swift
+++ b/native/apple/Sources/CallsKit/Media.swift
@@ -37,8 +37,10 @@ import LiveKitWebRTC
let source = factory.videoSource()
localVideo = factory.videoTrack(with: source, trackId: "camera")
localAudio = audio.makeTrack()
- let capture = supplied ?? CameraCapture()
+ let capture = supplied ?? CameraCapture.make()
self.capture = capture
+ offersCameras = capture.offersCameras
+ capture.observe { [weak self] in self?.call?.cameraChanged() }
try await capture.start(source: source)
#if os(iOS)
// A supplied audio device owns its session configuration (SDK contract).
@@ -61,9 +63,31 @@ import LiveKitWebRTC
/// The cameras the running capture can offer, and the one it is on.
/// Empty before prepare, and empty for the interop executable's
/// synthetic frames, which is how the picker knows not to appear.
- var cameras: [CameraDevice] { capture?.cameras ?? [] }
+ ///
+ /// With real cameras the list stays live after the capture has gone
+ /// (the call ended under the person), so the panel can still say what
+ /// is connected and a choice there is remembered for next time.
+ var cameras: [CameraDevice] {
+ if let capture { return capture.cameras }
+ return offersCameras ? CameraWatch.shared.cameras : []
+ }
var currentCamera: String? { capture?.currentCamera }
- func use(camera id: String) async throws { try await capture?.use(camera: id) }
+ /// Whether this call's camera is a real one, whose row the panel always
+ /// shows. Kept past the capture, for the same reason as `cameras`.
+ private(set) var offersCameras = false
+ var cameraStatus: CameraStatus? { capture?.cameraStatus }
+ var cameraNote: String { capture?.cameraNote ?? "" }
+ func use(camera id: String) async throws {
+ if let capture {
+ try await capture.use(camera: id)
+ } else if offersCameras, let device = CameraWatch.shared.cameras.first(where: { $0.id == id }) {
+ CameraPreference(id: device.id, name: device.name).save()
+ }
+ }
+ func retryCamera() async {
+ CameraWatch.shared.refresh()
+ await capture?.retryCamera()
+ }
func closePeers() {
for (_, p) in peers { p.close() }
diff --git a/native/apple/Sources/CallsKit/MiniWindow.swift b/native/apple/Sources/CallsKit/MiniWindow.swift
index bcf8172..d5c8fcf 100644
--- a/native/apple/Sources/CallsKit/MiniWindow.swift
+++ b/native/apple/Sources/CallsKit/MiniWindow.swift
@@ -287,7 +287,8 @@
muted: call.isMuted(peer), cameraOff: call.isCameraOff(peer))
case nil:
MacVideoTile(
- track: call.localTrack, label: "You", mirrored: true, cameraOff: call.cameraOff)
+ track: call.localTrack, label: "You", mirrored: true, cameraOff: call.cameraOff,
+ placeholder: call.selfPlaceholder)
}
}
// The tile is a picture, not a target: clicks and drags go through
diff --git a/native/apple/Sources/CallsKit/NoiseSuppression.swift b/native/apple/Sources/CallsKit/NoiseSuppression.swift
index 8cdb00e..fd37649 100644
--- a/native/apple/Sources/CallsKit/NoiseSuppression.swift
+++ b/native/apple/Sources/CallsKit/NoiseSuppression.swift
@@ -208,6 +208,9 @@ public final class CaptureDenoiser: NSObject, RTCAudioCustomProcessingDelegate,
var channels = 0
var initialised = false
var frames: UInt64 = 0
+ /// Every buffer WebRTC has handed over, processed or not: proof that
+ /// the microphone is capturing.
+ var buffers: UInt64 = 0
var busySeconds: Double = 0
var lastFrame: Double = -.infinity
var cpu: Double = 0
@@ -266,17 +269,25 @@ public final class CaptureDenoiser: NSObject, RTCAudioCustomProcessingDelegate,
// MARK: RTCAudioCustomProcessingDelegate
public func audioProcessingInitialize(sampleRate sampleRateHz: Int, channels: Int) {
- let changed = lock.withLock { s -> Bool in
- let changed = !s.initialised || s.rate != sampleRateHz
+ lock.withLock { s in
s.rate = sampleRateHz
s.channels = channels
s.initialised = true
- return changed
}
- if changed { rateChanged?(sampleRateHz) }
+ // Every time, not only on a change. WebRTC initialises its processing
+ // at 16 kHz when the module is made, long before any microphone opens,
+ // and that first call can arrive before anyone is listening; a
+ // Bluetooth headset that really does capture at 16 kHz then looked
+ // like "no change", and the plan never heard about it — leaving
+ // WebRTC's own suppressor off while the readout said it was standing
+ // in. The listener decides what is a change.
+ rateChanged?(sampleRateHz)
}
+ public var buffersSeen: UInt64 { lock.withLock { $0.buffers } }
+
public func audioProcessingProcess(audioBuffer: RTCAudioBuffer) {
+ lock.withLock { $0.buffers += 1 }
guard audioBuffer.channels == 1 else { return }
process(audioBuffer.rawBuffer(forChannel: 0), frames: audioBuffer.frames)
}
diff --git a/native/apple/Sources/CallsMac/CallsMac.swift b/native/apple/Sources/CallsMac/CallsMac.swift
index 250a39e..22234d2 100644
--- a/native/apple/Sources/CallsMac/CallsMac.swift
+++ b/native/apple/Sources/CallsMac/CallsMac.swift
@@ -52,6 +52,31 @@ final class CallsAppDelegate: NSObject, NSApplicationDelegate {
// and the consequence is simply no push.
}
+ /// Quitting mid-call leaves the call first. Without this the process
+ /// ended with its participation still in the room, inside the server's
+ /// thirty-second resume window, and the next launch — an update's
+ /// relaunch, most often — joined as a second participation of the same
+ /// session and was told "This meeting is open elsewhere". (The core now
+ /// takes its own place back in that case too; leaving is still the
+ /// right thing to do, because the others in the call see you go at
+ /// once rather than thirty seconds later.) The leave is given three
+ /// seconds; the core's own request gives up after two.
+ func applicationShouldTerminate(_ sender: NSApplication) -> NSApplication.TerminateReply {
+ guard let model = CallsAppDelegate.model, model.joining else { return .terminateNow }
+ var replied = false
+ let reply = { @MainActor in
+ guard !replied else { return }
+ replied = true
+ sender.reply(toApplicationShouldTerminate: true)
+ }
+ Task { @MainActor in
+ await model.leaveNow()
+ reply()
+ }
+ DispatchQueue.main.asyncAfter(deadline: .now() + 3) { MainActor.assumeIsolated { reply() } }
+ return .terminateLater
+ }
+
func applicationDidFinishLaunching(_ notification: Notification) {
if let url = Bundle.main.url(forResource: "MacIcon", withExtension: "png")
?? Bundle.module.url(forResource: "MacIcon", withExtension: "png")
@@ -66,7 +91,7 @@ struct ContentView: View {
var body: some View {
Group {
if model.joining {
- MacCallView(call: model.call, leave: model.leave).padding(24)
+ MacCallView(call: model.call, leave: model.leave, rejoin: model.join).padding(24)
} else {
CallsLobby(model: model)
}
diff --git a/native/apple/Tests/CallsKitTests/AudioHealthTests.swift b/native/apple/Tests/CallsKitTests/AudioHealthTests.swift
index 2597fe3..6c7f093 100644
--- a/native/apple/Tests/CallsKitTests/AudioHealthTests.swift
+++ b/native/apple/Tests/CallsKitTests/AudioHealthTests.swift
@@ -97,13 +97,20 @@ final class AudioHealthTests: XCTestCase {
final class AudioRoutesTests: XCTestCase {
@MainActor func testMacListsTheModuleDevicesAndCanChoose() throws {
#if os(macOS)
- let routes = AudioRoutes()
+ let suite = "calls-audio-module-\(UUID().uuidString)"
+ let store = UserDefaults(suiteName: suite)!
+ defer { store.removePersistentDomain(forName: suite) }
+ let routes = AudioRoutes(defaults: store)
XCTAssertTrue(routes.inputs.isEmpty, "no module yet, so nothing to list")
XCTAssertThrowsError(try routes.use(input: "anything"))
let audio = CallAudio(routes: routes, audioDevice: nil)
_ = audio
XCTAssertTrue(routes.selectable)
+ // What this machine's module lists, for the record: every input
+ // device macOS has, named as macOS names it, the default among them.
+ for device in routes.inputs { print("audio input: \(device.name) [\(device.id)]") }
+ for device in routes.outputs { print("audio output: \(device.name) [\(device.id)]") }
// A machine with no input at all is possible, so this asserts the
// shape rather than the presence.
for device in routes.inputs + routes.outputs {
@@ -127,10 +134,48 @@ final class AudioRoutesTests: XCTestCase {
XCTAssertNoThrow(try routes.use(input: other.id))
XCTAssertEqual(routes.currentInput, other.id)
routes.reset()
- XCTAssertEqual(routes.currentInput, routes.inputs.first?.id, "leaving goes back to the default")
+ XCTAssertEqual(routes.currentInput, other.id, "leaving keeps the choice for next time")
}
#else
throw XCTSkip("macOS only")
#endif
}
}
+
+/// The readout before the microphone has started. Paul's 0.3.1 screenshot,
+/// alone in a call, read "The microphone runs at 16 kHz … Echo
+/// cancellation: off. Noise suppression: off." Those are exactly what the
+/// module says before any microphone opens: WebRTC opens it when the first
+/// connection starts sending, and until then its processing sits at its
+/// construction default of 16 kHz with nothing applied.
+final class ProcessingReadoutTests: XCTestCase {
+ func testNotStartedSaysSoInsteadOfStaleValues() {
+ let report = ProcessingReport(
+ suppression: .unsupportedRate(16_000), echoCancellation: "off", noiseSuppression: "off",
+ gainControl: "off", refused: nil, started: false)
+ XCTAssertFalse(report.summary.contains("16 kHz"))
+ XCTAssertFalse(report.summary.contains("Echo cancellation: off"))
+ XCTAssertTrue(report.summary.hasPrefix("The microphone has not started yet."))
+ }
+
+ @MainActor func testAloneInACallTheMicrophoneHasNotStarted() throws {
+ #if os(macOS)
+ let suite = "calls-readout-\(UUID().uuidString)"
+ let store = UserDefaults(suiteName: suite)!
+ defer { store.removePersistentDomain(forName: suite) }
+ let audio = CallAudio(routes: AudioRoutes(defaults: store), audioDevice: nil)
+ let track = audio.makeTrack()
+ _ = track
+ let report = audio.report()
+ // The evidence: with a track made and nothing sending, the module is
+ // not recording, and the processing reports the 16 kHz default with
+ // echo cancellation off — the screenshot's numbers.
+ XCTAssertFalse(audio.capturing)
+ print("pre-start readout: suppression=\(audio.denoiser.state) ec=\(report.echoCancellation) ns=\(report.noiseSuppression) gain=\(report.gainControl)")
+ XCTAssertFalse(report.started)
+ XCTAssertTrue(report.summary.hasPrefix("The microphone has not started yet."))
+ #else
+ throw XCTSkip("macOS only")
+ #endif
+ }
+}
diff --git a/native/apple/Tests/CallsKitTests/AudioRoutesTests.swift b/native/apple/Tests/CallsKitTests/AudioRoutesTests.swift
index 9bd5101..90ec0db 100644
--- a/native/apple/Tests/CallsKitTests/AudioRoutesTests.swift
+++ b/native/apple/Tests/CallsKitTests/AudioRoutesTests.swift
@@ -2,7 +2,7 @@
import XCTest
@testable import CallsKit
- /// The Mac's per-call routing rules, held against a stand-in for the
+ /// The Mac's remembered routing rules, held against a stand-in for the
/// audio device module so no microphone is opened. The module itself
/// follows the system default and sample-rate changes; these hold what
/// AudioRoutes adds on top of it.
@@ -34,62 +34,82 @@
}
}
+ private var store: UserDefaults!
+ private var storeName = ""
+
+ override func setUp() {
+ super.setUp()
+ storeName = "calls-audio-tests-\(UUID().uuidString)"
+ store = UserDefaults(suiteName: storeName)
+ }
+ override func tearDown() {
+ store.removePersistentDomain(forName: storeName)
+ super.tearDown()
+ }
+
private func routes(_ module: Module) -> AudioRoutes {
- let routes = AudioRoutes()
+ let routes = AudioRoutes(defaults: store)
routes.attach(module.control())
return routes
}
- func testEveryCallStartsOnTheSystemDefault() {
- let routes = routes(Module())
+ func testFirstCallStartsOnTheSystemDefault() {
+ let module = Module()
+ let routes = routes(module)
XCTAssertEqual(routes.currentInput, "default")
XCTAssertEqual(routes.currentOutput, "default")
XCTAssertEqual(routes.inputs.first?.name, "System default (Built-in Microphone)")
XCTAssertEqual(routes.outputName, "System default (Built-in Speakers)")
+ XCTAssertTrue(module.selected.isEmpty, "the module already follows the default")
}
- func testSelectionsAreIndependentAndResetForNextCall() throws {
- let module = Module()
- let routes = routes(module)
- let other = self.routes(Module())
+ /// Paul, 2026-09-24: "it should remember the Scarlett between app
+ /// launches". A new AudioRoutes on the same defaults is a new launch.
+ func testChoicesAreRememberedAcrossCallsAndLaunches() throws {
+ let routes = routes(Module())
try routes.use(input: "usb")
try routes.use(output: "headset")
+ routes.reset() // leaving the call
XCTAssertEqual(routes.currentInput, "usb")
XCTAssertEqual(routes.currentOutput, "headset")
- XCTAssertEqual(module.selected.map(\.id), ["usb", "headset"])
- XCTAssertEqual(other.currentInput, "default")
- XCTAssertEqual(other.currentOutput, "default")
- routes.reset()
- XCTAssertEqual(routes.currentInput, "default")
- XCTAssertEqual(routes.currentOutput, "default")
- XCTAssertEqual(
- module.selected.suffix(2).map(\.id), ["default", "default"],
- "leaving puts the module back on the system default for the next call")
+
+ let next = Module()
+ let launch = self.routes(next)
+ XCTAssertEqual(launch.currentInput, "usb")
+ XCTAssertEqual(launch.currentOutput, "headset")
+ XCTAssertEqual(next.selected.map(\.id), ["usb", "headset"], "applied at join, before anyone asks")
+ XCTAssertEqual(store.string(forKey: "audioInputUID"), "usb")
+ XCTAssertEqual(store.string(forKey: "audioInputName"), "usb")
}
- func testEmptyIdIsTheSystemDefault() throws {
- let module = Module()
- let routes = routes(module)
+ func testSystemDefaultIsRememberedWhenPicked() throws {
+ let routes = routes(Module())
try routes.use(input: "usb")
try routes.use(input: "")
XCTAssertEqual(routes.currentInput, "default")
- XCTAssertEqual(module.selected.last?.id, "default")
+ XCTAssertEqual(store.string(forKey: "audioInputUID"), "", "the default is a choice of its own")
+ let next = Module()
+ let launch = self.routes(next)
+ XCTAssertEqual(launch.currentInput, "default")
+ XCTAssertTrue(next.selected.isEmpty)
}
- func testRefusedSwitchKeepsPreviousSelection() throws {
+ func testMissingDeviceAtJoinUsesTheDefaultSaysSoAndReturns() {
+ AudioPreference(id: "scarlett", name: "Scarlett 18i8").save(input: true, store)
let module = Module()
let routes = routes(module)
- try routes.use(input: "usb")
- XCTAssertThrowsError(try routes.use(input: "missing")) { error in
- XCTAssertEqual((error as NSError).code, 5)
- }
- XCTAssertEqual(routes.currentInput, "usb")
- XCTAssertThrowsError(try routes.use(output: "missing"))
- XCTAssertEqual(routes.currentOutput, "default")
- XCTAssertEqual(module.selected.count, 1, "a refused choice never reaches the module")
+ XCTAssertEqual(routes.currentInput, "default")
+ XCTAssertEqual(routes.routeNote, "Scarlett 18i8 isn’t connected; using Built-in Microphone.")
+ XCTAssertTrue(module.selected.isEmpty)
+ module.inputs.append("scarlett")
+ routes.refresh()
+ XCTAssertEqual(routes.currentInput, "scarlett")
+ XCTAssertEqual(module.selected.last?.id, "scarlett")
+ XCTAssertEqual(routes.routeNote, "")
+ XCTAssertEqual(AudioPreference.load(input: true, store)?.id, "scarlett", "the preference was kept")
}
- func testDisconnectedInputFallsBackWithoutChangingSelectedOutput() throws {
+ func testDisconnectMidCallFallsBackAndComesBackOnItsOwn() throws {
let module = Module()
let routes = routes(module)
try routes.use(input: "usb")
@@ -100,30 +120,39 @@
XCTAssertEqual(routes.currentOutput, "headset")
XCTAssertEqual(module.selected.last?.id, "default")
XCTAssertEqual(module.selected.last?.input, true)
- XCTAssertTrue(routes.routeNote.contains("microphone"))
- XCTAssertFalse(routes.routeNote.contains("speaker"))
- // The device coming back does not take the call back to it: the
- // fallback is where the call is now, and the note stays until the
- // person chooses again.
+ XCTAssertEqual(routes.routeNote, "usb isn’t connected; using Built-in Microphone.")
+ let count = module.selected.count
+ routes.refresh()
+ XCTAssertEqual(module.selected.count, count, "the module is moved once, not on every look")
module.inputs = ["built-in", "usb"]
routes.refresh()
- XCTAssertEqual(routes.currentInput, "default")
- XCTAssertFalse(routes.routeNote.isEmpty)
- try routes.use(input: "usb")
+ XCTAssertEqual(routes.currentInput, "usb")
+ XCTAssertEqual(module.selected.last?.id, "usb")
XCTAssertEqual(routes.routeNote, "")
}
+ func testRefusedSwitchKeepsPreviousSelection() throws {
+ let module = Module()
+ let routes = routes(module)
+ try routes.use(input: "usb")
+ XCTAssertThrowsError(try routes.use(input: "missing")) { error in
+ XCTAssertEqual((error as NSError).code, 5)
+ }
+ XCTAssertEqual(routes.currentInput, "usb")
+ XCTAssertThrowsError(try routes.use(output: "missing"))
+ XCTAssertEqual(routes.currentOutput, "default")
+ XCTAssertEqual(module.selected.count, 1, "a refused choice never reaches the module")
+ XCTAssertEqual(store.string(forKey: "audioInputUID"), "usb", "nor the memory")
+ }
+
func testSystemDefaultFollowsChangesButExplicitChoiceDoesNot() throws {
let module = Module()
let routes = routes(module)
- // Following: macOS moves its default and the entry follows it, with
- // nothing asked of the module (it follows the default itself).
module.defaultOutput = "Headset"
routes.refresh()
XCTAssertEqual(routes.currentOutput, "default")
XCTAssertEqual(routes.outputName, "System default (Headset)")
XCTAssertTrue(module.selected.isEmpty)
- // Chosen: a later change of default leaves the choice where it is.
try routes.use(output: "speakers")
let count = module.selected.count
module.defaultOutput = "Built-in Speakers"
@@ -133,7 +162,7 @@
}
func testNoModuleMeansNothingToChoose() {
- let routes = AudioRoutes()
+ let routes = AudioRoutes(defaults: store)
XCTAssertTrue(routes.inputs.isEmpty)
XCTAssertNil(routes.currentInput)
XCTAssertThrowsError(try routes.use(input: ""))
diff --git a/native/apple/Tests/CallsKitTests/CameraRecoveryTests.swift b/native/apple/Tests/CallsKitTests/CameraRecoveryTests.swift
new file mode 100644
index 0000000..db5a2f6
--- /dev/null
+++ b/native/apple/Tests/CallsKitTests/CameraRecoveryTests.swift
@@ -0,0 +1,329 @@
+import LiveKitWebRTC
+import XCTest
+
+@testable import CallsKit
+
+/// The camera rules from Paul's ATEM Mini report against 0.3.1: the camera
+/// remembered, followed as it comes and goes, and watched for frames.
+@MainActor final class CameraRecoveryTests: XCTestCase {
+ private let atem = CameraDevice(id: "atem", name: "ATEM Mini")
+ private let faceTime = CameraDevice(id: "facetime", name: "FaceTime HD Camera", builtIn: true)
+ private let phone = CameraDevice(id: "phone", name: "Paul’s iPhone Camera")
+
+ private func defaults() -> UserDefaults {
+ let name = "calls-camera-tests-\(UUID().uuidString)"
+ let defaults = UserDefaults(suiteName: name)!
+ addTeardownBlock { defaults.removePersistentDomain(forName: name) }
+ return defaults
+ }
+
+ private func director(_ defaults: UserDefaults, formats: Int = 2) -> CameraDirector {
+ CameraDirector(defaults: defaults, formats: { _ in formats })
+ }
+
+ // MARK: Choosing
+
+ func testJoinUsesTheRememberedCamera() {
+ let store = defaults()
+ CameraPreference(id: "atem", name: "ATEM Mini").save(store)
+ let d = director(store)
+ XCTAssertEqual(d.start(devices: [faceTime, atem]), .open("atem", format: 0))
+ XCTAssertEqual(d.status, .starting("ATEM Mini"))
+ XCTAssertEqual(d.note, "")
+ }
+
+ func testJoinFallsBackToTheBuiltInCameraAndSaysSo() {
+ let store = defaults()
+ CameraPreference(id: "atem", name: "ATEM Mini").save(store)
+ let d = director(store)
+ // The built-in camera, not merely the first listed.
+ XCTAssertEqual(d.start(devices: [phone, faceTime]), .open("facetime", format: 0))
+ XCTAssertEqual(d.note, "ATEM Mini isn’t connected; using FaceTime HD Camera")
+ }
+
+ func testWithNothingRememberedTheBuiltInCameraIsFirstChoice() {
+ let d = director(defaults())
+ XCTAssertEqual(d.start(devices: [atem, faceTime]), .open("facetime", format: 0))
+ XCTAssertEqual(d.note, "")
+ XCTAssertEqual(CameraDirector.choose([atem, phone], preferred: nil)?.id, "atem")
+ XCTAssertNil(CameraDirector.choose([], preferred: "atem"))
+ }
+
+ func testNoCameraAtAllIsSaidAndStopped() {
+ let d = director(defaults())
+ XCTAssertEqual(d.start(devices: []), .stop)
+ XCTAssertEqual(d.status, .noCamera)
+ XCTAssertEqual(d.status.tileText, "No camera found")
+ // One arriving later is used.
+ XCTAssertEqual(d.devicesChanged([atem]), .open("atem", format: 0))
+ }
+
+ func testChoosingACameraRemembersIt() {
+ let store = defaults()
+ let d = director(store)
+ _ = d.start(devices: [faceTime, atem])
+ XCTAssertEqual(d.choose("atem"), .open("atem", format: 0))
+ XCTAssertEqual(CameraPreference.load(store), CameraPreference(id: "atem", name: "ATEM Mini"))
+ XCTAssertNil(d.choose("missing"), "a camera that is not connected cannot be chosen")
+ }
+
+ // MARK: Coming and going
+
+ func testActiveCameraDisconnectingFallsBackAtOnceAndSaysSo() {
+ let store = defaults()
+ CameraPreference(id: "atem", name: "ATEM Mini").save(store)
+ let d = director(store)
+ _ = d.start(devices: [faceTime, atem])
+ d.opened("atem", at: 0, frames: 0)
+ XCTAssertEqual(d.devicesChanged([faceTime]), .open("facetime", format: 0))
+ XCTAssertEqual(d.note, "ATEM Mini disconnected; using FaceTime HD Camera")
+ }
+
+ func testRememberedCameraReconnectingIsReturnedTo() {
+ let store = defaults()
+ CameraPreference(id: "atem", name: "ATEM Mini").save(store)
+ let d = director(store)
+ _ = d.start(devices: [faceTime, atem])
+ _ = d.devicesChanged([faceTime])
+ XCTAssertEqual(d.active, "facetime")
+ // Something unrelated changing does not move the call.
+ XCTAssertNil(d.devicesChanged([faceTime, phone]))
+ XCTAssertEqual(d.devicesChanged([faceTime, phone, atem]), .open("atem", format: 0))
+ XCTAssertEqual(d.note, "", "the fallback note goes when the camera is back")
+ }
+
+ func testRememberedCameraAppearingLaterIsSwitchedTo() {
+ let store = defaults()
+ CameraPreference(id: "atem", name: "ATEM Mini").save(store)
+ let d = director(store)
+ XCTAssertEqual(d.start(devices: [faceTime]), .open("facetime", format: 0))
+ XCTAssertEqual(d.devicesChanged([faceTime, atem]), .open("atem", format: 0))
+ }
+
+ func testLastCameraDisconnectingStopsAndSaysDisconnected() {
+ let d = director(defaults())
+ _ = d.start(devices: [atem])
+ XCTAssertEqual(d.devicesChanged([]), .stop)
+ XCTAssertEqual(d.status, .disconnected("ATEM Mini"))
+ XCTAssertEqual(d.status.tileText, "Camera disconnected")
+ XCTAssertEqual(d.devicesChanged([atem]), .open("atem", format: 0), "and back when it is")
+ }
+
+ func testOpenFailureTriesAnotherCameraThenGivesUpWithRetry() {
+ let d = director(defaults())
+ _ = d.start(devices: [atem, faceTime]) // facetime (built in)
+ XCTAssertEqual(d.openFailed("facetime"), .open("atem", format: 0))
+ XCTAssertEqual(d.note, "FaceTime HD Camera would not start; using ATEM Mini")
+ XCTAssertNil(d.openFailed("atem"))
+ XCTAssertEqual(d.status, .failed("ATEM Mini"))
+ XCTAssertEqual(d.retry(), .open("facetime", format: 0), "Try again forgives both")
+ }
+
+ // MARK: The watchdog
+
+ func testWatchdogLadder() {
+ var dog = CameraWatchdog()
+ dog.opened(at: 0, frames: 0)
+ XCTAssertNil(dog.observe(frames: 0, at: 2.9))
+ XCTAssertEqual(dog.observe(frames: 0, at: 3), .restart)
+ XCTAssertNil(dog.observe(frames: 0, at: 10), "nothing more until reopened")
+ dog.opened(at: 10, frames: 0)
+ XCTAssertEqual(dog.observe(frames: 0, at: 13), .nextFormat)
+ dog.opened(at: 13, frames: 0)
+ XCTAssertEqual(dog.observe(frames: 0, at: 16), .fallback)
+ }
+
+ func testWatchdogStallAfterFramesAndRecovery() {
+ var dog = CameraWatchdog()
+ dog.opened(at: 0, frames: 0)
+ XCTAssertNil(dog.observe(frames: 5, at: 1))
+ XCTAssertTrue(dog.flowing)
+ XCTAssertNil(dog.observe(frames: 9, at: 2))
+ XCTAssertNil(dog.observe(frames: 9, at: 4.9))
+ XCTAssertEqual(dog.observe(frames: 9, at: 5), .restart)
+ dog.opened(at: 5, frames: 9)
+ XCTAssertNil(dog.observe(frames: 12, at: 6))
+ XCTAssertEqual(dog.level, 0, "frames put it back at the bottom of the ladder")
+ }
+
+ func testNoFirstFrameRestartsThenNextFormatThenFallsBack() {
+ let store = defaults()
+ CameraPreference(id: "atem", name: "ATEM Mini").save(store)
+ let d = director(store)
+ _ = d.start(devices: [faceTime, atem])
+ d.opened("atem", at: 0, frames: 0)
+ XCTAssertEqual(d.tick(frames: 0, at: 3), .open("atem", format: 0), "restart, same format")
+ d.opened("atem", at: 4, frames: 0)
+ XCTAssertEqual(d.tick(frames: 0, at: 7), .open("atem", format: 1), "then the next format")
+ d.opened("atem", at: 8, frames: 0)
+ XCTAssertEqual(d.tick(frames: 0, at: 11), .open("facetime", format: 0), "then another camera")
+ XCTAssertEqual(d.note, "No signal from ATEM Mini; using FaceTime HD Camera")
+ d.opened("facetime", at: 12, frames: 0)
+ XCTAssertNil(d.tick(frames: 3, at: 12.5))
+ XCTAssertEqual(d.status, .live("FaceTime HD Camera"))
+ // The silent ATEM is not returned to just because something else
+ // changed; only its reconnecting (a new session) forgives it.
+ XCTAssertNil(d.devicesChanged([faceTime, atem, phone]))
+ _ = d.devicesChanged([faceTime, phone])
+ XCTAssertEqual(d.devicesChanged([faceTime, phone, atem]), .open("atem", format: 0))
+ }
+
+ func testOneFormatSkipsStraightToFallback() {
+ let d = director(defaults(), formats: 1)
+ _ = d.start(devices: [atem])
+ d.opened("atem", at: 0, frames: 0)
+ XCTAssertEqual(d.tick(frames: 0, at: 3), .open("atem", format: 0))
+ d.opened("atem", at: 3, frames: 0)
+ // No next format, no other camera: give up, say so, keep listening.
+ XCTAssertNil(d.tick(frames: 0, at: 6))
+ XCTAssertEqual(d.status, .noSignal("ATEM Mini"))
+ XCTAssertEqual(d.status.tileText, "No signal from ATEM Mini — check its input")
+ XCTAssertNil(d.tick(frames: 0, at: 60), "no restarting forever")
+ XCTAssertNil(d.tick(frames: 1, at: 61))
+ XCTAssertEqual(d.status, .live("ATEM Mini"), "its signal coming back is noticed")
+ }
+
+ func testHeldClockDoesNotClimbTheLadder() {
+ let d = director(defaults())
+ _ = d.start(devices: [atem])
+ d.opened("atem", at: 0, frames: 0)
+ for second in stride(from: 1.0, through: 30, by: 1) { d.hold(frames: 0, at: second) }
+ XCTAssertNil(d.tick(frames: 0, at: 31), "the background is not a stall")
+ XCTAssertEqual(d.tick(frames: 0, at: 33), .open("atem", format: 0))
+ }
+
+ func testRestartsAreRateLimited() {
+ let d = director(defaults())
+ _ = d.start(devices: [atem])
+ XCTAssertEqual(d.restart(at: 100), .open("atem", format: 0))
+ XCTAssertNil(d.restart(at: 102))
+ XCTAssertEqual(d.restart(at: 105), .open("atem", format: 0))
+ }
+
+ // MARK: Formats
+
+ func testFrameRateIsAlwaysOneTheFormatSupports() {
+ XCTAssertEqual(CameraFormatChoice.rate(for: [1...30]), 30)
+ XCTAssertEqual(CameraFormatChoice.rate(for: [29.97...29.97]), 29.97)
+ XCTAssertEqual(CameraFormatChoice.rate(for: [25...25]), 25)
+ XCTAssertEqual(CameraFormatChoice.rate(for: [50...50, 59.94...59.94]), 50)
+ XCTAssertEqual(CameraFormatChoice.rate(for: [23.976...23.976, 60...60]), 23.976)
+ XCTAssertNil(CameraFormatChoice.rate(for: []))
+ }
+
+ func testATEMFormatsRankByRateNearestThirty() {
+ let rates: [Double] = [23.976, 24, 25, 29.97, 30, 50, 59.94, 60]
+ let formats = rates.map {
+ CameraFormatChoice.Candidate(width: 1920, height: 1080, ranges: [$0...$0])
+ }
+ let ranked = CameraFormatChoice.rank(formats).map { rates[$0] }
+ XCTAssertEqual(ranked.prefix(3), [30, 29.97, 25])
+ XCTAssertEqual(ranked.count, rates.count)
+ }
+
+ func testSmallerFormatsComeFirstAndDuplicatesGo() {
+ let formats = [
+ CameraFormatChoice.Candidate(width: 1920, height: 1080, ranges: [1...30]),
+ CameraFormatChoice.Candidate(width: 640, height: 480, ranges: [1...30]),
+ CameraFormatChoice.Candidate(width: 640, height: 480, ranges: [1...30]),
+ CameraFormatChoice.Candidate(width: 1280, height: 720, ranges: [1...60]),
+ ]
+ XCTAssertEqual(CameraFormatChoice.rank(formats), [1, 3, 0])
+ }
+
+ // MARK: The panel
+
+ func testCameraRowStates() {
+ XCTAssertEqual(CameraRowState(cameras: 0, offers: false, status: nil), .hidden)
+ XCTAssertEqual(CameraRowState(cameras: 0, offers: true, status: nil), .noCamera)
+ XCTAssertTrue(CameraRowState(cameras: 0, offers: true, status: .noCamera).offersRetry)
+ XCTAssertEqual(
+ CameraRowState(cameras: 2, offers: true, status: .live("ATEM Mini")), .picker(line: nil, retry: false))
+ XCTAssertEqual(
+ CameraRowState(cameras: 1, offers: true, status: .starting("ATEM Mini")),
+ .picker(line: "Starting ATEM Mini…", retry: false))
+ let silent = CameraRowState(cameras: 1, offers: true, status: .noSignal("ATEM Mini"))
+ XCTAssertTrue(silent.offersRetry)
+ XCTAssertEqual(
+ silent,
+ .picker(
+ line: "ATEM Mini is connected but sending no picture. Check its input, or choose another camera.",
+ retry: true))
+ XCTAssertTrue(CameraRowState(cameras: 1, offers: true, status: .failed("ATEM Mini")).offersRetry)
+ }
+
+ func testSelfTileSaysWhatItIsWaitingFor() {
+ XCTAssertEqual(CameraStatus.starting("ATEM Mini").tileText, "Starting camera…")
+ XCTAssertEqual(CameraStatus.disconnected("ATEM Mini").tileText, "Camera disconnected")
+ XCTAssertNil(CameraStatus.live("ATEM Mini").tileText)
+ }
+
+ // MARK: The managed camera, end to end with a fake backend
+
+ private final class FakeBackend: CameraBackend {
+ var devices: [CameraDevice]
+ var opened: [String] = []
+ var refuse: Set<String> = []
+ var devicesChanged: (@MainActor () -> Void)?
+ var trouble: (@MainActor () -> Void)?
+ init(_ devices: [CameraDevice]) { self.devices = devices }
+ func requestAccess() async throws {}
+ func formatCount(_ id: String) -> Int { 1 }
+ func open(_ id: String, format: Int, delegate: RTCVideoCapturerDelegate) async throws {
+ if refuse.contains(id) { throw NSError(domain: "test", code: 1) }
+ opened.append(id)
+ }
+ func close() async {}
+ }
+
+ func testManagedCameraFollowsDisconnectAndReturn() async throws {
+ let store = defaults()
+ CameraPreference(id: "atem", name: "ATEM Mini").save(store)
+ let backend = FakeBackend([faceTime, atem])
+ let camera = ManagedCamera(backend: backend, defaults: store)
+ var changes = 0
+ camera.observe { changes += 1 }
+ RTCInitializeSSL()
+ let factory = RTCPeerConnectionFactory()
+ try await camera.start(source: factory.videoSource())
+ XCTAssertEqual(camera.currentCamera, "atem")
+ XCTAssertEqual(camera.cameraStatus, .starting("ATEM Mini"))
+
+ backend.devices = [faceTime]
+ backend.devicesChanged?()
+ try await waitFor { camera.currentCamera == "facetime" }
+ XCTAssertEqual(camera.cameraNote, "ATEM Mini disconnected; using FaceTime HD Camera")
+
+ backend.devices = [faceTime, atem]
+ backend.devicesChanged?()
+ try await waitFor { camera.currentCamera == "atem" }
+ XCTAssertEqual(camera.cameraNote, "")
+ XCTAssertEqual(backend.opened, ["atem", "facetime", "atem"])
+ XCTAssertGreaterThan(changes, 0)
+ camera.stop()
+ XCTAssertNil(camera.cameraStatus)
+ }
+
+ func testManagedCameraRefusedOpenFallsBack() async throws {
+ let backend = FakeBackend([faceTime, atem])
+ backend.refuse = ["facetime"]
+ let camera = ManagedCamera(backend: backend, defaults: defaults())
+ RTCInitializeSSL()
+ try await camera.start(source: RTCPeerConnectionFactory().videoSource())
+ XCTAssertEqual(camera.currentCamera, "atem")
+ XCTAssertEqual(camera.cameraNote, "FaceTime HD Camera would not start; using ATEM Mini")
+ do {
+ try await camera.use(camera: "nothing")
+ XCTFail("an unknown camera is refused")
+ } catch {}
+ camera.stop()
+ }
+
+ private func waitFor(_ condition: @escaping @MainActor () -> Bool) async throws {
+ for _ in 0..<200 {
+ if condition() { return }
+ try await Task.sleep(for: .milliseconds(10))
+ }
+ XCTFail("condition never held")
+ }
+}
diff --git a/native/apple/Tests/CallsKitTests/NoiseSuppressionTests.swift b/native/apple/Tests/CallsKitTests/NoiseSuppressionTests.swift
index 8389351..6d0c267 100644
--- a/native/apple/Tests/CallsKitTests/NoiseSuppressionTests.swift
+++ b/native/apple/Tests/CallsKitTests/NoiseSuppressionTests.swift
@@ -103,14 +103,19 @@ final class CaptureDenoiserTests: XCTestCase {
XCTAssertEqual(fake.calls, 2)
}
- func testRateChangesAreReportedOnceEach() {
+ /// Every initialisation is reported, a repeat included: WebRTC's first
+ /// one is its 16 kHz construction default, before any microphone, and a
+ /// headset really capturing at 16 kHz repeats it. Reporting only changes
+ /// hid that second one, and the plan never put WebRTC's own suppressor
+ /// back on (Calls 0.3.1's "16 kHz … Noise suppression: off").
+ func testEveryInitialisationIsReported() {
let (denoiser, _, _) = adapter()
let seen = LockedRates()
denoiser.rateChanged = { seen.append($0) }
- denoiser.audioProcessingInitialize(sampleRate: 48_000, channels: 1)
- denoiser.audioProcessingInitialize(sampleRate: 48_000, channels: 1)
denoiser.audioProcessingInitialize(sampleRate: 16_000, channels: 1)
- XCTAssertEqual(seen.rates, [48_000, 16_000])
+ denoiser.audioProcessingInitialize(sampleRate: 16_000, channels: 1)
+ denoiser.audioProcessingInitialize(sampleRate: 48_000, channels: 1)
+ XCTAssertEqual(seen.rates, [16_000, 16_000, 48_000])
}
func testTheToggleAndVoiceIsolationBothPassThrough() {
diff --git a/native/ios/Sources/CallsPhone.swift b/native/ios/Sources/CallsPhone.swift
index db20ceb..24f96f6 100644
--- a/native/ios/Sources/CallsPhone.swift
+++ b/native/ios/Sources/CallsPhone.swift
@@ -158,7 +158,8 @@ struct PhoneCall: View {
.aspectRatio(4 / 3, contentMode: .fit)
}
PhoneVideoTile(
- track: call.localTrack, label: "You", mirrored: true, cameraOff: call.cameraOff
+ track: call.localTrack, label: "You", mirrored: true, cameraOff: call.cameraOff,
+ placeholder: call.selfPlaceholder
)
.aspectRatio(4 / 3, contentMode: .fit)
ForEach(peers) { peer in
@@ -191,7 +192,8 @@ struct PhoneCall: View {
.background(CallsAppearance.background, in: RoundedRectangle(cornerRadius: 18))
}
PhoneVideoTile(
- track: call.localTrack, label: "You", mirrored: true, cameraOff: call.cameraOff
+ track: call.localTrack, label: "You", mirrored: true, cameraOff: call.cameraOff,
+ placeholder: call.selfPlaceholder
)
.frame(
width: min(160, geometry.size.width * 0.32),
@@ -308,17 +310,25 @@ struct PhoneVideoTile: View {
/// more than anywhere, cropping a laptop's screen to a portrait tile
/// would leave the middle third of it.
var screen = false
+ /// Your own tile: what the camera is doing, in place of a picture that
+ /// is not arriving (see `Call.selfPlaceholder`). Nil shows the video.
+ var placeholder: String? = nil
var body: some View {
ZStack {
Color.black
- if let track, !cameraOff {
+ if let track, !cameraOff, placeholder == nil {
PhoneVideo(track: track, fit: screen).scaleEffect(x: mirrored ? -1 : 1, y: 1)
} else {
- Image(
- systemName: cameraOff
- ? "video.slash" : screen ? "rectangle.on.rectangle" : "person.crop.circle"
- )
- .font(.largeTitle).foregroundStyle(.white.opacity(0.8))
+ VStack(spacing: 6) {
+ Image(
+ systemName: cameraOff
+ ? "video.slash" : screen ? "rectangle.on.rectangle" : "person.crop.circle"
+ )
+ .font(.largeTitle)
+ if let placeholder, !cameraOff {
+ Text(placeholder).font(.caption).multilineTextAlignment(.center).padding(.horizontal, 6)
+ }
+ }.foregroundStyle(.white.opacity(0.8))
}
}
.overlay(alignment: .bottomLeading) {
@@ -333,7 +343,7 @@ struct PhoneVideoTile: View {
.accessibilityValue(
cameraOff
? "Camera off"
- : track == nil ? (screen ? "Waiting for the screen" : "Waiting for video") : "Video on")
+ : placeholder ?? (track == nil ? (screen ? "Waiting for the screen" : "Waiting for video") : "Video on"))
}
}
struct PhoneVideo: UIViewRepresentable {
diff --git a/native/ios/project.yml b/native/ios/project.yml
index 60fb255..8b8811b 100644
--- a/native/ios/project.yml
+++ b/native/ios/project.yml
@@ -10,7 +10,7 @@ settings:
base:
SWIFT_VERSION: '5.0'
CODE_SIGN_STYLE: Automatic
- MARKETING_VERSION: '0.3.1'
+ MARKETING_VERSION: '0.3.2'
CURRENT_PROJECT_VERSION: '1'
# The App Group the app and its broadcast extension share: the socket
# the iPhone's screen crosses lives in its container. One setting
diff --git a/test/browser/native.mjs b/test/browser/native.mjs
index 4415282..e7a19b5 100644
--- a/test/browser/native.mjs
+++ b/test/browser/native.mjs
@@ -87,6 +87,10 @@ try {
assert.equal(sample.stats.webrtcGainControl, 0, "WebRTC's gain control is off");
assert.equal(sample.stats.webrtcEchoCancellation, 1, "echo cancellation stays on");
console.log('PASS WebRTC noise suppression and gain off, echo cancellation on');
+ // With audio flowing, the readout reports what is running rather than
+ // "The microphone has not started yet".
+ assert.equal(sample.stats.audioReadoutStarted, 1, 'the audio readout knows the microphone has started');
+ console.log('PASS the audio readout reports a started microphone');
// Each remote video has its own decoded frame count; local preview alone
// cannot satisfy this check. Read the actual room test observation contract.
// Remote videos exclude the muted local preview. getVideoPlaybackQuality
@@ -153,6 +157,20 @@ try {
await wait(async () => await remoteFrames() > refusedFrames + 15, 'media survives a refused camera switch');
console.log('PASS camera switched mid-call without disturbing the connection');
}
+ // A camera that stops sending — an ATEM Mini whose input changed format
+ // — is noticed by the watchdog and restarted, on the same track, and the
+ // far side's picture comes back without anyone touching anything.
+ {
+ await wait(() => sample?.cameraStatus === 'live', 'the camera reports live before the stall');
+ const opensBefore = sample.cameraOpens;
+ native.stdin.write('camera-stall\n');
+ await wait(() => sample?.cameraOpens > opensBefore, 'the watchdog restarts a stalled camera', 20);
+ await wait(() => sample?.cameraStatus === 'live', 'the restarted camera reports live again', 20);
+ const recoveredFrames = await remoteFrames();
+ await wait(async () => await remoteFrames() > recoveredFrames + 15, 'browser decodes again after the camera recovers');
+ assert.equal(sample.stats.peerConnections, 1, 'recovering the camera keeps the same peer connection');
+ console.log('PASS a stalled camera was restarted and the far side kept its picture');
+ }
await page.locator('[data-chat-text]').fill('browser chat probe');
await page.locator('[data-chat-send]').click();
await wait(() => sample?.receivedChatProbe === true, 'native displays browser chat');