| index 21e814b..6d973ed 100644 |
| --- a/internal/meet/static/room.js |
| +++ b/internal/meet/static/room.js |
| @@ -142,6 +142,7 @@ |
| servers: [], // the RTCIceServer list, [] on the floor (§5.5) |
| expiresAt: 0, // the credential's hard deadline on the relay |
| renewAt: 0, // when the server asks us to come back |
| + margin: 0, // expiresAt - renewAt: a duration both clocks agree on |
| fetchedAt: 0, // when the list we hold arrived |
| failedSince: 0, // the first failure of the current run, or 0 |
| fetches: 0, // *successful* fetches, for the gate |
| @@ -1131,49 +1132,83 @@ |
| const peer = state.peer; |
| if (state.stopped || !peer) return false; |
| const path = '/m/' + encodeURIComponent(sqid) + '/ice?peer=' + encodeURIComponent(peer); |
| - let res; |
| - try { |
| - res = await probe(path, Date.now() + SEND_DEADLINE); |
| - } catch { |
| - icePrimed = true; |
| - iceFailed(); |
| - return false; |
| - } |
| - if (state.stopped || state.peer !== peer) return false; |
| - if (res.signedOut) { |
| - terminal('signedout'); |
| - return false; |
| - } |
| - if (res.status === 404) { |
| - // Not a relay failure: a 404 here says this participation is gone, |
| - // and /signal's own 404 says no more than that either. So it goes |
| - // through the same probe-first recovery — this browser's own |
| - // participation is the authority on what the 404 meant — and the |
| - // next roster fetches again, because whatever comes back from that |
| - // ladder is a call that has not got a list yet. |
| - icePrimed = false; |
| - firstIce = null; |
| - await recoverFromNotFound(res.data.boot); |
| - return false; |
| - } |
| - if (res.status !== 200 || !Array.isArray(res.data.iceServers)) { |
| + // One budget for the whole exchange, however many requests it takes |
| + // — the shape §4.6 already gives one outbound message, and the same |
| + // reason applies: /ice spends /signal's own per-session, per-room |
| + // bucket, so a renewal that lands inside a burst of signalling gets |
| + // a 429 and a Retry-After rather than an answer. |
| + const deadline = Date.now() + SEND_DEADLINE; |
| + for (;;) { |
| + let res; |
| + try { |
| + res = await probe(path, deadline); |
| + } catch { |
| + icePrimed = true; |
| + iceFailed(); |
| + return false; |
| + } |
| + if (state.stopped || state.peer !== peer) return false; |
| + if (res.signedOut) { |
| + terminal('signedout'); |
| + return false; |
| + } |
| + if (res.status === 429) { |
| + // Not a failure: the server said "not now, come back in n". Honour |
| + // it inside this budget and say nothing — a warning about the |
| + // relay would be untrue, and the person's own call is what filled |
| + // that bucket. Only a wait that runs out of budget is a failure. |
| + if (!(await waitWithin(retryAfter(res.retryAfter), deadline))) { |
| + icePrimed = true; |
| + iceFailed(); |
| + return false; |
| + } |
| + continue; |
| + } |
| + if (res.status === 404) { |
| + // Not a relay failure: a 404 here says this participation is |
| + // gone — a resume window that has lapsed included, now that |
| + // /ice gates on the same liveness /signal does — and /signal's |
| + // own 404 says no more than that either. So it goes through |
| + // the same probe-first recovery — this browser's own |
| + // participation is the authority on what the 404 meant — and the |
| + // next roster fetches again, because whatever comes back from that |
| + // ladder is a call that has not got a list yet. |
| + icePrimed = false; |
| + firstIce = null; |
| + await recoverFromNotFound(res.data.boot); |
| + // The ladder is allowed to decide nothing at all: a probe that |
| + // cannot answer inside its budget leaves this participation alone |
| + // (§4.5), and a call left with no list and no retry pending would |
| + // simply sit there. So unless it ended the call or entered afresh |
| + // — both of which change what is checked here — the minute retry |
| + // covers it. |
| + if (!state.stopped && state.peer === peer) iceFailed(); |
| + return false; |
| + } |
| + if (res.status !== 200 || !Array.isArray(res.data.iceServers)) { |
| + icePrimed = true; |
| + iceFailed(); |
| + return false; |
| + } |
| icePrimed = true; |
| - iceFailed(); |
| - return false; |
| + ice.servers = res.data.iceServers; |
| + ice.expiresAt = millis(res.data.expiresAt); |
| + ice.renewAt = millis(res.data.renewAt); |
| + // The margin the server left itself: expiry minus renew time, on |
| + // its clock, where the two subtract to a duration both machines |
| + // agree on. Everything below is scheduled from it and from |
| + // `fetchedAt`, which is this browser's own clock — see renewDelay. |
| + ice.margin = Math.max(0, ice.expiresAt - ice.renewAt); |
| + ice.fetchedAt = Date.now(); |
| + ice.lastFetchAt = ice.fetchedAt; |
| + ice.fetches += 1; |
| + ice.failedSince = 0; |
| + clearTimeout(iceRetryTimer); |
| + iceWarning(false); |
| + scheduleRenewal(); |
| + adoptServers(); |
| + return true; |
| } |
| - icePrimed = true; |
| - ice.servers = res.data.iceServers; |
| - ice.expiresAt = millis(res.data.expiresAt); |
| - ice.renewAt = millis(res.data.renewAt); |
| - ice.fetchedAt = Date.now(); |
| - ice.lastFetchAt = ice.fetchedAt; |
| - ice.fetches += 1; |
| - ice.failedSince = 0; |
| - clearTimeout(iceRetryTimer); |
| - iceWarning(false); |
| - scheduleRenewal(); |
| - adoptServers(); |
| - return true; |
| } |
| |
| // The wire carries unix seconds; everything in this file works in |
| @@ -1214,6 +1249,10 @@ |
| function iceWarning(show) { |
| const el = warningEl(); |
| if (!el) return; |
| + // On the floor the sentence would be untrue: there is no relay this |
| + // instance could reach, /ice answers an empty list, and a fetch that |
| + // fails there costs the call nothing a person could act on (§5.5). |
| + if (show && !relayConfigured) return; |
| if (show) { |
| // Unhidden first, then written: a live region announces the change |
| // it can see, and text set into a hidden element is not one. |
| @@ -1266,16 +1305,42 @@ |
| // `renewAt` already behind us runs at once. |
| function scheduleRenewal() { |
| clearTimeout(renewTimer); |
| - if (state.stopped || !ice.renewAt || !ice.servers.length) return; |
| - renewTimer = setTimeout(renew, Math.max(0, ice.renewAt - Date.now())); |
| + if (state.stopped || !ice.margin || !ice.servers.length) return; |
| + renewTimer = setTimeout(renew, Math.max(0, renewDelay() - (Date.now() - ice.fetchedAt))); |
| + } |
| + |
| + // renewDelay and mustSucceedBy are §5.3's two instants, measured as |
| + // durations from the moment this browser took the list — never as |
| + // points on the server's clock. A browser whose clock is minutes off |
| + // the server's would otherwise renew at the wrong time, and one far |
| + // enough ahead would decide the credential had expired before it ever |
| + // used it. What both machines do agree on is how long the credential |
| + // lasts: the server sends renewAt and expiresAt from one clock, and |
| + // their difference is the margin it left itself. The contract's own |
| + // shape — renew at five sixths of the life — makes the rest |
| + // arithmetic: the margin is a sixth of the life, so the delay is five |
| + // of them and the whole life is six. |
| + function renewDelay() { |
| + return ice.margin * 5; |
| + } |
| + |
| + function mustSucceedBy() { |
| + return ice.fetchedAt + ice.margin * 6; |
| } |
| |
| // §5.3's last paragraph: a suspended tab stops its timers, so a tab |
| // coming back to visible compares the clock with `renewAt` rather than |
| // trusting one, and runs the sequence at once if it is overdue. |
| function catchUpRenewal() { |
| - if (state.stopped || !ice.renewAt || !ice.servers.length) return; |
| - if (Date.now() >= ice.renewAt) renew(); |
| + if (state.stopped || !ice.margin || !ice.servers.length) return; |
| + // A fetch that is already failing has a retry of its own running on |
| + // its own minute, and a tab coming back to visible is not a reason |
| + // to jump that queue. The comparison is with the clock rather than |
| + // with the timer because that timer is exactly what a suspended tab |
| + // stops: a failure older than the retry interval means the minute |
| + // has passed with nothing to run it, and this is what runs it. |
| + if (ice.failedSince && Date.now() - ice.failedSince < ICE_RETRY) return; |
| + if (Date.now() - ice.fetchedAt >= renewDelay()) renew(); |
| } |
| |
| function renew() { |
| @@ -1286,23 +1351,27 @@ |
| |
| // renewAll is §5.3's sequence for the whole mesh: fetch, then renew |
| // every live connection. The deadline it hands each of them is the |
| - // *old* credential's expiry (step 3) — the one honest signal, because |
| - // the browser cannot see which credential an allocation is using — and |
| - // it survives the minute retries, because a failed fetch leaves |
| - // `ice.expiresAt` where it was. |
| + // *old* credential's — step 3's must-succeed-by, and the one honest |
| + // signal, because the browser cannot see which credential an |
| + // allocation is using — so both it and the stamp that names the |
| + // credential it belongs to are read *before* the fetch. A failed fetch |
| + // leaves neither where the watchdog could confuse them: `fetchedAt` |
| + // only ever advances on a success, which is exactly why reading it |
| + // afterwards made the watchdog unreachable (round-1 review, item 1). |
| async function renewAll() { |
| if (state.stopped || renewing) return; |
| renewing = true; |
| renewDue = true; |
| - const mustSucceedBy = ice.expiresAt; |
| - armMustSucceedBy(mustSucceedBy); |
| + const expiring = ice.fetchedAt; |
| + const deadline = mustSucceedBy(); |
| + armMustSucceedBy(deadline, expiring); |
| try { |
| // A failed fetch has already armed the warning and the minute |
| // retry, and that retry comes back here. |
| if (!(await fetchIce())) return; |
| renewDue = false; |
| for (const id of Array.from(pcs.keys())) { |
| - renewOne(id, mustSucceedBy).catch((err) => { |
| + renewOne(id, deadline).catch((err) => { |
| console.error('meet: renew', err); |
| }); |
| } |
| @@ -1318,19 +1387,28 @@ |
| // restart is still in flight is left alone — it is the thing that |
| // would clear this, and interrupting it would show the sentence to |
| // someone whose call is about to come back. |
| - function armMustSucceedBy(at) { |
| + function armMustSucceedBy(at, expiring) { |
| clearTimeout(expiryTimer); |
| if (!at) return; |
| - expiryTimer = setTimeout(expireUnrenewed, Math.max(0, at - Date.now())); |
| - } |
| - |
| - function expireUnrenewed() { |
| + expiryTimer = setTimeout(() => expireUnrenewed(expiring), Math.max(0, at - Date.now())); |
| + } |
| + |
| + // expireUnrenewed is the verdict itself. `expiring` is the stamp of |
| + // the credential that has just run out, so "newer than that" means |
| + // "built on a list this deadline was never about" — a peer who joined |
| + // after the fetch, whose connection has the current credential and |
| + // nothing to renew. It has to be the *expiring* stamp and not |
| + // `ice.fetchedAt`: on the path this exists for, the fetch is still |
| + // failing and `ice.fetchedAt` has not moved, so comparing against it |
| + // exempted every connection in the call and nothing was ever marked. |
| + function expireUnrenewed(expiring) { |
| if (state.stopped) return; |
| for (const [id, entry] of pcs) { |
| - // Built on the list we hold now: this connection never had the |
| - // credential that just expired. |
| - if (entry.listAt >= ice.fetchedAt) continue; |
| + if (entry.listAt > expiring) continue; |
| const record = renewals.get(id); |
| + // A restart still in flight is the thing that would clear this, |
| + // and interrupting it would show the sentence to someone whose |
| + // call is about to come back. |
| if (record && (record.renewed || record.running)) continue; |
| markRelayExpired(id); |
| } |
| @@ -1358,7 +1436,7 @@ |
| // the must-succeed-by is read *between* attempts rather than during |
| // one — cutting a restart that is about to land would show a person |
| // the expired sentence for a call that was coming back. |
| - async function renewOne(id, mustSucceedBy) { |
| + async function renewOne(id, deadline) { |
| const entry = pcs.get(id); |
| if (!entry || state.stopped) return false; |
| const record = renewalRecord(id); |
| @@ -1386,11 +1464,20 @@ |
| } catch { |
| // No restartIce at all: a fresh generation is the only way to |
| // put a new credential under this pair, and the `pair` that |
| - // answers rebuilds the connection from nothing. |
| + // answers rebuilds the connection from nothing — a rebuild |
| + // this record does not survive, which is what "the connection |
| + // is gone from under us" means below. If nothing has rebuilt |
| + // it by the end of the window, nothing is going to: the |
| + // connection is still on a credential that has run out, and |
| + // the person is owed the sentence rather than a tile that |
| + // looks fine and carries no media. |
| requestReset(id); |
| + await sleep(RENEW_WINDOW); |
| + if (state.stopped || pcs.get(id) !== entry) return false; |
| + markRelayExpired(id); |
| return false; |
| } |
| - const proven = await proveRenewed(entry, oldUfrag, from, Date.now() + RENEW_WINDOW); |
| + const proven = await proveRenewed(id, entry, oldUfrag, from, Date.now() + RENEW_WINDOW); |
| if (state.stopped || pcs.get(id) !== entry) return false; |
| if (proven !== null) { |
| record.renewed = true; |
| @@ -1401,7 +1488,10 @@ |
| renderTiles(); |
| return true; |
| } |
| - if (mustSucceedBy && Date.now() > mustSucceedBy) break; |
| + // Read between attempts, never during one: `mustSucceedBy` is |
| + // the whole call's deadline, and a restart that is about to |
| + // land is worth more than a sentence delivered on time. |
| + if (deadline && Date.now() > deadline) break; |
| } |
| markRelayExpired(id); |
| return false; |
| @@ -1415,18 +1505,38 @@ |
| // must-succeed-by — the deadline it would carry is already behind us, |
| // and a person who asked for this is owed the three attempts, not an |
| // instant repeat of the same sentence. |
| - async function retryRenewal(id) { |
| - if (!(await fetchIce())) return; |
| - await renewOne(id, 0); |
| + // |
| + // A fetch that fails is the case worth writing out. `restartIce` has |
| + // already cleared the sentence off the tile, so returning here left a |
| + // person looking at a tile that said nothing was wrong while nothing |
| + // had been done about it. The sentence goes back up — it is still |
| + // true — and the connection still gets the plain restart the ordinary |
| + // Retry would have given it, which is the one thing left that might |
| + // help. |
| + async function retryRenewal(id, wasExpired) { |
| + if (await fetchIce()) { |
| + await renewOne(id, 0); |
| + return; |
| + } |
| + if (wasExpired) { |
| + renewalRecord(id).relayExpired = true; |
| + renderTiles(); |
| + } |
| + const entry = pcs.get(id); |
| + if (entry) bareRestart(id, entry); |
| } |
| |
| // proveRenewed polls getStats until the decision below says the |
| // restart landed, or until the window runs out. Half a second between |
| // polls: an ICE restart is a round trip and a handful of candidate |
| // exchanges, and asking faster only costs the call CPU. |
| - async function proveRenewed(entry, oldUfrag, from, deadline) { |
| + async function proveRenewed(id, entry, oldUfrag, from, deadline) { |
| for (;;) { |
| - if (state.stopped) return null; |
| + // A connection replaced under us — a `pair`, a roster, an |
| + // unreachable target — is not the connection this proof is about, |
| + // and polling its stats for another thirty seconds proves nothing |
| + // about either of them. |
| + if (state.stopped || pcs.get(id) !== entry) return null; |
| const proven = renewedUfrag(await statsList(entry.pc), oldUfrag, |
| entry.pc.signalingState, entry.desc > from); |
| if (proven !== null) return proven; |
| @@ -1467,24 +1577,39 @@ |
| // username fragment on the local candidate of the selected, succeeded |
| // candidate pair: the fragment when the browser names it, '' when the |
| // pair is there and the fragment is not, and null when there is no |
| - // such pair at all. Chromium marks the pair through the transport's |
| - // selectedCandidatePairId and through `nominated`, Firefox through |
| - // `selected`; all three are read, so the same function answers on any |
| - // of them. Pure, like the decision above. |
| + // such pair at all. Pure, like the decision above. |
| + // |
| + // Where a browser names the selected pair itself — Chromium's |
| + // transport.selectedCandidatePairId — that answer wins outright, and a |
| + // pair it does not name is not consulted however succeeded and |
| + // nominated it looks. Chromium keeps the *previous* generation's pair |
| + // in the report for a while after a restart, still succeeded and still |
| + // nominated, and scanning for one of those was how a restart could |
| + // read as proven by the connection it had just replaced (round-1 |
| + // review, item 2). `nominated`/`selected` are the fallback for a |
| + // report with no transport in it, which is what Firefox sends. |
| function selectedPairUfrag(reports) { |
| const locals = new Map(); |
| const pairs = []; |
| - let selectedId = ''; |
| + const selected = new Set(); |
| + let transports = 0; |
| for (const r of reports || []) { |
| if (!r || !r.type) continue; |
| if (r.type === 'local-candidate') locals.set(r.id, r); |
| else if (r.type === 'candidate-pair') pairs.push(r); |
| - else if (r.type === 'transport' && r.selectedCandidatePairId) selectedId = r.selectedCandidatePairId; |
| + else if (r.type === 'transport') { |
| + transports += 1; |
| + if (r.selectedCandidatePairId) selected.add(r.selectedCandidatePairId); |
| + } |
| } |
| + // A transport that has not named a pair yet has not selected one: |
| + // `selected` is empty, nothing matches, and the answer is null. |
| + const chosen = transports |
| + ? pairs.filter((p) => selected.has(p.id)) |
| + : pairs.filter((p) => p.nominated || p.selected); |
| let found = null; |
| - for (const p of pairs) { |
| + for (const p of chosen) { |
| if (p.state !== 'succeeded') continue; |
| - if (!(p.nominated || p.selected || (selectedId && p.id === selectedId))) continue; |
| const local = locals.get(p.localCandidateId); |
| const ufrag = (local && local.usernameFragment) || ''; |
| // A pair that names the fragment is worth more than one that does |
| @@ -2046,7 +2171,12 @@ |
| const record = renewals.get(id); |
| if (record && record.relayExpired) return ICE_EXPIRED; |
| if (!entry.failed) return ''; |
| - return relayConfigured ? FAILED_RELAY : FAILED_PLAIN; |
| + // Both halves have to be true for the longer sentence: this |
| + // instance has a relay, and *this connection* was given it. A |
| + // connection still running on the empty list never went near one, |
| + // and saying it failed through the relay would be a claim about |
| + // something it never tried. |
| + return relayConfigured && !entry.floor ? FAILED_RELAY : FAILED_PLAIN; |
| } |
| |
| // tile builds or updates one <figure> per participant. Nothing here |
| @@ -2165,16 +2295,20 @@ |
| if (record) record.relayExpired = false; |
| renderTiles(); |
| if (expired || iceStale()) { |
| - retryRenewal(id).catch((err) => { |
| + retryRenewal(id, expired).catch((err) => { |
| console.error('meet: renew', err); |
| }); |
| return; |
| } |
| + bareRestart(id, entry); |
| + } |
| + |
| + // bareRestart is §5.1's retry, unchanged: the restart, and a fresh |
| + // generation from the hub for a browser that has no restartIce. |
| + function bareRestart(id, entry) { |
| try { |
| entry.pc.restartIce(); |
| } catch { |
| - // No restartIce: ask the hub for a fresh generation instead, |
| - // which rebuilds the connection from nothing. |
| requestReset(id); |
| } |
| } |
| @@ -2182,7 +2316,8 @@ |
| // iceStale: we hold a list, and the server's own renew time for it has |
| // passed. A retry then is a renewal, not a restart. |
| function iceStale() { |
| - return !!(ice.servers.length && ice.renewAt && Date.now() >= ice.renewAt); |
| + return !!(ice.servers.length && ice.margin && |
| + Date.now() - ice.fetchedAt >= renewDelay()); |
| } |
| |
| // applySink routes a remote tile's audio to the speaker the holding |
| @@ -2422,7 +2557,9 @@ |
| lastFetchAt: ice.lastFetchAt, |
| expiresAt: ice.expiresAt, |
| renewAt: ice.renewAt, |
| - warning: !!ice.failedSince, |
| + // The sentence that is actually showing: on the floor there |
| + // is no relay to fail and none is shown (§5.5). |
| + warning: !!(ice.failedSince && relayConfigured), |
| servers: ice.servers.length, |
| }, |
| // Per connection: how many restarts §5.3 step 2 has spent, and |
amadan