amadan
merge: canary TTL — expiry is visibility, never a gate (spec §5)
Every canary/* pointer carries an implicit 7-day lease measured from promoted_at; re-promoting refreshes it, so actively-updated canaries never expire. Expiry is computed at read time — no new storage, no expired bit — and surfaces as a console badge with age, a loud-null count on the fleet status feed, and one nag line per expired route per HOURLY adopt pass (the fast canary loop is forbidden from nagging, or a stationary expired canary floods the journal every 2-3s). Nothing stops serving, adopting or promoting: that contract is pinned at all three surfaces, including the console promote handler, where a mutation-tested gate previously failed zero tests.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
18 files changed,
+1266
−19
internal/console/icons.go+9 −0internal/console/ladder.go+57 −1internal/console/ladder_test.go+179 −0internal/console/templates/ladder.html+1 −0internal/console/templates/layout.html+4 −0internal/console/view.go+1 −1internal/host/agent.go+88 −1internal/host/host_test.go+208 −0internal/release/adopt.go+73 −6internal/release/adopt_test.go+252 −0internal/release/canary_ttl.go+85 −0internal/release/canary_ttl_test.go+116 −0internal/status/assets/index.html+18 −0internal/status/contract_test.go+7 −3internal/status/dryrun.go+9 −7internal/status/fleet.go+36 −0internal/status/snapshot.go+16 −0internal/status/status_test.go+107 −0
diff --git a/internal/console/icons.go b/internal/console/icons.go| index 6e5a22618..b16ac55cb 100644 |
| --- a/internal/console/icons.go |
| +++ b/internal/console/icons.go |
| @@ -84,6 +84,14 @@ const ( |
| `<path d="m12.83 2.18a2 2 0 0 0-1.66 0L2.6 6.08a1 1 0 0 0 0 1.83l8.58 3.91a2 2 0 0 0 1.66 0l8.58-3.9a1 1 0 0 0 0-1.83z"/>` + |
| `<path d="m6.08 9.5-3.5 1.6a1 1 0 0 0 0 1.81l8.58 3.91a2 2 0 0 0 1.66 0l8.58-3.9a1 1 0 0 0 0-1.83l-3.5-1.59"/>` + |
| `<path d="m6.08 14.5-3.5 1.6a1 1 0 0 0 0 1.81l8.58 3.91a2 2 0 0 0 1.66 0l8.58-3.9a1 1 0 0 0 0-1.83l-3.5-1.59"/></svg>` |
| + |
| + // A canary/* route whose lease has run out (spec §5 "Canaries expire") — |
| + // distinct from iconTriangleAlert's prod-wired warning, because expiry is |
| + // a clock running out, not a hazard (Lucide: hourglass). |
| + iconHourglass template.HTML = iconOpen + |
| + `<path d="M5 22h14"/><path d="M5 2h14"/>` + |
| + `<path d="M17 22v-4.172a2 2 0 0 0-.586-1.414L12 12l-4.414 4.414A2 2 0 0 0 7 17.828V22"/>` + |
| + `<path d="M7 2v4.172a2 2 0 0 0 .586 1.414L12 12l4.414-4.414A2 2 0 0 0 17 6.172V2"/></svg>` |
| ) |
| // icons maps each Lucide slug to its vendored markup. Slugs are the Lucide |
| @@ -96,6 +104,7 @@ var icons = map[string]template.HTML{ |
| "flask-conical": iconFlask, |
| "triangle-alert": iconTriangleAlert, |
| "layers": iconLayers, |
| + "hourglass": iconHourglass, |
| } |
| // icon renders one chrome icon by its Lucide slug: {{icon "chevron-down"}}. |
diff --git a/internal/console/ladder.go b/internal/console/ladder.go| index 2d7e290a9..ceadd0ab7 100644 |
| --- a/internal/console/ladder.go |
| +++ b/internal/console/ladder.go |
| @@ -19,6 +19,11 @@ package console |
| // and prodWiredCanaryHosts below answers "production" from that AND from |
| // which host is the app's apex: the environment says what glue a canary is |
| // wired to, the apex says whether it is in front of everybody. |
| +// |
| +// canaryExpiry answers a different question (spec §5 "Canaries expire"): |
| +// not whether a canary is dangerous, but whether it has been sitting there |
| +// so long it stopped being a canary and became infrastructure nobody |
| +// remembers deploying. Visibility only — an expired canary keeps serving. |
| import ( |
| "sort" |
| @@ -73,6 +78,15 @@ type LadderRoute struct { |
| // environment as one of the app's stable routes, OR sitting on the app's |
| // apex host. See prodWiredCanaryHosts for why it is both. |
| ProdWired bool |
| + // Expired is whether this route's canary/* channel has outlived |
| + // release.CanaryTTL, measured from the pointer's own PromotedAt (spec |
| + // §5 "Canaries expire"). Always false for a non-canary route. Visibility |
| + // only: an expired canary keeps serving, keeps adopting and stays |
| + // promotable — this badges it, nothing more. |
| + Expired bool |
| + // ExpiredFor is how long past the TTL the channel is, formatted for a |
| + // human ("9d", "6h") — valid only when Expired is true. |
| + ExpiredFor string |
| } |
| // LadderStep is the move from one rung to the one above it. |
| @@ -107,16 +121,20 @@ var gates = map[string]string{ |
| // lists it. A host following a rung channel that has no pointer DOES appear, |
| // as an error line: a host serving nothing is exactly the kind of thing this |
| // section exists to stop being invisible. |
| -func assembleLadder(channels []ChannelView, routes []RouteView) []LadderRung { |
| +func assembleLadder(channels []ChannelView, routes []RouteView, now time.Time) []LadderRung { |
| prod := prodWiredCanaryHosts(routes) |
| + expiredFor := canaryExpiry(channels, now) |
| byChannel := map[string][]LadderRoute{} |
| for _, rt := range routes { |
| rung, ok := release.Rung(rt.Channel) |
| + age := expiredFor[rt.Channel] |
| byChannel[rt.Channel] = append(byChannel[rt.Channel], LadderRoute{ |
| Host: rt.Host, |
| Environment: rt.Environment, |
| Canary: ok && rung == release.RungCanary, |
| ProdWired: prod[rt.Host], |
| + Expired: age != "", |
| + ExpiredFor: age, |
| }) |
| } |
| entries := map[string][]LadderChannel{} |
| @@ -250,3 +268,41 @@ func prodWiredCanaryHosts(routes []RouteView) map[string]bool { |
| } |
| return out |
| } |
| + |
| +// canaryExpiry maps each canary/* channel with a readable pointer to how |
| +// long past release.CanaryTTL it is, formatted for a human — spec §5 |
| +// "Canaries expire". A channel that has not (yet) expired, or whose pointer |
| +// carries an Err (assembleChannels degraded it to one bad row already), is |
| +// simply absent from the map, which is exactly what LadderRoute.Expired |
| +// reads as false. |
| +// |
| +// Built once per ladder assembly, keyed by channel name rather than host, |
| +// because expiry is a property of the CHANNEL's PromotedAt — every route |
| +// following the same canary shares one answer, computed once rather than |
| +// re-derived per host. |
| +// |
| +// Formatting goes through release.CanaryAge + release.FormatCanaryAge — |
| +// the SAME pair the adopt pass's nag calls (internal/release/adopt.go) — |
| +// rather than computing a duration here and rendering it locally. That used |
| +// to be two independent implementations that quietly disagreed by exactly |
| +// CanaryTTL (2026-08-03 review, IMPORTANT 2): sharing the formatter is what |
| +// makes "the nag and the badge cannot drift apart" a structural fact |
| +// instead of a coincidence two people have to keep re-verifying. |
| +func canaryExpiry(channels []ChannelView, now time.Time) map[string]string { |
| + out := map[string]string{} |
| + for _, cv := range channels { |
| + if cv.Err != "" { |
| + continue |
| + } |
| + // A synthetic pointer carrying only the two fields release.CanaryExpired |
| + // reads: the channel's name (so it recognizes canary/*) and the |
| + // promotion timestamp ChannelView already carries. No pointer re-read — |
| + // expiry is computed from data the console already has in hand. |
| + ptr := &release.ChannelPointer{Channel: cv.Name, PromotedAt: cv.PromotedAt} |
| + if !release.CanaryExpired(ptr, now) { |
| + continue |
| + } |
| + out[cv.Name] = release.FormatCanaryAge(release.CanaryAge(ptr, now)) |
| + } |
| + return out |
| +} |
diff --git a/internal/console/ladder_test.go b/internal/console/ladder_test.go| index 4742efb0d..2194c2768 100644 |
| --- a/internal/console/ladder_test.go |
| +++ b/internal/console/ladder_test.go |
| @@ -6,7 +6,11 @@ package console |
| // passkey, and a skip up the ladder is refused however it is posted. |
| import ( |
| + "bytes" |
| "context" |
| + "crypto/ed25519" |
| + "encoding/base64" |
| + "encoding/json" |
| "net/http" |
| "net/url" |
| "strings" |
| @@ -357,3 +361,178 @@ func TestLadderEnvironmentAndProdWiredVisibleToNonOwner(t *testing.T) { |
| t.Errorf("a member does not see route environment chrome:\n%s", stableBlock) |
| } |
| } |
| + |
| +// ---- Canary TTL (spec §5 "Canaries expire") -------------------------------- |
| + |
| +// A canary promoted moments ago — exactly what every real promotion through |
| +// this console produces, Promote always stamping "now" — must carry no |
| +// expired badge. This is the case CLAUDE.md's own console route is in on |
| +// every deploy (canary/console, re-promoted each time): a false positive |
| +// here would badge the console's own production traffic as neglected. |
| +func TestLadderFreshCanaryCarriesNoExpiredBadge(t *testing.T) { |
| + fx := newConsoleFixture(t) |
| + fx.addRoute(t, registry.Route{Host: "rc.kass.carlosapp.dev", Kind: "instance", Status: "ready", |
| + Version: "v2", App: "kass", Channel: "canary/x", Addr: "127.0.0.1:9002"}) |
| + seedChannel(t, fx, "kass", "canary/x", "v2") // stamps PromotedAt = now |
| + |
| + c := signInPaul(t, fx) |
| + _, _, body := get(t, c, fx.u("/apps/kass")) |
| + block := rungBlock(t, body, "canary") |
| + line := routeLine(t, block, "rc.kass.carlosapp.dev") |
| + |
| + if strings.Contains(line, "chip-expired") { |
| + t.Errorf("a freshly-promoted canary is badged expired:\n%s", line) |
| + } |
| + // This route binds the same (default) environment as the fixture's |
| + // stable route, so it is ALSO prod-wired (TestProdWiredCanaryIsBadgedDistinctly |
| + // covers that badge on its own) — chip-prod or chip-canary, either is the |
| + // "this is a canary" badge the expired flag would sit beside; the point |
| + // of this test is only that neither carries chip-expired. |
| + if !strings.Contains(line, "chip-prod") && !strings.Contains(line, "chip-canary") { |
| + t.Errorf("the fresh canary lost its canary-family badge entirely:\n%s", line) |
| + } |
| +} |
| + |
| +// assembleLadder is exercised directly here (no store, no signing) because |
| +// there is no way to get a genuinely aged, verifiably-signed pointer through |
| +// the public Promote path — it always stamps the real clock, deliberately |
| +// (release.TestSignStampsPromotedAt pins that). Constructing the ChannelView |
| +// by hand is the same input assembleChannels would have produced from a |
| +// pointer that really was promoted eight days ago. |
| +func TestLadderExpiredCanaryIsBadgedWithAge(t *testing.T) { |
| + now := time.Now() |
| + old := now.Add(-(release.CanaryTTL + 8*24*time.Hour)) // 8 days past the TTL |
| + channels := []ChannelView{{Name: "canary/x", Version: "v9", PromotedAt: old}} |
| + // A stable route on a DIFFERENT, non-apex, non-matching-environment host |
| + // keeps this canary out of BOTH prod-wired signals (prodWiredCanaryHosts |
| + // is exercised on its own elsewhere) — this test is about the expired |
| + // badge specifically, so it stays clear of the prod-wired one. |
| + routes := []RouteView{ |
| + {Host: "a.example.com", Channel: "stable", Environment: "prod-env"}, |
| + {Host: "rc.example.com", Channel: "canary/x"}, |
| + } |
| + |
| + rungs := assembleLadder(channels, routes, now) |
| + var canary *LadderRung |
| + for i := range rungs { |
| + if rungs[i].Rung == release.RungCanary { |
| + canary = &rungs[i] |
| + } |
| + } |
| + if canary == nil || len(canary.Channels) != 1 || len(canary.Channels[0].Routes) != 1 { |
| + t.Fatalf("canary rung = %+v, want one channel with one route", canary) |
| + } |
| + rt := canary.Channels[0].Routes[0] |
| + if !rt.Expired { |
| + t.Fatal("a canary promoted more than CanaryTTL ago must be marked Expired") |
| + } |
| + if rt.ExpiredFor != "8d" { |
| + t.Errorf("ExpiredFor = %q, want \"8d\"", rt.ExpiredFor) |
| + } |
| + if !rt.Canary { |
| + t.Error("an expired canary route must still carry the ordinary canary flag") |
| + } |
| + |
| + // Render it through the real template — proves the badge actually |
| + // reaches the page, with the icon and the age, beside the plain canary |
| + // badge rather than instead of it. |
| + p := page{Base: "/a/bab", Role: RoleOwner, CanPromote: true, |
| + Detail: &AppView{App: "demo", Ladder: rungs}} |
| + var buf bytes.Buffer |
| + if err := templates.ExecuteTemplate(&buf, "ladder", p); err != nil { |
| + t.Fatalf("render ladder: %v", err) |
| + } |
| + html := buf.String() |
| + line := routeLine(t, rungBlock(t, html, "canary"), "rc.example.com") |
| + if !strings.Contains(line, "chip-expired") { |
| + t.Errorf("rendered route carries no chip-expired badge:\n%s", line) |
| + } |
| + if !strings.Contains(line, "8d") { |
| + t.Errorf("rendered badge does not show the age:\n%s", line) |
| + } |
| + if !strings.Contains(line, "chip-canary") { |
| + t.Errorf("the expired badge replaced the ordinary canary badge instead of sitting beside it:\n%s", line) |
| + } |
| + if !strings.Contains(line, "<svg") { |
| + t.Error("the expired badge is a text glyph, not an icon") |
| + } |
| +} |
| + |
| +// A channel that has not yet reached CanaryTTL contributes nothing to |
| +// canaryExpiry's map — the direct unit-level twin of the fixture-based fresh |
| +// test above, pinned at the function boundary rather than through HTTP. |
| +func TestCanaryExpiryOmitsUnexpiredChannels(t *testing.T) { |
| + now := time.Now() |
| + channels := []ChannelView{ |
| + {Name: "canary/fresh", PromotedAt: now}, |
| + {Name: "canary/old", PromotedAt: now.Add(-(release.CanaryTTL + time.Hour))}, |
| + {Name: "stable", PromotedAt: now.Add(-365 * 24 * time.Hour)}, // no TTL at all |
| + {Name: "canary/broken", PromotedAt: now.Add(-365 * 24 * time.Hour), Err: "unreadable"}, |
| + } |
| + got := canaryExpiry(channels, now) |
| + if _, ok := got["canary/fresh"]; ok { |
| + t.Error("a fresh canary must not appear in the expiry map") |
| + } |
| + if _, ok := got["stable"]; ok { |
| + t.Error("stable carries no TTL and must never appear in the expiry map") |
| + } |
| + if _, ok := got["canary/broken"]; ok { |
| + t.Error("a channel whose pointer errored must not be guessed at as expired") |
| + } |
| + if _, ok := got["canary/old"]; !ok { |
| + t.Error("an old canary must appear in the expiry map") |
| + } |
| +} |
| + |
| +// seedExpiredCanary hand-signs a canary pointer whose lease ran out long |
| +// ago — every public promotion path stamps the real clock, so forging |
| +// PromotedAt in the wrapper is the only way to put an EXPIRED canary in |
| +// front of a console under test (the same technique, for the same reason, |
| +// as internal/release's writeBackdatedPointer, which is deliberately not |
| +// exported). |
| +func seedExpiredCanary(t *testing.T, fx *consoleFixture, app, channel, version string) { |
| + t.Helper() |
| + p := &release.ChannelPointer{Schema: 1, App: app, Channel: channel, |
| + Version: version, PromotedAt: time.Now().Add(-(release.CanaryTTL + 30*24*time.Hour))} |
| + payload, err := json.Marshal(p) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + b64 := base64.StdEncoding.EncodeToString(payload) |
| + wrapper, err := json.Marshal(struct { |
| + Payload string `json:"payload"` |
| + Sig string `json:"sig"` |
| + }{b64, base64.StdEncoding.EncodeToString(ed25519.Sign(fx.priv, []byte(b64)))}) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := fx.rel.Put(context.Background(), |
| + release.ChannelKey(release.OpsAccount, app, channel), wrapper, ""); err != nil { |
| + t.Fatal(err) |
| + } |
| +} |
| + |
| +// TestLadderPromotesOntoAnExpiredCanary pins spec §5's visibility-only |
| +// contract at the one surface the release-layer pins cannot reach: the |
| +// console's promote handler. An expired canary is exactly where the next |
| +// build lands when work on that branch resumes, and a "this canary expired" |
| +// gate here — however well-meant a confirmation might look — would turn a |
| +// guard rail into an obstacle at the moment it is least wanted. (2026-08-03 |
| +// review, IMPORTANT 1: a mutation gating handlePromote on expiry failed |
| +// zero tests before this one existed.) |
| +func TestLadderPromotesOntoAnExpiredCanary(t *testing.T) { |
| + fx := newConsoleFixture(t) |
| + shipVersion(t, fx, "kass", "v8") |
| + seedExpiredCanary(t, fx, "kass", "canary/old-branch", "v1") |
| + c := signInPaul(t, fx) |
| + csrf := sessionToken(t, fx, c) |
| + |
| + status, _, body := postForm(t, c, fx.u("/apps/kass/promote"), |
| + url.Values{"csrf": {csrf}, "channel": {"canary/old-branch"}, "version": {"v8"}}) |
| + if status != http.StatusSeeOther { |
| + t.Fatalf("promotion onto an expired canary = %d, want 303:\n%s", status, body) |
| + } |
| + if p := fx.channelVersion(t, "kass", "canary/old-branch"); p.Version != "v8" { |
| + t.Fatalf("expired canary still holds %s — expiry has become a gate", p.Version) |
| + } |
| +} |
diff --git a/internal/console/templates/ladder.html b/internal/console/templates/ladder.html| index 2f8da0342..72448ae20 100644 |
| --- a/internal/console/templates/ladder.html |
| +++ b/internal/console/templates/ladder.html |
| @@ -25,6 +25,7 @@ touching what any host is wired to.</p> |
| <span class="chip chip-env">{{icon "layers"}} {{if .Environment}}{{.Environment}}{{else}}default{{end}}</span> |
| {{if .ProdWired}}<span class="chip chip-prod">{{icon "triangle-alert"}} canary in front of everybody</span> |
| {{else if .Canary}}<span class="chip chip-canary">{{icon "flask-conical"}} canary</span>{{end}} |
| + {{if .Expired}}<span class="chip chip-expired">{{icon "hourglass"}} expired {{.ExpiredFor}}</span>{{end}} |
| </li> |
| {{end}} |
| </ul> |
diff --git a/internal/console/templates/layout.html b/internal/console/templates/layout.html| index 8d4185ec2..cf142d4d1 100644 |
| --- a/internal/console/templates/layout.html |
| +++ b/internal/console/templates/layout.html |
| @@ -146,6 +146,10 @@ |
| every user is the one thing on this page a reader must not skim past. */ |
| .chip-canary { border: 1px solid var(--accent); color: var(--accent); } |
| .chip-prod { background: var(--bad); color: var(--bg); } |
| + /* An expired canary (spec §5) keeps serving — this is "trust this less", |
| + the same bordered-only warning as .chip-stale, never a filled outage |
| + marker like .chip-prod. */ |
| + .chip-expired { border: 1px solid var(--warn, #a6790f); color: var(--warn, #a6790f); } |
| /* A route's bound environment — a fact, not a warning, so it is bordered |
| like .chip-canary but in a neutral tone: it sits on every route, not |
| just the ones worth a second look. */ |
diff --git a/internal/console/view.go b/internal/console/view.go| index 4da58704c..9b61c1ca2 100644 |
| --- a/internal/console/view.go |
| +++ b/internal/console/view.go |
| @@ -241,7 +241,7 @@ func Assemble(regPath string, s release.Store, pub ed25519.PublicKey, acct strin |
| Domains: domainsByApp[app], |
| Channels: channels, |
| Canaries: assembleCanaries(channels, byApp[app]), |
| - Ladder: assembleLadder(channels, byApp[app]), |
| + Ladder: assembleLadder(channels, byApp[app], now), |
| RouteAccount: mismatch, |
| }) |
| } |
diff --git a/internal/host/agent.go b/internal/host/agent.go| index 9e42508c9..7edbdcc9d 100644 |
| --- a/internal/host/agent.go |
| +++ b/internal/host/agent.go |
| @@ -64,6 +64,7 @@ package host |
| import ( |
| "context" |
| + "crypto/ed25519" |
| "errors" |
| "flag" |
| "io" |
| @@ -205,6 +206,13 @@ func adoptLoop(reg *registry.Registry) { |
| for { |
| time.Sleep(delay) |
| delay = time.Hour + rand.N(10*time.Minute) |
| + // DO NOT add release.NagExpiredCanaries(...) to this call. This loop |
| + // runs hourly, but it is NOT "the hourly root adopt pass" the option's |
| + // own doc comment names — that's RunAdopt (carlos-adopt.service) |
| + // below, root-privileged and the one place config sync also runs. Two |
| + // hourly callers both nagging would double the log line every hour; |
| + // worse, if this loop's cadence ever changes, a nag wired here changes |
| + // frequency with it silently. One caller, one nag. |
| if err := release.Pass(ctx, reg, store, pub, optDir, restartRoute); err != nil { |
| log.Printf("agent: adopt pass: %v", err) |
| } |
| @@ -283,6 +291,12 @@ func canaryAdoptLoop(reg *registry.Registry) { |
| log.Printf("agent: canary adopt loop armed (2s + jitter, canary/* routes only)") |
| for { |
| time.Sleep(2*time.Second + rand.N(time.Second)) |
| + // DO NOT add release.NagExpiredCanaries(...) to this call, ever. This |
| + // is THE loop release.NagExpiredCanaries' doc comment warns about by |
| + // name: every 2-3 seconds, canary/* only. An expired canary sitting |
| + // still would nag on every single tick — a journal-flooding poll storm |
| + // out of what is supposed to be an hourly, easy-to-skim log line. The |
| + // nag belongs to RunAdopt (carlos-adopt.service) alone. |
| if err := release.PassFilter(ctx, reg, store, pub, optDir, restartRoute, isCanaryRoute); err != nil { |
| log.Printf("agent: canary adopt pass: %v", err) |
| } |
| @@ -298,6 +312,54 @@ func isCanaryRoute(rt registry.Route) bool { |
| return strings.HasPrefix(rt.Channel, "canary/") |
| } |
| +// countExpiredCanaries counts routes (this box's own, platform aliases |
| +// excluded — same convention as CollectBox) on a canary/* channel whose |
| +// pointer has passed release.CanaryTTL, for the fleet status page's §5 |
| +// count (internal/status Probes.ExpiredCanaries). A canary route whose |
| +// pointer can't be resolved at all — store unavailable, pubkey unusable, or |
| +// the specific channel won't verify — nils the WHOLE count rather than |
| +// undercounting: "we couldn't check" and "there are none" are different |
| +// claims, and this function only ever makes the second one when it's sure. |
| +// No canary routes at all is a confident, known 0 — it needs no store to |
| +// answer "none" about routes that don't exist. |
| +// |
| +// Pointers are resolved at most once per distinct (account, app, channel) |
| +// among the routes given (the local `cache` map below), mirroring adopt.go's |
| +// Pass — this function is called once per 5-minute status tick |
| +// (statusLoop), so N routes on this box following the same canary channel |
| +// cost exactly ONE release.Channel() store read for that tick, not N |
| +// (2026-08-03 review: confirmed and pinned by |
| +// TestCountExpiredCanariesReadsEachChannelOnce). The cache is local to one |
| +// call/one tick — nothing persists across ticks, which is fine at a |
| +// 5-minute cadence. |
| +func countExpiredCanaries(ctx context.Context, routes []registry.Route, store release.Store, pub ed25519.PublicKey, now time.Time) *int { |
| + type key struct{ acct, app, channel string } |
| + cache := map[key]*release.ChannelPointer{} |
| + n := 0 |
| + for _, rt := range routes { |
| + if rt.Platform || !isCanaryRoute(rt) { |
| + continue |
| + } |
| + if store == nil || len(pub) != ed25519.PublicKeySize { |
| + return nil // a canary route exists but nothing here can resolve its pointer |
| + } |
| + k := key{release.RouteAccount(rt), rt.App, rt.Channel} |
| + ptr, cached := cache[k] |
| + if !cached { |
| + p, err := release.Channel(ctx, store, pub, k.acct, k.app, k.channel) |
| + if err != nil { |
| + return nil // this route's own pointer won't verify — unknown, not zero |
| + } |
| + ptr = p |
| + cache[k] = ptr |
| + } |
| + if release.CanaryExpired(ptr, now) { |
| + n++ |
| + } |
| + } |
| + return &n |
| +} |
| + |
| // peekAcmeCache pre-parses the agent args for -acme-cache the same way |
| // peekRegistryPath peeks -registry: the edge does the authoritative parse, |
| // this just lets the domains loop stamp cert expiry from the SAME cache dir |
| @@ -519,6 +581,21 @@ func statusLoop(reg *registry.Registry) { |
| // place an operator can confirm which bucket actually won. |
| log.Printf("agent: status loop armed (5min + jitter), publishing to bucket %s", bucket) |
| + // canaryStore/canaryPub resolve the release deployment config |
| + // countExpiredCanaries needs to answer §5's "how many canaries on this |
| + // box have expired" — the SAME env pair adoptLoop reads |
| + // (CARLOS_DEPLOYMENT_BUCKET/_DIR + CARLOS_RELEASE_PUBKEY), not |
| + // necessarily the status bucket above, which may deliberately be a |
| + // separate one. Either can come back unusable: no deployment store |
| + // configured, or no pubkey set. That is a normal, valid box shape (the |
| + // same "routes managed by hand" case adoptLoop's own doc comment |
| + // names) — it just means countExpiredCanaries can't answer and reports |
| + // nil, same as every other unmeasured probe in this loop. Resolved once |
| + // here, not re-read every tick, mirroring adoptLoop's own one-time |
| + // resolution. |
| + canaryStore, _ := openDeployStore(ctx) |
| + canaryPub, _ := release.ParsePublicKey(os.Getenv("CARLOS_RELEASE_PUBKEY"), "CARLOS_RELEASE_PUBKEY") |
| + |
| // Jittered on the same shape as adoptLoop's delay: every box on the same |
| // 5-minute cadence would all hit the lease and the bucket in the same |
| // second otherwise. |
| @@ -551,6 +628,9 @@ func statusLoop(reg *registry.Registry) { |
| // flagship's silent no-adopt state, which nothing on any page |
| // could see before this. |
| AdoptRunning: adoptTimerActive(), |
| + // ExpiredCanaries answers spec §5 for this box's own routes; see |
| + // countExpiredCanaries and the canaryStore/canaryPub comment above. |
| + ExpiredCanaries: countExpiredCanaries(ctx, routes, canaryStore, canaryPub, time.Now()), |
| // LagSecs stays nil: litestream's per-DB replication lag isn't |
| // surfaced up to the agent process anywhere yet — the |
| // replication machinery (internal/lease's Replicator) lives |
| @@ -691,7 +771,14 @@ func RunAdopt(args []string) { |
| } |
| } |
| - if err := release.Pass(ctx, reg, store, pub, release.OptDir(), restartRoute); err != nil { |
| + // NagExpiredCanaries(os.Stdout) is set HERE ONLY (2026-08-03 review) — the |
| + // canary-TTL nag (spec §5) belongs to the hourly root pass |
| + // (carlos-adopt.service, this verb's ExecStart), the same split PR #9's |
| + // config sync above uses. adoptLoop and canaryAdoptLoop in this same |
| + // package call release.Pass/PassFilter with no options at all, so an |
| + // expired canary nags once an hour from here, never every 2-3 seconds |
| + // from the in-process fast loop. |
| + if err := release.Pass(ctx, reg, store, pub, release.OptDir(), restartRoute, release.NagExpiredCanaries(os.Stdout)); err != nil { |
| log.Fatalf("adopt: %v", err) |
| } |
diff --git a/internal/host/host_test.go b/internal/host/host_test.go| index 4312df011..b2fb565f0 100644 |
| --- a/internal/host/host_test.go |
| +++ b/internal/host/host_test.go |
| @@ -2,6 +2,9 @@ package host |
| import ( |
| "context" |
| + "crypto/ed25519" |
| + "encoding/base64" |
| + "encoding/json" |
| "net" |
| "os" |
| "path/filepath" |
| @@ -485,3 +488,208 @@ func TestAdoptTimerStateReading(t *testing.T) { |
| } |
| func boolPtr(v bool) *bool { return &v } |
| + |
| +// ---- Pipelines: canary TTL, the status loop's expired-canary count -------- |
| + |
| +// wireWrapper mirrors release's unexported signedWrapper wire shape |
| +// (payload/sig) — duplicated here rather than imported (it isn't exported, |
| +// deliberately: forging a pointer's PromotedAt is not something production |
| +// code anywhere should be able to do) so this test can write a validly |
| +// SIGNED but arbitrarily-dated channel pointer straight into a DirStore, |
| +// the only way to get an actually-expired pointer under test since every |
| +// public promotion path (release.Promote) stamps the real clock. |
| +type wireWrapper struct { |
| + Payload string `json:"payload"` |
| + Sig string `json:"sig"` |
| +} |
| + |
| +// writeBackdatedPointer signs and writes a channel pointer with an |
| +// arbitrary PromotedAt directly to key release.ChannelKey(acct, app, |
| +// channel), bypassing Promote's clock stamp. |
| +func writeBackdatedPointer(t *testing.T, s release.Store, priv ed25519.PrivateKey, acct, app, channel, version string, promotedAt time.Time) { |
| + t.Helper() |
| + p := struct { |
| + Schema int `json:"schema"` |
| + App string `json:"app"` |
| + Channel string `json:"channel"` |
| + Version string `json:"version"` |
| + PromotedAt time.Time `json:"promoted_at"` |
| + }{Schema: 1, App: app, Channel: channel, Version: version, PromotedAt: promotedAt} |
| + payload, err := json.Marshal(p) |
| + if err != nil { |
| + t.Fatalf("marshal backdated pointer: %v", err) |
| + } |
| + b64 := base64.StdEncoding.EncodeToString(payload) |
| + sig := ed25519.Sign(priv, []byte(b64)) |
| + wrapper, err := json.Marshal(wireWrapper{Payload: b64, Sig: base64.StdEncoding.EncodeToString(sig)}) |
| + if err != nil { |
| + t.Fatalf("marshal backdated wrapper: %v", err) |
| + } |
| + if _, err := s.Put(context.Background(), release.ChannelKey(acct, app, channel), wrapper, ""); err != nil { |
| + t.Fatalf("write backdated pointer: %v", err) |
| + } |
| +} |
| + |
| +func testCanaryKeys(t *testing.T) (ed25519.PrivateKey, ed25519.PublicKey) { |
| + t.Helper() |
| + seedB64, pubB64, err := release.GenerateKey() |
| + if err != nil { |
| + t.Fatalf("GenerateKey: %v", err) |
| + } |
| + priv, err := release.ParseSigningKey(seedB64, "test") |
| + if err != nil { |
| + t.Fatalf("ParseSigningKey: %v", err) |
| + } |
| + pub, err := release.ParsePublicKey(pubB64, "test") |
| + if err != nil { |
| + t.Fatalf("ParsePublicKey: %v", err) |
| + } |
| + return priv, pub |
| +} |
| + |
| +// No canary routes at all needs no store to answer "none" — a confident, |
| +// known 0, not the unknown/nil path a missing store would otherwise imply. |
| +func TestCountExpiredCanariesNoCanaryRoutesIsZero(t *testing.T) { |
| + routes := []registry.Route{ |
| + {Host: "a.example.com", Kind: "instance", App: "myapp", Channel: "stable"}, |
| + {Host: "b.example.com", Kind: "static"}, |
| + } |
| + got := countExpiredCanaries(context.Background(), routes, nil, nil, time.Now()) |
| + if got == nil || *got != 0 { |
| + t.Errorf("count = %v, want 0 (no canary routes, no store needed)", got) |
| + } |
| +} |
| + |
| +// A canary route exists but there is no deployment store to resolve its |
| +// pointer against — the "routes managed by hand" box shape adoptLoop's own |
| +// doc comment names. That must publish unknown, never a guessed 0. |
| +func TestCountExpiredCanariesNilStoreIsUnknown(t *testing.T) { |
| + routes := []registry.Route{{Host: "rc.example.com", Kind: "instance", App: "myapp", Channel: "canary/x"}} |
| + if got := countExpiredCanaries(context.Background(), routes, nil, nil, time.Now()); got != nil { |
| + t.Errorf("count = %v, want nil (no store to resolve the pointer against)", *got) |
| + } |
| +} |
| + |
| +// A canary route whose channel was never actually promoted (no pointer at |
| +// that key) is the same "can't answer" case as no store at all — release. |
| +// Channel returns os.ErrNotExist, and that must null the count too, not |
| +// silently skip the route and undercount. |
| +func TestCountExpiredCanariesUnresolvablePointerIsUnknown(t *testing.T) { |
| + s := release.NewDirStore(t.TempDir()) |
| + _, pub := testCanaryKeys(t) |
| + routes := []registry.Route{{Host: "rc.example.com", Kind: "instance", App: "myapp", Channel: "canary/x", Account: "bab"}} |
| + if got := countExpiredCanaries(context.Background(), routes, s, pub, time.Now()); got != nil { |
| + t.Errorf("count = %v, want nil (channel was never promoted)", *got) |
| + } |
| +} |
| + |
| +// A fresh canary (real Promote, real clock) counts as 0 — not expired, and |
| +// known. |
| +func TestCountExpiredCanariesFreshCanaryIsZero(t *testing.T) { |
| + s := release.NewDirStore(t.TempDir()) |
| + ctx := context.Background() |
| + priv, pub := testCanaryKeys(t) |
| + shipVersion(t, s, "bab", "myapp", "v1") |
| + if err := release.Promote(ctx, s, priv, "bab", "myapp", "canary/x", "v1"); err != nil { |
| + t.Fatalf("Promote: %v", err) |
| + } |
| + routes := []registry.Route{{Host: "rc.example.com", Kind: "instance", App: "myapp", Channel: "canary/x", Account: "bab"}} |
| + got := countExpiredCanaries(ctx, routes, s, pub, time.Now()) |
| + if got == nil || *got != 0 { |
| + t.Errorf("count = %v, want 0 (freshly promoted)", got) |
| + } |
| +} |
| + |
| +// The core case: a backdated (expired) canary pointer counts as 1; a second, |
| +// unexpired canary route on a different channel does not add to it. |
| +func TestCountExpiredCanariesCountsExpiredOnly(t *testing.T) { |
| + s := release.NewDirStore(t.TempDir()) |
| + ctx := context.Background() |
| + priv, pub := testCanaryKeys(t) |
| + writeBackdatedPointer(t, s, priv, "bab", "myapp", "canary/old", "v1", |
| + time.Now().Add(-(release.CanaryTTL + 24*time.Hour))) |
| + shipVersion(t, s, "bab", "myapp", "v2") |
| + if err := release.Promote(ctx, s, priv, "bab", "myapp", "canary/fresh", "v2"); err != nil { |
| + t.Fatalf("Promote: %v", err) |
| + } |
| + routes := []registry.Route{ |
| + {Host: "old.example.com", Kind: "instance", App: "myapp", Channel: "canary/old", Account: "bab"}, |
| + {Host: "fresh.example.com", Kind: "instance", App: "myapp", Channel: "canary/fresh", Account: "bab"}, |
| + } |
| + got := countExpiredCanaries(ctx, routes, s, pub, time.Now()) |
| + if got == nil || *got != 1 { |
| + t.Errorf("count = %v, want 1 (only the old channel's route is expired)", got) |
| + } |
| +} |
| + |
| +// A platform-minted alias is a second NAME for a route already counted, not |
| +// a second canary route — same convention CollectBox uses to avoid |
| +// double-counting aliases (host_test.go's own |
| +// TestCollectBoxSkipsPlatformMintedRoutes in internal/status covers that |
| +// half; this pins the same rule for the expired-canary count). |
| +func TestCountExpiredCanariesSkipsPlatformAliases(t *testing.T) { |
| + s := release.NewDirStore(t.TempDir()) |
| + priv, pub := testCanaryKeys(t) |
| + writeBackdatedPointer(t, s, priv, "bab", "myapp", "canary/old", "v1", |
| + time.Now().Add(-(release.CanaryTTL + 24*time.Hour))) |
| + routes := []registry.Route{ |
| + {Host: "alias.example.com", Kind: "instance", App: "myapp", Channel: "canary/old", Account: "bab", Platform: true}, |
| + } |
| + got := countExpiredCanaries(context.Background(), routes, s, pub, time.Now()) |
| + if got == nil || *got != 0 { |
| + t.Errorf("count = %v, want 0 — a platform alias must not be counted", got) |
| + } |
| +} |
| + |
| +// getCountingStore wraps a real release.Store and counts Get calls — used to |
| +// prove countExpiredCanaries' per-call cache actually dedupes store reads, |
| +// not just channel-pointer resolutions in memory. |
| +type getCountingStore struct { |
| + release.Store |
| + gets int |
| +} |
| + |
| +func (s *getCountingStore) Get(ctx context.Context, key string) ([]byte, string, error) { |
| + s.gets++ |
| + return s.Store.Get(ctx, key) |
| +} |
| + |
| +// TestCountExpiredCanariesReadsEachChannelOnce pins the caching half of the |
| +// 2026-08-03 review: statusLoop calls countExpiredCanaries once per |
| +// 5-minute tick, and this function must resolve a given canary channel's |
| +// pointer AT MOST ONCE per call however many of this box's routes follow |
| +// it — N routes sharing one channel costs one store.Get, not N. |
| +func TestCountExpiredCanariesReadsEachChannelOnce(t *testing.T) { |
| + real := release.NewDirStore(t.TempDir()) |
| + priv, pub := testCanaryKeys(t) |
| + writeBackdatedPointer(t, real, priv, "bab", "myapp", "canary/old", "v1", |
| + time.Now().Add(-(release.CanaryTTL + 24*time.Hour))) |
| + counting := &getCountingStore{Store: real} |
| + |
| + routes := []registry.Route{ |
| + {Host: "a.example.com", Kind: "instance", App: "myapp", Channel: "canary/old", Account: "bab"}, |
| + {Host: "b.example.com", Kind: "instance", App: "myapp", Channel: "canary/old", Account: "bab"}, |
| + {Host: "c.example.com", Kind: "instance", App: "myapp", Channel: "canary/old", Account: "bab"}, |
| + } |
| + got := countExpiredCanaries(context.Background(), routes, counting, pub, time.Now()) |
| + if got == nil || *got != 3 { |
| + t.Fatalf("count = %v, want 3 (all three routes share the one expired channel)", got) |
| + } |
| + if counting.gets != 1 { |
| + t.Errorf("store.Get called %d times for 3 routes on ONE channel, want exactly 1 — the per-tick cache must dedupe by (account, app, channel)", counting.gets) |
| + } |
| +} |
| + |
| +// shipVersion ships a minimal one-artifact release so Promote has something |
| +// real to point at — release.Promote refuses to point a channel at a |
| +// version nothing shipped. |
| +func shipVersion(t *testing.T, s release.Store, acct, app, version string) { |
| + t.Helper() |
| + m := release.Manifest{ |
| + Schema: 1, App: app, Version: version, Kind: "binary", |
| + Artifacts: map[string]release.Artifact{"linux-amd64": {File: app}}, |
| + } |
| + if err := release.Ship(context.Background(), s, acct, m, map[string][]byte{"linux-amd64": []byte("bin")}); err != nil { |
| + t.Fatalf("Ship %s/%s: %v", app, version, err) |
| + } |
| +} |
diff --git a/internal/release/adopt.go b/internal/release/adopt.go| index dfe39e1c9..5f6ef1228 100644 |
| --- a/internal/release/adopt.go |
| +++ b/internal/release/adopt.go |
| @@ -9,6 +9,7 @@ import ( |
| "crypto/ed25519" |
| "errors" |
| "fmt" |
| + "io" |
| "os" |
| "runtime" |
| "time" |
| @@ -20,6 +21,43 @@ import ( |
| // tests can exercise resolution without faking runtime.GOOS/GOARCH. |
| var adoptPlatform = runtime.GOOS + "-" + runtime.GOARCH |
| +// PassOption configures one call to Pass or PassFilter beyond their required |
| +// arguments. Currently the only option is NagExpiredCanaries; the type |
| +// exists so a call site opts in explicitly and a future option composes |
| +// the same way, rather than every option becoming its own trailing |
| +// parameter. |
| +type PassOption func(*passConfig) |
| + |
| +type passConfig struct { |
| + // nag is where the canary-TTL nag (spec §5) is printed; nil (the |
| + // default — every call site that does not pass NagExpiredCanaries) |
| + // means the nag never fires at all, not even to a discarded writer. |
| + nag io.Writer |
| +} |
| + |
| +// NagExpiredCanaries opts a Pass/PassFilter call into spec §5's nag: one |
| +// line per expired canary/* route the call sees, written to w. |
| +// |
| +// DO NOT pass this to a fast/frequent adoption loop. It belongs to the |
| +// HOURLY ROOT adopt pass ALONE — carlos-adopt.service's RunAdopt |
| +// (internal/host/agent.go), the only caller that sets it (2026-08-03 |
| +// review), and every future caller must keep it that way. The same split |
| +// PR #9's config sync uses, for the same reason: the in-process adoptLoop |
| +// (hourly + jitter, inside the sandboxed carlos-edge.service) and |
| +// canaryAdoptLoop (every 2-3 SECONDS, canary/* routes only — the whole |
| +// point of which is fast pickup of a freshly-promoted canary) must NEVER |
| +// set this. An expired canary sitting still on that fast loop would |
| +// otherwise nag every 2-3 seconds forever, turning a visibility feature |
| +// into a journal-flooding one; gating it to the hourly root pass keeps the |
| +// nag journal-friendly — once an hour, not a poll-storm of log lines. This |
| +// is enforced by convention (which call sites pass the option), not by the |
| +// type system — see agent.go's adoptLoop/canaryAdoptLoop for the loud |
| +// "never add this" comments at the two places it would be easiest to add |
| +// by accident. |
| +func NagExpiredCanaries(w io.Writer) PassOption { |
| + return func(c *passConfig) { c.nag = w } |
| +} |
| + |
| // Pass is one full adoption sweep: group instance routes by (App, Channel), |
| // resolve each channel pointer once, verify signatures and artifact sha256s, |
| // download/stage missing versions, and repoint live/<host> symlinks. |
| @@ -38,8 +76,8 @@ var adoptPlatform = runtime.GOOS + "-" + runtime.GOARCH |
| // surfaces at wake, recovered by Rollback, which repoints back. Hibernating |
| // hosts are processed AFTER every awake host, so an awake host discovers a |
| // boot-broken build (and poisons it) before any sleeper is moved onto it. |
| -func Pass(ctx context.Context, reg *registry.Registry, s Store, pub ed25519.PublicKey, optDir string, restart func(host string) error) error { |
| - return PassFilter(ctx, reg, s, pub, optDir, restart, nil) |
| +func Pass(ctx context.Context, reg *registry.Registry, s Store, pub ed25519.PublicKey, optDir string, restart func(host string) error, opts ...PassOption) error { |
| + return PassFilter(ctx, reg, s, pub, optDir, restart, nil, opts...) |
| } |
| // PassFilter is Pass narrowed to the routes extra additionally accepts |
| @@ -52,7 +90,11 @@ func Pass(ctx context.Context, reg *registry.Registry, s Store, pub ed25519.Publ |
| // no window"), so sweeping just them every few seconds instead of hourly |
| // can't turn into a fleet-wide poll storm the way sweeping every route |
| // that fast would. |
| -func PassFilter(ctx context.Context, reg *registry.Registry, s Store, pub ed25519.PublicKey, optDir string, restart func(host string) error, extra func(registry.Route) bool) error { |
| +func PassFilter(ctx context.Context, reg *registry.Registry, s Store, pub ed25519.PublicKey, optDir string, restart func(host string) error, extra func(registry.Route) bool, opts ...PassOption) error { |
| + var cfg passConfig |
| + for _, opt := range opts { |
| + opt(&cfg) |
| + } |
| routes, err := reg.List() |
| if err != nil { |
| return err |
| @@ -127,12 +169,12 @@ func PassFilter(ctx context.Context, reg *registry.Registry, s Store, pub ed2551 |
| continue |
| } |
| ptr := pointers[groupKey{RouteAccount(rt), rt.App, routeChannel(rt)}] |
| - adoptHost(ctx, reg, s, optDir, bake, poisoned, rt, ptr, restart, now) |
| + adoptHost(ctx, reg, s, optDir, bake, poisoned, rt, ptr, restart, now, cfg.nag) |
| } |
| for _, rt := range hibernating { |
| ptr := pointers[groupKey{RouteAccount(rt), rt.App, routeChannel(rt)}] |
| // restart=nil: hibernating hosts get the pure repoint — never woken. |
| - adoptHost(ctx, reg, s, optDir, bake, poisoned, rt, ptr, nil, now) |
| + adoptHost(ctx, reg, s, optDir, bake, poisoned, rt, ptr, nil, now, cfg.nag) |
| } |
| return nil |
| } |
| @@ -182,10 +224,35 @@ func RouteAccount(rt registry.Route) string { |
| // pointer didn't resolve; skip quietly, the group phase already logged). |
| // restart==nil is the symlink-only path (hibernating hosts, or a caller with |
| // no proc manager): repoint and record, never wake, never health-gate. |
| -func adoptHost(ctx context.Context, reg *registry.Registry, s Store, optDir string, bake *FirstSeen, poisoned map[string]bool, rt registry.Route, ptr *ChannelPointer, restart func(host string) error, now time.Time) { |
| +// nag is passConfig.nag from the call's PassOptions — nil (the ordinary |
| +// case) means the canary-TTL nag below never fires, never even builds its |
| +// string; see NagExpiredCanaries for who sets it and why. |
| +func adoptHost(ctx context.Context, reg *registry.Registry, s Store, optDir string, bake *FirstSeen, poisoned map[string]bool, rt registry.Route, ptr *ChannelPointer, restart func(host string) error, now time.Time, nag io.Writer) { |
| if ptr == nil { |
| return |
| } |
| + // The nag (spec §5 "Canaries expire"): visibility only, and deliberately |
| + // BEFORE the up-to-date short-circuit below — an expired canary that |
| + // isn't moving (the common case: the branch shipped once and nobody |
| + // touched it since) must still be seen every pass it's part of, not just |
| + // the passes that happen to change something. Nothing here blocks adopt, |
| + // restart or promote; it only (optionally) prints. Gated on nag != nil |
| + // rather than always printing to os.Stdout: see NagExpiredCanaries — this |
| + // must stay silent on the 2-3-second canary fast loop and speak only on |
| + // the hourly root adopt pass, or an expired canary sitting still would |
| + // nag every few seconds forever instead of once an hour. |
| + if nag != nil && CanaryExpired(ptr, now) { |
| + // FormatCanaryAge(CanaryAge(...)), NOT a raw now.Sub(ptr.PromotedAt): |
| + // that would print the pointer's TOTAL age (CanaryAge + CanaryTTL), |
| + // overstating how long it's been expired by exactly one lease length |
| + // (2026-08-03 review, IMPORTANT 2) — and disagreeing with the |
| + // console's badge, which renders the same CanaryAge/FormatCanaryAge |
| + // pair (internal/console/ladder.go's canaryExpiry). One formatter, |
| + // shared, so the two operator-facing surfaces of this feature cannot |
| + // drift apart on what a duration means. |
| + fmt.Fprintf(nag, " %s: canary %s (%s) has been expired %s — still serving, nothing blocked\n", |
| + rt.Host, ptr.Channel, ptr.Version, FormatCanaryAge(CanaryAge(ptr, now))) |
| + } |
| channel := routeChannel(rt) |
| ver := ptr.Version |
| cur, err := LiveVersion(optDir, rt.Host) |
diff --git a/internal/release/adopt_test.go b/internal/release/adopt_test.go| index 624c67f38..ab541f310 100644 |
| --- a/internal/release/adopt_test.go |
| +++ b/internal/release/adopt_test.go |
| @@ -1,11 +1,17 @@ |
| package release |
| import ( |
| + "bytes" |
| "context" |
| + "crypto/ed25519" |
| + "encoding/base64" |
| + "encoding/json" |
| "errors" |
| "os" |
| "path/filepath" |
| + "strings" |
| "testing" |
| + "time" |
| "github.com/carlosframework/platform/internal/registry" |
| ) |
| @@ -370,3 +376,249 @@ func TestPassFilterOnlyTouchesRoutesExtraAccepts(t *testing.T) { |
| t.Errorf("stable.example.com live = %q, want unmoved — PassFilter must not touch a route extra rejects", v) |
| } |
| } |
| + |
| +// writeBackdatedPointer signs and writes a channel pointer with an arbitrary |
| +// PromotedAt, bypassing Promote/signPointer's own "stamp now" behavior — the |
| +// only way to get an EXPIRED canary pointer into a store under test, since |
| +// every public promotion path stamps the real clock. Kept in this internal |
| +// test package (never exported) precisely because forging a pointer's lease |
| +// is not something production code should ever be able to do. |
| +func writeBackdatedPointer(t *testing.T, s Store, priv ed25519.PrivateKey, acct, app, channel, version string, promotedAt time.Time) { |
| + t.Helper() |
| + p := &ChannelPointer{Schema: 1, App: app, Channel: channel, Version: version, PromotedAt: promotedAt} |
| + payload, err := json.MarshalIndent(p, "", " ") |
| + if err != nil { |
| + t.Fatalf("marshal backdated pointer: %v", err) |
| + } |
| + b64 := base64.StdEncoding.EncodeToString(payload) |
| + sig := ed25519.Sign(priv, []byte(b64)) |
| + wrapper, err := json.MarshalIndent(signedWrapper{ |
| + Payload: b64, Sig: base64.StdEncoding.EncodeToString(sig), |
| + }, "", " ") |
| + if err != nil { |
| + t.Fatalf("marshal backdated wrapper: %v", err) |
| + } |
| + if _, err := s.Put(context.Background(), ChannelKey(acct, app, channel), wrapper, ""); err != nil { |
| + t.Fatalf("write backdated pointer: %v", err) |
| + } |
| +} |
| + |
| +// TestPassAdoptsAnExpiredCanary pins spec §5's core promise: expiry is |
| +// VISIBILITY, not enforcement. A canary/* route whose pointer is well past |
| +// CanaryTTL must still adopt exactly like a fresh one — nothing about |
| +// expiry may block a repoint, a restart, or a rollback/promote (the same |
| +// signed-pointer plumbing every other channel uses; expiry checks nothing |
| +// about the ladder or the CAS path). |
| +func TestPassAdoptsAnExpiredCanary(t *testing.T) { |
| + s := NewDirStore(t.TempDir()) |
| + ctx := context.Background() |
| + priv, pub := testKeys(t) |
| + opt := t.TempDir() |
| + reg := testRegistry(t) |
| + |
| + shipRelease(t, s, "bab", "myapp", "v1", adoptPlatform, []byte("build one")) |
| + writeBackdatedPointer(t, s, priv, "bab", "myapp", "canary/x", "v1", |
| + time.Now().Add(-(CanaryTTL + 30*24*time.Hour))) // long expired |
| + addRoute(t, reg, "old-canary.example.com", "myapp", "canary/x", false) |
| + |
| + var restarted []string |
| + restart := func(host string) error { restarted = append(restarted, host); return nil } |
| + if err := Pass(ctx, reg, s, pub, opt, restart); err != nil { |
| + t.Fatalf("Pass: %v", err) |
| + } |
| + |
| + if v, _ := LiveVersion(opt, "old-canary.example.com"); v != "v1" { |
| + t.Fatalf("an expired canary route did not adopt: live = %q, want v1", v) |
| + } |
| + if len(restarted) != 1 || restarted[0] != "old-canary.example.com" { |
| + t.Fatalf("an expired canary route was not restarted: restarted = %v", restarted) |
| + } |
| + rt, err := reg.Get("old-canary.example.com") |
| + if err != nil || rt == nil || rt.Version != "v1" { |
| + t.Fatalf("registry version after adopting an expired canary = %+v, %v", rt, err) |
| + } |
| + |
| + // Promotion off (or onto) an expired canary channel is unaffected too — |
| + // expiry is computed from PromotedAt at read time, never stored, so |
| + // nothing about a pointer's own expiry state can gate a write to it. |
| + shipRelease(t, s, "bab", "myapp", "v2", adoptPlatform, []byte("build two")) |
| + if err := Promote(ctx, s, priv, "bab", "myapp", "canary/x", "v2"); err != nil { |
| + t.Fatalf("promoting a new version onto an expired canary channel: %v", err) |
| + } |
| + p, err := Channel(ctx, s, pub, "bab", "myapp", "canary/x") |
| + if err != nil { |
| + t.Fatalf("Channel after re-promoting an expired canary: %v", err) |
| + } |
| + if p.Version != "v2" { |
| + t.Fatalf("channel = %q after promote, want v2", p.Version) |
| + } |
| + // And the re-promotion refreshed the lease — CanaryExpired.go's whole |
| + // point: an actively-updated canary never expires. |
| + if CanaryExpired(p, time.Now()) { |
| + t.Error("re-promoting an expired canary must refresh its lease, not leave it expired") |
| + } |
| +} |
| + |
| +// TestPassFilterDoesNotNagWithoutTheOption pins the fast-loop half of the |
| +// nag's scoping (2026-08-03 review): internal/host/agent.go's canaryAdoptLoop |
| +// calls PassFilter every 2-3 seconds for canary/* routes only, and NEVER |
| +// passes NagExpiredCanaries — so an expired canary that is sitting still |
| +// must produce ZERO nag lines from that path, not one every few seconds. |
| +// This is PassFilter called exactly the way canaryAdoptLoop calls it (no |
| +// options) against the same expired fixture TestPassAdoptsAnExpiredCanary |
| +// uses. |
| +func TestPassFilterDoesNotNagWithoutTheOption(t *testing.T) { |
| + s := NewDirStore(t.TempDir()) |
| + ctx := context.Background() |
| + priv, pub := testKeys(t) |
| + opt := t.TempDir() |
| + reg := testRegistry(t) |
| + |
| + shipRelease(t, s, "bab", "myapp", "v1", adoptPlatform, []byte("build one")) |
| + writeBackdatedPointer(t, s, priv, "bab", "myapp", "canary/x", "v1", |
| + time.Now().Add(-(CanaryTTL + 30*24*time.Hour))) |
| + addRoute(t, reg, "old-canary.example.com", "myapp", "canary/x", false) |
| + |
| + var buf bytes.Buffer |
| + withCapturedStdout(t, &buf, func() { |
| + if err := PassFilter(ctx, reg, s, pub, opt, func(string) error { return nil }, isCanaryTestRoute); err != nil { |
| + t.Fatalf("PassFilter: %v", err) |
| + } |
| + }) |
| + if strings.Contains(buf.String(), "expired") { |
| + t.Errorf("PassFilter without NagExpiredCanaries printed a nag line:\n%s", buf.String()) |
| + } |
| +} |
| + |
| +// TestPassNagsOncePerExpiredCanaryRoute pins the root path's half: RunAdopt |
| +// (carlos-adopt.service, hourly) passes NagExpiredCanaries(os.Stdout) to |
| +// Pass, and that call must print ONE nag line PER EXPIRED CANARY ROUTE it |
| +// sees — not one per hourly pass regardless of how many routes follow the |
| +// expired channel. Three routes (two awake, one hibernating — the awake and |
| +// hibernating lists are walked separately in Pass/PassFilter, so this is |
| +// also the regression test for "hibernating routes get skipped" and |
| +// "a route is visited twice") on ONE expired canary channel must produce |
| +// exactly three lines, each naming its own host. |
| +// |
| +// (2026-08-03 review, MINOR 6: this test used to be named |
| +// TestPassNagsExactlyOnceWithTheOption and only ever exercised one route, |
| +// so "not once per host revisit" was asserted by name only, never proven.) |
| +func TestPassNagsOncePerExpiredCanaryRoute(t *testing.T) { |
| + s := NewDirStore(t.TempDir()) |
| + ctx := context.Background() |
| + priv, pub := testKeys(t) |
| + opt := t.TempDir() |
| + reg := testRegistry(t) |
| + |
| + shipRelease(t, s, "bab", "myapp", "v1", adoptPlatform, []byte("build one")) |
| + writeBackdatedPointer(t, s, priv, "bab", "myapp", "canary/x", "v1", |
| + time.Now().Add(-(CanaryTTL + 30*24*time.Hour))) |
| + addRoute(t, reg, "awake-a.example.com", "myapp", "canary/x", false) |
| + addRoute(t, reg, "awake-b.example.com", "myapp", "canary/x", false) |
| + addRoute(t, reg, "sleepy.example.com", "myapp", "canary/x", true) |
| + |
| + var buf bytes.Buffer |
| + if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }, NagExpiredCanaries(&buf)); err != nil { |
| + t.Fatalf("Pass: %v", err) |
| + } |
| + out := buf.String() |
| + if got := strings.Count(out, "expired"); got != 3 { |
| + t.Errorf("nag lines containing \"expired\" = %d, want exactly 3 (one per route, awake and hibernating alike):\n%s", got, out) |
| + } |
| + for _, host := range []string{"awake-a.example.com", "awake-b.example.com", "sleepy.example.com"} { |
| + if !strings.Contains(out, host) { |
| + t.Errorf("nag output does not name %s:\n%s", host, out) |
| + } |
| + } |
| +} |
| + |
| +// TestNagReportsTimePastTTLNotTotalAge pins the 2026-08-03 review's |
| +// IMPORTANT 2 fix: the nag must report how long PAST CanaryTTL a canary is, |
| +// not its total age. A pointer promoted CanaryTTL+1h ago has been EXPIRED |
| +// for about an hour, not 169 hours (7 days 1 hour, its total age) — the |
| +// second number is real but answers a different question, and printing it |
| +// under "has been expired" is the bug the review caught. It must also |
| +// disagree with nothing the console shows for the same pointer |
| +// (internal/console/ladder_test.go's TestLadderExpiredCanaryIsBadgedWithAge |
| +// exercises the same release.CanaryAge/FormatCanaryAge pair). |
| +func TestNagReportsTimePastTTLNotTotalAge(t *testing.T) { |
| + s := NewDirStore(t.TempDir()) |
| + ctx := context.Background() |
| + priv, pub := testKeys(t) |
| + opt := t.TempDir() |
| + reg := testRegistry(t) |
| + |
| + shipRelease(t, s, "bab", "myapp", "v1", adoptPlatform, []byte("build one")) |
| + writeBackdatedPointer(t, s, priv, "bab", "myapp", "canary/x", "v1", |
| + time.Now().Add(-(CanaryTTL + time.Hour))) |
| + addRoute(t, reg, "old-canary.example.com", "myapp", "canary/x", false) |
| + |
| + var buf bytes.Buffer |
| + if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }, NagExpiredCanaries(&buf)); err != nil { |
| + t.Fatalf("Pass: %v", err) |
| + } |
| + out := buf.String() |
| + if !strings.Contains(out, "1h") { |
| + t.Errorf("nag does not say 1h (time past TTL) for a canary expired ~1h ago:\n%s", out) |
| + } |
| + if strings.Contains(out, "169h") { |
| + t.Errorf("nag reports total age (169h) instead of time past TTL:\n%s", out) |
| + } |
| +} |
| + |
| +// TestPassDoesNotNagAFreshCanaryEvenWithTheOption: the option opts a CALL |
| +// into nagging; it does not turn every canary into a nag target. A fresh |
| +// canary must stay silent whether or not NagExpiredCanaries is set. |
| +func TestPassDoesNotNagAFreshCanaryEvenWithTheOption(t *testing.T) { |
| + s := NewDirStore(t.TempDir()) |
| + ctx := context.Background() |
| + priv, pub := testKeys(t) |
| + opt := t.TempDir() |
| + reg := testRegistry(t) |
| + |
| + shipRelease(t, s, "bab", "myapp", "v1", adoptPlatform, []byte("build one")) |
| + if err := Promote(ctx, s, priv, "bab", "myapp", "canary/x", "v1"); err != nil { |
| + t.Fatalf("Promote: %v", err) |
| + } |
| + addRoute(t, reg, "fresh-canary.example.com", "myapp", "canary/x", false) |
| + |
| + var buf bytes.Buffer |
| + if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }, NagExpiredCanaries(&buf)); err != nil { |
| + t.Fatalf("Pass: %v", err) |
| + } |
| + if strings.Contains(buf.String(), "expired") { |
| + t.Errorf("a freshly-promoted canary produced a nag line:\n%s", buf.String()) |
| + } |
| +} |
| + |
| +// isCanaryTestRoute mirrors host.isCanaryRoute (unexported, different |
| +// package) for the one fast-loop-shaped test above that needs a canary-only |
| +// filter without importing internal/host. |
| +func isCanaryTestRoute(rt registry.Route) bool { return strings.HasPrefix(rt.Channel, "canary/") } |
| + |
| +// withCapturedStdout redirects os.Stdout to buf for the duration of fn — |
| +// used ONLY to prove PassFilter's default (no NagExpiredCanaries) path |
| +// prints nothing canary-related even via the package's ordinary fmt.Printf |
| +// progress lines, which is the honest thing to assert against (adoptHost's |
| +// other progress lines still go to os.Stdout; NagExpiredCanaries is what |
| +// redirects the NAG line specifically to an injected writer instead). |
| +func withCapturedStdout(t *testing.T, buf *bytes.Buffer, fn func()) { |
| + t.Helper() |
| + r, w, err := os.Pipe() |
| + if err != nil { |
| + t.Fatalf("os.Pipe: %v", err) |
| + } |
| + orig := os.Stdout |
| + os.Stdout = w |
| + defer func() { os.Stdout = orig }() |
| + |
| + done := make(chan struct{}) |
| + go func() { |
| + buf.ReadFrom(r) |
| + close(done) |
| + }() |
| + |
| + fn() |
| + w.Close() |
| + <-done |
| +} |
diff --git a/internal/release/canary_ttl.go b/internal/release/canary_ttl.go| new file mode 100644 |
| index 000000000..9a3425300 |
| --- /dev/null |
| +++ b/internal/release/canary_ttl.go |
| @@ -0,0 +1,85 @@ |
| +// Canary TTL — the pipelines spec's second guard rail (§5 "Canaries |
| +// expire"). Every canary/* channel carries a default 7-day lease, measured |
| +// from the pointer's own PromotedAt: the pointer already carries that |
| +// field, so this needs no new storage, and re-promoting a canary refreshes |
| +// the lease, which is the right semantic — an actively-updated canary never |
| +// expires, however old the branch it started from. |
| +// |
| +// Expiry is visibility, not enforcement, matching §5's "prod-wired" guard |
| +// rail right above it: nothing here stops a route serving, nothing deletes |
| +// a pointer, and promotion to or from an expired canary stays allowed. The |
| +// hazard §5 names is not a canary in production — it's one that outlives |
| +// the branch it was testing and quietly becomes permanent infrastructure |
| +// nobody remembers deploying. Every caller that needs to know computes it |
| +// fresh from the pointer already in hand; nothing persists an "expired" bit |
| +// anywhere. |
| +package release |
| + |
| +import ( |
| + "fmt" |
| + "strings" |
| + "time" |
| +) |
| + |
| +// CanaryTTL is a canary/* channel's default lease length before it is |
| +// flagged expired. Spec §5: "default 7 days". |
| +const CanaryTTL = 7 * 24 * time.Hour |
| + |
| +// CanaryExpired reports whether p is a canary/* channel pointer whose lease |
| +// (PromotedAt + CanaryTTL) has passed as of now. Always false for a channel |
| +// outside canary/* — edge, beta and stable carry no TTL — and false for a |
| +// nil pointer or a zero PromotedAt (an unpromoted or not-yet-read pointer): |
| +// unknown age is never treated as proof of age. Exactly CanaryTTL old is |
| +// NOT yet expired; the TTL names the lease's length. |
| +func CanaryExpired(p *ChannelPointer, now time.Time) bool { |
| + if p == nil || !strings.HasPrefix(p.Channel, RungCanary+"/") { |
| + return false |
| + } |
| + if p.PromotedAt.IsZero() { |
| + return false |
| + } |
| + return now.Sub(p.PromotedAt) > CanaryTTL |
| +} |
| + |
| +// CanaryAge is how long PAST CanaryTTL an expired canary pointer is — the |
| +// number both operator-facing surfaces (the adopt pass's nag and the |
| +// console's badge) render, NOT the pointer's total age. Total age |
| +// (now.Sub(p.PromotedAt)) is always exactly CanaryAge + CanaryTTL, and |
| +// reporting IT as "how long has this been expired" overstates the lapse by |
| +// one full lease length — a canary that missed its lease by an hour would |
| +// read as a seven-day-old problem (2026-08-03 review, IMPORTANT 2). |
| +// |
| +// Meaningful only when CanaryExpired(p, now) is already true; call that |
| +// first. A caller that skips the check gets a negative or otherwise |
| +// nonsensical duration back for a fresh or non-canary pointer, the same as |
| +// any bare Sub on the wrong side of zero. |
| +func CanaryAge(p *ChannelPointer, now time.Time) time.Duration { |
| + return now.Sub(p.PromotedAt) - CanaryTTL |
| +} |
| + |
| +// FormatCanaryAge renders a CanaryAge duration for a human. It is the ONE |
| +// formatter both the adopt pass's nag (internal/release/adopt.go) and the |
| +// console's badge (internal/console/ladder.go) call, so the two |
| +// operator-facing surfaces of this feature can never drift into disagreeing |
| +// about what a duration means (2026-08-03 review, IMPORTANT 2) — the way |
| +// they briefly did when the console rounded-and-bucketed a duration the nag |
| +// printed raw and un-subtracted. |
| +// |
| +// "<1h" for anything under a full hour past the TTL — never "0h", which |
| +// reads as "not actually expired" to whoever sees it first (2026-08-03 |
| +// review, MINOR 7). "Nh" rounded to the nearest hour from one hour up to a |
| +// day. "Nd" TRUNCATED (not rounded) at a day and beyond: nobody needs |
| +// minute-level precision on a guard rail whose whole point is that nobody |
| +// was watching the clock, and losing up to 23 hours off the day count is |
| +// the deliberate coarseness that buys — nothing here mistakes a 47-hour |
| +// lapse for two days. |
| +func FormatCanaryAge(d time.Duration) string { |
| + if d < time.Hour { |
| + return "<1h" |
| + } |
| + d = d.Round(time.Hour) |
| + if d < 24*time.Hour { |
| + return fmt.Sprintf("%dh", int(d.Hours())) |
| + } |
| + return fmt.Sprintf("%dd", int(d.Hours())/24) |
| +} |
diff --git a/internal/release/canary_ttl_test.go b/internal/release/canary_ttl_test.go| new file mode 100644 |
| index 000000000..f598a079d |
| --- /dev/null |
| +++ b/internal/release/canary_ttl_test.go |
| @@ -0,0 +1,116 @@ |
| +package release |
| + |
| +import ( |
| + "testing" |
| + "time" |
| +) |
| + |
| +func TestCanaryExpiredFalseForFreshPromotion(t *testing.T) { |
| + p := &ChannelPointer{App: "myapp", Channel: "canary/x", Version: "v1", PromotedAt: time.Now()} |
| + if CanaryExpired(p, time.Now()) { |
| + t.Error("a canary promoted moments ago must not be expired") |
| + } |
| +} |
| + |
| +func TestCanaryExpiredTrueAfterTTL(t *testing.T) { |
| + old := time.Now().Add(-(CanaryTTL + time.Hour)) |
| + p := &ChannelPointer{App: "myapp", Channel: "canary/x", Version: "v1", PromotedAt: old} |
| + if !CanaryExpired(p, time.Now()) { |
| + t.Error("a canary promoted more than CanaryTTL ago must be expired") |
| + } |
| +} |
| + |
| +// Right at the boundary: exactly CanaryTTL old is NOT yet expired — the TTL |
| +// names the lease's length, not a moment strictly before it. |
| +func TestCanaryExpiredBoundary(t *testing.T) { |
| + now := time.Now() |
| + atBoundary := &ChannelPointer{Channel: "canary/x", PromotedAt: now.Add(-CanaryTTL)} |
| + if CanaryExpired(atBoundary, now) { |
| + t.Error("exactly CanaryTTL old must not be expired yet") |
| + } |
| + pastBoundary := &ChannelPointer{Channel: "canary/x", PromotedAt: now.Add(-CanaryTTL - time.Second)} |
| + if !CanaryExpired(pastBoundary, now) { |
| + t.Error("one second past CanaryTTL must be expired") |
| + } |
| +} |
| + |
| +// Re-promoting a canary refreshes PromotedAt, which is the whole point of |
| +// keying age off the pointer's own lease timestamp: an actively-updated |
| +// canary never expires, however old the channel name itself is. |
| +func TestCanaryExpiredRefreshesOnRepromotion(t *testing.T) { |
| + now := time.Now() |
| + stale := &ChannelPointer{Channel: "canary/x", PromotedAt: now.Add(-(CanaryTTL + 24*time.Hour))} |
| + if !CanaryExpired(stale, now) { |
| + t.Fatal("setup: this pointer should read as expired before the repromotion") |
| + } |
| + refreshed := &ChannelPointer{Channel: "canary/x", PromotedAt: now} |
| + if CanaryExpired(refreshed, now) { |
| + t.Error("a freshly re-promoted canary must not be expired, however old the channel is") |
| + } |
| +} |
| + |
| +// Only canary/* channels carry a TTL at all — edge, beta and stable are |
| +// never flagged, however old their PromotedAt is. |
| +func TestCanaryExpiredFalseForNonCanaryChannels(t *testing.T) { |
| + now := time.Now() |
| + ancient := now.Add(-365 * 24 * time.Hour) |
| + for _, ch := range []string{"stable", "beta", "edge", "canary"} { |
| + p := &ChannelPointer{Channel: ch, PromotedAt: ancient} |
| + if CanaryExpired(p, now) { |
| + t.Errorf("channel %q must never be flagged expired (not a canary/* channel)", ch) |
| + } |
| + } |
| +} |
| + |
| +// A pointer written before PromotedAt existed (or any zero value) must not |
| +// be flagged expired — unknown age is not proof of age, and validatePointer |
| +// already refuses a zero PromotedAt on anything read off the wire, so this |
| +// is defense for a directly-constructed pointer, not a live-fleet case. |
| +func TestCanaryExpiredFalseForZeroPromotedAt(t *testing.T) { |
| + p := &ChannelPointer{Channel: "canary/x"} |
| + if CanaryExpired(p, time.Now()) { |
| + t.Error("a zero PromotedAt must not be flagged expired") |
| + } |
| +} |
| + |
| +func TestCanaryExpiredNilPointer(t *testing.T) { |
| + if CanaryExpired(nil, time.Now()) { |
| + t.Error("a nil pointer must not be flagged expired") |
| + } |
| +} |
| + |
| +// ---- CanaryAge / FormatCanaryAge (2026-08-03 review, IMPORTANT 2) --------- |
| + |
| +// CanaryAge must report time PAST the TTL, not total age — a pointer |
| +// CanaryTTL+1h old has been expired for about an hour, not 7 days 1 hour. |
| +func TestCanaryAgeIsTimePastTTLNotTotalAge(t *testing.T) { |
| + now := time.Now() |
| + p := &ChannelPointer{Channel: "canary/x", PromotedAt: now.Add(-(CanaryTTL + time.Hour))} |
| + got := CanaryAge(p, now) |
| + if got < 59*time.Minute || got > 61*time.Minute { |
| + t.Errorf("CanaryAge = %s, want ~1h (total age minus CanaryTTL, not the 169h total age)", got) |
| + } |
| +} |
| + |
| +func TestFormatCanaryAge(t *testing.T) { |
| + cases := []struct { |
| + d time.Duration |
| + want string |
| + }{ |
| + {0, "<1h"}, // the instant it expires |
| + {24 * time.Minute, "<1h"}, // MINOR 7: must not read "0h" |
| + {59 * time.Minute, "<1h"}, |
| + {time.Hour, "1h"}, // review's own example: 7d1h old -> "1h" |
| + {90 * time.Minute, "2h"}, // rounds to nearest hour |
| + {23 * time.Hour, "23h"}, |
| + {24 * time.Hour, "1d"}, |
| + {47 * time.Hour, "1d"}, // deliberate coarseness: truncates, not rounds |
| + {8 * 24 * time.Hour, "8d"}, |
| + {30 * 24 * time.Hour, "30d"}, |
| + } |
| + for _, c := range cases { |
| + if got := FormatCanaryAge(c.d); got != c.want { |
| + t.Errorf("FormatCanaryAge(%s) = %q, want %q", c.d, got, c.want) |
| + } |
| + } |
| +} |
diff --git a/internal/status/assets/index.html b/internal/status/assets/index.html| index 4c3580849..dab99dfe0 100644 |
| --- a/internal/status/assets/index.html |
| +++ b/internal/status/assets/index.html |
| @@ -30,6 +30,10 @@ |
| <h2>Adoption</h2> |
| <p class="tile-value" id="tile-adoption"></p> |
| </div> |
| + <div class="tile" id="tile-expired-canaries-wrap" hidden> |
| + <h2>Expired canaries</h2> |
| + <p class="tile-value" id="tile-expired-canaries"></p> |
| + </div> |
| </section> |
| <section> |
| @@ -145,6 +149,20 @@ |
| // adoption timers aren't armed is one where nothing picks up a release on |
| // its own, which used to be invisible from anywhere. |
| document.getElementById("tile-adoption").replaceChildren(asBool(f.adoption_running, "running", "not running")); |
| + |
| + // The expired-canaries tile is deliberately absent, not zeroed, when |
| + // there is nothing to see (pipelines spec §5): a fleet with no expired |
| + // canaries — the ordinary, common case — should not carry a permanent |
| + // "0" tile competing for attention with the three that always matter. |
| + // Unknown (null) also stays hidden: "we couldn't check" is not the same |
| + // claim as "there are some", and this tile only ever asserts the latter. |
| + var wrap = document.getElementById("tile-expired-canaries-wrap"); |
| + if (typeof f.expired_canaries === "number" && f.expired_canaries > 0) { |
| + wrap.hidden = false; |
| + document.getElementById("tile-expired-canaries").replaceChildren(asText(f.expired_canaries)); |
| + } else { |
| + wrap.hidden = true; |
| + } |
| } |
| function renderBoxes(f) { |
diff --git a/internal/status/contract_test.go b/internal/status/contract_test.go| index 9243aa624..02cb6b135 100644 |
| --- a/internal/status/contract_test.go |
| +++ b/internal/status/contract_test.go |
| @@ -28,7 +28,11 @@ var allowedBoxKeys = map[string][]string{ |
| // says whether the platform's own release-adoption timer is armed on this |
| // box. It carries no tenant, hostname or machine identity — only whether |
| // an opaquely-labelled box is picking up its own releases. |
| - "": {"generated_at", "label", "region", "routes", "load", "capacity", "replication", "adoption_running"}, |
| + // |
| + // "expired_canaries" is likewise deliberate (pipelines spec §5 "Canaries |
| + // expire"): a COUNT of this box's own canary/* routes past their TTL, |
| + // never which ones — no hostname, no channel name, no tenant identity. |
| + "": {"generated_at", "label", "region", "routes", "load", "capacity", "replication", "adoption_running", "expired_canaries"}, |
| "routes": {"static", "resident", "hibernating", "awake", "asleep"}, |
| "replication": {"healthy", "lag_seconds"}, |
| } |
| @@ -36,8 +40,8 @@ var allowedBoxKeys = map[string][]string{ |
| // allowedFleetKeys are THE schema for Fleet's JSON output. Same friction as |
| // allowedBoxKeys: every new field is a deliberate publication decision. |
| var allowedFleetKeys = map[string][]string{ |
| - "": {"generated_at", "boxes", "instances", "capacity", "apps", "apps_unlisted", "services", "replication", "adoption_running"}, |
| - "boxes[]": {"label", "region", "load", "stale", "routes", "adoption_running"}, |
| + "": {"generated_at", "boxes", "instances", "capacity", "apps", "apps_unlisted", "services", "replication", "adoption_running", "expired_canaries"}, |
| + "boxes[]": {"label", "region", "load", "stale", "routes", "adoption_running", "expired_canaries"}, |
| "boxes[].routes": {"static", "resident", "hibernating", "awake", "asleep"}, |
| "instances": {"total", "static", "resident", "hibernating", "awake", "asleep"}, |
| "capacity": {"level"}, |
diff --git a/internal/status/dryrun.go b/internal/status/dryrun.go| index 5b75cdaac..288254cab 100644 |
| --- a/internal/status/dryrun.go |
| +++ b/internal/status/dryrun.go |
| @@ -87,14 +87,16 @@ func dryRunFixtures() ([]registry.Route, Probes) { |
| memFrac, diskFrac := 0.73, 0.95 |
| lag := int64(120) |
| - adopting := true // a known reading, so adoption_running publishes non-null and gets scanned |
| + adopting := true // a known reading, so adoption_running publishes non-null and gets scanned |
| + expiredCanaries := 2 // ditto for expired_canaries — a known count, not the unexercised nil path |
| probes := Probes{ |
| - MachineID: "ip-10-0-0-7", |
| - Region: "us-east-1", |
| - MemFrac: &memFrac, |
| - DiskFrac: &diskFrac, |
| - LagSecs: &lag, |
| - AdoptRunning: &adopting, |
| + MachineID: "ip-10-0-0-7", |
| + Region: "us-east-1", |
| + MemFrac: &memFrac, |
| + DiskFrac: &diskFrac, |
| + LagSecs: &lag, |
| + AdoptRunning: &adopting, |
| + ExpiredCanaries: &expiredCanaries, |
| // Every hibernating route reports "listening" so Awake/Asleep publish |
| // non-null — an unexercised null field is a field DryRun can't scan. |
| Listening: func(string) *bool { v := true; return &v }, |
diff --git a/internal/status/fleet.go b/internal/status/fleet.go| index f0b9af913..7f69d8e92 100644 |
| --- a/internal/status/fleet.go |
| +++ b/internal/status/fleet.go |
| @@ -42,6 +42,14 @@ type FleetBox struct { |
| // only whether the platform's own timer is armed on an opaquely-labelled |
| // machine. |
| AdoptionRunning *bool `json:"adoption_running"` |
| + // ExpiredCanaries appears per-box for the same reason AdoptionRunning |
| + // does: the fleet-wide sum says something is expired somewhere, this |
| + // says on which box, without naming which route. Published in the FEED |
| + // only for now — the rendered boxes table deliberately has no column |
| + // for it yet (2026-08-03 review, MINOR 4: this comment used to imply |
| + // the table showed it; it does not, and adding the column is deferred |
| + // until an expired canary has actually been hunted via the feed once). |
| + ExpiredCanaries *int `json:"expired_canaries"` |
| } |
| // Fleet is the merged feed: status/fleet.json. Every field here is either a |
| @@ -62,6 +70,13 @@ type Fleet struct { |
| } `json:"services"` |
| Replication Replication `json:"replication"` |
| AdoptionRunning *bool `json:"adoption_running"` |
| + // ExpiredCanaries is the fleet-wide sum of every box's own count — see |
| + // sumExpiredCanaries. Null-propagating: one box that couldn't check |
| + // nulls the whole aggregate, the same rule sumInstances applies to |
| + // Awake/Asleep, because a partial sum would read as more certain than |
| + // it is. The status page shows this only when it is a known, positive |
| + // count — see assets/index.html. |
| + ExpiredCanaries *int `json:"expired_canaries"` |
| } |
| // capacityRank orders the capacity band words worst-last, so "worst of a |
| @@ -105,6 +120,7 @@ func BuildFleet(ctx context.Context, s release.Store, routes []registry.Route, n |
| Stale: stale[i], |
| Routes: b.Routes, |
| AdoptionRunning: b.AdoptionRunning, |
| + ExpiredCanaries: b.ExpiredCanaries, |
| }) |
| } |
| @@ -112,6 +128,7 @@ func BuildFleet(ctx context.Context, s release.Store, routes []registry.Route, n |
| f.Capacity.Level = worstCapacity(boxes) |
| f.Replication = worstReplication(boxes, stale) |
| f.AdoptionRunning = worstAdoption(boxes, stale) |
| + f.ExpiredCanaries = sumExpiredCanaries(boxes) |
| apps, unlisted := buildApps(routes) |
| f.Apps = apps |
| @@ -192,6 +209,25 @@ func sumInstances(boxes []Box) map[string]*int { |
| return instances |
| } |
| +// sumExpiredCanaries adds each box's own ExpiredCanaries count — stale boxes |
| +// included, same reasoning as sumInstances: a box's last-known count is a |
| +// "trust this less" signal rendered loudly per-box (FleetBox.Stale), not a |
| +// reason to silently drop it from the fleet total. Per the null-propagation |
| +// rule every *int aggregate here follows, ANY box contributing nil (it |
| +// couldn't check) nils the whole sum — a partial count would read as more |
| +// certain than it is. No boxes at all sums to 0, not nil: there is nothing |
| +// unknown about an empty fleet. |
| +func sumExpiredCanaries(boxes []Box) *int { |
| + sum := 0 |
| + for _, b := range boxes { |
| + if b.ExpiredCanaries == nil { |
| + return nil |
| + } |
| + sum += *b.ExpiredCanaries |
| + } |
| + return intPtr(sum) |
| +} |
| + |
| // worstCapacity picks the worst capacity band among ALL boxes, stale ones |
| // included: a box that stopped reporting still had a last-known fill level, |
| // and dropping it from the vote just because its snapshot aged out would |
diff --git a/internal/status/snapshot.go b/internal/status/snapshot.go| index 534d2aac8..378e55f4d 100644 |
| --- a/internal/status/snapshot.go |
| +++ b/internal/status/snapshot.go |
| @@ -33,6 +33,16 @@ type Probes struct { |
| // "never asked / couldn't tell", and like every other nil here it must |
| // publish as null: a box that can't answer is not a box that is fine. |
| AdoptRunning *bool |
| + // ExpiredCanaries is how many of this box's own canary/* routes have |
| + // passed release.CanaryTTL (pipelines spec §5 "Canaries expire"), |
| + // pre-computed by the caller against the release store — CollectBox |
| + // stays pure and never resolves a channel pointer itself, the same |
| + // reason every other field here arrives pre-resolved. nil means this |
| + // box could not determine the answer (no deployment store configured, |
| + // or a channel pointer it needed wouldn't verify), never a guessed |
| + // zero; 0 is itself a real, confident answer when every canary route on |
| + // the box is fresh, or there are none at all. |
| + ExpiredCanaries *int |
| } |
| // RouteCounts is aggregate-only: no host, no socket path, just how many. |
| @@ -76,6 +86,11 @@ type Box struct { |
| // flagship spent weeks in the false state with no way to see it; this |
| // field is the way to see it. |
| AdoptionRunning *bool `json:"adoption_running"` |
| + // ExpiredCanaries publishes Probes.ExpiredCanaries straight through — a |
| + // count, never hostnames (pipelines spec §5): which canary route expired |
| + // is exactly the kind of identifying detail this package's whole design |
| + // refuses to carry, but how many is aggregate and safe. |
| + ExpiredCanaries *int `json:"expired_canaries"` |
| } |
| // replHealthyWithin is the lag under which replication counts as healthy. |
| @@ -240,6 +255,7 @@ func CollectBox(routes []registry.Route, p Probes, now time.Time) Box { |
| Capacity: bandWord(worst, capacityWords), |
| Replication: Replication{Healthy: healthy, LagSecs: p.LagSecs}, |
| AdoptionRunning: p.AdoptRunning, |
| + ExpiredCanaries: p.ExpiredCanaries, |
| } |
| } |
diff --git a/internal/status/status_test.go b/internal/status/status_test.go| index da4830b4c..f38b6d0ab 100644 |
| --- a/internal/status/status_test.go |
| +++ b/internal/status/status_test.go |
| @@ -635,6 +635,113 @@ func TestFleetAdoptionWorstOf(t *testing.T) { |
| // number says SOMETHING is not adopting, the boxes table says which label — |
| // without it, an operator reading "not running" on a two-box fleet has to go |
| // SSH into both to find out where the dead timer is. |
| +// ---- Pipelines Task: canary TTL (spec §5 "Canaries expire") --------------- |
| + |
| +// TestExpiredCanariesUnknownIsANull mirrors TestAdoptionUnknownIsANull: a box |
| +// whose probe never resolved its canary channel pointers (e.g. no deployment |
| +// store configured) publishes null, exactly like every other unmeasured |
| +// quantity here. A default 0 would read as "no expired canaries on this |
| +// box" when the honest answer is "this box couldn't check" — precisely the |
| +// reassuring-wrong-answer failure the rest of this package refuses to make. |
| +func TestExpiredCanariesUnknownIsANull(t *testing.T) { |
| + b := status.CollectBox(nil, status.Probes{MachineID: "m"}, time.Now()) // nil ExpiredCanaries |
| + if b.ExpiredCanaries != nil { |
| + t.Errorf("expired_canaries = %v, want null — an unasked question is not a healthy answer", *b.ExpiredCanaries) |
| + } |
| + for _, want := range []int{0, 3} { |
| + got := status.CollectBox(nil, status.Probes{MachineID: "m", ExpiredCanaries: &want}, time.Now()).ExpiredCanaries |
| + if got == nil || *got != want { |
| + t.Errorf("ExpiredCanaries=%d: expired_canaries = %v, want %d", want, got, want) |
| + } |
| + } |
| +} |
| + |
| +// TestFleetExpiredCanariesSums proves the fleet-wide count is the sum across |
| +// boxes when every box's own count is known — the aggregate an operator |
| +// actually wants ("how many expired canaries are running anywhere"), not a |
| +// per-box worst-of. |
| +func TestFleetExpiredCanariesSums(t *testing.T) { |
| + s := teststore(t) |
| + now := time.Now() |
| + two, one := 2, 1 |
| + for _, b := range []status.Box{ |
| + status.CollectBox(nil, status.Probes{MachineID: "a", ExpiredCanaries: &two}, now), |
| + status.CollectBox(nil, status.Probes{MachineID: "b", ExpiredCanaries: &one}, now), |
| + } { |
| + if err := status.WriteBox(context.Background(), s, b); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + f, err := status.BuildFleet(context.Background(), s, nil, now, 5*time.Minute) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if f.ExpiredCanaries == nil || *f.ExpiredCanaries != 3 { |
| + t.Errorf("fleet expired_canaries = %v, want 3", f.ExpiredCanaries) |
| + } |
| +} |
| + |
| +// TestFleetExpiredCanariesNullPropagates: one box that couldn't check nulls |
| +// the fleet-wide sum, the same null-propagation rule sumInstances applies to |
| +// Awake/Asleep — a partial sum would read as more certain than it is. |
| +func TestFleetExpiredCanariesNullPropagates(t *testing.T) { |
| + s := teststore(t) |
| + now := time.Now() |
| + two := 2 |
| + known := status.CollectBox(nil, status.Probes{MachineID: "known", ExpiredCanaries: &two}, now) |
| + unknown := status.CollectBox(nil, status.Probes{MachineID: "unknown"}, now) // nil probe |
| + for _, b := range []status.Box{known, unknown} { |
| + if err := status.WriteBox(context.Background(), s, b); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + f, err := status.BuildFleet(context.Background(), s, nil, now, 5*time.Minute) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if f.ExpiredCanaries != nil { |
| + t.Errorf("fleet expired_canaries = %v, want nil (one box could not check)", *f.ExpiredCanaries) |
| + } |
| +} |
| + |
| +// TestFleetEmptyExpiredCanariesIsZero: no boxes at all is not "unknown", the |
| +// same rule TestFleetEmptyIsValid pins for instances — there is nothing |
| +// unknown about an empty fleet. |
| +func TestFleetEmptyExpiredCanariesIsZero(t *testing.T) { |
| + s := teststore(t) |
| + f, err := status.BuildFleet(context.Background(), s, nil, time.Now(), 5*time.Minute) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if f.ExpiredCanaries == nil || *f.ExpiredCanaries != 0 { |
| + t.Errorf("expired_canaries = %v, want 0 with no boxes", f.ExpiredCanaries) |
| + } |
| +} |
| + |
| +// TestFleetBoxCarriesExpiredCanaries keeps the per-box row honest too — the |
| +// fleet-wide number says something is expired somewhere, the boxes table |
| +// says which box, the same reasoning TestFleetBoxCarriesAdoption pins for |
| +// the adoption-timer reading. |
| +func TestFleetBoxCarriesExpiredCanaries(t *testing.T) { |
| + s := teststore(t) |
| + now := time.Now() |
| + three := 3 |
| + if err := status.WriteBox(context.Background(), s, |
| + status.CollectBox(nil, status.Probes{MachineID: "leaky", ExpiredCanaries: &three}, now)); err != nil { |
| + t.Fatal(err) |
| + } |
| + f, err := status.BuildFleet(context.Background(), s, nil, now, 5*time.Minute) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if len(f.Boxes) != 1 { |
| + t.Fatalf("boxes = %d, want 1", len(f.Boxes)) |
| + } |
| + if f.Boxes[0].ExpiredCanaries == nil || *f.Boxes[0].ExpiredCanaries != 3 { |
| + t.Errorf("boxes[0].expired_canaries = %v, want 3", f.Boxes[0].ExpiredCanaries) |
| + } |
| +} |
| + |
| func TestFleetBoxCarriesAdoption(t *testing.T) { |
| s := teststore(t) |
| now := time.Now() |