amadan
console/status/host: badge, count and surface expired canaries (spec §5)
Wires release.CanaryExpired into the three places §5 promises: the console ladder badges an expired canary/* route with an hourglass and its age, beside the existing canary/prod-wired chips, never instead of them (TestLadderExpiredCanaryIsBadgedWithAge); a freshly-promoted canary — the console's own canary/console production route among them — never carries the badge (TestLadderFreshCanaryCarriesNoExpiredBadge). The fleet status page gets a per-box, loud-null count (Probes/Box/FleetBox.ExpiredCanaries — nil when a box's deployment store or pubkey can't resolve a canary route's pointer, never a guessed zero) summed fleet-wide with the same null-propagation rule sumInstances uses for Awake/Asleep; both contract_test.go allowlists are updated. The public page shows the tile only when the known count is greater than zero. internal/host/agent.go's statusLoop computes it via the new countExpiredCanaries, resolving each distinct canary channel's pointer at most once per tick.
14 files changed,
+619
−12
internal/console/icons.go+9 −0internal/console/ladder.go+62 −1internal/console/ladder_test.go+123 −0internal/console/templates/ladder.html+1 −0internal/console/templates/layout.html+4 −0internal/console/view.go+1 −1internal/host/agent.go+61 −0internal/host/host_test.go+169 −0internal/status/assets/index.html+18 −0internal/status/contract_test.go+7 −3internal/status/dryrun.go+9 −7internal/status/fleet.go+32 −0internal/status/snapshot.go+16 −0internal/status/status_test.go+107 −0
diff --git a/internal/console/icons.go b/internal/console/icons.go| index 6e5a22618..b16ac55cb 100644 |
| --- a/internal/console/icons.go |
| +++ b/internal/console/icons.go |
| @@ -84,6 +84,14 @@ const ( |
| `<path d="m12.83 2.18a2 2 0 0 0-1.66 0L2.6 6.08a1 1 0 0 0 0 1.83l8.58 3.91a2 2 0 0 0 1.66 0l8.58-3.9a1 1 0 0 0 0-1.83z"/>` + |
| `<path d="m6.08 9.5-3.5 1.6a1 1 0 0 0 0 1.81l8.58 3.91a2 2 0 0 0 1.66 0l8.58-3.9a1 1 0 0 0 0-1.83l-3.5-1.59"/>` + |
| `<path d="m6.08 14.5-3.5 1.6a1 1 0 0 0 0 1.81l8.58 3.91a2 2 0 0 0 1.66 0l8.58-3.9a1 1 0 0 0 0-1.83l-3.5-1.59"/></svg>` |
| + |
| + // A canary/* route whose lease has run out (spec §5 "Canaries expire") — |
| + // distinct from iconTriangleAlert's prod-wired warning, because expiry is |
| + // a clock running out, not a hazard (Lucide: hourglass). |
| + iconHourglass template.HTML = iconOpen + |
| + `<path d="M5 22h14"/><path d="M5 2h14"/>` + |
| + `<path d="M17 22v-4.172a2 2 0 0 0-.586-1.414L12 12l-4.414 4.414A2 2 0 0 0 7 17.828V22"/>` + |
| + `<path d="M7 2v4.172a2 2 0 0 0 .586 1.414L12 12l4.414-4.414A2 2 0 0 0 17 6.172V2"/></svg>` |
| ) |
| // icons maps each Lucide slug to its vendored markup. Slugs are the Lucide |
| @@ -96,6 +104,7 @@ var icons = map[string]template.HTML{ |
| "flask-conical": iconFlask, |
| "triangle-alert": iconTriangleAlert, |
| "layers": iconLayers, |
| + "hourglass": iconHourglass, |
| } |
| // icon renders one chrome icon by its Lucide slug: {{icon "chevron-down"}}. |
diff --git a/internal/console/ladder.go b/internal/console/ladder.go| index 2d7e290a9..562699e8f 100644 |
| --- a/internal/console/ladder.go |
| +++ b/internal/console/ladder.go |
| @@ -19,8 +19,14 @@ package console |
| // and prodWiredCanaryHosts below answers "production" from that AND from |
| // which host is the app's apex: the environment says what glue a canary is |
| // wired to, the apex says whether it is in front of everybody. |
| +// |
| +// canaryExpiry answers a different question (spec §5 "Canaries expire"): |
| +// not whether a canary is dangerous, but whether it has been sitting there |
| +// so long it stopped being a canary and became infrastructure nobody |
| +// remembers deploying. Visibility only — an expired canary keeps serving. |
| import ( |
| + "fmt" |
| "sort" |
| "strings" |
| "time" |
| @@ -73,6 +79,15 @@ type LadderRoute struct { |
| // environment as one of the app's stable routes, OR sitting on the app's |
| // apex host. See prodWiredCanaryHosts for why it is both. |
| ProdWired bool |
| + // Expired is whether this route's canary/* channel has outlived |
| + // release.CanaryTTL, measured from the pointer's own PromotedAt (spec |
| + // §5 "Canaries expire"). Always false for a non-canary route. Visibility |
| + // only: an expired canary keeps serving, keeps adopting and stays |
| + // promotable — this badges it, nothing more. |
| + Expired bool |
| + // ExpiredFor is how long past the TTL the channel is, formatted for a |
| + // human ("9d", "6h") — valid only when Expired is true. |
| + ExpiredFor string |
| } |
| // LadderStep is the move from one rung to the one above it. |
| @@ -107,16 +122,20 @@ var gates = map[string]string{ |
| // lists it. A host following a rung channel that has no pointer DOES appear, |
| // as an error line: a host serving nothing is exactly the kind of thing this |
| // section exists to stop being invisible. |
| -func assembleLadder(channels []ChannelView, routes []RouteView) []LadderRung { |
| +func assembleLadder(channels []ChannelView, routes []RouteView, now time.Time) []LadderRung { |
| prod := prodWiredCanaryHosts(routes) |
| + expiredFor := canaryExpiry(channels, now) |
| byChannel := map[string][]LadderRoute{} |
| for _, rt := range routes { |
| rung, ok := release.Rung(rt.Channel) |
| + age := expiredFor[rt.Channel] |
| byChannel[rt.Channel] = append(byChannel[rt.Channel], LadderRoute{ |
| Host: rt.Host, |
| Environment: rt.Environment, |
| Canary: ok && rung == release.RungCanary, |
| ProdWired: prod[rt.Host], |
| + Expired: age != "", |
| + ExpiredFor: age, |
| }) |
| } |
| entries := map[string][]LadderChannel{} |
| @@ -250,3 +269,45 @@ func prodWiredCanaryHosts(routes []RouteView) map[string]bool { |
| } |
| return out |
| } |
| + |
| +// canaryExpiry maps each canary/* channel with a readable pointer to how |
| +// long past release.CanaryTTL it is, formatted for a human — spec §5 |
| +// "Canaries expire". A channel that has not (yet) expired, or whose pointer |
| +// carries an Err (assembleChannels degraded it to one bad row already), is |
| +// simply absent from the map, which is exactly what LadderRoute.Expired |
| +// reads as false. |
| +// |
| +// Built once per ladder assembly, keyed by channel name rather than host, |
| +// because expiry is a property of the CHANNEL's PromotedAt — every route |
| +// following the same canary shares one answer, computed once rather than |
| +// re-derived per host. |
| +func canaryExpiry(channels []ChannelView, now time.Time) map[string]string { |
| + out := map[string]string{} |
| + for _, cv := range channels { |
| + if cv.Err != "" { |
| + continue |
| + } |
| + // A synthetic pointer carrying only the two fields release.CanaryExpired |
| + // reads: the channel's name (so it recognizes canary/*) and the |
| + // promotion timestamp ChannelView already carries. No pointer re-read — |
| + // expiry is computed from data the console already has in hand. |
| + ptr := &release.ChannelPointer{Channel: cv.Name, PromotedAt: cv.PromotedAt} |
| + if !release.CanaryExpired(ptr, now) { |
| + continue |
| + } |
| + out[cv.Name] = formatAge(now.Sub(cv.PromotedAt) - release.CanaryTTL) |
| + } |
| + return out |
| +} |
| + |
| +// formatAge renders a duration coarsely — days once it's at least one full |
| +// day, hours below that — because "expired 3d ago" is what a reader acts |
| +// on; nobody needs minute-level precision on a guard rail whose whole point |
| +// is that nobody was watching the clock. |
| +func formatAge(d time.Duration) string { |
| + d = d.Round(time.Hour) |
| + if d < 24*time.Hour { |
| + return fmt.Sprintf("%dh", int(d.Hours())) |
| + } |
| + return fmt.Sprintf("%dd", int(d.Hours())/24) |
| +} |
diff --git a/internal/console/ladder_test.go b/internal/console/ladder_test.go| index 4742efb0d..05e7cc388 100644 |
| --- a/internal/console/ladder_test.go |
| +++ b/internal/console/ladder_test.go |
| @@ -6,6 +6,7 @@ package console |
| // passkey, and a skip up the ladder is refused however it is posted. |
| import ( |
| + "bytes" |
| "context" |
| "net/http" |
| "net/url" |
| @@ -357,3 +358,125 @@ func TestLadderEnvironmentAndProdWiredVisibleToNonOwner(t *testing.T) { |
| t.Errorf("a member does not see route environment chrome:\n%s", stableBlock) |
| } |
| } |
| + |
| +// ---- Canary TTL (spec §5 "Canaries expire") -------------------------------- |
| + |
| +// A canary promoted moments ago — exactly what every real promotion through |
| +// this console produces, Promote always stamping "now" — must carry no |
| +// expired badge. This is the case CLAUDE.md's own console route is in on |
| +// every deploy (canary/console, re-promoted each time): a false positive |
| +// here would badge the console's own production traffic as neglected. |
| +func TestLadderFreshCanaryCarriesNoExpiredBadge(t *testing.T) { |
| + fx := newConsoleFixture(t) |
| + fx.addRoute(t, registry.Route{Host: "rc.kass.carlosapp.dev", Kind: "instance", Status: "ready", |
| + Version: "v2", App: "kass", Channel: "canary/x", Addr: "127.0.0.1:9002"}) |
| + seedChannel(t, fx, "kass", "canary/x", "v2") // stamps PromotedAt = now |
| + |
| + c := signInPaul(t, fx) |
| + _, _, body := get(t, c, fx.u("/apps/kass")) |
| + block := rungBlock(t, body, "canary") |
| + line := routeLine(t, block, "rc.kass.carlosapp.dev") |
| + |
| + if strings.Contains(line, "chip-expired") { |
| + t.Errorf("a freshly-promoted canary is badged expired:\n%s", line) |
| + } |
| + // This route binds the same (default) environment as the fixture's |
| + // stable route, so it is ALSO prod-wired (TestProdWiredCanaryIsBadgedDistinctly |
| + // covers that badge on its own) — chip-prod or chip-canary, either is the |
| + // "this is a canary" badge the expired flag would sit beside; the point |
| + // of this test is only that neither carries chip-expired. |
| + if !strings.Contains(line, "chip-prod") && !strings.Contains(line, "chip-canary") { |
| + t.Errorf("the fresh canary lost its canary-family badge entirely:\n%s", line) |
| + } |
| +} |
| + |
| +// assembleLadder is exercised directly here (no store, no signing) because |
| +// there is no way to get a genuinely aged, verifiably-signed pointer through |
| +// the public Promote path — it always stamps the real clock, deliberately |
| +// (release.TestSignStampsPromotedAt pins that). Constructing the ChannelView |
| +// by hand is the same input assembleChannels would have produced from a |
| +// pointer that really was promoted eight days ago. |
| +func TestLadderExpiredCanaryIsBadgedWithAge(t *testing.T) { |
| + now := time.Now() |
| + old := now.Add(-(release.CanaryTTL + 8*24*time.Hour)) // 8 days past the TTL |
| + channels := []ChannelView{{Name: "canary/x", Version: "v9", PromotedAt: old}} |
| + // A stable route on a DIFFERENT, non-apex, non-matching-environment host |
| + // keeps this canary out of BOTH prod-wired signals (prodWiredCanaryHosts |
| + // is exercised on its own elsewhere) — this test is about the expired |
| + // badge specifically, so it stays clear of the prod-wired one. |
| + routes := []RouteView{ |
| + {Host: "a.example.com", Channel: "stable", Environment: "prod-env"}, |
| + {Host: "rc.example.com", Channel: "canary/x"}, |
| + } |
| + |
| + rungs := assembleLadder(channels, routes, now) |
| + var canary *LadderRung |
| + for i := range rungs { |
| + if rungs[i].Rung == release.RungCanary { |
| + canary = &rungs[i] |
| + } |
| + } |
| + if canary == nil || len(canary.Channels) != 1 || len(canary.Channels[0].Routes) != 1 { |
| + t.Fatalf("canary rung = %+v, want one channel with one route", canary) |
| + } |
| + rt := canary.Channels[0].Routes[0] |
| + if !rt.Expired { |
| + t.Fatal("a canary promoted more than CanaryTTL ago must be marked Expired") |
| + } |
| + if rt.ExpiredFor != "8d" { |
| + t.Errorf("ExpiredFor = %q, want \"8d\"", rt.ExpiredFor) |
| + } |
| + if !rt.Canary { |
| + t.Error("an expired canary route must still carry the ordinary canary flag") |
| + } |
| + |
| + // Render it through the real template — proves the badge actually |
| + // reaches the page, with the icon and the age, beside the plain canary |
| + // badge rather than instead of it. |
| + p := page{Base: "/a/bab", Role: RoleOwner, CanPromote: true, |
| + Detail: &AppView{App: "demo", Ladder: rungs}} |
| + var buf bytes.Buffer |
| + if err := templates.ExecuteTemplate(&buf, "ladder", p); err != nil { |
| + t.Fatalf("render ladder: %v", err) |
| + } |
| + html := buf.String() |
| + line := routeLine(t, rungBlock(t, html, "canary"), "rc.example.com") |
| + if !strings.Contains(line, "chip-expired") { |
| + t.Errorf("rendered route carries no chip-expired badge:\n%s", line) |
| + } |
| + if !strings.Contains(line, "8d") { |
| + t.Errorf("rendered badge does not show the age:\n%s", line) |
| + } |
| + if !strings.Contains(line, "chip-canary") { |
| + t.Errorf("the expired badge replaced the ordinary canary badge instead of sitting beside it:\n%s", line) |
| + } |
| + if !strings.Contains(line, "<svg") { |
| + t.Error("the expired badge is a text glyph, not an icon") |
| + } |
| +} |
| + |
| +// A channel that has not yet reached CanaryTTL contributes nothing to |
| +// canaryExpiry's map — the direct unit-level twin of the fixture-based fresh |
| +// test above, pinned at the function boundary rather than through HTTP. |
| +func TestCanaryExpiryOmitsUnexpiredChannels(t *testing.T) { |
| + now := time.Now() |
| + channels := []ChannelView{ |
| + {Name: "canary/fresh", PromotedAt: now}, |
| + {Name: "canary/old", PromotedAt: now.Add(-(release.CanaryTTL + time.Hour))}, |
| + {Name: "stable", PromotedAt: now.Add(-365 * 24 * time.Hour)}, // no TTL at all |
| + {Name: "canary/broken", PromotedAt: now.Add(-365 * 24 * time.Hour), Err: "unreadable"}, |
| + } |
| + got := canaryExpiry(channels, now) |
| + if _, ok := got["canary/fresh"]; ok { |
| + t.Error("a fresh canary must not appear in the expiry map") |
| + } |
| + if _, ok := got["stable"]; ok { |
| + t.Error("stable carries no TTL and must never appear in the expiry map") |
| + } |
| + if _, ok := got["canary/broken"]; ok { |
| + t.Error("a channel whose pointer errored must not be guessed at as expired") |
| + } |
| + if _, ok := got["canary/old"]; !ok { |
| + t.Error("an old canary must appear in the expiry map") |
| + } |
| +} |
diff --git a/internal/console/templates/ladder.html b/internal/console/templates/ladder.html| index 2f8da0342..72448ae20 100644 |
| --- a/internal/console/templates/ladder.html |
| +++ b/internal/console/templates/ladder.html |
| @@ -25,6 +25,7 @@ touching what any host is wired to.</p> |
| <span class="chip chip-env">{{icon "layers"}} {{if .Environment}}{{.Environment}}{{else}}default{{end}}</span> |
| {{if .ProdWired}}<span class="chip chip-prod">{{icon "triangle-alert"}} canary in front of everybody</span> |
| {{else if .Canary}}<span class="chip chip-canary">{{icon "flask-conical"}} canary</span>{{end}} |
| + {{if .Expired}}<span class="chip chip-expired">{{icon "hourglass"}} expired {{.ExpiredFor}}</span>{{end}} |
| </li> |
| {{end}} |
| </ul> |
diff --git a/internal/console/templates/layout.html b/internal/console/templates/layout.html| index 8d4185ec2..cf142d4d1 100644 |
| --- a/internal/console/templates/layout.html |
| +++ b/internal/console/templates/layout.html |
| @@ -146,6 +146,10 @@ |
| every user is the one thing on this page a reader must not skim past. */ |
| .chip-canary { border: 1px solid var(--accent); color: var(--accent); } |
| .chip-prod { background: var(--bad); color: var(--bg); } |
| + /* An expired canary (spec §5) keeps serving — this is "trust this less", |
| + the same bordered-only warning as .chip-stale, never a filled outage |
| + marker like .chip-prod. */ |
| + .chip-expired { border: 1px solid var(--warn, #a6790f); color: var(--warn, #a6790f); } |
| /* A route's bound environment — a fact, not a warning, so it is bordered |
| like .chip-canary but in a neutral tone: it sits on every route, not |
| just the ones worth a second look. */ |
diff --git a/internal/console/view.go b/internal/console/view.go| index 4da58704c..9b61c1ca2 100644 |
| --- a/internal/console/view.go |
| +++ b/internal/console/view.go |
| @@ -241,7 +241,7 @@ func Assemble(regPath string, s release.Store, pub ed25519.PublicKey, acct strin |
| Domains: domainsByApp[app], |
| Channels: channels, |
| Canaries: assembleCanaries(channels, byApp[app]), |
| - Ladder: assembleLadder(channels, byApp[app]), |
| + Ladder: assembleLadder(channels, byApp[app], now), |
| RouteAccount: mismatch, |
| }) |
| } |
diff --git a/internal/host/agent.go b/internal/host/agent.go| index 07a1d6f55..c63999cbd 100644 |
| --- a/internal/host/agent.go |
| +++ b/internal/host/agent.go |
| @@ -64,6 +64,7 @@ package host |
| import ( |
| "context" |
| + "crypto/ed25519" |
| "errors" |
| "flag" |
| "io" |
| @@ -251,6 +252,48 @@ func isCanaryRoute(rt registry.Route) bool { |
| return strings.HasPrefix(rt.Channel, "canary/") |
| } |
| +// countExpiredCanaries counts routes (this box's own, platform aliases |
| +// excluded — same convention as CollectBox) on a canary/* channel whose |
| +// pointer has passed release.CanaryTTL, for the fleet status page's §5 |
| +// count (internal/status Probes.ExpiredCanaries). A canary route whose |
| +// pointer can't be resolved at all — store unavailable, pubkey unusable, or |
| +// the specific channel won't verify — nils the WHOLE count rather than |
| +// undercounting: "we couldn't check" and "there are none" are different |
| +// claims, and this function only ever makes the second one when it's sure. |
| +// No canary routes at all is a confident, known 0 — it needs no store to |
| +// answer "none" about routes that don't exist. |
| +// |
| +// Pointers are resolved at most once per distinct (account, app, channel) |
| +// among the routes given, mirroring adopt.go's Pass — cheap even when many |
| +// routes on this box share one canary channel. |
| +func countExpiredCanaries(ctx context.Context, routes []registry.Route, store release.Store, pub ed25519.PublicKey, now time.Time) *int { |
| + type key struct{ acct, app, channel string } |
| + cache := map[key]*release.ChannelPointer{} |
| + n := 0 |
| + for _, rt := range routes { |
| + if rt.Platform || !isCanaryRoute(rt) { |
| + continue |
| + } |
| + if store == nil || len(pub) != ed25519.PublicKeySize { |
| + return nil // a canary route exists but nothing here can resolve its pointer |
| + } |
| + k := key{release.RouteAccount(rt), rt.App, rt.Channel} |
| + ptr, cached := cache[k] |
| + if !cached { |
| + p, err := release.Channel(ctx, store, pub, k.acct, k.app, k.channel) |
| + if err != nil { |
| + return nil // this route's own pointer won't verify — unknown, not zero |
| + } |
| + ptr = p |
| + cache[k] = ptr |
| + } |
| + if release.CanaryExpired(ptr, now) { |
| + n++ |
| + } |
| + } |
| + return &n |
| +} |
| + |
| // peekAcmeCache pre-parses the agent args for -acme-cache the same way |
| // peekRegistryPath peeks -registry: the edge does the authoritative parse, |
| // this just lets the domains loop stamp cert expiry from the SAME cache dir |
| @@ -472,6 +515,21 @@ func statusLoop(reg *registry.Registry) { |
| // place an operator can confirm which bucket actually won. |
| log.Printf("agent: status loop armed (5min + jitter), publishing to bucket %s", bucket) |
| + // canaryStore/canaryPub resolve the release deployment config |
| + // countExpiredCanaries needs to answer §5's "how many canaries on this |
| + // box have expired" — the SAME env pair adoptLoop reads |
| + // (CARLOS_DEPLOYMENT_BUCKET/_DIR + CARLOS_RELEASE_PUBKEY), not |
| + // necessarily the status bucket above, which may deliberately be a |
| + // separate one. Either can come back unusable: no deployment store |
| + // configured, or no pubkey set. That is a normal, valid box shape (the |
| + // same "routes managed by hand" case adoptLoop's own doc comment |
| + // names) — it just means countExpiredCanaries can't answer and reports |
| + // nil, same as every other unmeasured probe in this loop. Resolved once |
| + // here, not re-read every tick, mirroring adoptLoop's own one-time |
| + // resolution. |
| + canaryStore, _ := openDeployStore(ctx) |
| + canaryPub, _ := release.ParsePublicKey(os.Getenv("CARLOS_RELEASE_PUBKEY"), "CARLOS_RELEASE_PUBKEY") |
| + |
| // Jittered on the same shape as adoptLoop's delay: every box on the same |
| // 5-minute cadence would all hit the lease and the bucket in the same |
| // second otherwise. |
| @@ -504,6 +562,9 @@ func statusLoop(reg *registry.Registry) { |
| // flagship's silent no-adopt state, which nothing on any page |
| // could see before this. |
| AdoptRunning: adoptTimerActive(), |
| + // ExpiredCanaries answers spec §5 for this box's own routes; see |
| + // countExpiredCanaries and the canaryStore/canaryPub comment above. |
| + ExpiredCanaries: countExpiredCanaries(ctx, routes, canaryStore, canaryPub, time.Now()), |
| // LagSecs stays nil: litestream's per-DB replication lag isn't |
| // surfaced up to the agent process anywhere yet — the |
| // replication machinery (internal/lease's Replicator) lives |
diff --git a/internal/host/host_test.go b/internal/host/host_test.go| index 4312df011..3916fcb67 100644 |
| --- a/internal/host/host_test.go |
| +++ b/internal/host/host_test.go |
| @@ -2,6 +2,9 @@ package host |
| import ( |
| "context" |
| + "crypto/ed25519" |
| + "encoding/base64" |
| + "encoding/json" |
| "net" |
| "os" |
| "path/filepath" |
| @@ -485,3 +488,169 @@ func TestAdoptTimerStateReading(t *testing.T) { |
| } |
| func boolPtr(v bool) *bool { return &v } |
| + |
| +// ---- Pipelines: canary TTL, the status loop's expired-canary count -------- |
| + |
| +// wireWrapper mirrors release's unexported signedWrapper wire shape |
| +// (payload/sig) — duplicated here rather than imported (it isn't exported, |
| +// deliberately: forging a pointer's PromotedAt is not something production |
| +// code anywhere should be able to do) so this test can write a validly |
| +// SIGNED but arbitrarily-dated channel pointer straight into a DirStore, |
| +// the only way to get an actually-expired pointer under test since every |
| +// public promotion path (release.Promote) stamps the real clock. |
| +type wireWrapper struct { |
| + Payload string `json:"payload"` |
| + Sig string `json:"sig"` |
| +} |
| + |
| +// writeBackdatedPointer signs and writes a channel pointer with an |
| +// arbitrary PromotedAt directly to key release.ChannelKey(acct, app, |
| +// channel), bypassing Promote's clock stamp. |
| +func writeBackdatedPointer(t *testing.T, s release.Store, priv ed25519.PrivateKey, acct, app, channel, version string, promotedAt time.Time) { |
| + t.Helper() |
| + p := struct { |
| + Schema int `json:"schema"` |
| + App string `json:"app"` |
| + Channel string `json:"channel"` |
| + Version string `json:"version"` |
| + PromotedAt time.Time `json:"promoted_at"` |
| + }{Schema: 1, App: app, Channel: channel, Version: version, PromotedAt: promotedAt} |
| + payload, err := json.Marshal(p) |
| + if err != nil { |
| + t.Fatalf("marshal backdated pointer: %v", err) |
| + } |
| + b64 := base64.StdEncoding.EncodeToString(payload) |
| + sig := ed25519.Sign(priv, []byte(b64)) |
| + wrapper, err := json.Marshal(wireWrapper{Payload: b64, Sig: base64.StdEncoding.EncodeToString(sig)}) |
| + if err != nil { |
| + t.Fatalf("marshal backdated wrapper: %v", err) |
| + } |
| + if _, err := s.Put(context.Background(), release.ChannelKey(acct, app, channel), wrapper, ""); err != nil { |
| + t.Fatalf("write backdated pointer: %v", err) |
| + } |
| +} |
| + |
| +func testCanaryKeys(t *testing.T) (ed25519.PrivateKey, ed25519.PublicKey) { |
| + t.Helper() |
| + seedB64, pubB64, err := release.GenerateKey() |
| + if err != nil { |
| + t.Fatalf("GenerateKey: %v", err) |
| + } |
| + priv, err := release.ParseSigningKey(seedB64, "test") |
| + if err != nil { |
| + t.Fatalf("ParseSigningKey: %v", err) |
| + } |
| + pub, err := release.ParsePublicKey(pubB64, "test") |
| + if err != nil { |
| + t.Fatalf("ParsePublicKey: %v", err) |
| + } |
| + return priv, pub |
| +} |
| + |
| +// No canary routes at all needs no store to answer "none" — a confident, |
| +// known 0, not the unknown/nil path a missing store would otherwise imply. |
| +func TestCountExpiredCanariesNoCanaryRoutesIsZero(t *testing.T) { |
| + routes := []registry.Route{ |
| + {Host: "a.example.com", Kind: "instance", App: "myapp", Channel: "stable"}, |
| + {Host: "b.example.com", Kind: "static"}, |
| + } |
| + got := countExpiredCanaries(context.Background(), routes, nil, nil, time.Now()) |
| + if got == nil || *got != 0 { |
| + t.Errorf("count = %v, want 0 (no canary routes, no store needed)", got) |
| + } |
| +} |
| + |
| +// A canary route exists but there is no deployment store to resolve its |
| +// pointer against — the "routes managed by hand" box shape adoptLoop's own |
| +// doc comment names. That must publish unknown, never a guessed 0. |
| +func TestCountExpiredCanariesNilStoreIsUnknown(t *testing.T) { |
| + routes := []registry.Route{{Host: "rc.example.com", Kind: "instance", App: "myapp", Channel: "canary/x"}} |
| + if got := countExpiredCanaries(context.Background(), routes, nil, nil, time.Now()); got != nil { |
| + t.Errorf("count = %v, want nil (no store to resolve the pointer against)", *got) |
| + } |
| +} |
| + |
| +// A canary route whose channel was never actually promoted (no pointer at |
| +// that key) is the same "can't answer" case as no store at all — release. |
| +// Channel returns os.ErrNotExist, and that must null the count too, not |
| +// silently skip the route and undercount. |
| +func TestCountExpiredCanariesUnresolvablePointerIsUnknown(t *testing.T) { |
| + s := release.NewDirStore(t.TempDir()) |
| + _, pub := testCanaryKeys(t) |
| + routes := []registry.Route{{Host: "rc.example.com", Kind: "instance", App: "myapp", Channel: "canary/x", Account: "bab"}} |
| + if got := countExpiredCanaries(context.Background(), routes, s, pub, time.Now()); got != nil { |
| + t.Errorf("count = %v, want nil (channel was never promoted)", *got) |
| + } |
| +} |
| + |
| +// A fresh canary (real Promote, real clock) counts as 0 — not expired, and |
| +// known. |
| +func TestCountExpiredCanariesFreshCanaryIsZero(t *testing.T) { |
| + s := release.NewDirStore(t.TempDir()) |
| + ctx := context.Background() |
| + priv, pub := testCanaryKeys(t) |
| + shipVersion(t, s, "bab", "myapp", "v1") |
| + if err := release.Promote(ctx, s, priv, "bab", "myapp", "canary/x", "v1"); err != nil { |
| + t.Fatalf("Promote: %v", err) |
| + } |
| + routes := []registry.Route{{Host: "rc.example.com", Kind: "instance", App: "myapp", Channel: "canary/x", Account: "bab"}} |
| + got := countExpiredCanaries(ctx, routes, s, pub, time.Now()) |
| + if got == nil || *got != 0 { |
| + t.Errorf("count = %v, want 0 (freshly promoted)", got) |
| + } |
| +} |
| + |
| +// The core case: a backdated (expired) canary pointer counts as 1; a second, |
| +// unexpired canary route on a different channel does not add to it. |
| +func TestCountExpiredCanariesCountsExpiredOnly(t *testing.T) { |
| + s := release.NewDirStore(t.TempDir()) |
| + ctx := context.Background() |
| + priv, pub := testCanaryKeys(t) |
| + writeBackdatedPointer(t, s, priv, "bab", "myapp", "canary/old", "v1", |
| + time.Now().Add(-(release.CanaryTTL + 24*time.Hour))) |
| + shipVersion(t, s, "bab", "myapp", "v2") |
| + if err := release.Promote(ctx, s, priv, "bab", "myapp", "canary/fresh", "v2"); err != nil { |
| + t.Fatalf("Promote: %v", err) |
| + } |
| + routes := []registry.Route{ |
| + {Host: "old.example.com", Kind: "instance", App: "myapp", Channel: "canary/old", Account: "bab"}, |
| + {Host: "fresh.example.com", Kind: "instance", App: "myapp", Channel: "canary/fresh", Account: "bab"}, |
| + } |
| + got := countExpiredCanaries(ctx, routes, s, pub, time.Now()) |
| + if got == nil || *got != 1 { |
| + t.Errorf("count = %v, want 1 (only the old channel's route is expired)", got) |
| + } |
| +} |
| + |
| +// A platform-minted alias is a second NAME for a route already counted, not |
| +// a second canary route — same convention CollectBox uses to avoid |
| +// double-counting aliases (host_test.go's own |
| +// TestCollectBoxSkipsPlatformMintedRoutes in internal/status covers that |
| +// half; this pins the same rule for the expired-canary count). |
| +func TestCountExpiredCanariesSkipsPlatformAliases(t *testing.T) { |
| + s := release.NewDirStore(t.TempDir()) |
| + priv, pub := testCanaryKeys(t) |
| + writeBackdatedPointer(t, s, priv, "bab", "myapp", "canary/old", "v1", |
| + time.Now().Add(-(release.CanaryTTL + 24*time.Hour))) |
| + routes := []registry.Route{ |
| + {Host: "alias.example.com", Kind: "instance", App: "myapp", Channel: "canary/old", Account: "bab", Platform: true}, |
| + } |
| + got := countExpiredCanaries(context.Background(), routes, s, pub, time.Now()) |
| + if got == nil || *got != 0 { |
| + t.Errorf("count = %v, want 0 — a platform alias must not be counted", got) |
| + } |
| +} |
| + |
| +// shipVersion ships a minimal one-artifact release so Promote has something |
| +// real to point at — release.Promote refuses to point a channel at a |
| +// version nothing shipped. |
| +func shipVersion(t *testing.T, s release.Store, acct, app, version string) { |
| + t.Helper() |
| + m := release.Manifest{ |
| + Schema: 1, App: app, Version: version, Kind: "binary", |
| + Artifacts: map[string]release.Artifact{"linux-amd64": {File: app}}, |
| + } |
| + if err := release.Ship(context.Background(), s, acct, m, map[string][]byte{"linux-amd64": []byte("bin")}); err != nil { |
| + t.Fatalf("Ship %s/%s: %v", app, version, err) |
| + } |
| +} |
diff --git a/internal/status/assets/index.html b/internal/status/assets/index.html| index 4c3580849..dab99dfe0 100644 |
| --- a/internal/status/assets/index.html |
| +++ b/internal/status/assets/index.html |
| @@ -30,6 +30,10 @@ |
| <h2>Adoption</h2> |
| <p class="tile-value" id="tile-adoption"></p> |
| </div> |
| + <div class="tile" id="tile-expired-canaries-wrap" hidden> |
| + <h2>Expired canaries</h2> |
| + <p class="tile-value" id="tile-expired-canaries"></p> |
| + </div> |
| </section> |
| <section> |
| @@ -145,6 +149,20 @@ |
| // adoption timers aren't armed is one where nothing picks up a release on |
| // its own, which used to be invisible from anywhere. |
| document.getElementById("tile-adoption").replaceChildren(asBool(f.adoption_running, "running", "not running")); |
| + |
| + // The expired-canaries tile is deliberately absent, not zeroed, when |
| + // there is nothing to see (pipelines spec §5): a fleet with no expired |
| + // canaries — the ordinary, common case — should not carry a permanent |
| + // "0" tile competing for attention with the three that always matter. |
| + // Unknown (null) also stays hidden: "we couldn't check" is not the same |
| + // claim as "there are some", and this tile only ever asserts the latter. |
| + var wrap = document.getElementById("tile-expired-canaries-wrap"); |
| + if (typeof f.expired_canaries === "number" && f.expired_canaries > 0) { |
| + wrap.hidden = false; |
| + document.getElementById("tile-expired-canaries").replaceChildren(asText(f.expired_canaries)); |
| + } else { |
| + wrap.hidden = true; |
| + } |
| } |
| function renderBoxes(f) { |
diff --git a/internal/status/contract_test.go b/internal/status/contract_test.go| index 9243aa624..02cb6b135 100644 |
| --- a/internal/status/contract_test.go |
| +++ b/internal/status/contract_test.go |
| @@ -28,7 +28,11 @@ var allowedBoxKeys = map[string][]string{ |
| // says whether the platform's own release-adoption timer is armed on this |
| // box. It carries no tenant, hostname or machine identity — only whether |
| // an opaquely-labelled box is picking up its own releases. |
| - "": {"generated_at", "label", "region", "routes", "load", "capacity", "replication", "adoption_running"}, |
| + // |
| + // "expired_canaries" is likewise deliberate (pipelines spec §5 "Canaries |
| + // expire"): a COUNT of this box's own canary/* routes past their TTL, |
| + // never which ones — no hostname, no channel name, no tenant identity. |
| + "": {"generated_at", "label", "region", "routes", "load", "capacity", "replication", "adoption_running", "expired_canaries"}, |
| "routes": {"static", "resident", "hibernating", "awake", "asleep"}, |
| "replication": {"healthy", "lag_seconds"}, |
| } |
| @@ -36,8 +40,8 @@ var allowedBoxKeys = map[string][]string{ |
| // allowedFleetKeys are THE schema for Fleet's JSON output. Same friction as |
| // allowedBoxKeys: every new field is a deliberate publication decision. |
| var allowedFleetKeys = map[string][]string{ |
| - "": {"generated_at", "boxes", "instances", "capacity", "apps", "apps_unlisted", "services", "replication", "adoption_running"}, |
| - "boxes[]": {"label", "region", "load", "stale", "routes", "adoption_running"}, |
| + "": {"generated_at", "boxes", "instances", "capacity", "apps", "apps_unlisted", "services", "replication", "adoption_running", "expired_canaries"}, |
| + "boxes[]": {"label", "region", "load", "stale", "routes", "adoption_running", "expired_canaries"}, |
| "boxes[].routes": {"static", "resident", "hibernating", "awake", "asleep"}, |
| "instances": {"total", "static", "resident", "hibernating", "awake", "asleep"}, |
| "capacity": {"level"}, |
diff --git a/internal/status/dryrun.go b/internal/status/dryrun.go| index 5b75cdaac..288254cab 100644 |
| --- a/internal/status/dryrun.go |
| +++ b/internal/status/dryrun.go |
| @@ -87,14 +87,16 @@ func dryRunFixtures() ([]registry.Route, Probes) { |
| memFrac, diskFrac := 0.73, 0.95 |
| lag := int64(120) |
| - adopting := true // a known reading, so adoption_running publishes non-null and gets scanned |
| + adopting := true // a known reading, so adoption_running publishes non-null and gets scanned |
| + expiredCanaries := 2 // ditto for expired_canaries — a known count, not the unexercised nil path |
| probes := Probes{ |
| - MachineID: "ip-10-0-0-7", |
| - Region: "us-east-1", |
| - MemFrac: &memFrac, |
| - DiskFrac: &diskFrac, |
| - LagSecs: &lag, |
| - AdoptRunning: &adopting, |
| + MachineID: "ip-10-0-0-7", |
| + Region: "us-east-1", |
| + MemFrac: &memFrac, |
| + DiskFrac: &diskFrac, |
| + LagSecs: &lag, |
| + AdoptRunning: &adopting, |
| + ExpiredCanaries: &expiredCanaries, |
| // Every hibernating route reports "listening" so Awake/Asleep publish |
| // non-null — an unexercised null field is a field DryRun can't scan. |
| Listening: func(string) *bool { v := true; return &v }, |
diff --git a/internal/status/fleet.go b/internal/status/fleet.go| index f0b9af913..f5d6387cb 100644 |
| --- a/internal/status/fleet.go |
| +++ b/internal/status/fleet.go |
| @@ -42,6 +42,10 @@ type FleetBox struct { |
| // only whether the platform's own timer is armed on an opaquely-labelled |
| // machine. |
| AdoptionRunning *bool `json:"adoption_running"` |
| + // ExpiredCanaries appears per-box for the same reason AdoptionRunning |
| + // does: the fleet-wide sum says something is expired somewhere, this |
| + // says on which box, without naming which route. |
| + ExpiredCanaries *int `json:"expired_canaries"` |
| } |
| // Fleet is the merged feed: status/fleet.json. Every field here is either a |
| @@ -62,6 +66,13 @@ type Fleet struct { |
| } `json:"services"` |
| Replication Replication `json:"replication"` |
| AdoptionRunning *bool `json:"adoption_running"` |
| + // ExpiredCanaries is the fleet-wide sum of every box's own count — see |
| + // sumExpiredCanaries. Null-propagating: one box that couldn't check |
| + // nulls the whole aggregate, the same rule sumInstances applies to |
| + // Awake/Asleep, because a partial sum would read as more certain than |
| + // it is. The status page shows this only when it is a known, positive |
| + // count — see assets/index.html. |
| + ExpiredCanaries *int `json:"expired_canaries"` |
| } |
| // capacityRank orders the capacity band words worst-last, so "worst of a |
| @@ -105,6 +116,7 @@ func BuildFleet(ctx context.Context, s release.Store, routes []registry.Route, n |
| Stale: stale[i], |
| Routes: b.Routes, |
| AdoptionRunning: b.AdoptionRunning, |
| + ExpiredCanaries: b.ExpiredCanaries, |
| }) |
| } |
| @@ -112,6 +124,7 @@ func BuildFleet(ctx context.Context, s release.Store, routes []registry.Route, n |
| f.Capacity.Level = worstCapacity(boxes) |
| f.Replication = worstReplication(boxes, stale) |
| f.AdoptionRunning = worstAdoption(boxes, stale) |
| + f.ExpiredCanaries = sumExpiredCanaries(boxes) |
| apps, unlisted := buildApps(routes) |
| f.Apps = apps |
| @@ -192,6 +205,25 @@ func sumInstances(boxes []Box) map[string]*int { |
| return instances |
| } |
| +// sumExpiredCanaries adds each box's own ExpiredCanaries count — stale boxes |
| +// included, same reasoning as sumInstances: a box's last-known count is a |
| +// "trust this less" signal rendered loudly per-box (FleetBox.Stale), not a |
| +// reason to silently drop it from the fleet total. Per the null-propagation |
| +// rule every *int aggregate here follows, ANY box contributing nil (it |
| +// couldn't check) nils the whole sum — a partial count would read as more |
| +// certain than it is. No boxes at all sums to 0, not nil: there is nothing |
| +// unknown about an empty fleet. |
| +func sumExpiredCanaries(boxes []Box) *int { |
| + sum := 0 |
| + for _, b := range boxes { |
| + if b.ExpiredCanaries == nil { |
| + return nil |
| + } |
| + sum += *b.ExpiredCanaries |
| + } |
| + return intPtr(sum) |
| +} |
| + |
| // worstCapacity picks the worst capacity band among ALL boxes, stale ones |
| // included: a box that stopped reporting still had a last-known fill level, |
| // and dropping it from the vote just because its snapshot aged out would |
diff --git a/internal/status/snapshot.go b/internal/status/snapshot.go| index 534d2aac8..378e55f4d 100644 |
| --- a/internal/status/snapshot.go |
| +++ b/internal/status/snapshot.go |
| @@ -33,6 +33,16 @@ type Probes struct { |
| // "never asked / couldn't tell", and like every other nil here it must |
| // publish as null: a box that can't answer is not a box that is fine. |
| AdoptRunning *bool |
| + // ExpiredCanaries is how many of this box's own canary/* routes have |
| + // passed release.CanaryTTL (pipelines spec §5 "Canaries expire"), |
| + // pre-computed by the caller against the release store — CollectBox |
| + // stays pure and never resolves a channel pointer itself, the same |
| + // reason every other field here arrives pre-resolved. nil means this |
| + // box could not determine the answer (no deployment store configured, |
| + // or a channel pointer it needed wouldn't verify), never a guessed |
| + // zero; 0 is itself a real, confident answer when every canary route on |
| + // the box is fresh, or there are none at all. |
| + ExpiredCanaries *int |
| } |
| // RouteCounts is aggregate-only: no host, no socket path, just how many. |
| @@ -76,6 +86,11 @@ type Box struct { |
| // flagship spent weeks in the false state with no way to see it; this |
| // field is the way to see it. |
| AdoptionRunning *bool `json:"adoption_running"` |
| + // ExpiredCanaries publishes Probes.ExpiredCanaries straight through — a |
| + // count, never hostnames (pipelines spec §5): which canary route expired |
| + // is exactly the kind of identifying detail this package's whole design |
| + // refuses to carry, but how many is aggregate and safe. |
| + ExpiredCanaries *int `json:"expired_canaries"` |
| } |
| // replHealthyWithin is the lag under which replication counts as healthy. |
| @@ -240,6 +255,7 @@ func CollectBox(routes []registry.Route, p Probes, now time.Time) Box { |
| Capacity: bandWord(worst, capacityWords), |
| Replication: Replication{Healthy: healthy, LagSecs: p.LagSecs}, |
| AdoptionRunning: p.AdoptRunning, |
| + ExpiredCanaries: p.ExpiredCanaries, |
| } |
| } |
diff --git a/internal/status/status_test.go b/internal/status/status_test.go| index da4830b4c..f38b6d0ab 100644 |
| --- a/internal/status/status_test.go |
| +++ b/internal/status/status_test.go |
| @@ -635,6 +635,113 @@ func TestFleetAdoptionWorstOf(t *testing.T) { |
| // number says SOMETHING is not adopting, the boxes table says which label — |
| // without it, an operator reading "not running" on a two-box fleet has to go |
| // SSH into both to find out where the dead timer is. |
| +// ---- Pipelines Task: canary TTL (spec §5 "Canaries expire") --------------- |
| + |
| +// TestExpiredCanariesUnknownIsANull mirrors TestAdoptionUnknownIsANull: a box |
| +// whose probe never resolved its canary channel pointers (e.g. no deployment |
| +// store configured) publishes null, exactly like every other unmeasured |
| +// quantity here. A default 0 would read as "no expired canaries on this |
| +// box" when the honest answer is "this box couldn't check" — precisely the |
| +// reassuring-wrong-answer failure the rest of this package refuses to make. |
| +func TestExpiredCanariesUnknownIsANull(t *testing.T) { |
| + b := status.CollectBox(nil, status.Probes{MachineID: "m"}, time.Now()) // nil ExpiredCanaries |
| + if b.ExpiredCanaries != nil { |
| + t.Errorf("expired_canaries = %v, want null — an unasked question is not a healthy answer", *b.ExpiredCanaries) |
| + } |
| + for _, want := range []int{0, 3} { |
| + got := status.CollectBox(nil, status.Probes{MachineID: "m", ExpiredCanaries: &want}, time.Now()).ExpiredCanaries |
| + if got == nil || *got != want { |
| + t.Errorf("ExpiredCanaries=%d: expired_canaries = %v, want %d", want, got, want) |
| + } |
| + } |
| +} |
| + |
| +// TestFleetExpiredCanariesSums proves the fleet-wide count is the sum across |
| +// boxes when every box's own count is known — the aggregate an operator |
| +// actually wants ("how many expired canaries are running anywhere"), not a |
| +// per-box worst-of. |
| +func TestFleetExpiredCanariesSums(t *testing.T) { |
| + s := teststore(t) |
| + now := time.Now() |
| + two, one := 2, 1 |
| + for _, b := range []status.Box{ |
| + status.CollectBox(nil, status.Probes{MachineID: "a", ExpiredCanaries: &two}, now), |
| + status.CollectBox(nil, status.Probes{MachineID: "b", ExpiredCanaries: &one}, now), |
| + } { |
| + if err := status.WriteBox(context.Background(), s, b); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + f, err := status.BuildFleet(context.Background(), s, nil, now, 5*time.Minute) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if f.ExpiredCanaries == nil || *f.ExpiredCanaries != 3 { |
| + t.Errorf("fleet expired_canaries = %v, want 3", f.ExpiredCanaries) |
| + } |
| +} |
| + |
| +// TestFleetExpiredCanariesNullPropagates: one box that couldn't check nulls |
| +// the fleet-wide sum, the same null-propagation rule sumInstances applies to |
| +// Awake/Asleep — a partial sum would read as more certain than it is. |
| +func TestFleetExpiredCanariesNullPropagates(t *testing.T) { |
| + s := teststore(t) |
| + now := time.Now() |
| + two := 2 |
| + known := status.CollectBox(nil, status.Probes{MachineID: "known", ExpiredCanaries: &two}, now) |
| + unknown := status.CollectBox(nil, status.Probes{MachineID: "unknown"}, now) // nil probe |
| + for _, b := range []status.Box{known, unknown} { |
| + if err := status.WriteBox(context.Background(), s, b); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + f, err := status.BuildFleet(context.Background(), s, nil, now, 5*time.Minute) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if f.ExpiredCanaries != nil { |
| + t.Errorf("fleet expired_canaries = %v, want nil (one box could not check)", *f.ExpiredCanaries) |
| + } |
| +} |
| + |
| +// TestFleetEmptyExpiredCanariesIsZero: no boxes at all is not "unknown", the |
| +// same rule TestFleetEmptyIsValid pins for instances — there is nothing |
| +// unknown about an empty fleet. |
| +func TestFleetEmptyExpiredCanariesIsZero(t *testing.T) { |
| + s := teststore(t) |
| + f, err := status.BuildFleet(context.Background(), s, nil, time.Now(), 5*time.Minute) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if f.ExpiredCanaries == nil || *f.ExpiredCanaries != 0 { |
| + t.Errorf("expired_canaries = %v, want 0 with no boxes", f.ExpiredCanaries) |
| + } |
| +} |
| + |
| +// TestFleetBoxCarriesExpiredCanaries keeps the per-box row honest too — the |
| +// fleet-wide number says something is expired somewhere, the boxes table |
| +// says which box, the same reasoning TestFleetBoxCarriesAdoption pins for |
| +// the adoption-timer reading. |
| +func TestFleetBoxCarriesExpiredCanaries(t *testing.T) { |
| + s := teststore(t) |
| + now := time.Now() |
| + three := 3 |
| + if err := status.WriteBox(context.Background(), s, |
| + status.CollectBox(nil, status.Probes{MachineID: "leaky", ExpiredCanaries: &three}, now)); err != nil { |
| + t.Fatal(err) |
| + } |
| + f, err := status.BuildFleet(context.Background(), s, nil, now, 5*time.Minute) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if len(f.Boxes) != 1 { |
| + t.Fatalf("boxes = %d, want 1", len(f.Boxes)) |
| + } |
| + if f.Boxes[0].ExpiredCanaries == nil || *f.Boxes[0].ExpiredCanaries != 3 { |
| + t.Errorf("boxes[0].expired_canaries = %v, want 3", f.Boxes[0].ExpiredCanaries) |
| + } |
| +} |
| + |
| func TestFleetBoxCarriesAdoption(t *testing.T) { |
| s := teststore(t) |
| now := time.Now() |