| 1 | package status |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "errors" |
| 7 | "log" |
| 8 | "time" |
| 9 | |
| 10 | "github.com/carlosframework/platform/internal/registry" |
| 11 | "github.com/carlosframework/platform/internal/release" |
| 12 | ) |
| 13 | |
| 14 | // FleetApp is one opted-in app's public row — name, count, health, and the |
| 15 | // release it's running. Nothing here exists for an app that hasn't set |
| 16 | // StatusNamed; see BuildFleet's apps grouping. |
| 17 | type FleetApp struct { |
| 18 | Name string `json:"name"` |
| 19 | Instances int `json:"instances"` |
| 20 | Healthy bool `json:"healthy"` |
| 21 | Channel string `json:"channel"` |
| 22 | Version string `json:"version"` // short sha as stored on the route |
| 23 | } |
| 24 | |
| 25 | // FleetBox is a Box's public row inside the merged feed: everything a Box |
| 26 | // carries except its own Replication (which only ever votes into the |
| 27 | // fleet-wide Replication, never appears per-box) and its own Capacity |
| 28 | // (same — it only ever votes into Fleet.Capacity), plus Stale, which only |
| 29 | // exists at merge time since a lone Box has no "how long ago" to compare |
| 30 | // against. |
| 31 | type FleetBox struct { |
| 32 | Label string `json:"label"` |
| 33 | Region string `json:"region,omitempty"` |
| 34 | Load *string `json:"load"` |
| 35 | Stale bool `json:"stale"` |
| 36 | Routes RouteCounts `json:"routes"` |
| 37 | // AdoptionRunning DOES appear per-box, unlike Replication and Capacity |
| 38 | // above, because it is the one reading whose whole value is knowing WHICH |
| 39 | // box has it wrong: "something in the fleet isn't adopting" sends an |
| 40 | // operator to ssh into every box in turn, which is barely better than the |
| 41 | // silence this field exists to end. It carries nothing about a tenant — |
| 42 | // only whether the platform's own timer is armed on an opaquely-labelled |
| 43 | // machine. |
| 44 | AdoptionRunning *bool `json:"adoption_running"` |
| 45 | // ConfigPathRunning appears per-box for exactly the reason |
| 46 | // AdoptionRunning does: the value of the reading is knowing WHICH box |
| 47 | // lost its config-sync trigger, since "somewhere in the fleet config is |
| 48 | // an hour late" is a search, not an answer. Deliberately NOT rolled up |
| 49 | // into a fleet-wide field: worst-of aggregates exist here only where the |
| 50 | // page shows a tile for them, and this one is a per-box operational |
| 51 | // fact, not a headline about the fleet. |
| 52 | ConfigPathRunning *bool `json:"config_path_running"` |
| 53 | // RestartPathRunning appears per-box for exactly the reason |
| 54 | // ConfigPathRunning does, and is not rolled up for the same reason: it is a |
| 55 | // per-box operational fact ("this box's restart trigger is not watching"), |
| 56 | // not a headline about the fleet. |
| 57 | RestartPathRunning *bool `json:"restart_path_running"` |
| 58 | // ExpiredCanaries appears per-box for the same reason AdoptionRunning |
| 59 | // does: the fleet-wide sum says something is expired somewhere, this |
| 60 | // says on which box, without naming which route. Published in the FEED |
| 61 | // only for now — the rendered boxes table deliberately has no column |
| 62 | // for it yet (2026-08-03 review, MINOR 4: this comment used to imply |
| 63 | // the table showed it; it does not, and adding the column is deferred |
| 64 | // until an expired canary has actually been hunted via the feed once). |
| 65 | ExpiredCanaries *int `json:"expired_canaries"` |
| 66 | } |
| 67 | |
| 68 | // Fleet is the merged feed: status/fleet.json. Every field here is either a |
| 69 | // fleet-wide aggregate, a band word, or behind an app's own opt-in — see the |
| 70 | // design spec's "rule for new fields". |
| 71 | type Fleet struct { |
| 72 | GeneratedAt time.Time `json:"generated_at"` |
| 73 | Boxes []FleetBox `json:"boxes"` |
| 74 | Instances map[string]*int `json:"instances"` // total/static/resident/hibernating/awake/asleep |
| 75 | Capacity struct { |
| 76 | Level *string `json:"level"` |
| 77 | } `json:"capacity"` |
| 78 | Apps []FleetApp `json:"apps"` |
| 79 | AppsUnlisted int `json:"apps_unlisted"` |
| 80 | Services []struct { |
| 81 | Name string `json:"name"` |
| 82 | Up bool `json:"up"` |
| 83 | } `json:"services"` |
| 84 | Replication Replication `json:"replication"` |
| 85 | AdoptionRunning *bool `json:"adoption_running"` |
| 86 | // ExpiredCanaries is the fleet-wide sum of every box's own count — see |
| 87 | // sumExpiredCanaries. Null-propagating: one box that couldn't check |
| 88 | // nulls the whole aggregate, the same rule sumInstances applies to |
| 89 | // Awake/Asleep, because a partial sum would read as more certain than |
| 90 | // it is. The status page shows this only when it is a known, positive |
| 91 | // count — see assets/index.html. |
| 92 | ExpiredCanaries *int `json:"expired_canaries"` |
| 93 | } |
| 94 | |
| 95 | // capacityRank orders the capacity band words worst-last, so "worst of a |
| 96 | // set of boxes" is just "the box with the highest rank" — built off |
| 97 | // snapshot.go's capacityWords so the two never drift apart into disagreeing |
| 98 | // about which word beats which. |
| 99 | var capacityRank = map[string]int{ |
| 100 | capacityWords[0]: 0, |
| 101 | capacityWords[1]: 1, |
| 102 | capacityWords[2]: 2, |
| 103 | } |
| 104 | |
| 105 | // intPtr is the one-line "give me a pointer to this value" a lot of the |
| 106 | // Instances map construction below needs — Go has no address-of-literal |
| 107 | // operator, so every *int field needs a local to take the address of |
| 108 | // without this helper. |
| 109 | func intPtr(v int) *int { return &v } |
| 110 | |
| 111 | // SnapshotPeriod is how often every box republishes its own |
| 112 | // status/boxes/<label>.json (the agent's status loop tick, 5 minutes + |
| 113 | // jitter). It lives here rather than at each reader because it is the unit |
| 114 | // BoxStale measures staleness in: the console's private host detail and |
| 115 | // `carlos status -live` both have to answer "is this box dark?" without an |
| 116 | // agent's local const to borrow, and a reader that guessed a different period |
| 117 | // would call a box unreachable at a different moment than the published |
| 118 | // boxes table does. |
| 119 | const SnapshotPeriod = 5 * time.Minute |
| 120 | |
| 121 | // staleFactor is how many missed snapshot periods make a box stale. Three |
| 122 | // consecutive misses: past any single hiccup (a slow S3 write, one skipped |
| 123 | // tick), and the point at which a box has plainly stopped self-reporting. |
| 124 | // One constant, one rule — BuildFleet's published FleetBox.Stale and every |
| 125 | // private reader go through BoxStale below, so nothing can drift into calling |
| 126 | // the same box fresh on one page and dark on another. |
| 127 | const staleFactor = 3 |
| 128 | |
| 129 | // BoxStale is the fleet's one staleness rule: a snapshot older than |
| 130 | // staleFactor×period means the box stopped reporting. Every caller — the |
| 131 | // public merge, the console's private host table, the CLI — asks this |
| 132 | // function, never its own subtraction. |
| 133 | // |
| 134 | // Callers that render must treat true as "this box's numbers are history": |
| 135 | // per the design spec an unreachable host shows how long it has been dark and |
| 136 | // NOTHING else, never its last happy readings. |
| 137 | func BoxStale(b Box, now time.Time, period time.Duration) bool { |
| 138 | return now.Sub(b.GeneratedAt) > staleFactor*period |
| 139 | } |
| 140 | |
| 141 | // LoadBoxes reads every per-box snapshot in the bucket — the same read |
| 142 | // BuildFleet does, exported for the private readers (the console's operator |
| 143 | // fleet page, `carlos status -live`) so there is exactly ONE parser of the |
| 144 | // status/boxes/ shape and one skip-corrupt rule, rather than a second loader |
| 145 | // that could disagree about which snapshots count. |
| 146 | // |
| 147 | // Unlike the merged feed, what comes back here still carries Box.Metrics: raw |
| 148 | // numbers that must reach a private surface only (bucket credentials or a |
| 149 | // session-gated operator page), never fleet.json. See the package comment. |
| 150 | func LoadBoxes(ctx context.Context, s release.Store) ([]Box, error) { |
| 151 | return loadBoxes(ctx, s) |
| 152 | } |
| 153 | |
| 154 | // BuildFleet merges every status/boxes/*.json (stale = GeneratedAt older |
| 155 | // than staleFactor×period — see BoxStale), folds the local registry's routes |
| 156 | // into apps/services, and never invents a number a snapshot didn't carry. |
| 157 | func BuildFleet(ctx context.Context, s release.Store, routes []registry.Route, now time.Time, period time.Duration) (*Fleet, error) { |
| 158 | boxes, err := loadBoxes(ctx, s) |
| 159 | if err != nil { |
| 160 | return nil, err |
| 161 | } |
| 162 | |
| 163 | f := &Fleet{GeneratedAt: now} |
| 164 | |
| 165 | // stale tracks each box's staleness in lockstep with boxes — computed |
| 166 | // once here and reused by both the per-box Stale field and the |
| 167 | // worst-of/replication aggregates below, which must vote only using |
| 168 | // fresh boxes. |
| 169 | stale := make([]bool, len(boxes)) |
| 170 | for i, b := range boxes { |
| 171 | stale[i] = BoxStale(b, now, period) |
| 172 | f.Boxes = append(f.Boxes, FleetBox{ |
| 173 | Label: b.Label, |
| 174 | Region: b.Region, |
| 175 | Load: b.Load, |
| 176 | Stale: stale[i], |
| 177 | Routes: b.Routes, |
| 178 | AdoptionRunning: b.AdoptionRunning, |
| 179 | ConfigPathRunning: b.ConfigPathRunning, |
| 180 | RestartPathRunning: b.RestartPathRunning, |
| 181 | ExpiredCanaries: b.ExpiredCanaries, |
| 182 | }) |
| 183 | } |
| 184 | |
| 185 | f.Instances = sumInstances(boxes) |
| 186 | f.Capacity.Level = worstCapacity(boxes) |
| 187 | f.Replication = worstReplication(boxes, stale) |
| 188 | f.AdoptionRunning = worstAdoption(boxes, stale) |
| 189 | f.ExpiredCanaries = sumExpiredCanaries(boxes) |
| 190 | |
| 191 | apps, unlisted := buildApps(routes) |
| 192 | f.Apps = apps |
| 193 | f.AppsUnlisted = unlisted |
| 194 | f.Services = buildServices(routes) |
| 195 | |
| 196 | return f, nil |
| 197 | } |
| 198 | |
| 199 | // loadBoxes reads every status/boxes/*.json snapshot it can. A snapshot |
| 200 | // that fails to Get or fails to unmarshal is logged and skipped — one box's |
| 201 | // bad write must never take the whole fleet page down, per the plan's |
| 202 | // "corrupt snapshot = logged skip, never fatal" rule. |
| 203 | // maxFleetBoxKeys bounds how many objects loadBoxes will ever process from |
| 204 | // one prefix listing, and maxFleetBoxBodyBytes bounds how large any single |
| 205 | // one of those objects may be before its body is read. Both exist for the |
| 206 | // SAME reason: the prefix loadBoxes reads (status/boxes/, and — via |
| 207 | // release.NewPrefixStore — a customer fleet's own fleets/<name>/status/boxes/) |
| 208 | // is written to by boxes this operator does not control in the fleet case. |
| 209 | // A hostile or badly-misbehaving box could park tens of thousands of keys, |
| 210 | // or one huge body, under its own prefix; without a cap this function would |
| 211 | // list/fetch/unmarshal all of it on every call, feeding both the operator |
| 212 | // page (platform prefix) and every fleet's own page (customer prefix) — |
| 213 | // unbounded memory from data this process does not trust. |
| 214 | const ( |
| 215 | maxFleetBoxKeys = 200 |
| 216 | maxFleetBoxBodyBytes = 1 << 20 // 1MB |
| 217 | ) |
| 218 | |
| 219 | func loadBoxes(ctx context.Context, s release.Store) ([]Box, error) { |
| 220 | boxes, _, err := loadBoxesReport(ctx, s) |
| 221 | return boxes, err |
| 222 | } |
| 223 | |
| 224 | // LoadBoxesComplete is LoadBoxes that also reports whether EVERY listed |
| 225 | // snapshot was read: complete is false when any object's read failed, any |
| 226 | // body was skipped as oversize or corrupt, or the listing was capped. The |
| 227 | // boxes returned are exactly what LoadBoxes would return, and the skips are |
| 228 | // logged the same way; the only addition is the accounting. It exists for |
| 229 | // a reader that must fail CLOSED on partial evidence (the console's |
| 230 | // automatic region placement: a box whose snapshot could not be read has |
| 231 | // UNKNOWN liveness, which is not the same as stale), where LoadBoxes' |
| 232 | // "logged skip, never fatal" posture would quietly drop the one box that |
| 233 | // mattered. |
| 234 | func LoadBoxesComplete(ctx context.Context, s release.Store) (boxes []Box, complete bool, err error) { |
| 235 | return loadBoxesReport(ctx, s) |
| 236 | } |
| 237 | |
| 238 | // statusListingHorizon is how long ago a report's object may last have |
| 239 | // been written and still be listed as a box (region HA plan 2b). Each box |
| 240 | // rewrites its own report every SnapshotPeriod, so an object older than a |
| 241 | // day is a box that has gone: with automatic replacement such reports pile |
| 242 | // up, one per replaced box, and could fill the key cap below, hiding the |
| 243 | // boxes that are running. They are dropped from the listing before any |
| 244 | // read and before the cap, so the cap bounds current reports only. This is |
| 245 | // an optimisation, never authority: every reader still judges each |
| 246 | // report's own GeneratedAt as before (BoxStale, the console's |
| 247 | // VerifyReport), and a day keeps every box an operator would still look |
| 248 | // for as stale. A store that cannot say when an object was written is |
| 249 | // listed whole, as before: one that is not a release.ModLister, and one |
| 250 | // that is but whose ListMod fails. The second is the claim's boundary |
| 251 | // stores (hostops' mutation boundary and conditional record store, the |
| 252 | // mail claim gate): they always have a ListMod method and answer it with |
| 253 | // their own refusal when the store they wrap has none, while their List |
| 254 | // still reads. Because the horizon is only an optimisation, a failed |
| 255 | // ListMod never fails the read: List answers, with main's errors. |
| 256 | const statusListingHorizon = 24 * time.Hour |
| 257 | |
| 258 | // listBoxKeys lists status/boxes/, without the reports the horizon drops. |
| 259 | func listBoxKeys(ctx context.Context, s release.Store) ([]string, error) { |
| 260 | ml, ok := s.(release.ModLister) |
| 261 | if !ok { |
| 262 | return s.List(ctx, "status/boxes/") |
| 263 | } |
| 264 | mods, err := ml.ListMod(ctx, "status/boxes/") |
| 265 | if err != nil { |
| 266 | return s.List(ctx, "status/boxes/") |
| 267 | } |
| 268 | cut := time.Now().Add(-statusListingHorizon) |
| 269 | keys := make([]string, 0, len(mods)) |
| 270 | for _, m := range mods { |
| 271 | if m.Mod.IsZero() || !m.Mod.Before(cut) { |
| 272 | keys = append(keys, m.Key) |
| 273 | } |
| 274 | } |
| 275 | return keys, nil |
| 276 | } |
| 277 | |
| 278 | func loadBoxesReport(ctx context.Context, s release.Store) (boxes []Box, complete bool, err error) { |
| 279 | keys, err := listBoxKeys(ctx, s) |
| 280 | if err != nil { |
| 281 | return nil, false, err |
| 282 | } |
| 283 | complete = true |
| 284 | if len(keys) > maxFleetBoxKeys { |
| 285 | log.Printf("status: fleet: %d keys under status/boxes/, capping at %d", len(keys), maxFleetBoxKeys) |
| 286 | keys = keys[:maxFleetBoxKeys] |
| 287 | complete = false |
| 288 | } |
| 289 | for _, key := range keys { |
| 290 | body, _, err := s.Get(ctx, key) |
| 291 | if err != nil { |
| 292 | log.Printf("status: fleet: skipping %s: read failed: %v", key, err) |
| 293 | complete = false |
| 294 | continue |
| 295 | } |
| 296 | if len(body) > maxFleetBoxBodyBytes { |
| 297 | log.Printf("status: fleet: skipping %s: body %d bytes exceeds %d byte cap", key, len(body), maxFleetBoxBodyBytes) |
| 298 | complete = false |
| 299 | continue |
| 300 | } |
| 301 | var b Box |
| 302 | if err := json.Unmarshal(body, &b); err != nil { |
| 303 | log.Printf("status: fleet: skipping %s: corrupt snapshot: %v", key, err) |
| 304 | complete = false |
| 305 | continue |
| 306 | } |
| 307 | boxes = append(boxes, b) |
| 308 | } |
| 309 | return boxes, complete, nil |
| 310 | } |
| 311 | |
| 312 | // sumInstances adds each RouteCounts component across every box — stale |
| 313 | // boxes included, since staleness is a "trust this less" signal rendered |
| 314 | // loudly per-box, not a reason to silently drop a box's last-known counts |
| 315 | // from the fleet total. Static/Resident/Hibernating are plain ints on Box |
| 316 | // (always known), so their sums are always known too. Awake/Asleep are |
| 317 | // *int on Box (null when a box's socket table was unreadable), so per the |
| 318 | // plan's null-propagation rule, ANY box contributing null nulls the whole |
| 319 | // summed field — a partial sum would read as more certain than it is. |
| 320 | func sumInstances(boxes []Box) map[string]*int { |
| 321 | var totalStatic, totalResident, totalHibernating, awakeSum, asleepSum int |
| 322 | awakeKnown, asleepKnown := true, true |
| 323 | for _, b := range boxes { |
| 324 | totalStatic += b.Routes.Static |
| 325 | totalResident += b.Routes.Resident |
| 326 | totalHibernating += b.Routes.Hibernating |
| 327 | if b.Routes.Awake == nil { |
| 328 | awakeKnown = false |
| 329 | } else { |
| 330 | awakeSum += *b.Routes.Awake |
| 331 | } |
| 332 | if b.Routes.Asleep == nil { |
| 333 | asleepKnown = false |
| 334 | } else { |
| 335 | asleepSum += *b.Routes.Asleep |
| 336 | } |
| 337 | } |
| 338 | instances := map[string]*int{ |
| 339 | "total": intPtr(totalStatic + totalResident + totalHibernating), |
| 340 | "static": intPtr(totalStatic), |
| 341 | "resident": intPtr(totalResident), |
| 342 | "hibernating": intPtr(totalHibernating), |
| 343 | } |
| 344 | if awakeKnown { |
| 345 | instances["awake"] = intPtr(awakeSum) |
| 346 | } else { |
| 347 | instances["awake"] = nil |
| 348 | } |
| 349 | if asleepKnown { |
| 350 | instances["asleep"] = intPtr(asleepSum) |
| 351 | } else { |
| 352 | instances["asleep"] = nil |
| 353 | } |
| 354 | return instances |
| 355 | } |
| 356 | |
| 357 | // sumExpiredCanaries adds each box's own ExpiredCanaries count — stale boxes |
| 358 | // included, same reasoning as sumInstances: a box's last-known count is a |
| 359 | // "trust this less" signal rendered loudly per-box (FleetBox.Stale), not a |
| 360 | // reason to silently drop it from the fleet total. Per the null-propagation |
| 361 | // rule every *int aggregate here follows, ANY box contributing nil (it |
| 362 | // couldn't check) nils the whole sum — a partial count would read as more |
| 363 | // certain than it is. No boxes at all sums to 0, not nil: there is nothing |
| 364 | // unknown about an empty fleet. |
| 365 | func sumExpiredCanaries(boxes []Box) *int { |
| 366 | sum := 0 |
| 367 | for _, b := range boxes { |
| 368 | if b.ExpiredCanaries == nil { |
| 369 | return nil |
| 370 | } |
| 371 | sum += *b.ExpiredCanaries |
| 372 | } |
| 373 | return intPtr(sum) |
| 374 | } |
| 375 | |
| 376 | // worstCapacity picks the worst capacity band among ALL boxes, stale ones |
| 377 | // included: a box that stopped reporting still had a last-known fill level, |
| 378 | // and dropping it from the vote just because its snapshot aged out would |
| 379 | // hide a real "getting full" or "at capacity" reading behind the newer, |
| 380 | // possibly-rosier boxes — exactly the silent-omission failure the plan's |
| 381 | // privacy rules refuse to allow, just pointed at capacity instead of a |
| 382 | // hostname. (Contrast worstReplication below, which the plan explicitly |
| 383 | // scopes to fresh boxes — staleness there means "we don't know", not "we |
| 384 | // know it was bad".) A box with an unknown (nil) band doesn't vote — nil |
| 385 | // never invents a word. All-nil input yields nil. |
| 386 | func worstCapacity(boxes []Box) *string { |
| 387 | var worst *string |
| 388 | worstRank := -1 |
| 389 | for _, b := range boxes { |
| 390 | if b.Capacity == nil { |
| 391 | continue |
| 392 | } |
| 393 | rank, ok := capacityRank[*b.Capacity] |
| 394 | if !ok { |
| 395 | continue // defensive: an unrecognized band word doesn't vote |
| 396 | } |
| 397 | if rank > worstRank { |
| 398 | worstRank = rank |
| 399 | v := *b.Capacity |
| 400 | worst = &v |
| 401 | } |
| 402 | } |
| 403 | return worst |
| 404 | } |
| 405 | |
| 406 | // worstReplication picks the worst replication reading among non-stale |
| 407 | // boxes: unhealthy beats healthy beats unknown, and ties within a band |
| 408 | // break toward the larger lag (still the worse reading). Nothing fresh |
| 409 | // with a known reading yields the zero Replication (both fields nil) — |
| 410 | // unknown, not a guessed "healthy". |
| 411 | func worstReplication(boxes []Box, stale []bool) Replication { |
| 412 | var worst Replication |
| 413 | haveWorst := false |
| 414 | for i, b := range boxes { |
| 415 | if stale[i] || b.Replication.Healthy == nil { |
| 416 | continue |
| 417 | } |
| 418 | switch { |
| 419 | case !haveWorst: |
| 420 | worst = b.Replication |
| 421 | haveWorst = true |
| 422 | case !*b.Replication.Healthy && *worst.Healthy: |
| 423 | // This box is unhealthy and the current worst isn't — a real |
| 424 | // outage always outranks "just some lag". |
| 425 | worst = b.Replication |
| 426 | case *b.Replication.Healthy == *worst.Healthy && |
| 427 | b.Replication.LagSecs != nil && worst.LagSecs != nil && |
| 428 | *b.Replication.LagSecs > *worst.LagSecs: |
| 429 | // Same health band; the larger lag is the worse reading. |
| 430 | worst = b.Replication |
| 431 | } |
| 432 | } |
| 433 | return worst |
| 434 | } |
| 435 | |
| 436 | // worstAdoption folds the boxes' adoption readings into one fleet-wide |
| 437 | // answer, worst-first: a single box that is KNOWN not to be adopting makes the |
| 438 | // fleet false, because a real defect must never be averaged away by the |
| 439 | // healthy boxes around it. Failing that, an unknown box nulls the aggregate — |
| 440 | // "all the boxes we could ask said yes" is not the same claim as "the fleet is |
| 441 | // adopting", and this package does not make the stronger claim from the weaker |
| 442 | // evidence. Only when every voting box is known-true is the fleet true. |
| 443 | // |
| 444 | // Stale boxes don't vote at all (same rule as worstReplication): a snapshot |
| 445 | // that stopped arriving means "we no longer know what that box is doing", and |
| 446 | // a reading from twenty minutes ago is not evidence about now. That is a real |
| 447 | // blind spot rather than a solved problem — but staleness is itself rendered |
| 448 | // loudly per-box, so it is a visible blind spot, which is the whole point of |
| 449 | // this field. |
| 450 | // The order of the checks matters and is not the order the boxes happen to |
| 451 | // arrive in: a known false is decided BEFORE any unknown is allowed to null |
| 452 | // the result, so one unreachable box can never mask a box that reported a dead |
| 453 | // timer. |
| 454 | func worstAdoption(boxes []Box, stale []bool) *bool { |
| 455 | anyUnknown, anyTrue := false, false |
| 456 | for i, b := range boxes { |
| 457 | if stale[i] { |
| 458 | continue |
| 459 | } |
| 460 | switch { |
| 461 | case b.AdoptionRunning == nil: |
| 462 | anyUnknown = true |
| 463 | case !*b.AdoptionRunning: |
| 464 | no := false |
| 465 | return &no |
| 466 | default: |
| 467 | anyTrue = true |
| 468 | } |
| 469 | } |
| 470 | if anyUnknown || !anyTrue { |
| 471 | return nil |
| 472 | } |
| 473 | yes := true |
| 474 | return &yes |
| 475 | } |
| 476 | |
| 477 | // routeGroup collects one App name's routes in the order they appeared in |
| 478 | // the input slice — used for both the apps and services groupings below, |
| 479 | // which differ only in which Kind they filter on and how they reduce a |
| 480 | // group down to one published row. |
| 481 | type routeGroup struct { |
| 482 | name string |
| 483 | routes []registry.Route |
| 484 | } |
| 485 | |
| 486 | // groupRoutesByApp buckets routes matching kind into per-app groups, |
| 487 | // preserving first-seen order so output is deterministic without needing a |
| 488 | // separate sort pass. |
| 489 | // |
| 490 | // The group key is (ACCOUNT, app), not app: app names are unique within an |
| 491 | // account, never globally, so two tenants can both run a "blog". Keying on |
| 492 | // the name alone would merge them — publishing one tenant's outage under the |
| 493 | // other's name, and letting either tenant's StatusNamed opt-in decide whether |
| 494 | // the other appears at all. The published NAME stays the app name; the |
| 495 | // account is only ever the grouping key, never anything the public page |
| 496 | // reveals. |
| 497 | func groupRoutesByApp(routes []registry.Route, kind string) []routeGroup { |
| 498 | type key struct{ acct, app string } |
| 499 | index := map[key]int{} |
| 500 | var groups []routeGroup |
| 501 | for _, r := range routes { |
| 502 | if r.Kind != kind { |
| 503 | continue |
| 504 | } |
| 505 | k := key{release.RouteAccount(r), r.App} |
| 506 | i, ok := index[k] |
| 507 | if !ok { |
| 508 | i = len(groups) |
| 509 | index[k] = i |
| 510 | groups = append(groups, routeGroup{name: r.App}) |
| 511 | } |
| 512 | groups[i].routes = append(groups[i].routes, r) |
| 513 | } |
| 514 | return groups |
| 515 | } |
| 516 | |
| 517 | // buildApps groups the registry's "instance" routes by App and publishes |
| 518 | // only groups where EVERY route opted in via StatusNamed — one un-opted |
| 519 | // route keeps its whole app dark, never a partial reveal. Everything else |
| 520 | // folds into the unlisted count so the aggregate is still honest about how |
| 521 | // many apps exist without naming the ones that didn't ask to be named. |
| 522 | func buildApps(routes []registry.Route) ([]FleetApp, int) { |
| 523 | groups := groupRoutesByApp(routes, "instance") |
| 524 | |
| 525 | var apps []FleetApp |
| 526 | for _, g := range groups { |
| 527 | allNamed := true |
| 528 | healthy := true |
| 529 | for _, r := range g.routes { |
| 530 | if !r.StatusNamed { |
| 531 | allNamed = false |
| 532 | } |
| 533 | if r.Status != "ready" { |
| 534 | healthy = false |
| 535 | } |
| 536 | } |
| 537 | if !allNamed { |
| 538 | continue |
| 539 | } |
| 540 | first := g.routes[0] |
| 541 | for _, r := range g.routes[1:] { |
| 542 | if r.Channel != first.Channel || r.Version != first.Version { |
| 543 | log.Printf("status: fleet: app %q routes disagree on channel/version (using first route's %q/%q)", |
| 544 | g.name, first.Channel, first.Version) |
| 545 | break |
| 546 | } |
| 547 | } |
| 548 | apps = append(apps, FleetApp{ |
| 549 | Name: g.name, |
| 550 | Instances: len(g.routes), |
| 551 | Healthy: healthy, |
| 552 | Channel: first.Channel, |
| 553 | Version: first.Version, |
| 554 | }) |
| 555 | } |
| 556 | return apps, len(groups) - len(apps) |
| 557 | } |
| 558 | |
| 559 | // buildServices groups the registry's "service" routes by App and reports |
| 560 | // each as up only if every route in the group is ready. Service names are |
| 561 | // platform-reserved (svc/*), not tenant hostnames, so — unlike apps — |
| 562 | // there's no opt-in gate here. |
| 563 | func buildServices(routes []registry.Route) []struct { |
| 564 | Name string `json:"name"` |
| 565 | Up bool `json:"up"` |
| 566 | } { |
| 567 | groups := groupRoutesByApp(routes, "service") |
| 568 | |
| 569 | var services []struct { |
| 570 | Name string `json:"name"` |
| 571 | Up bool `json:"up"` |
| 572 | } |
| 573 | for _, g := range groups { |
| 574 | up := true |
| 575 | for _, r := range g.routes { |
| 576 | if r.Status != "ready" { |
| 577 | up = false |
| 578 | } |
| 579 | } |
| 580 | services = append(services, struct { |
| 581 | Name string `json:"name"` |
| 582 | Up bool `json:"up"` |
| 583 | }{Name: g.name, Up: up}) |
| 584 | } |
| 585 | return services |
| 586 | } |
| 587 | |
| 588 | // pageAssets is the fixed publish order for assetsFS's contents: css before |
| 589 | // html, both before fleet.json (see PublishFleet). It is NOT fs.ReadDir |
| 590 | // order — "index.html" sorts alphabetically before "status.css", which |
| 591 | // would flip the required css-before-html order — so the sequence is |
| 592 | // spelled out explicitly rather than derived from a directory walk. |
| 593 | var pageAssets = []string{"status.css", "index.html"} |
| 594 | |
| 595 | // PublishFleet writes the whole viewer: the embedded page assets first, |
| 596 | // then fleet.json last. Order mirrors Eleven's upload-styles-first rule — |
| 597 | // fleet.json is the object readers poll most often, so a reader who fetches |
| 598 | // it the instant after a publish must never be able to load a page that |
| 599 | // references a stylesheet or markup that hasn't landed yet. Every object |
| 600 | // goes through putOverwrite: PublishFleet republishes all three keys on |
| 601 | // every pass (every 5 minutes, per the plan), so a create-only Put would |
| 602 | // fail from the second pass onward. |
| 603 | func PublishFleet(ctx context.Context, s release.Store, f *Fleet) error { |
| 604 | for _, name := range pageAssets { |
| 605 | body, err := assetsFS.ReadFile("assets/" + name) |
| 606 | if err != nil { |
| 607 | return err |
| 608 | } |
| 609 | if err := putOverwrite(ctx, s, "status/"+name, body); err != nil { |
| 610 | return err |
| 611 | } |
| 612 | } |
| 613 | |
| 614 | body, err := json.MarshalIndent(f, "", " ") |
| 615 | if err != nil { |
| 616 | return err |
| 617 | } |
| 618 | return putOverwrite(ctx, s, "status/fleet.json", body) |
| 619 | } |
| 620 | |
| 621 | // putOverwrite lands body at key with last-writer-wins semantics — the same |
| 622 | // read-etag-then-Put dance WriteBox uses, for the same reason: every |
| 623 | // status/ object PublishFleet writes gets republished on the same key each |
| 624 | // pass (fleet.json every 5 minutes, alongside the boxes), and |
| 625 | // release.Store's Put is CAS-only, so a bare create-only Put would fail |
| 626 | // every tick after the first with ErrCASConflict. One retry on a lost race; |
| 627 | // a second conflict is surfaced rather than retried again, since the status |
| 628 | // lease is supposed to make PublishFleet the only writer touching these |
| 629 | // keys — two losses in a row past the lease means something else is wrong. |
| 630 | func putOverwrite(ctx context.Context, s release.Store, key string, body []byte) error { |
| 631 | etag, err := currentEtag(ctx, s, key) |
| 632 | if err != nil { |
| 633 | return err |
| 634 | } |
| 635 | if _, err := s.Put(ctx, key, body, etag); err != nil { |
| 636 | if !errors.Is(err, release.ErrCASConflict) { |
| 637 | return err |
| 638 | } |
| 639 | etag, err = currentEtag(ctx, s, key) |
| 640 | if err != nil { |
| 641 | return err |
| 642 | } |
| 643 | if _, err := s.Put(ctx, key, body, etag); err != nil { |
| 644 | return err |
| 645 | } |
| 646 | } |
| 647 | return nil |
| 648 | } |
| 649 | |
amadan