| 1 | // Package status is the fleet status page's writer half: per-box snapshots |
| 2 | // that carry only what the public commitments allow — aggregate counts and |
| 3 | // band words, never ratios, hostnames, or machine ids. Privacy is enforced |
| 4 | // HERE, in the process that does the counting and banding, not downstream — |
| 5 | // a merge step or a page template can only leak what this package already |
| 6 | // let through. |
| 7 | // |
| 8 | // Box.Metrics is the one deliberate exception to that rule, not a hole in |
| 9 | // it: a raw, unbanded block (real byte counts, a load average, the running |
| 10 | // build, tunnel state) that CollectBox attaches to the Box it returns, which |
| 11 | // WriteBox lands in status/boxes/<label>.json alongside everything else this |
| 12 | // package computes. That file is bucket-private — the deployment bucket |
| 13 | // blocks all public access (infra/modules/deployment-buckets), and the edge |
| 14 | // process's fixed three-object menu (internal/edge/status.go) only ever |
| 15 | // serves "/", "/fleet.json", and "/status.css"; a request for |
| 16 | // status/boxes/<label>.json 404s the same as any other unlisted path. Only |
| 17 | // fleet-status machinery (loadBoxes in fleet.go) and operators with bucket |
| 18 | // credentials ever read it. The public merge step, fleet.go's BuildFleet, |
| 19 | // copies Box's fields into FleetBox one at a time by field name and simply |
| 20 | // never names Metrics — so the banded-words-only commitment on fleet.json |
| 21 | // holds by construction, not by a filter that could be forgotten. See |
| 22 | // dryrun.go's leak fixtures, which assert a raw metric value survives into |
| 23 | // the box object but never into fleet.json or index.html. |
| 24 | package status |
| 25 | |
| 26 | import ( |
| 27 | "context" |
| 28 | "crypto/sha256" |
| 29 | "encoding/binary" |
| 30 | "encoding/json" |
| 31 | "errors" |
| 32 | "os" |
| 33 | "time" |
| 34 | |
| 35 | "github.com/carlosframework/platform/internal/registry" |
| 36 | "github.com/carlosframework/platform/internal/release" |
| 37 | ) |
| 38 | |
| 39 | // Probes carries everything CollectBox may not read from the registry — |
| 40 | // each nil means "unknown", and unknown publishes as null, never a guess. |
| 41 | type Probes struct { |
| 42 | ConsoleProcesses []ConsoleProcess |
| 43 | ActivationID string |
| 44 | ActivationComplete bool |
| 45 | Enrollment string |
| 46 | MovesProtocol int |
| 47 | ReleaseTag string |
| 48 | BinarySHA256 string |
| 49 | BootID string |
| 50 | FloorSHA256 string |
| 51 | FloorGuardReady bool |
| 52 | HostClaimEnabled bool |
| 53 | RetirementReceiptSHA256 string |
| 54 | |
| 55 | MachineID string // hashed into the label; never serialized |
| 56 | // InstanceID is the cloud machine this box runs on (status.InstanceID), |
| 57 | // or "" for hardware no cloud owns. It is the console architecture |
| 58 | // page's join key — see BoxMetrics.InstanceID. |
| 59 | InstanceID string |
| 60 | Region string // operator-set, coarse; "" omits the field |
| 61 | MemFrac *float64 // 0..1 memory utilization |
| 62 | DiskFrac *float64 // 0..1 proximity to the disk floor |
| 63 | Listening func(sock string) *bool // kernel socket table read; nil fn = unknown |
| 64 | LagSecs *int64 // seconds since litestream last progressed |
| 65 | // AdoptRunning is whether release adoption is actually armed on this box |
| 66 | // — the carlos-adopt.timer being ACTIVE, not merely enabled. nil is |
| 67 | // "never asked / couldn't tell", and like every other nil here it must |
| 68 | // publish as null: a box that can't answer is not a box that is fine. |
| 69 | AdoptRunning *bool |
| 70 | // ConfigPathRunning is whether this box's config-sync trigger unit |
| 71 | // (carlos-adopt.path) is ACTIVE — the issue #44 sibling of AdoptRunning, |
| 72 | // with the identical failure mode behind it: enabled writes a symlink, |
| 73 | // only started arms the watch. An unarmed path unit degrades silently |
| 74 | // (the agent writes its marker, nothing reads it, unit-backed config |
| 75 | // waits out the hourly floor), which is precisely the class of quiet |
| 76 | // half-working this field exists to end. nil is "never asked / couldn't |
| 77 | // tell" and publishes as null, same as every other probe here. |
| 78 | ConfigPathRunning *bool |
| 79 | // RestartPathRunning is whether this box's restart trigger unit |
| 80 | // (carlos-restart.path) is ACTIVE — ConfigPathRunning's issue #113 sibling, |
| 81 | // same failure mode, sharper consequence: with it unarmed, a member who |
| 82 | // runs `carlos restart` against a wedged instance is told "requested" and |
| 83 | // waits out the hourly pass for a cycle they were led to expect in |
| 84 | // seconds. nil is "never asked / couldn't tell" and publishes as null. |
| 85 | RestartPathRunning *bool |
| 86 | // ExpiredCanaries is how many of this box's own canary/* routes have |
| 87 | // passed release.CanaryTTL (pipelines spec §5 "Canaries expire"), |
| 88 | // pre-computed by the caller against the release store — CollectBox |
| 89 | // stays pure and never resolves a channel pointer itself, the same |
| 90 | // reason every other field here arrives pre-resolved. nil means this |
| 91 | // box could not determine the answer (no deployment store configured, |
| 92 | // or a channel pointer it needed wouldn't verify), never a guessed |
| 93 | // zero; 0 is itself a real, confident answer when every canary route on |
| 94 | // the box is fresh, or there are none at all. |
| 95 | ExpiredCanaries *int |
| 96 | // Alerts is the box's alert-sink answer (see AlertDelivery), carried |
| 97 | // into Box.Metrics. nil means "never asked". |
| 98 | Alerts *AlertDelivery |
| 99 | |
| 100 | // MemTotalBytes, MemAvailBytes, Load1, and DiskFreeBytes are the raw |
| 101 | // numbers MemFrac/DiskFrac above are already computed FROM — CollectBox |
| 102 | // still bands MemFrac/DiskFrac into Load/Capacity as always, but also |
| 103 | // carries these through unbanded into Box.Metrics, the package's one |
| 104 | // documented private exception (see the package doc comment). nil means |
| 105 | // unknown, same as every other pointer field here. |
| 106 | MemTotalBytes *int64 |
| 107 | MemAvailBytes *int64 |
| 108 | Load1 *float64 |
| 109 | DiskFreeBytes *int64 |
| 110 | // Release is the running binary's version.Build (a short sha or similar), |
| 111 | // and Tunnel is one of ""|"connected"|"reconnecting" — both land in |
| 112 | // Box.Metrics unbanded, like the four numeric fields above. "" means |
| 113 | // unknown/not-applicable for both, never a guess. |
| 114 | Release string |
| 115 | Tunnel string |
| 116 | // GeoIP is the edge's GeoIP mirror state (geoip spec §5.5); nil when |
| 117 | // the edge has not published one (no edge yet, or an older binary). |
| 118 | GeoIP *GeoIPState |
| 119 | // Platform is runtime.GOOS-runtime.GOARCH, so the console knows which |
| 120 | // artifacts a platform promote must mirror. SystemChannel is the box's |
| 121 | // CARLOS_SYSTEM_CHANNEL, so the console knows which boxes a promote |
| 122 | // reaches. System is root's last self-update outcome (spec §5, §6). |
| 123 | Platform string |
| 124 | SystemChannel string |
| 125 | System *SystemState |
| 126 | // Upstreams is this box's edge probe table — route host -> per-upstream |
| 127 | // state — read WHOLE from the prober on every tick (internal/edge's |
| 128 | // Edge.UpstreamStates), never accumulated across ticks. That rebuild is |
| 129 | // the field's staleness contract: a route the box has stopped probing |
| 130 | // (deleted, or its addrs cleared) simply isn't in the next tick's map, so |
| 131 | // no ghost entry can outlive the route that produced it. Empty/nil means |
| 132 | // this box probes nothing — the shape of every box with no multi-upstream |
| 133 | // route, which is most of them — and publishes as an absent key, not an |
| 134 | // empty object. |
| 135 | Upstreams map[string][]UpstreamState |
| 136 | |
| 137 | // Aliases is this box's route_aliases rows (registry.RouteAlias), the |
| 138 | // agent's reg.ListAliases() read at statusTick — carried through raw so |
| 139 | // CollectBox can fold each row's DNS/cert stamp into Box.Metrics.Aliases |
| 140 | // alongside RouteVersions above. A failed read is the caller's problem |
| 141 | // (log-and-continue, see agent.go's statusTick), never CollectBox's: nil |
| 142 | // here means "no aliases known this tick," identical on the wire to "no |
| 143 | // alias rows exist." |
| 144 | Aliases []registry.RouteAlias |
| 145 | |
| 146 | // DBBytes sizes one route's SQLite database on local disk: the file at |
| 147 | // registry.Route.DBPath plus its -wal. CollectBox calls it for every |
| 148 | // non-platform, non-static route with a DBPath. A nil func, or a nil |
| 149 | // answer (no file yet, stat failed), means unknown and the host is left |
| 150 | // out of Metrics.RouteDBBytes rather than published as 0. |
| 151 | DBBytes func(dbPath string) *int64 |
| 152 | |
| 153 | // Layout is the box's layout report (isolated-exec design §8.4): which |
| 154 | // mode the box last converged to and, per instance-kind route, the word |
| 155 | // the layout pass's own Classify gave it. The caller (internal/host's |
| 156 | // statusLoop) builds it from the armed layout.Pass and the routes it |
| 157 | // already listed; nil means no pass is armed on this box, which |
| 158 | // CollectBox carries straight through as "nothing to report" rather |
| 159 | // than guess at a mode. Hostname-keyed, like Aliases and RouteVersions |
| 160 | // above, which is why it lands in Box.Metrics and nowhere else. |
| 161 | Layout *LayoutReport |
| 162 | // ConfigRefusals is the instances this box is CURRENTLY refusing to take |
| 163 | // because it cannot provide their config (amadan #213), as the host's |
| 164 | // config guard reports them; nil when it refuses nothing. They land in |
| 165 | // the private Metrics block only: they name hosts. ConfigRefusalsTotal is |
| 166 | // how many are refused in all when the guard capped the list. |
| 167 | ConfigRefusals []ConfigRefusal |
| 168 | ConfigRefusalsTotal int |
| 169 | // ReleaseRefusals is the builds this box rolled back and is now refusing |
| 170 | // to re-adopt (D1); nil when there are none. Private Metrics only. |
| 171 | ReleaseRefusals []ReleaseRefusal |
| 172 | // ProofWaits is the hosts held on their current build waiting for a dev |
| 173 | // proof (D3). Private Metrics only: they name hosts. |
| 174 | ProofWaits []ProofWait |
| 175 | // Rollout is the rollout scheduler's report (zero-downtime spec §8.4, |
| 176 | // §9.1); nil with CARLOS_SWAP off. Private Metrics only: it names hosts. |
| 177 | Rollout *RolloutReport |
| 178 | // ConfigGuard is "" in the normal mode and a words-only note when the |
| 179 | // config guard's explicit bypass is in effect on this box (amadan #213). |
| 180 | ConfigGuard string |
| 181 | // BuildStates is the build-state builds of the instances this box hosts |
| 182 | // (settings spec rev 11 §4.4), and BuildStatesErr says the box could not |
| 183 | // collect them. Private Metrics only: they name hosts. |
| 184 | BuildStates map[string]BuildStatePins |
| 185 | BuildStatesErr string |
| 186 | // UpdateSettings is the update-settings protocol this box enforces |
| 187 | // (BoxMetrics.UpdateSettings): UpdateSettingsProtocol, or 0 when the box |
| 188 | // cannot honour pin, manual and major. Updates is root's last decision per |
| 189 | // host. Private Metrics only: Updates names hosts and builds. |
| 190 | UpdateSettings int |
| 191 | Updates map[string]HostUpdates |
| 192 | } |
| 193 | |
| 194 | // HostUpdates is root's last update decision for one instance, for the |
| 195 | // console's list and instance page (settings spec §4.3). Everything is |
| 196 | // omitempty: a field root has nothing to say about is absent. |
| 197 | type HostUpdates struct { |
| 198 | // Desired is the build root last chose for the instance, and Reason why |
| 199 | // it differs from the channel head (a release.TargetReason word). Detail |
| 200 | // is a build version when root names one (the major hold's head). |
| 201 | Desired string `json:"desired,omitempty"` |
| 202 | Reason string `json:"reason,omitempty"` |
| 203 | Detail string `json:"detail,omitempty"` |
| 204 | // Baking is the head a pending request waits on, reported beside Reason. |
| 205 | Baking string `json:"baking,omitempty"` |
| 206 | // Major is the instance's effective major hold, so a list can name one |
| 207 | // the instance inherits. |
| 208 | Major *int `json:"major,omitempty"` |
| 209 | // Pending is the install request not yet completed; Completed is the |
| 210 | // highest completed request; CompletedFailed says that one failed. |
| 211 | Pending uint64 `json:"pending,omitempty"` |
| 212 | Completed uint64 `json:"completed,omitempty"` |
| 213 | CompletedFailed bool `json:"completed_failed,omitempty"` |
| 214 | // Account and App are the instance's tenant as the box's route names it |
| 215 | // (HostRollout.Account and .App's rule), so the console can join the |
| 216 | // report to a tenant without the rollout report (Astra r1, R18). Absent |
| 217 | // from a box that predates them: the console then joins only whether the |
| 218 | // box enforces settings, nothing the report says about the instance. |
| 219 | Account string `json:"account,omitempty"` |
| 220 | App string `json:"app,omitempty"` |
| 221 | } |
| 222 | |
| 223 | // RowLayout is one instance-kind route's layout, as the box's layout pass |
| 224 | // classified it (activator.Classify): Layout is one of flat, nested, |
| 225 | // fenced, conflict, foreign or external, and Reason carries the detail for |
| 226 | // anything but flat and nested — a path, or another row's host — which is |
| 227 | // why it is never shown on the console's instance page (see |
| 228 | // instancepage.go's handleAppInstance): a member sees the word, an operator |
| 229 | // reading status/boxes/<label>.json sees the reason too. |
| 230 | type RowLayout struct { |
| 231 | Layout string `json:"layout"` |
| 232 | Reason string `json:"reason,omitempty"` |
| 233 | // Note is the last event the pass recorded for the host that was not |
| 234 | // a refusal: what a move quarantined out of the instance directory or |
| 235 | // kept aside, and when. Paths, so private like Reason. |
| 236 | Note string `json:"note,omitempty"` |
| 237 | } |
| 238 | |
| 239 | // LayoutReport is one box's whole layout picture: Mode is the intent the |
| 240 | // box last converged to ("none" before any is recorded, "unreadable" when |
| 241 | // the last read of it failed or named a mode this binary does not know, |
| 242 | // otherwise "flat" or "nested"), and Rows is every instance-kind route on |
| 243 | // the box, keyed by host. It carries real hostnames, so — like RowLayout |
| 244 | // above — it is private: BoxMetrics.Layout, never a field on Box itself. |
| 245 | type LayoutReport struct { |
| 246 | Mode string `json:"mode"` |
| 247 | Rows map[string]RowLayout `json:"rows"` |
| 248 | // Default is set when Mode is nested by the deployment's default |
| 249 | // rather than by a recorded intent (design §8.6, "Nested by |
| 250 | // default"); DefaultBlocked is why the default is not in force on a |
| 251 | // box whose Mode is none. Both additive; an old reader ignores them. |
| 252 | Default bool `json:"default,omitempty"` |
| 253 | DefaultBlocked string `json:"default_blocked,omitempty"` |
| 254 | } |
| 255 | |
| 256 | // ConfigRefusal is one host this box is refusing to take because it cannot |
| 257 | // provide that instance's config: no CARLOS_SECRETS_KEY, one that does not |
| 258 | // parse, one that does not open the sealed config, or a store read it could |
| 259 | // not complete (a box that cannot tell refuses). Reason is a closed word |
| 260 | // (no_key, key_unparseable, cannot_open, store_error), never error text, so |
| 261 | // nothing from the store or the crypto reaches a snapshot. Since is when the |
| 262 | // host was first refused in this stretch, Last the most recent refusal, and |
| 263 | // Count how many wake or admission attempts were refused. |
| 264 | // |
| 265 | // PRIVATE, like everything under Box.Metrics: the host is a real route name. |
| 266 | // It is declared here, with the other wire types of the snapshot, so the |
| 267 | // readers (console, operators) need not import the host agent to decode it. |
| 268 | type ConfigRefusal struct { |
| 269 | Host string `json:"host"` |
| 270 | Reason string `json:"reason"` |
| 271 | Since time.Time `json:"since"` |
| 272 | Last time.Time `json:"last"` |
| 273 | Count int `json:"count"` |
| 274 | } |
| 275 | |
| 276 | // ReleaseRefusal is a build this box rolled back because its unit did not |
| 277 | // come up (D1, release.Refusal). PRIVATE like the rest of Box.Metrics: Host |
| 278 | // is a real route name. |
| 279 | type ReleaseRefusal struct { |
| 280 | Host string `json:"host"` |
| 281 | Version string `json:"version"` |
| 282 | Reason string `json:"reason"` |
| 283 | RefusedAt time.Time `json:"refused_at"` |
| 284 | // Kind is release.Refusal's: "" for a unit that did not come up, |
| 285 | // "wrong-program" for a build refused before it was ever pointed live. |
| 286 | Kind string `json:"kind,omitempty"` |
| 287 | // RecoveryError is set when the rollback itself did not come up (the |
| 288 | // record's Recovered is false): why. "" is a recovered rollback. |
| 289 | RecoveryError string `json:"recovery_error,omitempty"` |
| 290 | } |
| 291 | |
| 292 | // RolloutReport is this box's rollout scheduler state (zero-downtime spec |
| 293 | // §8.4, §9.1), published only with CARLOS_SWAP=overlap. It names hosts, so |
| 294 | // it lives in the private Metrics block alone. |
| 295 | type RolloutReport struct { |
| 296 | Hosts map[string]HostRollout `json:"hosts,omitempty"` |
| 297 | Apps map[string]AppRollout `json:"apps,omitempty"` // keyed "<acct>/<app>" |
| 298 | Failures []RolloutFailure `json:"failures,omitempty"` |
| 299 | // Unavailable, when set, is a reason code (not member-facing text) for |
| 300 | // why CARLOS_SWAP=overlap is on but the scheduler could not arm. Either |
| 301 | // the box cannot contain generations ("no-cgroup-delegation", |
| 302 | // "no-generation-control", "generations-off"), has host claims on |
| 303 | // ("host-claims-on"), or failed its arm-time test placement |
| 304 | // ("cgroup-placement-failed"), and stays on today's |
| 305 | // paths, or its required state is unavailable ("no-deploy-store", |
| 306 | // "no-release-key", "failure-book-unreadable"), and every managed start |
| 307 | // is refused meanwhile (Task 14, internal/host's swapArming and |
| 308 | // refuseManaged). The rest of the report is then empty. Plan 6 words it. |
| 309 | Unavailable string `json:"unavailable,omitempty"` |
| 310 | // FailureBook is set only beside a decline code (the first five): what a |
| 311 | // box that cannot contain generations does with the failure records an |
| 312 | // earlier armed run left (final review F6). "honoured": a managed wake, |
| 313 | // or a bounce, onto a build they hold for that instance is refused. |
| 314 | // "failure-book-unreadable": every managed start is refused and the |
| 315 | // bounce legs stand down. Empty: there are no records, and the box is on |
| 316 | // today's paths. A code, like Unavailable. |
| 317 | FailureBook string `json:"failure_book,omitempty"` |
| 318 | } |
| 319 | |
| 320 | // HostRollout is one managed instance. Attempt and Reason are the closed |
| 321 | // words below; plan 6 renders them. |
| 322 | type HostRollout struct { |
| 323 | Account string `json:"account"` |
| 324 | App string `json:"app"` |
| 325 | Installed string `json:"installed"` |
| 326 | Serving string `json:"serving,omitempty"` |
| 327 | Good string `json:"good,omitempty"` |
| 328 | Awake bool `json:"awake"` |
| 329 | Attempt string `json:"attempt,omitempty"` |
| 330 | AttemptBuild string `json:"attempt_build,omitempty"` |
| 331 | HeldBy string `json:"held_by,omitempty"` |
| 332 | Reason string `json:"reason,omitempty"` |
| 333 | ConfigPending bool `json:"config_pending,omitempty"` |
| 334 | // ConfigError is a reason word (Reason*), never raw error text. |
| 335 | ConfigError string `json:"config_error,omitempty"` |
| 336 | // HeldEpoch is the generation of HeldBy's restriction of this instance |
| 337 | // as published (rollout.FailureBook.PublishedEpoch): the console signs |
| 338 | // it into a `carlos instances retry` grant, which the box honours only |
| 339 | // while the app's restrictions are unchanged. 0 on a held instance means |
| 340 | // the box's failure book is not saved yet; absent from an older box, |
| 341 | // which is 0 too. |
| 342 | HeldEpoch uint64 `json:"held_epoch,omitempty"` |
| 343 | // HeldBook is the identity of the failure book HeldEpoch counts in |
| 344 | // (rollout.FailureBook.ID): epochs are a counter every book starts |
| 345 | // alike, so the console signs both into its grant and the box clears |
| 346 | // only for its own book (final review round 1, finding 3). Absent from |
| 347 | // an older box: a grant signed without it clears nothing. |
| 348 | HeldBook string `json:"held_book,omitempty"` |
| 349 | // RetryV is the `carlos instances retry` protocol this box takes, set |
| 350 | // beside HeldBook: HostRetryV is a root-signed, book-bound grant in the |
| 351 | // host-retry log (release.HostRetryKey). Absent (0) from a box built |
| 352 | // before that log, which reads no grant there, so the console writes |
| 353 | // it none and says the box is too old (readiness round 2, Task 3). |
| 354 | RetryV int `json:"retry_v,omitempty"` |
| 355 | // Transient is set when the activator owns the host's process but has no |
| 356 | // current generation for it: the process is starting or stopping, so the |
| 357 | // instance is neither asleep nor awake and no serving build is reported. |
| 358 | // The deploy watch keeps such a host pending (Astra R3 section 5 item 10). |
| 359 | // Absent from an older box, which is false. A reader that ignores the |
| 360 | // field sees a transient host as Serving "" and Awake false, that is, |
| 361 | // asleep. |
| 362 | Transient bool `json:"transient,omitempty"` |
| 363 | } |
| 364 | |
| 365 | // AppRollout counts one app's attempts. Settling is canary settles and |
| 366 | // first-wake observations under way. |
| 367 | type AppRollout struct { |
| 368 | Queued int `json:"queued"` |
| 369 | Running int `json:"running"` |
| 370 | Settling int `json:"settling"` |
| 371 | } |
| 372 | |
| 373 | // RolloutFailure is one failure record (spec §8.4). Reason is a reason word. |
| 374 | type RolloutFailure struct { |
| 375 | Account string `json:"account"` |
| 376 | App string `json:"app"` |
| 377 | Build string `json:"build"` |
| 378 | Host string `json:"host"` |
| 379 | Mode string `json:"mode,omitempty"` |
| 380 | Reason string `json:"reason"` |
| 381 | PromotedAt time.Time `json:"promoted_at"` |
| 382 | At time.Time `json:"at"` |
| 383 | HeldDown []string `json:"held_down,omitempty"` |
| 384 | } |
| 385 | |
| 386 | // The attempt words. |
| 387 | const ( |
| 388 | AttemptQueued = "queued" |
| 389 | AttemptRunning = "running" |
| 390 | AttemptSettling = "settling" |
| 391 | AttemptFailed = "failed" |
| 392 | AttemptDown = "down" |
| 393 | ) |
| 394 | |
| 395 | // HostRetryV is HostRollout.RetryV for a box that reads root-signed, |
| 396 | // book-bound retry grants from the host-retry log and honours a legacy |
| 397 | // signed one in the restart-scope log with the same checks. |
| 398 | const HostRetryV = 1 |
| 399 | |
| 400 | // The reason words: the closed vocabulary a member-facing event or status |
| 401 | // line may give for a failure. internal/rollout maps every diagnostic text |
| 402 | // onto one of them (reasonWord). |
| 403 | const ( |
| 404 | ReasonHealth = "health" |
| 405 | ReasonSettle = "settle" |
| 406 | ReasonExit = "exit" |
| 407 | ReasonStaging = "staging" |
| 408 | ReasonReused = "reused" |
| 409 | ) |
| 410 | |
| 411 | // ProofWait is a host this box is holding on its current build because its |
| 412 | // channel head has no dev proof yet (D3, release.ProofGate), or because the |
| 413 | // box's proof policy file does not load. PRIVATE like the rest of |
| 414 | // Box.Metrics: Host is a real route name. With no Prover the wait is about |
| 415 | // the policy, and Reason is the policy's own load error. |
| 416 | type ProofWait struct { |
| 417 | Host string `json:"host"` |
| 418 | Version string `json:"version"` |
| 419 | SHA256 string `json:"sha256,omitempty"` |
| 420 | Prover string `json:"prover,omitempty"` |
| 421 | Reason string `json:"reason"` |
| 422 | Since time.Time `json:"since"` |
| 423 | } |
| 424 | |
| 425 | // UpstreamState is one upstream's health as the edge's prober currently sees |
| 426 | // it: what it believes, when that belief last changed, and why. |
| 427 | // |
| 428 | // It lives HERE, in the writer of the snapshot, rather than in internal/edge |
| 429 | // where the prober that fills it lives, for one reason: this is a WIRE type on |
| 430 | // status/boxes/<label>.json, and its readers are the console |
| 431 | // (internal/console's box-detail views, which already parse BoxMetrics) and |
| 432 | // operators. Declaring it in internal/edge would force every one of those |
| 433 | // readers to import the edge — the whole ACME/DNS/proxy stack — just to name |
| 434 | // the shape of a struct it decodes. internal/edge aliases this type instead |
| 435 | // (type UpstreamState = status.UpstreamState), so the prober keeps writing |
| 436 | // UpstreamState in its own package with no conversion at the boundary, and the |
| 437 | // dependency runs edge -> status, the direction that leaves internal/status a |
| 438 | // leaf over registry/release. |
| 439 | // |
| 440 | // Like everything else under Box.Metrics this is PRIVATE: Addr is a real |
| 441 | // backend address and the map key is a real route host, safe here only because |
| 442 | // BuildFleet never copies Metrics into the public fleet.json (see the package |
| 443 | // doc comment). |
| 444 | type UpstreamState struct { |
| 445 | Addr string `json:"addr"` |
| 446 | Up bool `json:"up"` |
| 447 | Since time.Time `json:"since"` // last transition (or the sync that started this prober) |
| 448 | Fails int `json:"fails"` // consecutive probe failures |
| 449 | LastError string `json:"last_error,omitempty"` // dial vs status, for the log/dashboard |
| 450 | } |
| 451 | |
| 452 | // SystemState is the last platform self-update outcome root recorded on the |
| 453 | // box, published so `carlos system status` can show a rolled-back box with |
| 454 | // its reason within one heartbeat. |
| 455 | type SystemState struct { |
| 456 | Channel string `json:"channel"` |
| 457 | Version string `json:"version"` |
| 458 | PromotedAt time.Time `json:"promoted_at"` |
| 459 | Outcome string `json:"outcome"` |
| 460 | Reason string `json:"reason,omitempty"` |
| 461 | At time.Time `json:"at"` |
| 462 | } |
| 463 | |
| 464 | // BoxMetrics is the raw, PRIVATE block: real numbers, never banded. It lives |
| 465 | // in status/boxes/<label>.json — which only fleet-status machinery and |
| 466 | // operators read — and is deliberately NOT copied into FleetBox, so the |
| 467 | // publicly served fleet.json keeps the banded-words-only commitment. See the |
| 468 | // package doc comment for why this block is safe to exist at all. |
| 469 | type BoxMetrics struct { |
| 470 | ConsoleProcesses []ConsoleProcess `json:"console_processes,omitempty"` |
| 471 | ActivationID string `json:"activation_id,omitempty"` |
| 472 | ActivationComplete bool `json:"activation_complete,omitempty"` |
| 473 | Enrollment string `json:"enrollment,omitempty"` |
| 474 | MovesProtocol int `json:"moves_protocol,omitempty"` |
| 475 | ReleaseTag string `json:"release_tag,omitempty"` |
| 476 | BinarySHA256 string `json:"binary_sha256,omitempty"` |
| 477 | BootID string `json:"boot_id,omitempty"` |
| 478 | FloorSHA256 string `json:"floor_sha256,omitempty"` |
| 479 | FloorGuardReady bool `json:"floor_guard_ready,omitempty"` |
| 480 | HostClaimEnabled bool `json:"host_claim_enabled,omitempty"` |
| 481 | RetirementReceiptSHA256 string `json:"retirement_receipt_sha256,omitempty"` |
| 482 | |
| 483 | MemTotalBytes *int64 `json:"mem_total_bytes,omitempty"` |
| 484 | MemAvailBytes *int64 `json:"mem_avail_bytes,omitempty"` |
| 485 | Load1 *float64 `json:"load1,omitempty"` |
| 486 | DiskFreeBytes *int64 `json:"disk_free_bytes,omitempty"` |
| 487 | Release string `json:"release,omitempty"` // running binary's version.Build |
| 488 | Platform string `json:"platform,omitempty"` // runtime.GOOS-runtime.GOARCH |
| 489 | SystemChannel string `json:"system_channel,omitempty"` // CARLOS_SYSTEM_CHANNEL |
| 490 | System *SystemState `json:"system,omitempty"` // last self-update outcome |
| 491 | // Alerts is where this box's own alert rules (metrics.Evaluate) can send |
| 492 | // a fire or clear, and how many sends have failed since the agent |
| 493 | // started. nil means the box did not say (a snapshot from before the |
| 494 | // field existed), which is not the same as an AlertDelivery whose sinks |
| 495 | // are both off: that is a box that says it has nowhere to deliver. |
| 496 | // Private, like the rest of this block: BuildFleet never copies Metrics. |
| 497 | Alerts *AlertDelivery `json:"alerts,omitempty"` |
| 498 | // InstanceID is the cloud machine this box IS — read from the instance |
| 499 | // metadata service by the box itself (status.InstanceID). "" means no |
| 500 | // cloud machine owns it: a Pi, a laptop, someone else's hardware. |
| 501 | // |
| 502 | // It lives here, in the private per-box block, for the reason every |
| 503 | // hostname-bearing field here does: status/boxes/<label>.json is |
| 504 | // bucket-private and BuildFleet never copies Metrics into the public |
| 505 | // feed, so an instance id cannot reach fleet.json. |
| 506 | // |
| 507 | // The console's architecture page joins boxes to machines on this and |
| 508 | // nothing else matters as much: the first version matched an instance's |
| 509 | // Name tag against the box label, and on the flagship those namespaces |
| 510 | // are disjoint (carlos-flagship-ie versus annie-welding), so the join |
| 511 | // found nothing at all and every box read as bring-your-own. A box |
| 512 | // asking the metadata service which machine it is cannot drift from a |
| 513 | // naming convention, because it is not using one. |
| 514 | InstanceID string `json:"instance_id,omitempty"` |
| 515 | Tunnel string `json:"tunnel,omitempty"` // ""|"connected"|"reconnecting" |
| 516 | // GeoIP is present on every box running a GeoIP-aware binary (geoip |
| 517 | // spec §5.5); its absence is how the rollout verdict tells an older |
| 518 | // binary, which does not strip forged geo headers. |
| 519 | GeoIP *GeoIPState `json:"geoip,omitempty"` |
| 520 | // UpdateSettings is the instance-update-settings protocol this binary |
| 521 | // implements: it applies pin, manual and major, and its retention |
| 522 | // protects the builds they name (settings spec §4.4). Absent (0) is a |
| 523 | // binary that ignores them; the console refuses those settings until |
| 524 | // every box reports UpdateSettingsProtocol. |
| 525 | UpdateSettings int `json:"update_settings,omitempty"` |
| 526 | // BuildStates is, per instance this box hosts that keeps a build state, |
| 527 | // the builds that build state names: last_executed and a resolved |
| 528 | // request's target (settings spec rev 11 §4.4). The housekeeper protects |
| 529 | // them, because it cannot read the regional data buckets the objects |
| 530 | // live in. Host-keyed and private like RouteVersions; omitted when empty. |
| 531 | BuildStates map[string]BuildStatePins `json:"build_states,omitempty"` |
| 532 | // BuildStatesErr is set when this box could not collect BuildStates at |
| 533 | // all (its registry list failed). The housekeeper prunes nothing while a |
| 534 | // fresh report says so (Astra I2 #5): an absent map is never "nothing to |
| 535 | // protect". |
| 536 | BuildStatesErr string `json:"build_states_err,omitempty"` |
| 537 | // Updates is, per instance this box hosts that has non-default settings |
| 538 | // or a decision of root's, root's last decision (settings spec §4.3). |
| 539 | // Host-keyed and private like RouteVersions: it names hosts and builds, |
| 540 | // so it lives in this block and never reaches the public feed. Omitted |
| 541 | // when empty. |
| 542 | Updates map[string]HostUpdates `json:"updates,omitempty"` |
| 543 | // RouteVersions is host -> registry.Route.Version for every non-platform |
| 544 | // route this box carries with a Version the adopt pass has stamped |
| 545 | // (issue #78's staged_version chip) — a hostname-keyed map, which is |
| 546 | // exactly why it lives here and nowhere else: BuildFleet copies Box's |
| 547 | // fields into FleetBox one at a time by name and never names Metrics, so |
| 548 | // this map can carry real hostnames without the public fleet.json ever |
| 549 | // seeing one. It is the box's OWN routes only (CollectBox is called |
| 550 | // per-box), never a fleet-wide join. Omitted entirely when this box has |
| 551 | // no versioned route, same "absent, not empty" rule as the rest of this |
| 552 | // block. |
| 553 | RouteVersions map[string]string `json:"route_versions,omitempty"` |
| 554 | // RouteServing is host -> the build the agent last started for that |
| 555 | // route (registry ServingVersion, zero-downtime plan 5), for this box's |
| 556 | // own non-platform routes that have one. RouteVersions is root's install |
| 557 | // stamp and can name a build nothing runs yet; this is what runs. Private |
| 558 | // and omitted when empty, for RouteVersions' reasons. |
| 559 | RouteServing map[string]string `json:"route_serving,omitempty"` |
| 560 | // Upstreams is the box's per-upstream probe table, route host -> the |
| 561 | // UpstreamState of each addr behind it (multi-upstream routes only; a |
| 562 | // single-addr route with no health config is never probed and never |
| 563 | // appears). It is RouteVersions' twin in every way that matters here: a |
| 564 | // hostname-keyed map that belongs in this block and nowhere else, carried |
| 565 | // straight through from Probes.Upstreams with no banding, and omitted |
| 566 | // entirely — not present-and-empty — on a box that probes nothing, so |
| 567 | // every consumer written before this field parses such a box unchanged. |
| 568 | // |
| 569 | // Size is bounded by (probed routes × addrs per route) — a handful of |
| 570 | // entries on the busiest box today, and self-limiting: only routes an |
| 571 | // operator gave an addr list AND a health block are in it at all. |
| 572 | Upstreams map[string][]UpstreamState `json:"upstreams,omitempty"` |
| 573 | // Aliases is issue #151's per-hostname DNS/cert status, keyed by the |
| 574 | // alias hostname — the box's own route_aliases rows only (CollectBox is |
| 575 | // called per-box, same scope as RouteVersions above), never a fleet-wide |
| 576 | // join. It lives here for the identical reason RouteVersions does: |
| 577 | // status/boxes/<label>.json is bucket-private (this package's own doc |
| 578 | // comment, and dryrun.go's leak-scan self-check, which guards the PUBLIC |
| 579 | // feed only — fleet.json, index.html, status.css — and deliberately |
| 580 | // EXPECTS raw values inside status/boxes/*.json). BuildFleet copies Box's |
| 581 | // fields into FleetBox one at a time by name and never names Metrics, so |
| 582 | // these hostnames can never reach the public aggregation. Task 9's |
| 583 | // console list door is the intended reader. Omitted entirely (not a |
| 584 | // present-but-empty map) when this box has no alias rows, same |
| 585 | // "absent, not empty" rule as RouteVersions. |
| 586 | Aliases map[string]AliasReading `json:"aliases,omitempty"` |
| 587 | // RouteDBBytes is host -> on-disk size of that route's SQLite database |
| 588 | // (file plus -wal), for this box's own routes with a DBPath. It is |
| 589 | // RouteVersions' twin: hostname-keyed, so it lives in this private block |
| 590 | // and nowhere else. The console's instance and route listings are the |
| 591 | // readers. A host whose file could not be sized is absent, never 0; the |
| 592 | // whole map is omitted when no route could be sized. |
| 593 | RouteDBBytes map[string]int64 `json:"route_db_bytes,omitempty"` |
| 594 | // Layout is Probes.Layout carried straight through — the box's layout |
| 595 | // report, hostname-keyed like RouteVersions and RouteDBBytes above, and |
| 596 | // private for the identical reason: BuildFleet copies Box's fields into |
| 597 | // FleetBox one at a time by name and never names Metrics, so this can |
| 598 | // never reach the public fleet.json. nil (the field absent) means the |
| 599 | // box has no layout pass armed, not "every route reports flat". |
| 600 | Layout *LayoutReport `json:"layout,omitempty"` |
| 601 | // ConfigRefusals is the hosts this box is refusing to take because it |
| 602 | // cannot provide their config (amadan #213). It is how an operator sees |
| 603 | // that a box is keyless, or holds the wrong key, without reading its |
| 604 | // journal: a keyed sibling takes these instances, and this list says why |
| 605 | // this box did not. Hostname-bearing, so it lives in this private block |
| 606 | // and nowhere else (BuildFleet never names Metrics). Omitted when empty. |
| 607 | ConfigRefusals []ConfigRefusal `json:"config_refusals,omitempty"` |
| 608 | // ConfigRefusalsTotal is the count of refused hosts when ConfigRefusals |
| 609 | // was capped, so a long list reads as long. Omitted when zero. |
| 610 | ConfigRefusalsTotal int `json:"config_refusals_total,omitempty"` |
| 611 | // ReleaseRefusals is the builds this box rolled back because their unit |
| 612 | // did not come up (D1), one per host. Hostname-bearing, so private like |
| 613 | // ConfigRefusals. Omitted when empty. |
| 614 | ReleaseRefusals []ReleaseRefusal `json:"release_refusals,omitempty"` |
| 615 | // ProofWaits is the hosts this box is holding on their current build |
| 616 | // until a dev proof for the new head arrives (D3). Hostname-bearing, so |
| 617 | // it lives in this private block only. Omitted when empty. |
| 618 | ProofWaits []ProofWait `json:"proof_waits,omitempty"` |
| 619 | // Rollout is the rollout scheduler's report (zero-downtime spec §8.4, |
| 620 | // §9.1). Hostname-bearing, so it lives in this private block only. |
| 621 | // Omitted when the scheduler is off. |
| 622 | Rollout *RolloutReport `json:"rollout,omitempty"` |
| 623 | // ConfigGuard says the box's admission guard is bypassed, e.g. |
| 624 | // "bypassed (CARLOS_NO_SEALED_CONFIG)": it admits placed exec instances |
| 625 | // without checking their sealed config. Omitted in the normal mode, so an |
| 626 | // operator scanning snapshots sees only the boxes that are exposed. A |
| 627 | // fixed phrase, never a value from the environment. |
| 628 | ConfigGuard string `json:"config_guard,omitempty"` |
| 629 | } |
| 630 | |
| 631 | // UpdateSettingsProtocol is the BoxMetrics.UpdateSettings a binary reports |
| 632 | // once it applies update settings and protects the builds they name. |
| 633 | const UpdateSettingsProtocol = 1 |
| 634 | |
| 635 | // BuildStatePins is one instance's build-state builds in a box report. |
| 636 | // Complete is false when this box could not read the instance's build state |
| 637 | // (never read, a failed refresh, an unparseable copy): the housekeeper then |
| 638 | // prunes nothing for the app (Astra I1 #6). |
| 639 | type BuildStatePins struct { |
| 640 | Account string `json:"account"` |
| 641 | App string `json:"app"` |
| 642 | Builds []string `json:"builds"` |
| 643 | Complete bool `json:"complete"` |
| 644 | } |
| 645 | |
| 646 | // AliasReading is one alias hostname's status as CollectBox saw it this |
| 647 | // tick: Route is the route it points at (registry.RouteAlias.Host), DNSStatus |
| 648 | // and CertExpiresAt are the box-stamped health columns (Task 2/5), carried |
| 649 | // through unbanded like the rest of BoxMetrics. Route has no omitempty — an |
| 650 | // alias with no route would be a data bug worth seeing, not a field worth |
| 651 | // hiding; DNSStatus/CertExpiresAt omit when unstamped ("" / 0), same |
| 652 | // unknown-stays-absent convention as every other Metrics field. |
| 653 | type AliasReading struct { |
| 654 | Route string `json:"route"` |
| 655 | DNSStatus string `json:"dns_status,omitempty"` |
| 656 | // DNSCheckedAt is when this box last RESOLVED the hostname (registry |
| 657 | // dns_checked_at). Carried because a stamp without its age cannot be |
| 658 | // judged: a box that has stopped stamping keeps publishing fresh |
| 659 | // SNAPSHOTS full of frozen readings, and BoxStale — which measures the |
| 660 | // snapshot, not the stamp — cannot see it. Omitempty: unstamped stays |
| 661 | // absent, never a zero that reads as 1970. |
| 662 | DNSCheckedAt int64 `json:"dns_checked_at,omitempty"` |
| 663 | CertExpiresAt int64 `json:"cert_expires_at,omitempty"` |
| 664 | } |
| 665 | |
| 666 | // AlertDelivery is the box's own answer to "if a rule fires, does anyone |
| 667 | // hear it": which sinks the agent has, and how many sends to them have |
| 668 | // failed. It exists because an unset CARLOS_ALERT_WEBHOOK used to be |
| 669 | // silent, and the flagship's disk, memory and cpu alerts went nowhere for |
| 670 | // weeks (discussion #192). Webhook is CARLOS_ALERT_WEBHOOK set. SNS is |
| 671 | // CARLOS_ALERT_SNS_TOPIC_ARN set to a topic ARN the agent can use; a |
| 672 | // malformed value reads false, because it is not a sink. SNS is the state |
| 673 | // of the sink the agent actually built, not of the env: when a topic ARN is |
| 674 | // set but the agent could not build its SNS client (no region, a missing |
| 675 | // profile), SNS is false and SNSUnavailable is true, so the box never reads |
| 676 | // as having a sink it cannot use. Failures count sends (and that failed |
| 677 | // build) since the agent process started, so a role without sns:Publish |
| 678 | // shows up here and not only in the journal. |
| 679 | type AlertDelivery struct { |
| 680 | Webhook bool `json:"webhook"` |
| 681 | SNS bool `json:"sns"` |
| 682 | SNSUnavailable bool `json:"sns_unavailable,omitempty"` |
| 683 | WebhookFailures int64 `json:"webhook_failures,omitempty"` |
| 684 | SNSFailures int64 `json:"sns_failures,omitempty"` |
| 685 | } |
| 686 | |
| 687 | // Delivered is whether at least one sink is configured. |
| 688 | func (a AlertDelivery) Delivered() bool { return a.Webhook || a.SNS } |
| 689 | |
| 690 | // RouteCounts is aggregate-only: no host, no socket path, just how many. |
| 691 | // Awake/Asleep stay null together whenever the socket table can't be read in |
| 692 | // full — partial knowledge published as a count would read as more certain |
| 693 | // than it is. |
| 694 | type RouteCounts struct { |
| 695 | Static int `json:"static"` |
| 696 | Resident int `json:"resident"` |
| 697 | Hibernating int `json:"hibernating"` |
| 698 | Awake *int `json:"awake"` // null when the socket table is unreadable |
| 699 | Asleep *int `json:"asleep"` // null when the socket table is unreadable |
| 700 | } |
| 701 | |
| 702 | // Replication summarizes litestream's furthest-behind lag across this box's |
| 703 | // routes, banded to a healthy bool rather than the raw seconds — the raw |
| 704 | // number is still carried for the console's private view, but the public |
| 705 | // commitment is only ever "healthy" or "not". |
| 706 | type Replication struct { |
| 707 | Healthy *bool `json:"healthy"` |
| 708 | LagSecs *int64 `json:"lag_seconds"` |
| 709 | } |
| 710 | |
| 711 | // Box is one box's self-report: what CollectBox produces and WriteBox |
| 712 | // serializes. Every field here has already had its needles pulled — there is |
| 713 | // no hostname, machine id, or raw ratio anywhere in this type — with two |
| 714 | // deliberate exceptions: Metrics, the raw/private block documented on the |
| 715 | // package comment and on BoxMetrics itself, and Sidecars, an aggregate count |
| 716 | // that is private for a different reason (a sidecar's existence is |
| 717 | // account-visible, not public, not because the number itself is sensitive). |
| 718 | // Every OTHER field on Box is still bound by the no-raw-numbers rule and IS |
| 719 | // meant for the public page; Metrics and Sidecars are the two places that |
| 720 | // default is intentionally suspended, gated by staying bucket-private (see |
| 721 | // the package doc) and excluded from FleetBox by construction (fleet.go's |
| 722 | // BuildFleet copies fields one at a time and never names either). |
| 723 | type Box struct { |
| 724 | GeneratedAt time.Time `json:"generated_at"` |
| 725 | Label string `json:"label"` |
| 726 | Region string `json:"region,omitempty"` |
| 727 | Routes RouteCounts `json:"routes"` |
| 728 | Load *string `json:"load"` // comfortable|busy|critical |
| 729 | Capacity *string `json:"capacity"` // plenty|getting-full|at-capacity |
| 730 | Replication Replication `json:"replication"` |
| 731 | // AdoptionRunning is whether this box is actually picking up releases: |
| 732 | // true = the adoption timer is armed and will fire, false = it is not and |
| 733 | // nothing on this box will ever adopt on its own, null = unknown. It is a |
| 734 | // bare bool rather than a band word because there is no spectrum here and |
| 735 | // nothing private in the answer — a box either runs the platform's own |
| 736 | // adoption pass or it doesn't, which says nothing about any tenant. The |
| 737 | // flagship spent weeks in the false state with no way to see it; this |
| 738 | // field is the way to see it. |
| 739 | AdoptionRunning *bool `json:"adoption_running"` |
| 740 | // ConfigPathRunning is whether config reaches this box's UNIT-BACKED |
| 741 | // routes in seconds (carlos-adopt.path armed) or only on the hourly |
| 742 | // carlos-adopt.timer floor: true = armed, false = the trigger is |
| 743 | // installed but not watching, null = unknown. Published for the same |
| 744 | // reason as AdoptionRunning above and with the same content — the |
| 745 | // liveness of one of the platform's OWN units on an opaquely-labelled |
| 746 | // box, which says nothing about any tenant. Snapshots written before |
| 747 | // issue #44 carry no such key at all and decode to null, which is the |
| 748 | // honest reading of a box that never reported it. |
| 749 | ConfigPathRunning *bool `json:"config_path_running"` |
| 750 | // RestartPathRunning is whether a member's `carlos restart` reaches this |
| 751 | // box's units in seconds (carlos-restart.path armed) or only on the hourly |
| 752 | // floor: true = armed, false = installed but not watching, null = unknown. |
| 753 | // Same content and same privacy argument as ConfigPathRunning above — the |
| 754 | // liveness of one of the platform's own units on an opaquely-labelled box. |
| 755 | // Snapshots written before issue #113 carry no such key and decode to null, |
| 756 | // which is the honest reading of a box that never reported it. |
| 757 | RestartPathRunning *bool `json:"restart_path_running"` |
| 758 | // ExpiredCanaries publishes Probes.ExpiredCanaries straight through — a |
| 759 | // count, never hostnames (pipelines spec §5): which canary route expired |
| 760 | // is exactly the kind of identifying detail this package's whole design |
| 761 | // refuses to carry, but how many is aggregate and safe. |
| 762 | ExpiredCanaries *int `json:"expired_canaries"` |
| 763 | // Metrics is the raw/private exception documented above and on |
| 764 | // BoxMetrics — omitted entirely (not a present-but-empty object) when |
| 765 | // CollectBox has nothing to report, so an old parser reading a box this |
| 766 | // package wrote before Metrics existed, and a box that genuinely can't |
| 767 | // answer any of the six fields, produce the identical absent-key shape. |
| 768 | Metrics *BoxMetrics `json:"metrics,omitempty"` |
| 769 | // Sidecars is how many of this box's routes are standalone workers |
| 770 | // (registry.Route.IsSidecar) — counted by CollectBox in their own |
| 771 | // branch, never as Resident, Static or Hibernating (its Hibernate comes |
| 772 | // from the app manifest, so a sidecar may legitimately be either, and |
| 773 | // either way it does not belong in the public instance count). Present |
| 774 | // in the private per-box snapshot (bucket-private, read only by |
| 775 | // fleet-status machinery and credentialed operators) but — like |
| 776 | // Metrics — deliberately NEVER copied into FleetBox: a sidecar's |
| 777 | // existence is account-visible, not public, so even the aggregate count |
| 778 | // must not reach fleet.json. omitempty keeps a box with no sidecars |
| 779 | // byte-identical to one written before this field existed. |
| 780 | Sidecars int `json:"sidecars,omitempty"` |
| 781 | } |
| 782 | |
| 783 | // replHealthyWithin is the lag under which replication counts as healthy. |
| 784 | // 15 minutes: litestream's own replicate interval is seconds, so anything |
| 785 | // beyond a quarter hour behind means the replica loop has actually stalled |
| 786 | // (network partition, disk full, credential rot) rather than just being |
| 787 | // mid-batch — a threshold loose enough not to flap on ordinary jitter, tight |
| 788 | // enough to catch a real outage inside one on-call check-in. |
| 789 | const replHealthyWithin = 15 * time.Minute |
| 790 | |
| 791 | // loadWords and capacityWords band a fraction into a public word — never a |
| 792 | // number, per the fleet status page's public commitments (see the plan's |
| 793 | // privacy rules). The three words are ordered low, medium, high. |
| 794 | var loadWords = [3]string{"comfortable", "busy", "critical"} |
| 795 | var capacityWords = [3]string{"plenty", "getting-full", "at-capacity"} |
| 796 | |
| 797 | // bandBoundaryLow and bandBoundaryHigh are the two cut points shared by every |
| 798 | // banded fraction: below the low boundary is the first word, at-or-below the |
| 799 | // high boundary is the second, above it the third. Named constants so the |
| 800 | // 0.70/0.90 split — a decision the operator made once — isn't a magic number |
| 801 | // re-typed at each call site. |
| 802 | const ( |
| 803 | bandBoundaryLow = 0.70 |
| 804 | bandBoundaryHigh = 0.90 |
| 805 | ) |
| 806 | |
| 807 | // bandWord turns a possibly-unknown fraction into one of three words. nil in |
| 808 | // (unknown) means nil out — CollectBox must never invent a band for a |
| 809 | // quantity it never measured. |
| 810 | func bandWord(f *float64, words [3]string) *string { |
| 811 | if f == nil { |
| 812 | return nil |
| 813 | } |
| 814 | var w string |
| 815 | switch { |
| 816 | case *f < bandBoundaryLow: |
| 817 | w = words[0] |
| 818 | case *f <= bandBoundaryHigh: |
| 819 | w = words[1] |
| 820 | default: |
| 821 | w = words[2] |
| 822 | } |
| 823 | return &w |
| 824 | } |
| 825 | |
| 826 | // labelNames and labelActions are the two word lists Label indexes into — |
| 827 | // Eleven's "ada-stargazing" shape: a first name and a gerund action, picked |
| 828 | // to read as pleasant and give away nothing about the machine they name. |
| 829 | // Deliberately small const slices (not generated, not fetched) so the label |
| 830 | // space and its stability are auditable at a glance. |
| 831 | var labelNames = [40]string{ |
| 832 | "ada", "grace", "alan", "linus", "margaret", "dennis", "barbara", "ken", |
| 833 | "radia", "vint", "tim", "hedy", "kathleen", "edsger", "donald", "john", |
| 834 | "claude", "marvin", "shafi", "leslie", "whitfield", "adi", "ron", "sophie", |
| 835 | "joan", "frances", "betty", "jean", "mary", "annie", "katherine", "dorothy", |
| 836 | "nadia", "elena", "priya", "yuki", "noor", "amara", "leo", "iris", |
| 837 | } |
| 838 | var labelActions = [40]string{ |
| 839 | "stargazing", "birdwatching", "gardening", "sketching", "kayaking", |
| 840 | "orienteering", "foraging", "woodworking", "beekeeping", "cycling", |
| 841 | "origami", "brewing", "quilting", "hiking", "juggling", "puzzling", |
| 842 | "weaving", "sailing", "climbing", "cartography", "fermenting", "welding", |
| 843 | "knitting", "pottery", "birding", "spelunking", "kitesurfing", "canoeing", |
| 844 | "tinkering", "letterpress", "printmaking", "bookbinding", "beachcombing", |
| 845 | "fossicking", "aquascaping", "papermaking", "clockmaking", "lockpicking", |
| 846 | "stenciling", "marquetry", |
| 847 | } |
| 848 | |
| 849 | // Label derives a stable, opaque "name-gerund" label from a machine id — the |
| 850 | // same shape as Eleven's ada-stargazing box names. sha256 rather than a |
| 851 | // weaker hash: the machine id must never be recoverable from the label by |
| 852 | // brute force over any plausible id space (hostnames, instance ids), and the |
| 853 | // label is public. |
| 854 | // |
| 855 | // Each index is drawn from a 4-byte (32-bit) chunk of the digest rather than |
| 856 | // a single byte: 256 % 40 == 16, so indexing with one raw byte would leave |
| 857 | // names 0-15 roughly 1.4x likelier than names 16-39. A uint32 % 40 has a bias |
| 858 | // under 1 in 100 million — small enough to treat the label as uniform. |
| 859 | func Label(machineID string) string { |
| 860 | sum := sha256.Sum256([]byte(machineID)) |
| 861 | nameIdx := binary.BigEndian.Uint32(sum[0:4]) % uint32(len(labelNames)) |
| 862 | actionIdx := binary.BigEndian.Uint32(sum[4:8]) % uint32(len(labelActions)) |
| 863 | return labelNames[nameIdx] + "-" + labelActions[actionIdx] |
| 864 | } |
| 865 | |
| 866 | // CollectBox turns a box's live registry rows and probe results into the |
| 867 | // public Box for that box. It is the ONLY place fractions get banded to |
| 868 | // words, sockets get counted instead of named, and the machine id gets |
| 869 | // hashed instead of shipped — every value this function returns is already |
| 870 | // safe to publish as-is. |
| 871 | func CollectBox(routes []registry.Route, p Probes, now time.Time) Box { |
| 872 | var counts RouteCounts |
| 873 | var awakeKnown = true |
| 874 | awake, asleep := 0, 0 |
| 875 | // sidecars counts standalone workers (registry.Route.IsSidecar) apart |
| 876 | // from RouteCounts entirely — see Box.Sidecars' doc for why it is never |
| 877 | // public. |
| 878 | var sidecars int |
| 879 | // routeVersions is issue #78's staged_version chip: host -> Version for |
| 880 | // this box's own non-platform routes that have one stamped. Built |
| 881 | // alongside the counts above rather than in a second pass over routes, |
| 882 | // and nil (not an empty map) until the first entry, so an all-unversioned |
| 883 | // box's Metrics attachment check below sees exactly "nothing to report" |
| 884 | // rather than a present-but-empty map. |
| 885 | var routeVersions map[string]string |
| 886 | // routeServing is plan 5's ServingVersion: host -> the build the agent |
| 887 | // last started, for this box's own non-platform routes that have one. |
| 888 | // Same nil-until-first-entry convention as routeVersions. |
| 889 | var routeServing map[string]string |
| 890 | // aliasReadings is issue #151's Metrics.Aliases source. Declared here, |
| 891 | // above the route loop, so the route loop below (a first-route claim's |
| 892 | // ROUTE readings) and the p.Aliases loop after it (genuine alias rows) |
| 893 | // both fold into the same map — nil (not an empty map) until the first |
| 894 | // entry, same "nothing to report" convention as routeVersions. |
| 895 | var aliasReadings map[string]AliasReading |
| 896 | // routeDBBytes is Metrics.RouteDBBytes, built in the same loop and nil |
| 897 | // until the first entry, like routeVersions. |
| 898 | var routeDBBytes map[string]int64 |
| 899 | |
| 900 | for _, r := range routes { |
| 901 | // Platform-minted routes (internal/domains) are additional NAMES for |
| 902 | // routes already counted below, not additional instances — counting |
| 903 | // both would roughly double every fleet-wide number the moment a |
| 904 | // deployment sets PLATFORM_APPS_DOMAIN, publishing a fleet larger |
| 905 | // than the one that exists. Same reasoning as the opt-in and |
| 906 | // null-propagation rules: the aggregate never overstates. It is also |
| 907 | // why routeVersions skips them here: a platform alias's Version is |
| 908 | // the same route's, under a second name, not a second instance's. |
| 909 | if r.Platform { |
| 910 | continue |
| 911 | } |
| 912 | if r.Version != "" { |
| 913 | if routeVersions == nil { |
| 914 | routeVersions = map[string]string{} |
| 915 | } |
| 916 | routeVersions[r.Host] = r.Version |
| 917 | } |
| 918 | if r.ServingVersion != "" { |
| 919 | if routeServing == nil { |
| 920 | routeServing = map[string]string{} |
| 921 | } |
| 922 | routeServing[r.Host] = r.ServingVersion |
| 923 | } |
| 924 | if r.Kind != "static" && r.DBPath != "" && p.DBBytes != nil { |
| 925 | if n := p.DBBytes(r.DBPath); n != nil { |
| 926 | if routeDBBytes == nil { |
| 927 | routeDBBytes = map[string]int64{} |
| 928 | } |
| 929 | routeDBBytes[r.Host] = *n |
| 930 | } |
| 931 | } |
| 932 | // Non-platform routes carry hostname readings too. A first-route |
| 933 | // claim (claims.go: "an empty route means this host is its own |
| 934 | // primary") materializes a ROUTE, not an alias row, so without this |
| 935 | // the domains surface can never take it past "pending" — which is |
| 936 | // exactly what a static app deployed to its own custom domain has |
| 937 | // always shown. Route is the host itself, which is what the claim's |
| 938 | // empty Route field means. |
| 939 | if r.DNSStatus != "" || r.CertExpiresAt != 0 { |
| 940 | if aliasReadings == nil { |
| 941 | aliasReadings = map[string]AliasReading{} |
| 942 | } |
| 943 | aliasReadings[r.Host] = AliasReading{ |
| 944 | Route: r.Host, DNSStatus: r.DNSStatus, |
| 945 | DNSCheckedAt: r.DNSCheckedAt, CertExpiresAt: r.CertExpiresAt, |
| 946 | } |
| 947 | } |
| 948 | switch { |
| 949 | case r.IsSidecar(): |
| 950 | // Checked first, ahead of Hibernate: a sidecar's Hibernate comes |
| 951 | // from the app manifest and may legitimately be true, but |
| 952 | // either way it is not the public Hibernating count — its own |
| 953 | // Socket is empty by construction (no upstream to dial), so the |
| 954 | // awake/asleep probe below would be meaningless for it too. |
| 955 | sidecars++ |
| 956 | case r.Kind == "static": |
| 957 | counts.Static++ |
| 958 | case r.Hibernate: |
| 959 | counts.Hibernating++ |
| 960 | if p.Listening == nil { |
| 961 | awakeKnown = false |
| 962 | continue |
| 963 | } |
| 964 | isListening := p.Listening(r.Socket) |
| 965 | if isListening == nil { |
| 966 | awakeKnown = false |
| 967 | continue |
| 968 | } |
| 969 | if *isListening { |
| 970 | awake++ |
| 971 | } else { |
| 972 | asleep++ |
| 973 | } |
| 974 | default: |
| 975 | counts.Resident++ |
| 976 | } |
| 977 | } |
| 978 | // p.Aliases (genuine route_aliases rows) runs AFTER the route loop above, |
| 979 | // so a real alias wins its own key over the route-loop's fallback entry |
| 980 | // for the same host. |
| 981 | for _, a := range p.Aliases { |
| 982 | if aliasReadings == nil { |
| 983 | aliasReadings = map[string]AliasReading{} |
| 984 | } |
| 985 | aliasReadings[a.Alias] = AliasReading{ |
| 986 | Route: a.Host, DNSStatus: a.DNSStatus, |
| 987 | DNSCheckedAt: a.DNSCheckedAt, CertExpiresAt: a.CertExpiresAt, |
| 988 | } |
| 989 | } |
| 990 | |
| 991 | if awakeKnown && p.Listening != nil { |
| 992 | counts.Awake, counts.Asleep = &awake, &asleep |
| 993 | } |
| 994 | |
| 995 | // Capacity is the worst-of DiskFrac/MemFrac — whichever known fraction is |
| 996 | // closer to full drives the word, because a box that's fine on memory but |
| 997 | // out of disk is still a box about to have a bad day. |
| 998 | var worst *float64 |
| 999 | switch { |
| 1000 | case p.DiskFrac != nil && p.MemFrac != nil: |
| 1001 | if *p.DiskFrac >= *p.MemFrac { |
| 1002 | worst = p.DiskFrac |
| 1003 | } else { |
| 1004 | worst = p.MemFrac |
| 1005 | } |
| 1006 | case p.DiskFrac != nil: |
| 1007 | worst = p.DiskFrac |
| 1008 | case p.MemFrac != nil: |
| 1009 | worst = p.MemFrac |
| 1010 | } |
| 1011 | |
| 1012 | var healthy *bool |
| 1013 | if p.LagSecs != nil { |
| 1014 | h := *p.LagSecs < int64(replHealthyWithin.Seconds()) |
| 1015 | healthy = &h |
| 1016 | } |
| 1017 | |
| 1018 | box := Box{ |
| 1019 | GeneratedAt: now, |
| 1020 | Label: Label(p.MachineID), |
| 1021 | Region: p.Region, |
| 1022 | Routes: counts, |
| 1023 | Load: bandWord(p.MemFrac, loadWords), |
| 1024 | Capacity: bandWord(worst, capacityWords), |
| 1025 | Replication: Replication{Healthy: healthy, LagSecs: p.LagSecs}, |
| 1026 | AdoptionRunning: p.AdoptRunning, |
| 1027 | ConfigPathRunning: p.ConfigPathRunning, |
| 1028 | RestartPathRunning: p.RestartPathRunning, |
| 1029 | ExpiredCanaries: p.ExpiredCanaries, |
| 1030 | Sidecars: sidecars, |
| 1031 | } |
| 1032 | |
| 1033 | // Metrics attaches only when something is known — an all-unknown Probes, |
| 1034 | // no versioned route, no alias reading AND nothing probed publishes no |
| 1035 | // block at all, the same "unknown stays absent, never a guess" rule every |
| 1036 | // other field on Box follows, and it keeps a box that can't answer any of |
| 1037 | // the eight fields indistinguishable on the wire from a box running code |
| 1038 | // that predates Metrics entirely. An EMPTY (non-nil) probe or alias table |
| 1039 | // counts as nothing known for exactly the same reason an empty |
| 1040 | // routeVersions map does: a box with no multi-upstream route (or no |
| 1041 | // alias) has nothing to say, not something empty to say. |
| 1042 | if p.MemTotalBytes != nil || p.MemAvailBytes != nil || p.Load1 != nil || |
| 1043 | p.DiskFreeBytes != nil || p.Release != "" || p.Tunnel != "" || |
| 1044 | p.InstanceID != "" || p.Platform != "" || p.SystemChannel != "" || p.System != nil || |
| 1045 | len(routeVersions) > 0 || len(routeServing) > 0 || len(aliasReadings) > 0 || len(p.Upstreams) > 0 || |
| 1046 | len(routeDBBytes) > 0 || p.Layout != nil || len(p.ConfigRefusals) > 0 || len(p.ReleaseRefusals) > 0 || len(p.ProofWaits) > 0 || p.Rollout != nil || p.ConfigGuard != "" || |
| 1047 | p.Alerts != nil || p.GeoIP != nil || len(p.BuildStates) > 0 || p.BuildStatesErr != "" || |
| 1048 | p.UpdateSettings != 0 || len(p.Updates) > 0 { |
| 1049 | box.Metrics = &BoxMetrics{ |
| 1050 | ConsoleProcesses: p.ConsoleProcesses, |
| 1051 | ActivationID: p.ActivationID, ActivationComplete: p.ActivationComplete, |
| 1052 | Enrollment: p.Enrollment, |
| 1053 | MovesProtocol: p.MovesProtocol, |
| 1054 | ReleaseTag: p.ReleaseTag, |
| 1055 | BinarySHA256: p.BinarySHA256, |
| 1056 | BootID: p.BootID, |
| 1057 | FloorSHA256: p.FloorSHA256, |
| 1058 | FloorGuardReady: p.FloorGuardReady, |
| 1059 | HostClaimEnabled: p.HostClaimEnabled, |
| 1060 | RetirementReceiptSHA256: p.RetirementReceiptSHA256, |
| 1061 | |
| 1062 | MemTotalBytes: p.MemTotalBytes, MemAvailBytes: p.MemAvailBytes, |
| 1063 | Load1: p.Load1, DiskFreeBytes: p.DiskFreeBytes, |
| 1064 | Release: p.Release, Tunnel: p.Tunnel, GeoIP: p.GeoIP, |
| 1065 | Platform: p.Platform, SystemChannel: p.SystemChannel, System: p.System, |
| 1066 | InstanceID: p.InstanceID, |
| 1067 | Alerts: p.Alerts, |
| 1068 | RouteVersions: routeVersions, |
| 1069 | RouteServing: routeServing, |
| 1070 | Aliases: aliasReadings, |
| 1071 | // Carried through as the prober handed it over: proberSet.State |
| 1072 | // already returns a fresh copy per call, so there is nothing here |
| 1073 | // for a later probe tick to mutate under the marshaller. An empty |
| 1074 | // map (rather than nil) reaching this line can only happen |
| 1075 | // alongside another known field above, and omitempty drops it. |
| 1076 | Upstreams: p.Upstreams, |
| 1077 | RouteDBBytes: routeDBBytes, |
| 1078 | Layout: p.Layout, |
| 1079 | // A fresh copy: the guard's own slice is its to mutate. |
| 1080 | ConfigRefusals: append([]ConfigRefusal(nil), p.ConfigRefusals...), |
| 1081 | ConfigRefusalsTotal: p.ConfigRefusalsTotal, |
| 1082 | ReleaseRefusals: append([]ReleaseRefusal(nil), p.ReleaseRefusals...), |
| 1083 | ProofWaits: append([]ProofWait(nil), p.ProofWaits...), |
| 1084 | Rollout: p.Rollout, |
| 1085 | ConfigGuard: p.ConfigGuard, |
| 1086 | // The caller builds a fresh map every tick. |
| 1087 | BuildStates: p.BuildStates, |
| 1088 | BuildStatesErr: p.BuildStatesErr, |
| 1089 | UpdateSettings: p.UpdateSettings, |
| 1090 | Updates: p.Updates, |
| 1091 | } |
| 1092 | } |
| 1093 | |
| 1094 | return box |
| 1095 | } |
| 1096 | |
| 1097 | // WriteBox marshals a Box indented (human-diffable in the bucket) and writes |
| 1098 | // it to its label-derived key, last-writer-wins. Each box republishes its own |
| 1099 | // snapshot every 5 minutes to the SAME key, so a bare create-only Put (ifMatch |
| 1100 | // "") would fail every tick after the first with ErrCASConflict — that's the |
| 1101 | // bug this retry loop exists to avoid. The read-etag-then-Put dance is the |
| 1102 | // only way to express "replace whatever is there, or create it" on top of |
| 1103 | // release.Store's CAS-only Put: Get the current etag (a not-found just means |
| 1104 | // "absent", so ifMatch stays "" for a create), Put with that etag, and if the |
| 1105 | // Put itself loses the race (something else wrote in between), re-read once |
| 1106 | // and retry with the fresh etag. A second conflict is NOT retried again and |
| 1107 | // is returned to the caller: this box is the only writer that should ever be |
| 1108 | // targeting its own label's key, so two conflicts in a row means something |
| 1109 | // else collided on this label (clock-skewed duplicate machine id, a bug |
| 1110 | // elsewhere) — that deserves a loud failure, not a silent third attempt. |
| 1111 | func WriteBox(ctx context.Context, s release.Store, b Box) error { |
| 1112 | body, err := json.MarshalIndent(b, "", " ") |
| 1113 | if err != nil { |
| 1114 | return err |
| 1115 | } |
| 1116 | key := "status/boxes/" + b.Label + ".json" |
| 1117 | |
| 1118 | etag, err := currentEtag(ctx, s, key) |
| 1119 | if err != nil { |
| 1120 | return err |
| 1121 | } |
| 1122 | if _, err := s.Put(ctx, key, body, etag); err != nil { |
| 1123 | if !errors.Is(err, release.ErrCASConflict) { |
| 1124 | return err |
| 1125 | } |
| 1126 | etag, err = currentEtag(ctx, s, key) |
| 1127 | if err != nil { |
| 1128 | return err |
| 1129 | } |
| 1130 | if _, err := s.Put(ctx, key, body, etag); err != nil { |
| 1131 | return err |
| 1132 | } |
| 1133 | } |
| 1134 | return nil |
| 1135 | } |
| 1136 | |
| 1137 | // currentEtag reads a key's etag for a replace-only Put, folding "doesn't |
| 1138 | // exist yet" into "" (create-only) — the same absent-means-empty-ifMatch |
| 1139 | // convention release.Store's own CAS contract uses. |
| 1140 | func currentEtag(ctx context.Context, s release.Store, key string) (string, error) { |
| 1141 | _, etag, err := s.Get(ctx, key) |
| 1142 | if err != nil { |
| 1143 | if errors.Is(err, os.ErrNotExist) { |
| 1144 | return "", nil |
| 1145 | } |
| 1146 | return "", err |
| 1147 | } |
| 1148 | return etag, nil |
| 1149 | } |
| 1150 | |
| 1151 | // GeoIPState is one box's GeoIP mirror (geoip spec §5.5). |
| 1152 | type GeoIPState struct { |
| 1153 | Mode string `json:"mode"` // mirror | tunnel | none |
| 1154 | SHA256 string `json:"sha256,omitempty"` |
| 1155 | Type string `json:"type,omitempty"` |
| 1156 | Error string `json:"error,omitempty"` |
| 1157 | } |
| 1158 | |
amadan