| 1 | // The box-side half of the fleet status page's Probes contract |
| 2 | // (internal/status.Probes): every function here reads local kernel/OS state |
| 3 | // and MUST fail toward nil, never a guessed zero — status.CollectBox already |
| 4 | // treats a nil probe as "unknown, publish null" everywhere, so a probe that |
| 5 | // invents a number on failure would silently turn an honest unknown into a |
| 6 | // false reading on the public page. Deliberately small, single-purpose |
| 7 | // functions (rather than one combined "gather everything" call) so |
| 8 | // host_test.go's statusTick tests never have to touch /proc or a live |
| 9 | // syscall — they inject a status.Probes fixture directly and never call |
| 10 | // anything in this file. |
| 11 | package host |
| 12 | |
| 13 | import ( |
| 14 | "bufio" |
| 15 | "io" |
| 16 | "os" |
| 17 | "os/exec" |
| 18 | "strconv" |
| 19 | "strings" |
| 20 | "sync" |
| 21 | |
| 22 | "github.com/carlosframework/platform/internal/status" |
| 23 | |
| 24 | "github.com/carlosframework/platform/internal/registry" |
| 25 | ) |
| 26 | |
| 27 | // adoptTimerUnit is the unit whose liveness IS "this box adopts releases on |
| 28 | // its own" — the same name bootstrap installs and arms (see bootstrap.go's |
| 29 | // enabledUnits). |
| 30 | const adoptTimerUnit = "carlos-adopt.timer" |
| 31 | |
| 32 | // adoptTimerActive asks systemd whether the adoption timer is ACTIVE, not |
| 33 | // merely enabled. The distinction is the whole point: `systemctl enable` |
| 34 | // writes a .wants symlink and nothing more, so a timer can sit enabled and |
| 35 | // never armed until the box next reboots — which is exactly how the flagship |
| 36 | // ran for weeks without a single automatic adoption. `is-active` is the only |
| 37 | // question that distinguishes the two. |
| 38 | // |
| 39 | // The exit status is deliberately ignored in favour of the printed word: |
| 40 | // is-active exits non-zero for every not-active state while still printing |
| 41 | // which state it is, so the error would tell us less than the output already |
| 42 | // does. A systemctl that couldn't run at all prints nothing, and nothing maps |
| 43 | // to nil. |
| 44 | func adoptTimerActive() *bool { |
| 45 | out, _ := exec.Command("systemctl", "is-active", adoptTimerUnit).Output() |
| 46 | return parseIsActive(string(out)) |
| 47 | } |
| 48 | |
| 49 | // configPathUnit is the unit whose liveness IS "config reaches this box's |
| 50 | // unit-backed routes in seconds" (issue #44): the path unit watching |
| 51 | // /data/config-sync-requested, which the agent touches and which starts |
| 52 | // carlos-config-sync.service. Installed and armed by bootstrap alongside the |
| 53 | // timer (see enabledUnits). |
| 54 | const configPathUnit = "carlos-adopt.path" |
| 55 | |
| 56 | // configPathActive asks the same question adoptTimerActive asks, about the |
| 57 | // unit with the same failure mode: `systemctl enable` on a path unit writes a |
| 58 | // .wants symlink and watches nothing until the box next reboots. An unarmed |
| 59 | // path unit is invisible from inside the agent — it writes its marker file |
| 60 | // happily, and nothing ever reads it — so this probe is the only thing that |
| 61 | // can tell an operator the box quietly fell back to the hourly config floor. |
| 62 | func configPathActive() *bool { |
| 63 | out, _ := exec.Command("systemctl", "is-active", configPathUnit).Output() |
| 64 | return parseIsActive(string(out)) |
| 65 | } |
| 66 | |
| 67 | // restartPathUnit is the unit whose liveness IS "a member's `carlos restart` |
| 68 | // reaches this box's units in seconds" (issue #113): the path unit watching |
| 69 | // /data/restart-requested, which the agent touches and which starts |
| 70 | // carlos-restart.service. Installed and armed by bootstrap alongside the other |
| 71 | // two (see enabledUnits). |
| 72 | const restartPathUnit = "carlos-restart.path" |
| 73 | |
| 74 | // restartPathActive asks the same question configPathActive asks, about a unit |
| 75 | // with the same silent failure mode — and one worse consequence. An unarmed |
| 76 | // config trigger costs an env var an hour; an unarmed restart trigger means |
| 77 | // someone typed `carlos restart` at a wedged instance, was told "requested", |
| 78 | // and will stand there watching a process that will not cycle until the hourly |
| 79 | // pass comes round. Nothing inside the agent can see the difference: it writes |
| 80 | // its marker either way. |
| 81 | func restartPathActive() *bool { |
| 82 | out, _ := exec.Command("systemctl", "is-active", restartPathUnit).Output() |
| 83 | return parseIsActive(string(out)) |
| 84 | } |
| 85 | |
| 86 | // systemUpdatePathUnit is the unit whose liveness IS "an agent's fetched |
| 87 | // platform candidate reaches a root apply" (spec §4): the path unit watching |
| 88 | // /data/system-update-requested. Installed and armed by bootstrap alongside |
| 89 | // the other path units (see enabledUnits). |
| 90 | const systemUpdatePathUnit = "carlos-system-update.path" |
| 91 | |
| 92 | // systemUpdatePathActive asks the same question restartPathActive asks, about |
| 93 | // the self-update relay: an unarmed path here means an agent that fetched and |
| 94 | // verified a new build touches its marker and nothing ever picks it up. |
| 95 | func systemUpdatePathActive() *bool { |
| 96 | out, _ := exec.Command("systemctl", "is-active", systemUpdatePathUnit).Output() |
| 97 | return parseIsActive(string(out)) |
| 98 | } |
| 99 | |
| 100 | // parseIsActive maps systemd's is-active vocabulary onto the tri-state the |
| 101 | // status page publishes. "unknown" — systemd's answer when it has no such unit |
| 102 | // loaded — is a genuine false: a box with no adoption timer adopts nothing. |
| 103 | // Anything this function does not recognise (empty output because systemctl |
| 104 | // isn't there, a word a future systemd invents) is nil, because the one answer |
| 105 | // this probe must never invent is a reassuring one. |
| 106 | func parseIsActive(out string) *bool { |
| 107 | yes, no := true, false |
| 108 | switch strings.TrimSpace(out) { |
| 109 | case "active": |
| 110 | return &yes |
| 111 | case "inactive", "failed", "deactivating", "unknown": |
| 112 | return &no |
| 113 | } |
| 114 | return nil |
| 115 | } |
| 116 | |
| 117 | // cachedInstanceID reads this box's cloud machine id once and remembers it |
| 118 | // — the id cannot change under a running process, and the status loop asks |
| 119 | // every five minutes. |
| 120 | // |
| 121 | // Only a SUCCESSFUL read is cached. An empty answer is retried next tick, |
| 122 | // because the two reasons for one are not alike: off-cloud it stays empty |
| 123 | // forever and costs a couple of unroutable milliseconds a tick, while on a |
| 124 | // box whose metadata service was briefly unreachable at boot, caching the |
| 125 | // empty would cost the console's box-to-machine join for the whole life of |
| 126 | // the agent — cured only by a restart nobody would know to perform. |
| 127 | // |
| 128 | // This is the one probe in this file that reaches off the box, and it obeys |
| 129 | // the same rule as the rest: it fails toward "", which CollectBox publishes |
| 130 | // as absence, never a guess. |
| 131 | var ( |
| 132 | instanceIDMu sync.Mutex |
| 133 | instanceIDSeen string |
| 134 | ) |
| 135 | |
| 136 | func cachedInstanceID() string { |
| 137 | instanceIDMu.Lock() |
| 138 | defer instanceIDMu.Unlock() |
| 139 | if instanceIDSeen != "" { |
| 140 | return instanceIDSeen |
| 141 | } |
| 142 | instanceIDSeen = status.InstanceID() |
| 143 | return instanceIDSeen |
| 144 | } |
| 145 | |
| 146 | // readMachineID reads /etc/machine-id, trimmed. Unreadable (missing file, |
| 147 | // permission denied — plausible inside some containers) -> "", NOT a |
| 148 | // hash-of-hostname fallback. A hostname-derived id would look exactly like a |
| 149 | // real one to status.Label, silently manufacturing a stable-looking label |
| 150 | // for a box that never actually reported a machine id; "" is honest instead |
| 151 | // — it still hashes to a deterministic Label(""), it just always means "this |
| 152 | // probe never worked," which is the true state of affairs. |
| 153 | func readMachineID() string { |
| 154 | b, err := os.ReadFile("/etc/machine-id") |
| 155 | if err != nil { |
| 156 | return "" |
| 157 | } |
| 158 | return strings.TrimSpace(string(b)) |
| 159 | } |
| 160 | |
| 161 | // readMemFrac reads /proc/meminfo's MemAvailable/MemTotal (both in kB) and |
| 162 | // returns the used fraction (1 - available/total). MemAvailable (not |
| 163 | // MemFree) is the kernel's own "how much could a new process actually get |
| 164 | // without swapping" estimate — it already accounts for reclaimable |
| 165 | // page/slab cache, which MemFree does not, so MemFree would read as |
| 166 | // permanently "critical" on any box with a healthy page cache. Either field |
| 167 | // missing or unparsable -> nil. |
| 168 | func readMemFrac() *float64 { |
| 169 | f, err := os.Open("/proc/meminfo") |
| 170 | if err != nil { |
| 171 | return nil |
| 172 | } |
| 173 | defer f.Close() |
| 174 | |
| 175 | var totalKB, availKB int64 |
| 176 | haveTotal, haveAvail := false, false |
| 177 | scanner := bufio.NewScanner(f) |
| 178 | for scanner.Scan() { |
| 179 | fields := strings.Fields(scanner.Text()) |
| 180 | if len(fields) < 2 { |
| 181 | continue |
| 182 | } |
| 183 | v, err := strconv.ParseInt(fields[1], 10, 64) |
| 184 | if err != nil { |
| 185 | continue |
| 186 | } |
| 187 | switch fields[0] { |
| 188 | case "MemTotal:": |
| 189 | totalKB, haveTotal = v, true |
| 190 | case "MemAvailable:": |
| 191 | availKB, haveAvail = v, true |
| 192 | } |
| 193 | } |
| 194 | if err := scanner.Err(); err != nil || !haveTotal || !haveAvail || totalKB <= 0 { |
| 195 | return nil |
| 196 | } |
| 197 | frac := float64(totalKB-availKB) / float64(totalKB) |
| 198 | return &frac |
| 199 | } |
| 200 | |
| 201 | // readDiskFrac reports how full the data volume is (1 - available/total), |
| 202 | // via diskBytes on dataDir() — /data by default, CARLOS_DATA_DIR when |
| 203 | // set, matching every other on-box path in this package. There is no |
| 204 | // existing "floor" config to mirror: ops/litestream (grepped before writing |
| 205 | // this) carries no reserved-space setting, so this is the plain fraction, |
| 206 | // not a fraction-of-headroom-above-a-floor. Statfs failure (unmounted, |
| 207 | // missing dir) -> nil; a zero-block filesystem (shouldn't happen, but would |
| 208 | // divide by zero) also -> nil rather than a crash. |
| 209 | func readDiskFrac() *float64 { |
| 210 | total, avail, err := diskBytes(dataDir()) |
| 211 | if err != nil || total == 0 { |
| 212 | return nil |
| 213 | } |
| 214 | frac := 1 - float64(avail)/float64(total) |
| 215 | return &frac |
| 216 | } |
| 217 | |
| 218 | // listeningProbe reads /proc/net/unix — the kernel's own table of bound |
| 219 | // unix-domain sockets — rather than dialing sock. Dialing would itself be a |
| 220 | // connection attempt on a hibernating tenant's socket, which is exactly the |
| 221 | // event that wakes it: the probe would falsify the very thing it's trying to |
| 222 | // observe. Presence of sock's path in the table is the awake signal, absence |
| 223 | // is asleep; only a failure to read the table at all -> nil ("unknown"). |
| 224 | // |
| 225 | // This reading is correct ONLY for exec-backed routes, where the socket only |
| 226 | // exists in the kernel while the spawned child holds it open — asleep, the |
| 227 | // child is gone and the entry is gone with it. It is CONFIDENTLY WRONG for |
| 228 | // systemd-socket-activated (registry.BackingUnit) routes: the carlos-app@.socket |
| 229 | // unit holds the listener open in the kernel table at all times, awake or |
| 230 | // asleep, specifically so systemd can catch the first connection and spawn |
| 231 | // the service — so this function alone would report "awake" forever for a |
| 232 | // sleeping unit-backed tenant. Callers MUST route unit-backed sockets through |
| 233 | // wrapListeningProbe below rather than calling this directly; statusLoop is |
| 234 | // the only caller and does exactly that. |
| 235 | func listeningProbe(sock string) *bool { |
| 236 | f, err := os.Open("/proc/net/unix") |
| 237 | if err != nil { |
| 238 | return nil |
| 239 | } |
| 240 | defer f.Close() |
| 241 | |
| 242 | found := false |
| 243 | scanner := bufio.NewScanner(f) |
| 244 | scanner.Scan() // header line ("Num RefCount Protocol Flags Type St Inode Path") — not a data row |
| 245 | for scanner.Scan() { |
| 246 | fields := strings.Fields(scanner.Text()) |
| 247 | if len(fields) == 0 { |
| 248 | continue |
| 249 | } |
| 250 | // Path, when present, is always the last column; sockets with no |
| 251 | // Path (anonymous socketpairs) simply never match sock. |
| 252 | if fields[len(fields)-1] == sock { |
| 253 | found = true |
| 254 | break |
| 255 | } |
| 256 | } |
| 257 | if err := scanner.Err(); err != nil { |
| 258 | return nil |
| 259 | } |
| 260 | return &found |
| 261 | } |
| 262 | |
| 263 | // unitBackedSockets returns the set of socket paths belonging to routes |
| 264 | // whose Backing is registry.BackingUnit — the systemd-socket-activated |
| 265 | // whole-app tenants that make listeningProbe's kernel-table reading |
| 266 | // unreliable (see its doc comment). Pure and routes-only so it and |
| 267 | // wrapListeningProbe below are each independently testable at the seam |
| 268 | // host_test.go exercises, with no /proc read anywhere near the test. |
| 269 | func unitBackedSockets(routes []registry.Route) map[string]bool { |
| 270 | set := map[string]bool{} |
| 271 | for _, r := range routes { |
| 272 | if r.Backing == registry.BackingUnit { |
| 273 | set[r.Socket] = true |
| 274 | } |
| 275 | } |
| 276 | return set |
| 277 | } |
| 278 | |
| 279 | // wrapListeningProbe wraps probe so a unit-backed socket (per unitBacked) |
| 280 | // always reports nil — unknown — instead of whatever probe would say. |
| 281 | // systemd's .socket unit holds a unit-backed tenant's listener open in the |
| 282 | // kernel table at ALL times, awake or asleep; a raw kernel-table read can |
| 283 | // therefore never tell the two states apart for these routes, and reporting |
| 284 | // "awake" from that alone would be a confidently wrong answer where the |
| 285 | // status page's commitment demands an honest null. This stays true until the |
| 286 | // agent learns to ask systemd itself (e.g. `systemctl is-active |
| 287 | // carlos-app@<host>.service`) — a real fix, not implemented here, tracked as |
| 288 | // future work rather than guessed at. Exec-backed sockets (not in the set) |
| 289 | // pass straight through to probe unchanged, since listeningProbe's |
| 290 | // kernel-table reading IS reliable for them. |
| 291 | func wrapListeningProbe(unitBacked map[string]bool, probe func(string) *bool) func(string) *bool { |
| 292 | return func(sock string) *bool { |
| 293 | if unitBacked[sock] { |
| 294 | return nil |
| 295 | } |
| 296 | return probe(sock) |
| 297 | } |
| 298 | } |
| 299 | |
| 300 | // currentSocketProbe points probe at each activator-managed route's CURRENT |
| 301 | // generation socket (zero-downtime spec §7.1): with generations on, nothing |
| 302 | // binds a managed route's canonical path, so the kernel table would call |
| 303 | // every awake instance asleep. A route the activator reports no current |
| 304 | // generation for (asleep) is probed at its canonical path, where nothing |
| 305 | // listens: asleep, which is the truth. Any other socket passes straight |
| 306 | // through. appSocket is the activator's AppSocket. |
| 307 | func currentSocketProbe(routes []registry.Route, appSocket func(host string) (string, bool), probe func(string) *bool) func(string) *bool { |
| 308 | hostOf := map[string]string{} |
| 309 | for _, r := range routes { |
| 310 | if r.Kind == "instance" && r.Hibernate && r.Socket != "" && r.Backing != registry.BackingUnit { |
| 311 | hostOf[r.Socket] = r.Host |
| 312 | } |
| 313 | } |
| 314 | return func(sock string) *bool { |
| 315 | if host, ok := hostOf[sock]; ok { |
| 316 | if cur, ok := appSocket(host); ok { |
| 317 | return probe(cur) |
| 318 | } |
| 319 | } |
| 320 | return probe(sock) |
| 321 | } |
| 322 | } |
| 323 | |
| 324 | // parseMeminfoBytes reads /proc/meminfo's MemTotal and MemAvailable (both in |
| 325 | // kB) and returns them converted to bytes. Either field missing or |
| 326 | // unparsable -> nil for both. |
| 327 | func parseMeminfoBytes(r io.Reader) (*int64, *int64) { |
| 328 | var totalKB, availKB int64 |
| 329 | haveTotal, haveAvail := false, false |
| 330 | scanner := bufio.NewScanner(r) |
| 331 | for scanner.Scan() { |
| 332 | fields := strings.Fields(scanner.Text()) |
| 333 | if len(fields) < 2 { |
| 334 | continue |
| 335 | } |
| 336 | v, err := strconv.ParseInt(fields[1], 10, 64) |
| 337 | if err != nil { |
| 338 | continue |
| 339 | } |
| 340 | switch fields[0] { |
| 341 | case "MemTotal:": |
| 342 | totalKB, haveTotal = v, true |
| 343 | case "MemAvailable:": |
| 344 | availKB, haveAvail = v, true |
| 345 | } |
| 346 | } |
| 347 | if err := scanner.Err(); err != nil || !haveTotal || !haveAvail || totalKB <= 0 { |
| 348 | return nil, nil |
| 349 | } |
| 350 | totalBytes := totalKB * 1024 |
| 351 | availBytes := availKB * 1024 |
| 352 | return &totalBytes, &availBytes |
| 353 | } |
| 354 | |
| 355 | // readMemBytes reads /proc/meminfo and returns MemTotal and MemAvailable as |
| 356 | // bytes. Unreadable or unparseable -> nil, nil. |
| 357 | func readMemBytes() (*int64, *int64) { |
| 358 | f, err := os.Open("/proc/meminfo") |
| 359 | if err != nil { |
| 360 | return nil, nil |
| 361 | } |
| 362 | defer f.Close() |
| 363 | return parseMeminfoBytes(f) |
| 364 | } |
| 365 | |
| 366 | // parseLoad1 parses /proc/loadavg's first field (1-minute load average) from |
| 367 | // a string. The file format is: "load1 load5 load15 running/total lastpid". |
| 368 | // Empty or unparseable -> nil. |
| 369 | func parseLoad1(s string) *float64 { |
| 370 | fields := strings.Fields(s) |
| 371 | if len(fields) == 0 { |
| 372 | return nil |
| 373 | } |
| 374 | l, err := strconv.ParseFloat(fields[0], 64) |
| 375 | if err != nil { |
| 376 | return nil |
| 377 | } |
| 378 | return &l |
| 379 | } |
| 380 | |
| 381 | // readLoad1 reads /proc/loadavg and returns the 1-minute load average. |
| 382 | // Unreadable or unparseable -> nil. |
| 383 | func readLoad1() *float64 { |
| 384 | b, err := os.ReadFile("/proc/loadavg") |
| 385 | if err != nil { |
| 386 | return nil |
| 387 | } |
| 388 | return parseLoad1(string(b)) |
| 389 | } |
| 390 | |
| 391 | // readDiskFreeBytes reports how much free space is available on the data |
| 392 | // volume, via diskBytes on dataDir() — /data by default, CARLOS_DATA_DIR |
| 393 | // when set. Statfs failure or a zero-block filesystem -> nil rather than a |
| 394 | // crash. |
| 395 | func readDiskFreeBytes() *int64 { |
| 396 | total, avail, err := diskBytes(dataDir()) |
| 397 | if err != nil || total == 0 { |
| 398 | return nil |
| 399 | } |
| 400 | freeBytes := int64(avail) |
| 401 | return &freeBytes |
| 402 | } |
| 403 | |
| 404 | // dbFileBytes is status.Probes.DBBytes: the on-disk size of a route's SQLite |
| 405 | // database, the main file plus its -wal. The WAL counts because in WAL mode |
| 406 | // recent writes live there until a checkpoint, so the main file alone |
| 407 | // understates a busy database. A missing or unreadable main file is nil |
| 408 | // (unknown), never 0: a route that has not created its database yet has no |
| 409 | // size. A missing -wal is normal after a checkpoint and adds nothing. A |
| 410 | // hibernating route's file stays on disk while it sleeps (the activator keeps |
| 411 | // it as a warm cache), so asleep routes are sized too. |
| 412 | func dbFileBytes(dbPath string) *int64 { |
| 413 | fi, err := os.Stat(dbPath) |
| 414 | if err != nil || !fi.Mode().IsRegular() { |
| 415 | return nil |
| 416 | } |
| 417 | n := fi.Size() |
| 418 | if wal, err := os.Stat(dbPath + "-wal"); err == nil && wal.Mode().IsRegular() { |
| 419 | n += wal.Size() |
| 420 | } |
| 421 | return &n |
| 422 | } |
| 423 | |
amadan