| 1 | // The box-side half of the metrics sampler: every function here reads local |
| 2 | // kernel/OS state (cgroup v2 and /proc files) and MUST fail toward nil, never |
| 3 | // a guessed zero. This file is thin OS glue — os.ReadFile and os.ReadDir |
| 4 | // wrapping the pure parsers in internal/metrics — and the readers are |
| 5 | // deliberately untested by the same convention as probes.go readers; |
| 6 | // internal/metrics tests exercise the parsers with fixtures instead. The |
| 7 | // exceptions are listCarlosUnits, listStampedUnits and listExecCgroups: they are directory |
| 8 | // WALKs, not single reads, and which directories they walk is a decision that |
| 9 | // has already been wrong once (see listCarlosUnits' doc comment), so they take |
| 10 | // their roots as arguments and are pinned by metrics_probes_test.go against |
| 11 | // temp-dir fixtures. |
| 12 | package host |
| 13 | |
| 14 | import ( |
| 15 | "os" |
| 16 | "path/filepath" |
| 17 | "strings" |
| 18 | |
| 19 | "github.com/carlosframework/platform/internal/cgroup" |
| 20 | "github.com/carlosframework/platform/internal/metrics" |
| 21 | ) |
| 22 | |
| 23 | // readProcStat reads /proc/stat's aggregate cpu line and parses its counters. |
| 24 | // Read failure or parse failure -> nil. |
| 25 | func readProcStat() *metrics.CPUCounters { |
| 26 | b, err := os.ReadFile("/proc/stat") |
| 27 | if err != nil { |
| 28 | return nil |
| 29 | } |
| 30 | return metrics.ParseProcStat(string(b)) |
| 31 | } |
| 32 | |
| 33 | // readLoadAvg reads /proc/loadavg and parses the 1/5/15-minute load averages. |
| 34 | // Any read or parse failure (too few fields, non-numeric values) -> all three |
| 35 | // results nil, since a partial load average is not a value worth publishing |
| 36 | // on its own. |
| 37 | func readLoadAvg() (l1, l5, l15 *float64) { |
| 38 | b, err := os.ReadFile("/proc/loadavg") |
| 39 | if err != nil { |
| 40 | return nil, nil, nil |
| 41 | } |
| 42 | return metrics.ParseLoadAvg(string(b)) |
| 43 | } |
| 44 | |
| 45 | // cgroupSliceDir is the cgroup v2 directory where CARLOS units report their |
| 46 | // resource usage. It is the default root for listCarlosUnits, which takes the |
| 47 | // root as an argument only so the discovery walk below can be tested against |
| 48 | // a temp-dir fixture instead of the live kernel. |
| 49 | const cgroupSliceDir = "/sys/fs/cgroup/system.slice" |
| 50 | |
| 51 | // cgroupCarlosSliceDir is the second root listCarlosUnits and |
| 52 | // listStampedUnits must scan: carlos-edge.service now lives in carlos.slice, |
| 53 | // not system.slice, because the isolated-exec design (§4.3) moved the edge |
| 54 | // there to share its memory wall with the exec-child instances it supervises |
| 55 | // (see internal/host/units/carlos-edge.service's Slice=carlos.slice). On a |
| 56 | // flat fleet — no tenant instances yet delegated their own cgroups — the |
| 57 | // edge's own leaf is still the hierarchical total of everything under it, |
| 58 | // delegated exec children included, so publishing that same |
| 59 | // memory.current/cpu.stat/io.stat reading under the unchanged unit-name key |
| 60 | // "carlos-edge.service" preserves the number operators already look at; only |
| 61 | // the directory that number is read from has moved. |
| 62 | const cgroupCarlosSliceDir = "/sys/fs/cgroup/carlos.slice" |
| 63 | |
| 64 | // unitCgroup is one discovered CARLOS unit: its systemd unit name (the key |
| 65 | // everything downstream — sample maps, the console's activator.UnitName |
| 66 | // match — uses) and the cgroup directory that unit's files actually live in. |
| 67 | // The two are NOT interchangeable: only flat units sit at |
| 68 | // <root>/<name>, and rebuilding the path from the name would miss every |
| 69 | // tenant instance (see listCarlosUnits). |
| 70 | type unitCgroup struct { |
| 71 | name string |
| 72 | dir string |
| 73 | } |
| 74 | |
| 75 | // listCarlosUnits lists all active CARLOS systemd units under root by |
| 76 | // filtering directory names for the "carlos-" prefix and ".service" suffix. |
| 77 | // Read failure -> nil. |
| 78 | // |
| 79 | // Two levels are scanned, because systemd puts CARLOS's two kinds of unit in |
| 80 | // two different places: |
| 81 | // |
| 82 | // - Flat, directly under root: the platform's fixed units |
| 83 | // (carlos-edge.service, carlos-litestream, carlos-adopt, |
| 84 | // carlos-restore-verify) — plain non-templated services land in |
| 85 | // system.slice itself. |
| 86 | // - Nested one level down, inside a "system-*.slice" child: INSTANTIATED |
| 87 | // units. carlos-app@<host>.service sets no Slice= (see |
| 88 | // internal/host/units/carlos-app@.service), so systemd places every |
| 89 | // instance of the template in the implicit per-template slice |
| 90 | // system.slice/system-carlos\x2dapp.slice/. Scanning only the top level |
| 91 | // therefore finds every platform unit and not a single tenant — the exact |
| 92 | // set the per-unit table and the owner app-metrics page exist to show. |
| 93 | // |
| 94 | // Only these two levels are walked: a unit's own cgroup directory has |
| 95 | // children of its own on some systems, and nothing of ours ever nests deeper |
| 96 | // than one slice. |
| 97 | func listCarlosUnits(root string) []unitCgroup { |
| 98 | entries, err := os.ReadDir(root) |
| 99 | if err != nil { |
| 100 | return nil |
| 101 | } |
| 102 | |
| 103 | var units []unitCgroup |
| 104 | for _, entry := range entries { |
| 105 | if !entry.IsDir() { |
| 106 | continue |
| 107 | } |
| 108 | name := entry.Name() |
| 109 | if isCarlosUnitName(name) { |
| 110 | units = append(units, unitCgroup{name: name, dir: filepath.Join(root, name)}) |
| 111 | continue |
| 112 | } |
| 113 | if !strings.HasPrefix(name, "system-") || !strings.HasSuffix(name, ".slice") { |
| 114 | continue |
| 115 | } |
| 116 | sliceDir := filepath.Join(root, name) |
| 117 | nested, err := os.ReadDir(sliceDir) |
| 118 | if err != nil { |
| 119 | continue // an unreadable slice loses its units, never the whole scan |
| 120 | } |
| 121 | for _, child := range nested { |
| 122 | if !child.IsDir() || !isCarlosUnitName(child.Name()) { |
| 123 | continue |
| 124 | } |
| 125 | units = append(units, unitCgroup{name: child.Name(), dir: filepath.Join(sliceDir, child.Name())}) |
| 126 | } |
| 127 | } |
| 128 | return units |
| 129 | } |
| 130 | |
| 131 | // listStampedUnits finds the cgroup directories of operator-stamped bespoke |
| 132 | // units — registry Route.Unit (#110), the routes the carlos- naming |
| 133 | // convention above cannot see (an operator's console.service, tito's |
| 134 | // titogo-canary@<tenant>.service). Read failure -> nil, same as |
| 135 | // listCarlosUnits. |
| 136 | // |
| 137 | // Same two-level walk, for the same reason: a flat unit sits under root, a |
| 138 | // template instance inside its implicit system-*.slice. The stamped NAME says |
| 139 | // nothing about which of the two systemd chose, so both levels are matched |
| 140 | // against the same wanted set. |
| 141 | // |
| 142 | // A stamped unit with no cgroup dir under root is simply not returned — no |
| 143 | // reading is the honest report for a unit that isn't running, and the console |
| 144 | // renders that absence rather than a guessed zero. Names arrive from the |
| 145 | // registry already shaped by registry.ValidUnitName (the door validates; see |
| 146 | // SetUnit), so nothing here re-validates them: an unmatchable name matches |
| 147 | // nothing, which is the same outcome as a stopped unit. |
| 148 | func listStampedUnits(root string, stamped []string) []unitCgroup { |
| 149 | if len(stamped) == 0 { |
| 150 | return nil |
| 151 | } |
| 152 | want := make(map[string]bool, len(stamped)) |
| 153 | for _, u := range stamped { |
| 154 | want[u] = true |
| 155 | } |
| 156 | |
| 157 | entries, err := os.ReadDir(root) |
| 158 | if err != nil { |
| 159 | return nil |
| 160 | } |
| 161 | |
| 162 | var units []unitCgroup |
| 163 | for _, entry := range entries { |
| 164 | if !entry.IsDir() { |
| 165 | continue |
| 166 | } |
| 167 | name := entry.Name() |
| 168 | if want[name] { |
| 169 | units = append(units, unitCgroup{name: name, dir: filepath.Join(root, name)}) |
| 170 | continue |
| 171 | } |
| 172 | if !strings.HasPrefix(name, "system-") || !strings.HasSuffix(name, ".slice") { |
| 173 | continue |
| 174 | } |
| 175 | sliceDir := filepath.Join(root, name) |
| 176 | nested, err := os.ReadDir(sliceDir) |
| 177 | if err != nil { |
| 178 | continue // an unreadable slice loses its units, never the whole scan |
| 179 | } |
| 180 | for _, child := range nested { |
| 181 | if !child.IsDir() || !want[child.Name()] { |
| 182 | continue |
| 183 | } |
| 184 | units = append(units, unitCgroup{name: child.Name(), dir: filepath.Join(sliceDir, child.Name())}) |
| 185 | } |
| 186 | } |
| 187 | return units |
| 188 | } |
| 189 | |
| 190 | // isCarlosUnitName reports whether a cgroup directory name is one of ours: |
| 191 | // the "carlos-" prefix and the ".service" suffix, with at least one character |
| 192 | // between them. |
| 193 | func isCarlosUnitName(name string) bool { |
| 194 | return len(name) > len("carlos-.service") && |
| 195 | strings.HasPrefix(name, "carlos-") && |
| 196 | strings.HasSuffix(name, ".service") |
| 197 | } |
| 198 | |
| 199 | // unitRaw is a single reading of a systemd unit's resource usage from its |
| 200 | // cgroup v2 files. Each field represents a metric that may have failed to |
| 201 | // read independently. |
| 202 | type unitRaw struct { |
| 203 | cpuUsec *int64 |
| 204 | memBytes *int64 |
| 205 | memPeakBytes *int64 |
| 206 | readBytes *int64 |
| 207 | writeBytes *int64 |
| 208 | } |
| 209 | |
| 210 | // readUnitRaw reads a unit's resource usage from the cgroup v2 files in |
| 211 | // unitPath — the directory listCarlosUnits reported for that unit, which for |
| 212 | // an instantiated unit is NOT <cgroupSliceDir>/<name>. Each file read failure |
| 213 | // sets only that field to nil; the others still read independently. Parse |
| 214 | // failures also yield nil for that field only. |
| 215 | func readUnitRaw(unitPath string) unitRaw { |
| 216 | // Read cpu.stat for usage_usec |
| 217 | raw := unitRaw{} |
| 218 | if b, err := os.ReadFile(filepath.Join(unitPath, "cpu.stat")); err == nil { |
| 219 | raw.cpuUsec = metrics.ParseCgroupCPUUsec(string(b)) |
| 220 | } |
| 221 | |
| 222 | // Read memory.current |
| 223 | if b, err := os.ReadFile(filepath.Join(unitPath, "memory.current")); err == nil { |
| 224 | raw.memBytes = metrics.ParseCgroupInt(string(b)) |
| 225 | } |
| 226 | |
| 227 | // Read memory.peak |
| 228 | if b, err := os.ReadFile(filepath.Join(unitPath, "memory.peak")); err == nil { |
| 229 | raw.memPeakBytes = metrics.ParseCgroupInt(string(b)) |
| 230 | } |
| 231 | |
| 232 | // Read io.stat for rbytes and wbytes |
| 233 | if b, err := os.ReadFile(filepath.Join(unitPath, "io.stat")); err == nil { |
| 234 | rbytes, wbytes := metrics.ParseCgroupIO(string(b)) |
| 235 | raw.readBytes = rbytes |
| 236 | raw.writeBytes = wbytes |
| 237 | } |
| 238 | |
| 239 | return raw |
| 240 | } |
| 241 | |
| 242 | // listExecCgroups lists the per-host exec-child cgroups under the agent's |
| 243 | // delegated subtree — the third scan root, alongside listCarlosUnits' two |
| 244 | // levels. An empty root (no delegation armed) or a read failure -> nil. The |
| 245 | // agent's own supervise leaf fails IsExecName and is skipped: that process |
| 246 | // is already reported as carlos-edge.service. A dir whose cgroup.procs is |
| 247 | // empty or unreadable is skipped too: a failed seed or an aborted wake |
| 248 | // leaves the dir behind (until a later exit's rmdir or the next agent |
| 249 | // start's sweep), and reporting it would render a host whose wake FAILED |
| 250 | // as a live instance at zero — absence is the data, never a guessed zero. |
| 251 | // readUnitRaw reads the included dirs unchanged — same cgroup v2 files, |
| 252 | // same fail-toward-nil fields. |
| 253 | func listExecCgroups(root string) []unitCgroup { |
| 254 | if root == "" { |
| 255 | return nil |
| 256 | } |
| 257 | entries, err := os.ReadDir(root) |
| 258 | if err != nil { |
| 259 | return nil |
| 260 | } |
| 261 | var units []unitCgroup |
| 262 | for _, entry := range entries { |
| 263 | if !entry.IsDir() || !cgroup.IsExecName(entry.Name()) { |
| 264 | continue |
| 265 | } |
| 266 | dir := filepath.Join(root, entry.Name()) |
| 267 | procs, err := os.ReadFile(filepath.Join(dir, "cgroup.procs")) |
| 268 | if err != nil { |
| 269 | continue |
| 270 | } |
| 271 | // A generation parent (zero-downtime spec §7.3) holds no process of |
| 272 | // its own; its generations do. cgroup.events "populated 1" is the |
| 273 | // kernel's word that something under it is alive. |
| 274 | if len(strings.TrimSpace(string(procs))) == 0 && !subtreePopulated(dir) { |
| 275 | continue |
| 276 | } |
| 277 | units = append(units, unitCgroup{name: entry.Name(), dir: dir}) |
| 278 | } |
| 279 | return units |
| 280 | } |
| 281 | |
| 282 | // subtreePopulated reports whether dir's cgroup.events says a process is |
| 283 | // alive anywhere under it. Unreadable is no: absence is the data. |
| 284 | func subtreePopulated(dir string) bool { |
| 285 | b, err := os.ReadFile(filepath.Join(dir, "cgroup.events")) |
| 286 | return err == nil && strings.Contains(string(b), "populated 1") |
| 287 | } |
| 288 | |
amadan