| 1 | // Adopt-pass advisories: warnings about registry rows whose claims |
| 2 | // disagree with what systemd is actually running. Advisory only — the |
| 3 | // pass never mutates a route to "fix" it; the operator decides (see |
| 4 | // platform issue #14 for the kass incident that motivated this). |
| 5 | package host |
| 6 | |
| 7 | import ( |
| 8 | "log" |
| 9 | "os/exec" |
| 10 | "os/user" |
| 11 | "strconv" |
| 12 | "strings" |
| 13 | |
| 14 | "github.com/carlosframework/platform/internal/caps" |
| 15 | "github.com/carlosframework/platform/internal/registry" |
| 16 | "github.com/carlosframework/platform/internal/release" |
| 17 | ) |
| 18 | |
| 19 | // warnBackingMismatch reports hosts whose route claims exec backing |
| 20 | // ("") while a carlos-app@<host> unit is actively running — the state |
| 21 | // where release adoption still works but config sync silently skips the |
| 22 | // route. unitsActive answers for ALL candidates in one probe (one |
| 23 | // systemctl invocation, not one per route). Returns the warned hosts; |
| 24 | // the caller logs them. |
| 25 | func warnBackingMismatch(routes []registry.Route, unitsActive func(hosts []string) map[string]bool) []string { |
| 26 | var candidates []string |
| 27 | for _, rt := range routes { |
| 28 | if rt.Kind != "instance" || rt.Platform || rt.Backing == registry.BackingUnit { |
| 29 | continue |
| 30 | } |
| 31 | candidates = append(candidates, rt.Host) |
| 32 | } |
| 33 | if len(candidates) == 0 { |
| 34 | return nil |
| 35 | } |
| 36 | active := unitsActive(candidates) |
| 37 | var warned []string |
| 38 | for _, h := range candidates { |
| 39 | if active[h] { |
| 40 | warned = append(warned, h) |
| 41 | } |
| 42 | } |
| 43 | return warned |
| 44 | } |
| 45 | |
| 46 | // unitsAreActive: the real batched systemd probe. `systemctl is-active` |
| 47 | // prints one state line per named unit, positionally, and exits non-zero |
| 48 | // when ANY unit is not active — the exit code is meaningless for a batch, |
| 49 | // so only stdout is read. |
| 50 | func unitsAreActive(hosts []string) map[string]bool { |
| 51 | args := []string{"is-active"} |
| 52 | for _, h := range hosts { |
| 53 | args = append(args, "carlos-app@"+h+".service") |
| 54 | } |
| 55 | out, _ := exec.Command("systemctl", args...).Output() |
| 56 | return parseIsActiveBatch(out, hosts) |
| 57 | } |
| 58 | |
| 59 | // parseIsActiveBatch maps `systemctl is-active` stdout onto hosts |
| 60 | // positionally (named distinctly from probes.go's single-unit parseIsActive, |
| 61 | // whose tri-state *bool shape this is not). A mismatched line count cannot |
| 62 | // be attributed positionally — a short batch would misattribute "active" to |
| 63 | // the wrong host, and the resulting warning tells an operator to delete the |
| 64 | // wrong host's decrypted secrets file. No warnings beats wrong warnings, so |
| 65 | // a length mismatch returns an empty map. |
| 66 | func parseIsActiveBatch(out []byte, hosts []string) map[string]bool { |
| 67 | lines := strings.Split(strings.TrimRight(string(out), "\n"), "\n") |
| 68 | if len(lines) != len(hosts) { |
| 69 | return map[string]bool{} |
| 70 | } |
| 71 | active := make(map[string]bool, len(hosts)) |
| 72 | for i, h := range hosts { |
| 73 | switch strings.TrimSpace(lines[i]) { |
| 74 | case "active", "reloading": |
| 75 | active[h] = true |
| 76 | } |
| 77 | } |
| 78 | return active |
| 79 | } |
| 80 | |
| 81 | // adviseBackingMismatches runs the advisory against live systemd and |
| 82 | // logs each finding — called from RunAdopt, root-side, where systemctl |
| 83 | // is reachable and the hourly log is read. |
| 84 | func adviseBackingMismatches(routes []registry.Route) { |
| 85 | for _, h := range warnBackingMismatch(routes, unitsAreActive) { |
| 86 | log.Printf("adopt: route %s claims exec backing but carlos-app@%s is ACTIVE — stamp backing='unit' (see platform issue #14), then remove the stray /data/%s.config.env the exec loop materialized — the unit reads /etc/carlos/apps instead", h, h, h) |
| 87 | } |
| 88 | } |
| 89 | |
| 90 | // imdsAddr is the link-local metadata address every cloud instance role is |
| 91 | // reached through, and the thing the box's nftables gate is keyed on. A rule |
| 92 | // that does not name it is not part of that gate (a box can have plenty of |
| 93 | // other skuid rules), so it is the scope of the parse below as well as a |
| 94 | // string in the printed command. |
| 95 | const imdsAddr = "169.254.169.254" |
| 96 | |
| 97 | // adviseHostRoleGateMissing: the app holds the host_role grant, so it runs as |
| 98 | // the static user carlos-<app> — but that uid is not in this box's IMDS |
| 99 | // nftables gate, and the observed failure mode of that gap is SILENCE, not an |
| 100 | // error (keymail's rehearsal finding F1: the restore came up empty and |
| 101 | // served). So it is named every pass for as long as the gap exists. |
| 102 | // |
| 103 | // The platform NEVER writes the firewall rule. The firewall is operator |
| 104 | // territory; this prints the exact command and stops there (spec §4b, §5a). |
| 105 | // |
| 106 | // Two seams, two distinct claims, deliberately not folded together: |
| 107 | // |
| 108 | // - uidOf answers whether the static user EXISTS on this box at all. It is |
| 109 | // also the quiet gate on the whole advisory: converge creates that user |
| 110 | // when it first materializes host_role, so a user that does not resolve |
| 111 | // means nothing is running under it yet and there is nothing for the gate |
| 112 | // to be missing. Advising then would be an hourly line about a uid that |
| 113 | // does not exist. |
| 114 | // - gateUIDs answers (uids, ok), and three different things can be true. |
| 115 | // ok == false is "I could not read a gate". A readable ruleset with NO |
| 116 | // IMDS gate in it at all is a third claim again — the box is ungated, and |
| 117 | // the fix is not "add a uid" but "harden this box first" (tito has no |
| 118 | // gate today). Each gets its own line, because acting on the wrong one of |
| 119 | // the three sends an operator to edit a firewall that is fine, or to add |
| 120 | // a uid to a gate that does not exist. |
| 121 | // |
| 122 | // A route whose grants column does not parse is skipped in silence here: |
| 123 | // buildPlans already says so once, naming the row and the fix, and repeating |
| 124 | // it in a second voice would double every line of an already-broken row. |
| 125 | func adviseHostRoleGateMissing(routes []registry.Route, uidOf func(user string) (int, bool), gateUIDs func() (map[int]bool, bool)) { |
| 126 | var cands []registry.Route |
| 127 | for _, rt := range routes { |
| 128 | // Only a route that can actually materialize host_role: the same |
| 129 | // predicate the converge passes use. A grant on a route that cannot |
| 130 | // materialize is `carlos route -grant`'s problem (grantRail refuses |
| 131 | // it), not the firewall's. |
| 132 | if !release.CapabilityRoute(rt) { |
| 133 | continue |
| 134 | } |
| 135 | g, err := caps.ParseGrants(rt.Caps) |
| 136 | if err != nil || !g.HostRole { |
| 137 | continue |
| 138 | } |
| 139 | cands = append(cands, rt) |
| 140 | } |
| 141 | if len(cands) == 0 { |
| 142 | return // no candidates, no `nft` invocation at all |
| 143 | } |
| 144 | uids, ok := gateUIDs() // ONE read of the ruleset for the whole pass |
| 145 | for _, rt := range cands { |
| 146 | u := caps.AppUser(rt.App) |
| 147 | uid, exists := uidOf(u) |
| 148 | if !exists { |
| 149 | continue |
| 150 | } |
| 151 | if !ok { |
| 152 | log.Printf("adopt: %s holds the host_role grant on %s but this box's nft ruleset could not be read (nft absent, or this pass is not root) — confirm by hand that uid %d is allowed to reach %s (see the app-capabilities design §4b)", u, rt.Host, uid, imdsAddr) |
| 153 | continue |
| 154 | } |
| 155 | if len(uids) == 0 { |
| 156 | log.Printf("adopt: %s (uid %d) holds the host_role grant on %s but no IMDS uid gate found in the nft ruleset — this box does not gate %s at all, so every process on it can reach the instance role; run platform-infrastructure scripts/harden-host.sh on this box, then add uid %d", u, uid, rt.Host, imdsAddr, uid) |
| 157 | continue |
| 158 | } |
| 159 | if !uids[uid] { |
| 160 | log.Printf("adopt: %s (uid %d) holds the host_role grant on %s but is not in this box's IMDS gate — its AWS clients will be starved SILENTLY; add it: nft add rule inet filter output ip daddr %s skuid %d accept (and add the uid to harden-host.sh so a re-run does not strip it)", u, uid, rt.Host, imdsAddr, uid) |
| 161 | } |
| 162 | } |
| 163 | } |
| 164 | |
| 165 | // lookupUID resolves a system user to its numeric uid — the same account the |
| 166 | // converge pass creates for a host_role app, looked up the same way |
| 167 | // caps.Ops.ensureUser proves its own useradd resolved. |
| 168 | func lookupUID(name string) (int, bool) { |
| 169 | u, err := user.Lookup(name) |
| 170 | if err != nil { |
| 171 | return 0, false |
| 172 | } |
| 173 | uid, err := strconv.Atoi(u.Uid) |
| 174 | if err != nil { |
| 175 | return 0, false |
| 176 | } |
| 177 | return uid, true |
| 178 | } |
| 179 | |
| 180 | // nftGateUIDs is the real read of the box's IMDS gate: the OS glue half of |
| 181 | // the parseNftSkuids split, exactly as unitsAreActive is to parseIsActiveBatch. |
| 182 | // |
| 183 | // `-n` is load-bearing, not tidiness: it makes nft print uids numerically, so |
| 184 | // the parse never has to resolve a printed account NAME back to a number and |
| 185 | // can never mistake one for the other. |
| 186 | // |
| 187 | // ok is false ONLY when the ruleset could not be read at all (absent binary, |
| 188 | // not root, nft error). A ruleset that reads fine and carries no IMDS gate is |
| 189 | // (empty, true) — a different fact, with a different fix, and one this |
| 190 | // advisory must not go quiet about: a host_role app on a box with no IMDS |
| 191 | // gate is not safe, it is ungated, which is F1's exposure with nothing in |
| 192 | // front of it. |
| 193 | func nftGateUIDs() (map[int]bool, bool) { |
| 194 | out, err := exec.Command("nft", "-n", "list", "ruleset").Output() |
| 195 | if err != nil { |
| 196 | return nil, false |
| 197 | } |
| 198 | return parseNftSkuids(out), true |
| 199 | } |
| 200 | |
| 201 | // parseNftSkuids reads the uids ALLOWED to reach the metadata address out of |
| 202 | // `nft -n list ruleset` output. |
| 203 | // |
| 204 | // The production gate is inline and numeric — `ip daddr 169.254.169.254 meta |
| 205 | // skuid <n> accept`, one rule per allowed uid, written by the infra repo's |
| 206 | // scripts/harden-host.sh, with no named sets today — so that is the shape |
| 207 | // this reads, plus nft's `skuid { a, b }` set spelling for free. |
| 208 | // |
| 209 | // Two rails, both in the fail-CLOSED direction, because every mistake this |
| 210 | // function can make ends in an advisory that stays quiet about a silent |
| 211 | // starvation: |
| 212 | // |
| 213 | // - scoped to lines naming imdsAddr: a box has other skuid rules (SSH, a |
| 214 | // backup agent) and counting those would report a gate allowing uids it |
| 215 | // does not. |
| 216 | // - scoped to lines whose VERDICT accepts. `... meta skuid 998 drop` is the |
| 217 | // opposite of an allowlist entry, and reading it as one would report the |
| 218 | // gate as open for exactly the uid it blocks. A jump/goto/return is |
| 219 | // counted as accepting: it hands the packet to another chain rather than |
| 220 | // dropping it here, and over-reporting a uid as gated is the direction |
| 221 | // that costs one dismissed advisory line rather than a silent outage. |
| 222 | // |
| 223 | // Negated and range matches (`skuid != 0`, `skuid 100-200`) are deliberately |
| 224 | // not interpreted: neither is an allowlist of one uid, and guessing would be |
| 225 | // the fail-open direction again. |
| 226 | func parseNftSkuids(out []byte) map[int]bool { |
| 227 | uids := map[int]bool{} |
| 228 | for _, line := range strings.Split(string(out), "\n") { |
| 229 | if !strings.Contains(line, imdsAddr) { |
| 230 | continue |
| 231 | } |
| 232 | fields := strings.Fields(line) |
| 233 | if !acceptingVerdict(fields) { |
| 234 | continue |
| 235 | } |
| 236 | for i := 0; i < len(fields); i++ { |
| 237 | if fields[i] != "skuid" || i+1 >= len(fields) { |
| 238 | continue |
| 239 | } |
| 240 | if fields[i+1] == "{" { |
| 241 | j := i + 2 |
| 242 | for ; j < len(fields) && fields[j] != "}"; j++ { |
| 243 | addUID(uids, fields[j]) |
| 244 | } |
| 245 | i = j |
| 246 | continue |
| 247 | } |
| 248 | addUID(uids, fields[i+1]) |
| 249 | i++ |
| 250 | } |
| 251 | } |
| 252 | return uids |
| 253 | } |
| 254 | |
| 255 | // acceptingVerdict reports whether a rule's verdict lets the packet through. |
| 256 | // A rule that drops or rejects is never read for allowed uids, whatever else |
| 257 | // it says — and a rule with no verdict at all (a counter, a comment) allows |
| 258 | // nothing either. |
| 259 | func acceptingVerdict(fields []string) bool { |
| 260 | accepting := false |
| 261 | for _, f := range fields { |
| 262 | switch strings.Trim(f, `",;`) { |
| 263 | case "drop", "reject": |
| 264 | return false |
| 265 | case "accept", "return", "goto", "jump": |
| 266 | accepting = true |
| 267 | } |
| 268 | } |
| 269 | return accepting |
| 270 | } |
| 271 | |
| 272 | // addUID records one uid token, tolerating the punctuation nft's set syntax |
| 273 | // carries ("998," inside braces) and ignoring anything that is not a plain |
| 274 | // non-negative number. |
| 275 | func addUID(uids map[int]bool, tok string) { |
| 276 | tok = strings.Trim(tok, `",;{}`) |
| 277 | n, err := strconv.Atoi(tok) |
| 278 | if err != nil || n < 0 { |
| 279 | return |
| 280 | } |
| 281 | uids[n] = true |
| 282 | } |
| 283 | |
amadan