| 1 | // The grant-drift capability pass: RunAdopt's step 2 (spec §4d). |
| 2 | // |
| 3 | // This is the survey half of the capability design. adoptHost's converge |
| 4 | // (release.WithCapabilities) is what a VERSION change moves through; this is |
| 5 | // what a GRANT change moves through — and, because it converges every |
| 6 | // adoptable unit route from the staged manifest of the version that is |
| 7 | // ACTUALLY LIVE, it is also the repair path for everything else that can |
| 8 | // leave a box's unit shape disagreeing with what the platform believes: |
| 9 | // |
| 10 | // - a grant added or revoked with `carlos route -grant/-revoke`; |
| 11 | // - a drop-in deleted by hand, or lost with an /etc restore; |
| 12 | // - a socket unit that is not enabled, or a daemon that is not running; |
| 13 | // - a FIRST adoption whose restart failed: the symlink rolled back to |
| 14 | // nothing (there was no previous version), so the drop-ins this pass |
| 15 | // re-verifies are the live version's and stand as converged; |
| 16 | // - any stranded shape left by a pass that died between two writes. |
| 17 | // |
| 18 | // Both entry points call the same caps.Converge, and running both in one |
| 19 | // pass is idempotent: the second sees byte-identical content, writes nothing, |
| 20 | // reloads nothing and restarts nothing. |
| 21 | package host |
| 22 | |
| 23 | import ( |
| 24 | "fmt" |
| 25 | "log" |
| 26 | "path/filepath" |
| 27 | "sort" |
| 28 | "strings" |
| 29 | |
| 30 | "github.com/carlosframework/platform/internal/caps" |
| 31 | "github.com/carlosframework/platform/internal/registry" |
| 32 | "github.com/carlosframework/platform/internal/release" |
| 33 | ) |
| 34 | |
| 35 | // capabilityPass converges every adoptable unit route on this box and stamps |
| 36 | // what it learned about their databases onto the registry. |
| 37 | // |
| 38 | // Pure enough to test: the registry, the opt dir and the caps.Ops seams are |
| 39 | // all arguments, so its tests need no systemd, no root and no /etc. |
| 40 | // |
| 41 | // Its error return is for FOUNDATION failures only (the registry would not |
| 42 | // list). A per-route refusal — an unreadable manifest, an unparseable grants |
| 43 | // column, a port another route already claims — logs and continues: this pass |
| 44 | // surveys a whole box, and one misconfigured route must not stop every other |
| 45 | // route's drift from being repaired. |
| 46 | func capabilityPass(reg *registry.Registry, optDir string, o caps.Ops) error { |
| 47 | routes, err := reg.List() |
| 48 | if err != nil { |
| 49 | return err |
| 50 | } |
| 51 | // An empty route list is never authority to do anything, exactly as in |
| 52 | // appconfig.Sync: registry.Open silently creates an empty database when |
| 53 | // handed a wrong path or an unmounted /data, and a pass that believed it |
| 54 | // would see every route on the box as removed. Nothing here deletes on |
| 55 | // absence today — a drop-in is only ever removed for a route this pass |
| 56 | // still sees — but the guard is cheap and keeps that true by construction. |
| 57 | if len(routes) == 0 { |
| 58 | return nil |
| 59 | } |
| 60 | |
| 61 | plans, advice := buildPlans(routes, stagedManifestOfLiveVersion(optDir)) |
| 62 | for _, line := range advice { |
| 63 | log.Print(line) |
| 64 | } |
| 65 | |
| 66 | // Box-wide exclusivity across every plan at once, BEFORE anything is |
| 67 | // written: two routes granted the same privileged port, or the same |
| 68 | // runtime_dir. Converge repeats the check against what is on DISK; this |
| 69 | // one is the only half that can see the other plans. |
| 70 | refused := caps.CheckClaims(plans) |
| 71 | for _, p := range plans { |
| 72 | if err := refused[p.Host]; err != nil { |
| 73 | log.Printf("adopt: %s: %v — nothing written for it this pass", p.Host, err) |
| 74 | continue |
| 75 | } |
| 76 | // Converge has already logged both a refusal's operator fix and a |
| 77 | // failure's cause, in its own house style, so nothing is re-logged |
| 78 | // here. DeferRestart stays FALSE (caps.Plan's contract): restarting |
| 79 | // the live binary to pick up a new unit shape is the entire point of |
| 80 | // this pass, and nothing here follows up with a Restart. |
| 81 | // |
| 82 | // Both returns are discarded ON PURPOSE, and that is correct FOR THIS |
| 83 | // CALLER only. Result.Changed together with an error means bytes are |
| 84 | // still moved and Converge will not move them back — which a |
| 85 | // version-change caller must answer with caps.Restore, because there |
| 86 | // the moved bytes are the NEW version's shape under the OLD binary. |
| 87 | // Here there is no version change and no old binary: the bytes moved |
| 88 | // are this route's own live shape, which is the steady state this |
| 89 | // pass is converging TOWARD, and the next pass simply finds whatever |
| 90 | // diff remains and retries it. Adding a Restore here would undo a |
| 91 | // correct repair every time a post-reload verb failed. |
| 92 | _, _ = caps.Converge(o, p) |
| 93 | } |
| 94 | |
| 95 | stampDBPaths(reg, routes, plans, o) |
| 96 | return nil |
| 97 | } |
| 98 | |
| 99 | // stagedManifestOfLiveVersion is the production loader: the staged sibling |
| 100 | // manifest of the version a route's live symlink CURRENTLY names. |
| 101 | // |
| 102 | // The live version, never the newest staged one — a box that has staged a |
| 103 | // version it has not adopted (a bake window, a poisoned build, a rollback) |
| 104 | // must keep the unit shape of what it is actually running. |
| 105 | // |
| 106 | // (nil, nil) — no live symlink, or no sibling beside the binary — is the |
| 107 | // leave-alone state, and it is deliberately NOT healed here: this pass does |
| 108 | // not fetch from the bucket. release.adoptHost's back-fill re-stages the live |
| 109 | // version's sibling on every pass, so a sibling that failed to write once |
| 110 | // heals there and this pass picks it up next hour. |
| 111 | func stagedManifestOfLiveVersion(optDir string) func(registry.Route) (*release.Manifest, error) { |
| 112 | return func(rt registry.Route) (*release.Manifest, error) { |
| 113 | ver, err := release.LiveVersion(optDir, rt.Host) |
| 114 | if err != nil { |
| 115 | return nil, err |
| 116 | } |
| 117 | if ver == "" { |
| 118 | return nil, nil |
| 119 | } |
| 120 | return release.ReadStagedManifest(optDir, release.RouteAccount(rt), ver, rt.App) |
| 121 | } |
| 122 | } |
| 123 | |
| 124 | // buildPlans is the pure half: routes plus their live manifests in, converge |
| 125 | // plans plus the advisory lines that fall out of reading them out. Extracted |
| 126 | // so every word an operator reads in the hourly log is table-testable without |
| 127 | // touching disk, systemd or the network. |
| 128 | // |
| 129 | // Three states, and they are not interchangeable (spec §4d): |
| 130 | // |
| 131 | // - load returns (nil, nil) — no live version, or no staged sibling for it: |
| 132 | // a plan with a NIL Set, which converge reads as "leave every drop-in |
| 133 | // exactly as it is" and logs once. Not a removal: a bucket hiccup must |
| 134 | // not strip :25 off a running mail server. |
| 135 | // - load returns an error (a sibling that exists but does not parse): the |
| 136 | // route gets NO plan at all — the same leave-alone outcome — plus one |
| 137 | // line naming what could not be read. |
| 138 | // - load returns a manifest: a plan whose Set is non-nil even when the |
| 139 | // manifest declares nothing, so that a capability DROPPED from a manifest |
| 140 | // is actually removed. Manifest.Capabilities is a nil pointer for both a |
| 141 | // legacy manifest and a fetch that never happened, so it is never passed |
| 142 | // through verbatim. |
| 143 | // |
| 144 | // An unparseable grants column refuses that route's converge and says so, |
| 145 | // rather than defaulting to no grants: "no grants" is a real instruction that |
| 146 | // drops :25, and a column this binary cannot read is not evidence that an |
| 147 | // operator asked for that. |
| 148 | func buildPlans(routes []registry.Route, load func(rt registry.Route) (*release.Manifest, error)) ([]caps.Plan, []string) { |
| 149 | var plans []caps.Plan |
| 150 | var advice []string |
| 151 | for _, rt := range routes { |
| 152 | unitRoute := release.CapabilityRoute(rt) |
| 153 | // The exec-backed candidate set is warnBackingMismatch's plus one |
| 154 | // condition: an instance route that is not a platform alias and is |
| 155 | // not unit-backed, AND that names an app — without an app there is no |
| 156 | // release, so nothing could declare a capability to advise about. It |
| 157 | // materializes nothing (release.CapabilityRoute excludes it) and is |
| 158 | // told so once per pass. |
| 159 | execCandidate := !unitRoute && rt.Kind == "instance" && rt.App != "" && !rt.Platform && rt.Backing != registry.BackingUnit |
| 160 | if !unitRoute && !execCandidate { |
| 161 | continue |
| 162 | } |
| 163 | |
| 164 | m, err := load(rt) |
| 165 | if err != nil { |
| 166 | // Two different consequences, so two different sentences: a unit |
| 167 | // route HAS drop-ins that are being left alone, an exec-backed |
| 168 | // route has none and never did — telling its operator that |
| 169 | // something is being left as it is would describe a file that |
| 170 | // does not exist. |
| 171 | if unitRoute { |
| 172 | advice = append(advice, fmt.Sprintf("adopt: %s: the staged manifest of its live version could not be read (%v) — its capability drop-ins are left exactly as they are until it can be", rt.Host, err)) |
| 173 | } else { |
| 174 | advice = append(advice, fmt.Sprintf("adopt: %s: the staged manifest of its live version could not be read (%v) — this pass cannot tell whether it declares capabilities; it is exec-backed, so nothing is materialized for it either way", rt.Host, err)) |
| 175 | } |
| 176 | continue |
| 177 | } |
| 178 | |
| 179 | if execCandidate { |
| 180 | if m != nil && m.Capabilities != nil { |
| 181 | advice = append(advice, fmt.Sprintf("adopt: route %s declares capabilities but is exec-backed — nothing is materialized; the row has to be stamped backing='unit' before any capability can be written (see the app-capabilities design §5 for the stop → copy → start migration)", rt.Host)) |
| 182 | } |
| 183 | continue |
| 184 | } |
| 185 | |
| 186 | g, err := caps.ParseGrants(rt.Caps) |
| 187 | if err != nil { |
| 188 | advice = append(advice, fmt.Sprintf("adopt: %s: %v — refusing to converge this route rather than guess which grants were meant; fix the row with `carlos route --host %s --grant/--revoke` from a known-good state", rt.Host, err, rt.Host)) |
| 189 | continue |
| 190 | } |
| 191 | |
| 192 | p := caps.Plan{Host: rt.Host, App: rt.App, Grants: g, Hibernate: rt.Hibernate} |
| 193 | if m == nil { |
| 194 | // Set stays nil: leave-alone, and converge logs the one line. |
| 195 | plans = append(plans, p) |
| 196 | continue |
| 197 | } |
| 198 | p.Set = m.Capabilities |
| 199 | if p.Set == nil { |
| 200 | p.Set = &caps.Set{V: caps.Version} |
| 201 | } |
| 202 | plans = append(plans, p) |
| 203 | advice = append(advice, ungrantedAdvice(rt, p.Set, g)...) |
| 204 | advice = append(advice, undeclaredGrantAdvice(rt, p.Set, g)...) |
| 205 | advice = append(advice, dataAdvice(rt, p.Set)...) |
| 206 | } |
| 207 | return plans, advice |
| 208 | } |
| 209 | |
| 210 | // ungrantedAdvice: one line per (route, operator-consented capability) that |
| 211 | // the app DECLARES and this box has not GRANTED. Not a failure and not an |
| 212 | // adoption failure — the drop-in simply does not carry it — so the line names |
| 213 | // the consequence and the exact command that fixes it. |
| 214 | // |
| 215 | // Deliberately silent for the self-service capabilities: `command`, `daemon`, |
| 216 | // `children`, `runtime_dir` and `data` need no grant, so there is nothing for |
| 217 | // an operator to do about them and a line would be noise. On a healthy fleet |
| 218 | // (every declared grant granted) this advisory produces nothing at all. |
| 219 | func ungrantedAdvice(rt registry.Route, s *caps.Set, g caps.Grants) []string { |
| 220 | var out []string |
| 221 | ports := append([]caps.Port(nil), s.Ports...) |
| 222 | sort.Slice(ports, func(i, j int) bool { return ports[i].Port < ports[j].Port }) |
| 223 | for _, decl := range ports { |
| 224 | if g.Ports[decl.Port] { |
| 225 | continue |
| 226 | } |
| 227 | out = append(out, fmt.Sprintf("adopt: %s declares capability %s but this box has not granted it — nothing is bound on :%d; grant it with: carlos route --host %s --grant %s", |
| 228 | rt.Host, caps.PortToken(decl.Port), decl.Port, rt.Host, caps.PortToken(decl.Port))) |
| 229 | } |
| 230 | if s.HostRole && !g.HostRole { |
| 231 | out = append(out, fmt.Sprintf("adopt: %s declares capability %s but this box has not granted it — the unit keeps DynamicUser and cannot use the box's instance role; grant it with: carlos route --host %s --grant %s", |
| 232 | rt.Host, caps.HostRoleToken, rt.Host, caps.HostRoleToken)) |
| 233 | } |
| 234 | if s.DocEngine && !g.DocEngine { |
| 235 | out = append(out, fmt.Sprintf("adopt: %s declares capability %s but this box has not granted it — the unit keeps MemoryDenyWriteExecute and the default memory tier, so a Chromium or LibreOffice engine will not start (and would be OOM-killed if it did); grant it with: carlos route --host %s --grant %s", |
| 236 | rt.Host, caps.DocEngineToken, rt.Host, caps.DocEngineToken)) |
| 237 | } |
| 238 | return out |
| 239 | } |
| 240 | |
| 241 | // undeclaredGrantAdvice is ungrantedAdvice's mirror, and the half that was |
| 242 | // missing (2026-08-03 review, M5): a token this box HAS granted that the live |
| 243 | // release does not declare. |
| 244 | // |
| 245 | // Not a refusal, and not a security hole — an operator-consented capability |
| 246 | // materializes only where the manifest asks for it, so an undeclared grant |
| 247 | // writes nothing and reaches nothing. What it IS is a lie in the registry: a |
| 248 | // row that reads "this box has handed :25 to keymail" while nothing on the box |
| 249 | // binds :25, which is exactly the state an operator looks at during an |
| 250 | // incident. It also names the two ways it happens — a grant staged for a |
| 251 | // release that was never shipped, and a release that DROPPED a capability it |
| 252 | // used to declare, which removes the drop-in and leaves the grant behind. |
| 253 | // |
| 254 | // Silent for a leave-alone route: this reads p.Set, which the caller only |
| 255 | // fills from a manifest it actually read, so a bucket hiccup cannot make this |
| 256 | // print "granted but not declared" about a release nobody could see. Sorted |
| 257 | // by port for the same reason ungrantedAdvice sorts — an hourly log line whose |
| 258 | // order shuffles per pass reads as churn. |
| 259 | func undeclaredGrantAdvice(rt registry.Route, s *caps.Set, g caps.Grants) []string { |
| 260 | declared := make(map[int]bool, len(s.Ports)) |
| 261 | for _, p := range s.Ports { |
| 262 | declared[p.Port] = true |
| 263 | } |
| 264 | ports := make([]int, 0, len(g.Ports)) |
| 265 | for port := range g.Ports { |
| 266 | if !declared[port] { |
| 267 | ports = append(ports, port) |
| 268 | } |
| 269 | } |
| 270 | sort.Ints(ports) |
| 271 | var out []string |
| 272 | for _, port := range ports { |
| 273 | out = append(out, fmt.Sprintf("adopt: %s is granted %s but the live release does not declare it — nothing is materialized for it; revoke it with `carlos route --host %s --revoke %s`, or ship a release that declares it", |
| 274 | rt.Host, caps.PortToken(port), rt.Host, caps.PortToken(port))) |
| 275 | } |
| 276 | if g.HostRole && !s.HostRole { |
| 277 | out = append(out, fmt.Sprintf("adopt: %s is granted %s but the live release does not declare it — the unit keeps DynamicUser and reaches nothing of the box's instance role; revoke it with `carlos route --host %s --revoke %s`, or ship a release that declares it", |
| 278 | rt.Host, caps.HostRoleToken, rt.Host, caps.HostRoleToken)) |
| 279 | } |
| 280 | if g.DocEngine && !s.DocEngine { |
| 281 | out = append(out, fmt.Sprintf("adopt: %s is granted %s but the live release does not declare it — the unit keeps the default resource tier and its normal egress; revoke it with `carlos route --host %s --revoke %s`, or ship a release that declares it", |
| 282 | rt.Host, caps.DocEngineToken, rt.Host, caps.DocEngineToken)) |
| 283 | } |
| 284 | return out |
| 285 | } |
| 286 | |
| 287 | // dataAdvice is what the DBPath stamp cannot say by stamping (spec §6). |
| 288 | // |
| 289 | // Route.DBPath is single-valued and data.dbs is a list, so only the first |
| 290 | // name (sorted) is replicated and the rest are named rather than quietly |
| 291 | // dropped. And a route whose app streams its own WAL keeps whatever DBPath an |
| 292 | // operator set — the platform never clears it: two replicators on one WAL |
| 293 | // fight over checkpointing and corrupt the replica, and deciding which one to |
| 294 | // switch off is the operator's call, not an hourly timer's. |
| 295 | func dataAdvice(rt registry.Route, s *caps.Set) []string { |
| 296 | if s.Data == nil { |
| 297 | return nil |
| 298 | } |
| 299 | var out []string |
| 300 | if s.Data.Replicate == caps.DataReplicatePlatform && len(s.Data.DBs) > 1 { |
| 301 | for _, extra := range s.Data.DBs[1:] { |
| 302 | out = append(out, fmt.Sprintf("adopt: %s declares data.dbs %q, but a route replicates exactly one database (DBPath is single-valued) — %q is NOT replicated by the box-wide litestream unit", |
| 303 | rt.Host, extra, extra)) |
| 304 | } |
| 305 | } |
| 306 | if s.Data.Replicate == caps.DataReplicateApp && rt.DBPath != "" { |
| 307 | out = append(out, fmt.Sprintf("adopt: %s declares data.replicate=%q — the app streams its own WAL — but its route still names db_path %s, so the box-wide litestream unit replicates it too; two replicators on one WAL fight over checkpointing and corrupt the replica. The platform will not clear the column for you: which replicator stops is an operator decision", |
| 308 | rt.Host, caps.DataReplicateApp, rt.DBPath)) |
| 309 | } |
| 310 | return out |
| 311 | } |
| 312 | |
| 313 | // stampDBPaths is how ops.FleetDBs learns a unit tenant's database, instead |
| 314 | // of depending on an operator having typed `-db` at `carlos add` — which the |
| 315 | // current code documents as "silently replicated by nobody" (spec §6). |
| 316 | // |
| 317 | // Only `data.replicate: "platform"` with a non-empty dbs list stamps, only |
| 318 | // dbs[0] AS DECLARED, and only when the value would actually change: this |
| 319 | // runs hourly, and a write per pass would be a registry heartbeat nobody |
| 320 | // asked for. |
| 321 | // |
| 322 | // dbs[0] as declared, NOT sorted (orchestrator ruling, 2026-08-03, |
| 323 | // overruling the plan): an author writing ["console.db", "audit.db"] means |
| 324 | // console.db is the one that matters, and lexicographic order would have |
| 325 | // replicated audit.db instead — the platform silently picking a different |
| 326 | // database than the app named. Declaration order is the app's own statement |
| 327 | // of which database is primary; nothing else in the manifest carries it. |
| 328 | // |
| 329 | // The overwrite rule is the sharp one, and it is deliberately conservative |
| 330 | // (2026-08-03 review, I3). An existing NON-EMPTY db_path this pass does not |
| 331 | // recognise — anything not already under this box's state root for this host |
| 332 | // — is an operator's `-db`, pointing at where the data lives RIGHT NOW. On a |
| 333 | // pre-migration route (design §5) that is /data/<host>.db, the file the |
| 334 | // box-wide replicator is streaming today; overwriting it with the |
| 335 | // post-migration path would point litestream at a file that does not exist |
| 336 | // yet and silently stop replicating a live database. So it is advised and |
| 337 | // left alone, and the stamp follows the operator's data copy. |
| 338 | // |
| 339 | // It is deliberately NOT gated on the route's converge succeeding. The state |
| 340 | // directory is the unchanged template's own StateDirectory=carlos-app/%i, so |
| 341 | // the path is right whether or not a drop-in was written this pass, and a |
| 342 | // route refused for an unrelated reason (a port another route already claims) |
| 343 | // must still have its database replicated — a capability refusal is not a |
| 344 | // reason to stop backing up somebody's data. |
| 345 | // |
| 346 | // The state root comes from the same Ops the converge used, so a test's temp |
| 347 | // dir is stamped rather than a production path a test never wrote to. |
| 348 | func stampDBPaths(reg *registry.Registry, routes []registry.Route, plans []caps.Plan, o caps.Ops) { |
| 349 | root := o.StateRoot |
| 350 | if root == "" { |
| 351 | root = caps.DefaultStateRoot |
| 352 | } |
| 353 | cur := make(map[string]string, len(routes)) |
| 354 | for _, rt := range routes { |
| 355 | cur[rt.Host] = rt.DBPath |
| 356 | } |
| 357 | for _, p := range plans { |
| 358 | if p.Set == nil || p.Set.Data == nil { |
| 359 | continue |
| 360 | } |
| 361 | d := p.Set.Data |
| 362 | if d.Replicate != caps.DataReplicatePlatform || len(d.DBs) == 0 { |
| 363 | continue |
| 364 | } |
| 365 | want := filepath.Join(root, p.Host, d.DBs[0]) |
| 366 | from := cur[p.Host] |
| 367 | if from == want { |
| 368 | continue |
| 369 | } |
| 370 | if from != "" && !ownStateDirPath(from, root, p.Host) { |
| 371 | log.Printf("adopt: %s carries db_path %s, which is not under this box's state directory for it; the manifest expects %s — leaving the operator's value alone, because repointing the replicator at a file the §5 data copy has not created yet would silently stop replicating a live database. The stamp follows the copy", |
| 372 | p.Host, from, want) |
| 373 | continue |
| 374 | } |
| 375 | if err := reg.SetDBPath(p.Host, from, want); err != nil { |
| 376 | log.Printf("adopt: %s: recording db_path %s: %v", p.Host, want, err) |
| 377 | continue |
| 378 | } |
| 379 | if from == "" { |
| 380 | log.Printf("adopt: %s: db_path recorded as %s from the app's data.dbs — the box-wide litestream unit replicates it from the next `carlos ops litestream` run", p.Host, want) |
| 381 | } else { |
| 382 | log.Printf("adopt: %s: db_path %s -> %s, following the app's data.dbs — the box-wide litestream unit picks the new file up on the next `carlos ops litestream` run", p.Host, from, want) |
| 383 | } |
| 384 | } |
| 385 | } |
| 386 | |
| 387 | // ownStateDirPath reports whether path is a file this box's own state |
| 388 | // directory for host owns — i.e. a value some pass of this function wrote, |
| 389 | // or would write. Compared as a cleaned path prefix so a trailing slash or a |
| 390 | // "." segment cannot dodge it, and never as a substring: /var/lib/carlos-app/ |
| 391 | // <host>-old/x must not read as this host's. |
| 392 | func ownStateDirPath(path, root, host string) bool { |
| 393 | dir := filepath.Clean(filepath.Join(root, host)) |
| 394 | rel, err := filepath.Rel(dir, filepath.Clean(path)) |
| 395 | return err == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) |
| 396 | } |
| 397 | |
amadan