| 1 | // Package host is the box side of the platform: `carlos bootstrap` provisions |
| 2 | // a fresh machine (units, dirs, service user), `carlos agent` runs the |
| 3 | // single-box role (edge + activator + release adoption in one process), and |
| 4 | // `carlos adopt` is one immediate adoption pass for the timer unit. |
| 5 | // |
| 6 | // Bootstrap ported from Eleven's bootstrap.go (elevenmessenger/messenger: |
| 7 | // bootstrap.go), radically slimmed: carlos has one long-running unit (the |
| 8 | // edge/agent) plus timers — no sibling-service zoo, no collectors, no SSH |
| 9 | // hardening verbs (those stay app-level concerns). |
| 10 | package host |
| 11 | |
| 12 | import ( |
| 13 | "github.com/carlosframework/platform/internal/cliflag" |
| 14 | |
| 15 | "bytes" |
| 16 | "embed" |
| 17 | "encoding/json" |
| 18 | "errors" |
| 19 | "flag" |
| 20 | "fmt" |
| 21 | "io" |
| 22 | "log" |
| 23 | "os" |
| 24 | "os/exec" |
| 25 | "os/user" |
| 26 | "path/filepath" |
| 27 | "strconv" |
| 28 | "strings" |
| 29 | ) |
| 30 | |
| 31 | // The systemd units ship inside the binary (source of truth: |
| 32 | // internal/host/units), so `carlos bootstrap` can provision a fresh box with |
| 33 | // no separate files to copy. Ported from Eleven's deploy/systemd (renamed, |
| 34 | // same hardening) and titogo's deploy/systemd (litestream + restore-verify). |
| 35 | // |
| 36 | //go:embed units/*.service units/*.timer units/*.socket units/*.path units/*.slice units/*.mount |
| 37 | var unitsFS embed.FS |
| 38 | |
| 39 | // serviceUser is the unprivileged account the edge unit runs as; bootstrap |
| 40 | // creates it and hands it the data + runtime dirs. The edge keeps |
| 41 | // CAP_NET_BIND_SERVICE to still bind :80/:443. |
| 42 | const serviceUser = "carlos" |
| 43 | |
| 44 | // socketsGroup is the blessed cross-user consumers group for exec-backed |
| 45 | // instance sockets (issue #103): a host app's own process (titogo's router |
| 46 | // was the finding) enrolled here may dial every instance socket on the box. |
| 47 | // A DEDICATED group, never serviceUser's own: group `carlos` is also the |
| 48 | // group on /data (0750), so enrolling a consumer there would grant |
| 49 | // group-read of every instance database and env file — every |
| 50 | // CARLOS_ADMIN_TOKEN on the box — when the grant's whole meaning is "may |
| 51 | // dial sockets". |
| 52 | const socketsGroup = "carlos-sockets" |
| 53 | |
| 54 | // tmpfilesConf keeps /run/carlos across reboots (/run is tmpfs). Owned by the |
| 55 | // service user so the non-root units can bind their sockets there; group is |
| 56 | // the consumers group with SETGID so every socket an exec child binds inside |
| 57 | // inherits it — that inheritance is what lets the activator's post-health |
| 58 | // 0660 normalization (backend_exec.go) grant exactly the enrolled consumers |
| 59 | // and nobody else, with no app aware the group exists. The apps subdir holds |
| 60 | // the carlos-app@ tenant sockets: root systemd binds them (SocketUser=carlos, |
| 61 | // 0600), tenants never need path access (the fd is passed in) — it stays |
| 62 | // carlos:carlos, so consumer enrollment deliberately does not reach unit |
| 63 | // tenants. other=0 on both dirs keeps every unenrolled uid out. |
| 64 | const tmpfilesConf = "d /data/.parked 2770 " + serviceUser + " " + serviceUser + " -\n" + |
| 65 | "d /run/carlos 2750 " + serviceUser + " " + socketsGroup + " -\n" + |
| 66 | "d /run/carlos/apps 0750 " + serviceUser + " " + serviceUser + " -\n" + |
| 67 | "d /run/carlos/hostguard 2770 " + serviceUser + " " + serviceUser + " -\n" + |
| 68 | "d /data/hostguard 2770 " + serviceUser + " " + serviceUser + " -\n" + |
| 69 | // The isolated-exec trees (design §3.1, §3.2). Each protected view is |
| 70 | // a nosymfollow bind of the dotted raw directory beside it (the mount |
| 71 | // units); tmpfiles creates both every boot because /run is tmpfs and |
| 72 | // because the mounts are ordered after it. The socket trees keep the |
| 73 | // consumers group and setgid, exactly like /run/carlos above, so an |
| 74 | // enrolled router can still dial a migrated instance (issue #103). |
| 75 | "d /data/.instances 0700 " + serviceUser + " " + serviceUser + " -\n" + |
| 76 | // /data/instances is a nosymfollow bind mount OF /data/.instances, so |
| 77 | // once the mount is active the two paths name ONE inode: a mode that |
| 78 | // differed between the two lines would flip back and forth on every |
| 79 | // tmpfiles run, since each write lands on the same inode the other |
| 80 | // line just set. Both are 0700 so re-runs converge instead of racing. |
| 81 | "d /data/instances 0700 " + serviceUser + " " + serviceUser + " -\n" + |
| 82 | "d /data/layout 0700 " + serviceUser + " " + serviceUser + " -\n" + |
| 83 | // The relay's one writable state dir (carlos-relay.service's |
| 84 | // ReadWritePaths). It is in bootstrap's dirs list too, but that list |
| 85 | // only ever runs against a FRESH box or a staging root — a self-update |
| 86 | // apply creates no directories, it re-runs tmpfiles. Without this line |
| 87 | // a box bootstrapped before the relay existed gets the unit, the |
| 88 | // binary and the master file and still fails to start with "Failed to |
| 89 | // set up mount namespacing: /data/relay: No such file or directory" |
| 90 | // (discussion #118, hit on the Ireland box 2026-09-07). |
| 91 | "d /data/relay 0750 " + serviceUser + " " + serviceUser + " -\n" + |
| 92 | "d /run/carlos/.instances 2750 " + serviceUser + " " + socketsGroup + " -\n" + |
| 93 | "d /run/carlos/instances 2750 " + serviceUser + " " + socketsGroup + " -\n" + |
| 94 | // The readiness probe's own instance directory (design §8.5): a name |
| 95 | // no host can have, created every boot so the probe unit's bind |
| 96 | // sources exist before it starts. |
| 97 | "d /data/.instances/_probe 0700 " + serviceUser + " " + serviceUser + " -\n" + |
| 98 | "d /run/carlos/.instances/_probe 2750 " + serviceUser + " " + socketsGroup + " -\n" + |
| 99 | "d /run/carlos/.logs 0700 " + serviceUser + " " + serviceUser + " -\n" + |
| 100 | "d /run/carlos/logs 0700 " + serviceUser + " " + serviceUser + " -\n" + |
| 101 | // The root instance-control channel's directory (design §5.2): root's, |
| 102 | // so no carlos process can create, rename or replace anything in it; |
| 103 | // group carlos with search so the agent can connect. |
| 104 | "d /run/carlos-control 0750 root " + serviceUser + " -\n" + |
| 105 | // The readiness probe's fixture (design §8.5, codex R11-1): a file |
| 106 | // root:carlos 0640 that an isolated instance must NOT be able to read. |
| 107 | // Written here, not by the control service, whose ProtectSystem=strict |
| 108 | // makes /etc read-only to it. |
| 109 | "f /etc/carlos/probe-secret 0640 root " + serviceUser + " - probe\n" + |
| 110 | // And one in the host's /dev/shm, which an isolated instance must NOT |
| 111 | // see: its own /dev/shm is a private tmpfs (#126). Without a file |
| 112 | // here, an empty listing inside the probe would prove nothing. |
| 113 | "f /dev/shm/carlos-probe 0644 root root - probe\n" |
| 114 | |
| 115 | // journaldConfName is the journald drop-in every box gets (discussion #111). |
| 116 | // |
| 117 | // The name matches the one placed by hand on the flagship during the |
| 118 | // 2026-09-06 relief, and must keep matching it: drop-ins are read in name |
| 119 | // order and the LAST assignment wins, so a differently-named file would sit |
| 120 | // beside the hand-placed one and whichever sorted later would decide the cap. |
| 121 | // Same name means this file supersedes it on every box that has it, and the |
| 122 | // hand-placed copy can be deleted or left alone with the same result. |
| 123 | const journaldConfName = "50-cap.conf" |
| 124 | |
| 125 | // journaldConf caps the journal. Uncapped, journald's default is 10% of the |
| 126 | // filesystem — 2 GB of a 20 GB root, of which the flagship had spent 1.2 GB |
| 127 | // by the time it filled. A cap is not a fix for a chatty box, it is the |
| 128 | // difference between a chatty box losing old log lines and a chatty box |
| 129 | // losing the ability to accept a push. |
| 130 | // |
| 131 | // Both halves are set explicitly. SystemMaxUse governs /var/log/journal, the |
| 132 | // persistent journal, which is the one that filled; RuntimeMaxUse governs |
| 133 | // /run/log/journal, which is tmpfs — so its default 10% is 10% of RAM, and on |
| 134 | // a small box a journal storm there is memory pressure rather than disk. One |
| 135 | // drop-in that leaves the volatile half at the default would have to be |
| 136 | // reasoned about again the first time that matters. |
| 137 | const journaldConf = "# Written by carlos bootstrap (discussion #111). Do not edit by hand:\n" + |
| 138 | "# the next bootstrap or self-update apply rewrites this file.\n" + |
| 139 | "[Journal]\n" + |
| 140 | "SystemMaxUse=256M\n" + |
| 141 | "RuntimeMaxUse=64M\n" |
| 142 | |
| 143 | // enabledUnits are what a live bootstrap turns on: the always-on front door, |
| 144 | // the hourly adoption timer, and the path units that let the sandboxed |
| 145 | // agent ask a root pass for the halves it cannot do itself: carlos-adopt.path |
| 146 | // (config fast convergence, issue #44), carlos-stage.path (release staging |
| 147 | // — the agent sees a promotion in seconds but cannot write /opt), |
| 148 | // carlos-restart.path (a member's `carlos restart`, issue #113 — the agent |
| 149 | // bounces its own exec children but cannot cycle a unit) and |
| 150 | // carlos-system-update.path (the platform self-update relay, spec §4 — the |
| 151 | // agent can fetch and verify a candidate but cannot write /opt or /etc). |
| 152 | // carlos-system-recover.service is enabled (not armed by a path — it also |
| 153 | // runs Before=carlos-edge.service on every boot, and on the update oneshot's |
| 154 | // own OnFailure=) so recovery from an interrupted self-update never depends |
| 155 | // on a marker file surviving a crash. The litestream + restore-verify units install |
| 156 | // everywhere but stay inert until an operator stages /data/litestream.env and |
| 157 | // generates /etc/litestream.yml (`carlos ops litestream`) — a box with no |
| 158 | // replication bucket is unaffected. carlos-relay.service (relay spec §3) is |
| 159 | // enabled AND started fleet-wide, unlike the edge: its |
| 160 | // ConditionPathExists=/etc/carlos/relay-masters makes the start a no-op on |
| 161 | // every box the operator has not armed with a region master. |
| 162 | // |
| 163 | // relayUnitName is spelled once because two places must agree on it: this |
| 164 | // list and the systemctl plan's start set. |
| 165 | const relayUnitName = "carlos-relay.service" |
| 166 | |
| 167 | var enabledUnits = []string{"carlos-edge.service", relayUnitName, "carlos-adopt.timer", "carlos-adopt.path", "carlos-stage.path", "carlos-restart.path", |
| 168 | "carlos-system-update.path", "carlos-system-recover.service", |
| 169 | // Region HA joining (spec 2 §5 step 5.3): root's half, on the agent's |
| 170 | // request. Idle on every box that is not joining. |
| 171 | "carlos-system-join.path", |
| 172 | // The isolated-exec mounts (design §3.2): the edge Requires= them. |
| 173 | "data-instances.mount", "run-carlos-instances.mount", "run-carlos-logs.mount", |
| 174 | // The root instance-control channel (design §5.2); socket-activated, so |
| 175 | // only the socket is enabled and started. |
| 176 | "carlos-instance-control.socket"} |
| 177 | |
| 178 | // inertUnits are installed everywhere and deliberately never armed by |
| 179 | // bootstrap or by a self-update apply: restore-verify until an operator |
| 180 | // stages litestream, hibernate-verify by long-standing choice. |
| 181 | var inertUnits = []string{"carlos-restore-verify.timer", "carlos-hibernate-verify.timer"} |
| 182 | |
| 183 | // bootstrapPlan is what `bootstrap -root <dir> -plan-json` prints: the |
| 184 | // arming list of THIS binary, so a self-update apply arms what the new |
| 185 | // binary's bootstrap would arm and never a list an older binary remembers. |
| 186 | // |
| 187 | // Armed is the other half of that promise, and it was missing until |
| 188 | // discussion #118. Enabled alone says which units to `enable`; WHICH of them |
| 189 | // also get a `start` was decided by the applier's own systemctlPlan — code |
| 190 | // from the OLD binary, which cannot know about a unit the new build just |
| 191 | // introduced. That is how the relay landed on the Ireland box enabled but |
| 192 | // never started on 2026-09-07: the applying build predated |
| 193 | // carlos-relay.service, so its systemctlPlan had no reason to start it. |
| 194 | // Carrying the list the TARGET build would start closes that gap for every |
| 195 | // future unit, not just the relay. |
| 196 | // |
| 197 | // Additive on purpose: a plan.json written by a build older than this field |
| 198 | // (a rollback stages carlos.prev's plan) carries no "armed" key, and arm() |
| 199 | // then decides locally exactly as it always did. The key is NOT omitempty: |
| 200 | // nil ("this build is too old to say") and empty ("start nothing") are |
| 201 | // different answers, and omitempty would collapse them into the same absent |
| 202 | // key, making "start nothing" impossible for a build to express. |
| 203 | type bootstrapPlan struct { |
| 204 | Enabled []string `json:"enabled"` |
| 205 | Inert []string `json:"inert"` |
| 206 | Armed []string `json:"armed"` |
| 207 | } |
| 208 | |
| 209 | func currentBootstrapPlan() bootstrapPlan { |
| 210 | return bootstrapPlan{ |
| 211 | Enabled: append([]string(nil), enabledUnits...), |
| 212 | Inert: append([]string(nil), inertUnits...), |
| 213 | Armed: armedUnits(enabledUnits), |
| 214 | } |
| 215 | } |
| 216 | |
| 217 | // armedUnits is the "and start these too" rule in one place: the unit types |
| 218 | // whose enable-without-start is a silent no-op until the next boot, plus |
| 219 | // carlos-relay.service, whose ConditionPathExists makes starting it safe on a |
| 220 | // box with no region master. See systemctlPlan for the long version. |
| 221 | func armedUnits(units []string) []string { |
| 222 | var armed []string |
| 223 | for _, u := range units { |
| 224 | if strings.HasSuffix(u, ".timer") || strings.HasSuffix(u, ".path") || |
| 225 | strings.HasSuffix(u, ".socket") || strings.HasSuffix(u, ".mount") || u == relayUnitName { |
| 226 | armed = append(armed, u) |
| 227 | } |
| 228 | } |
| 229 | return armed |
| 230 | } |
| 231 | |
| 232 | // systemctlPlan is the exact systemctl sequence a live bootstrap runs — and |
| 233 | // the same sequence a staging run (-root <dir>) PRINTS instead of running, so |
| 234 | // the two can never drift into describing different behaviour. |
| 235 | // |
| 236 | // Timers and path units get an explicit `start` after the `enable`, and that |
| 237 | // line is the point of this function: `systemctl enable` writes a .wants |
| 238 | // symlink and stops there, which arms the unit at the NEXT boot and not a |
| 239 | // moment sooner. carlos-adopt.timer sat enabled-but-inactive on the flagship |
| 240 | // for weeks for exactly this reason — symlink present, nothing rebooted, no |
| 241 | // release ever adopted itself, every deploy silently needing a hand-run |
| 242 | // adoption pass. A path unit fails the same way and just as quietly: an |
| 243 | // unstarted carlos-adopt.path watches nothing, so the agent's config-sync |
| 244 | // marker lands in a file no one is reading. A re-run is harmless: starting an |
| 245 | // already-running timer or path unit is a no-op, which keeps bootstrap |
| 246 | // idempotent. |
| 247 | // |
| 248 | // Services are deliberately NOT started, with one exception. carlos-edge |
| 249 | // needs its binary at /opt/carlos/carlos and /etc/carlos/host.env staged |
| 250 | // first — neither of which bootstrap can do for the operator, which is why |
| 251 | // it ends by telling them to. Starting timers is safe in a way starting |
| 252 | // services is not: a timer that fires before the binary lands just fails one |
| 253 | // adoption pass and tries again on the next tick. |
| 254 | // |
| 255 | // carlos-relay.service is the exception (relay spec §3). It carries |
| 256 | // ConditionPathExists=/etc/carlos/relay-masters, so on a box the operator |
| 257 | // has not armed with a region master the start is a no-op systemd records as |
| 258 | // a failed condition, not a failure — which is what makes starting it safe |
| 259 | // fleet-wide. On a box that IS armed, starting it here is what saves a |
| 260 | // reboot. |
| 261 | // |
| 262 | // Socket and mount units are armed the same way (isolated-exec design |
| 263 | // §8.6, codex R1-14): an enabled-but-unstarted socket is a control channel |
| 264 | // nobody is listening on, and an enabled-but-unmounted bind is an instance |
| 265 | // tree with no protection, until the next reboot. Starting either is |
| 266 | // idempotent. |
| 267 | func systemctlPlan(units []string) [][]string { |
| 268 | return systemctlPlanArming(units, armedUnits(units)) |
| 269 | } |
| 270 | |
| 271 | // systemctlPlanArming is systemctlPlan with the start list supplied rather |
| 272 | // than derived — the shape a self-update apply needs, where the units come |
| 273 | // from the target build's plan.json and so does the decision about which of |
| 274 | // them to start (discussion #118). |
| 275 | func systemctlPlanArming(units, armed []string) [][]string { |
| 276 | plan := [][]string{ |
| 277 | {"daemon-reload"}, |
| 278 | append([]string{"enable"}, units...), |
| 279 | } |
| 280 | if len(armed) > 0 { |
| 281 | plan = append(plan, append([]string{"start"}, armed...)) |
| 282 | } |
| 283 | return plan |
| 284 | } |
| 285 | |
| 286 | // journaldReload is the one systemctl call that makes a freshly written |
| 287 | // journald drop-in take effect. Kept out of systemctlPlan on purpose: that |
| 288 | // plan is "enable these units and arm the triggers among them", and it is |
| 289 | // also what the self-update relay's arm step and ensureSystemUnits run over |
| 290 | // narrow unit lists — neither of which should be bouncing the box's logging |
| 291 | // daemon. Both places that install the drop-in call this explicitly instead. |
| 292 | var journaldReload = []string{"restart", "systemd-journald"} |
| 293 | |
| 294 | // RunBootstrap is the `carlos bootstrap` verb. Idempotent — safe to re-run |
| 295 | // (which is also how a box enrolls a new socket consumer after the fact). |
| 296 | func RunBootstrap(args []string) { |
| 297 | fs := flag.NewFlagSet("bootstrap", flag.ExitOnError) |
| 298 | root := fs.String("root", "/", "install root; a staging dir writes files only (no user creation, no systemctl)") |
| 299 | offcloud := fs.Bool("offcloud", false, "off-cloud box: require a staged 0600 credentials file instead of an EC2 instance profile") |
| 300 | planJSON := fs.Bool("plan-json", false, "with -root: print the arming plan as JSON on stdout instead of the human summary") |
| 301 | var consumers repeatableFlag |
| 302 | fs.Var(&consumers, "socket-consumer", "host-app user to enroll in the "+socketsGroup+" group (repeatable); members may dial every instance socket on the box") |
| 303 | fs.Usage = cliflag.Usage(fs, "") |
| 304 | _ = fs.Parse(args) |
| 305 | if *planJSON && filepath.Clean(*root) == "/" { |
| 306 | log.Fatal("bootstrap: -plan-json is for a staging root") |
| 307 | } |
| 308 | if *offcloud { |
| 309 | if err := bootstrapOffcloud(*root, consumers); err != nil { |
| 310 | log.Fatalf("bootstrap: %v", err) |
| 311 | } |
| 312 | return |
| 313 | } |
| 314 | if err := bootstrapWith(*root, consumers, *planJSON); err != nil { |
| 315 | log.Fatalf("bootstrap: %v", err) |
| 316 | } |
| 317 | } |
| 318 | |
| 319 | // repeatableFlag collects every occurrence of a string flag. |
| 320 | type repeatableFlag []string |
| 321 | |
| 322 | func (f *repeatableFlag) String() string { return strings.Join(*f, ",") } |
| 323 | func (f *repeatableFlag) Set(v string) error { |
| 324 | *f = append(*f, v) |
| 325 | return nil |
| 326 | } |
| 327 | |
| 328 | // bootstrapOffcloud is bootstrap for a box with no EC2 instance profile |
| 329 | // (issue #53 point 4): identical structural work, plus the credential |
| 330 | // preconditions — a staged 0600 credentials file scoped (at IAM) to the |
| 331 | // deployment bucket + this host's own region bucket and nothing else, so |
| 332 | // wrong-region-fails-at-IAM holds off-cloud too. The flag is explicit; no |
| 333 | // IMDS sniffing (the agent's one IMDS probe is already gated for non-EC2). |
| 334 | // |
| 335 | // Credential shape and mode checks precede bootstrap mutations. A rejected |
| 336 | // credential precondition leaves the box untouched. |
| 337 | func bootstrapOffcloud(root string, consumers []string) error { |
| 338 | cred := filepath.Join(root, "etc", "carlos", "credentials") |
| 339 | fi, err := checkOffcloudCredentials(cred) |
| 340 | if err != nil { |
| 341 | return fmt.Errorf("offcloud: stage the scoped credentials file first: %s (0600, see infra/remote-host.md): %w", cred, err) |
| 342 | } |
| 343 | if fi.Mode().Perm() != 0o600 && fi.Mode().Perm() != 0o640 { |
| 344 | return fmt.Errorf("offcloud: %s must be mode 0600 or 0640, is %o", cred, fi.Mode().Perm()) |
| 345 | } |
| 346 | if err := bootstrap(root, consumers); err != nil { |
| 347 | return err |
| 348 | } |
| 349 | fmt.Printf("offcloud bootstrap complete.\n ensure /etc/carlos/host.env sets:\n AWS_SHARED_CREDENTIALS_FILE=/etc/carlos/credentials\n then stage /opt/carlos/carlos and start carlos-edge.service (see infra/remote-host.md)\n") |
| 350 | return nil |
| 351 | } |
| 352 | |
| 353 | // bootstrap provisions (or re-converges) a box under root: units into |
| 354 | // <root>/etc/systemd/system, the standard dirs, the tmpfiles.d entry, and — |
| 355 | // only when root is "/" — the service user, ownership, and the systemctl |
| 356 | // sequence (daemon-reload, enable, and arming the timers; see systemctlPlan). |
| 357 | // A non-"/" root is the staging/test path: files only, with the privileged |
| 358 | // steps printed instead of run. |
| 359 | func bootstrap(root string, consumers []string) error { |
| 360 | return bootstrapWith(root, consumers, false) |
| 361 | } |
| 362 | |
| 363 | // bootstrapWith is bootstrap's body, plus the -plan-json staging path: when |
| 364 | // planJSON is set (staging root only — RunBootstrap refuses it with -root /), |
| 365 | // every "wrote"/"created" line below is redirected to stderr so the JSON |
| 366 | // plan currentBootstrapPlan() prints is alone on stdout. |
| 367 | func bootstrapWith(root string, consumers []string, planJSON bool) error { |
| 368 | live := filepath.Clean(root) == "/" |
| 369 | hostEnv := filepath.Join(root, "etc", "carlos", "host.env") |
| 370 | if live { |
| 371 | hostEnv = claimBootstrapHostEnv |
| 372 | } |
| 373 | claimSettings, settingsErr := readClaimUnitSettings(hostEnv) |
| 374 | if settingsErr != nil { |
| 375 | fmt.Fprintf(os.Stderr, "bootstrap: instance-control unit settings: %v\n", settingsErr) |
| 376 | claimSettings = claimUnitSettings{} |
| 377 | } |
| 378 | beforeControl, err := os.ReadFile(filepath.Join(root, "etc", "systemd", "system", claimControlUnit)) |
| 379 | if err != nil && !errors.Is(err, os.ErrNotExist) { |
| 380 | return err |
| 381 | } |
| 382 | // A host.env this build cannot parse says nothing about the claim |
| 383 | // (6b-N2). A control unit already rendered for an enabled box stays as |
| 384 | // it is, unrestarted, as adopt leaves it; a flag-off one is rendered as |
| 385 | // main renders it. |
| 386 | keepControl := settingsErr != nil && bytes.Contains(beforeControl, []byte(claimControlEnvironment)) |
| 387 | var afterControl []byte |
| 388 | out := io.Writer(os.Stdout) |
| 389 | if planJSON { |
| 390 | out = os.Stderr |
| 391 | } |
| 392 | if live { |
| 393 | // Validate consumer users BEFORE any mutation (the offcloud |
| 394 | // precondition's discipline): a typo in -socket-consumer must leave |
| 395 | // the box untouched, not abort bootstrap halfway with units written |
| 396 | // but timers never armed. |
| 397 | for _, u := range consumers { |
| 398 | if _, err := user.Lookup(u); err != nil { |
| 399 | return fmt.Errorf("--socket-consumer %s: no such user (create the host app's user first): %w", u, err) |
| 400 | } |
| 401 | } |
| 402 | } |
| 403 | unitDir := filepath.Join(root, "etc", "systemd", "system") |
| 404 | dataDir := filepath.Join(root, "data") |
| 405 | runDir := filepath.Join(root, "run", "carlos") |
| 406 | |
| 407 | type dir struct { |
| 408 | path string |
| 409 | mode os.FileMode |
| 410 | } |
| 411 | dirs := []dir{ |
| 412 | {dataDir, 0o750}, |
| 413 | {filepath.Join(dataDir, "acme"), 0o750}, |
| 414 | // The relay's one writable state dir (carlos-relay.service's |
| 415 | // ReadWritePaths). Created on every box, armed or not, so the unit |
| 416 | // never fails for want of a directory the moment a master lands. |
| 417 | {filepath.Join(dataDir, "relay"), 0o750}, |
| 418 | // Setgid: sockets bound inside inherit the dir's group — the |
| 419 | // consumers group, once a live run (or the tmpfiles line) sets it. |
| 420 | // See socketsGroup/tmpfilesConf. |
| 421 | {runDir, 0o750 | os.ModeSetgid}, |
| 422 | // The carlos-app@ tenant sockets (see tmpfilesConf for the perms |
| 423 | // rationale). |
| 424 | {filepath.Join(runDir, "apps"), 0o750}, |
| 425 | // The isolated-exec trees (design §3.1). The dotted names are the |
| 426 | // raw directories the mount units bind onto the undotted views; |
| 427 | // see tmpfilesConf for the modes and why both exist. |
| 428 | {filepath.Join(dataDir, ".instances"), 0o700}, |
| 429 | // One inode once the bind is active (see tmpfilesConf): both this |
| 430 | // entry and the tmpfiles line for /data/instances must agree, or |
| 431 | // the mode flips on every tmpfiles run. |
| 432 | {filepath.Join(dataDir, "instances"), 0o700}, |
| 433 | {filepath.Join(dataDir, "layout"), 0o700}, |
| 434 | {filepath.Join(runDir, ".instances"), 0o750 | os.ModeSetgid}, |
| 435 | {filepath.Join(runDir, "instances"), 0o750 | os.ModeSetgid}, |
| 436 | // The readiness probe's own instance directory (design §8.5): a |
| 437 | // name no host can have, created here so a staging root has it |
| 438 | // too (see tmpfilesConf for the modes and why both exist). |
| 439 | {filepath.Join(dataDir, ".instances", "_probe"), 0o700}, |
| 440 | {filepath.Join(runDir, ".instances", "_probe"), 0o750 | os.ModeSetgid}, |
| 441 | {filepath.Join(runDir, ".logs"), 0o700}, |
| 442 | {filepath.Join(runDir, "logs"), 0o700}, |
| 443 | // Root-owned on a live box (privilegedPlan chowns /run/carlos |
| 444 | // recursively to the service user; this one sits BESIDE it, not |
| 445 | // under it, precisely so that chown never reaches it). |
| 446 | {filepath.Join(root, "run", "carlos-control"), 0o750}, |
| 447 | // The version store + per-host live symlinks (release/versions.go). |
| 448 | // World-readable/executable so sandboxed instance units can exec their |
| 449 | // binary; only root (the adopt pass) writes here. |
| 450 | {filepath.Join(root, "opt", "carlos", "versions"), 0o755}, |
| 451 | {filepath.Join(root, "opt", "carlos", "live"), 0o755}, |
| 452 | // Host-wide config (host.env, 0600 root-owned, staged by the operator). |
| 453 | {filepath.Join(root, "etc", "carlos"), 0o755}, |
| 454 | // Per-app env files for carlos-app@ tenants (systemd reads them as |
| 455 | // root before dropping to the unit's DynamicUser, so 0600 root works). |
| 456 | {filepath.Join(root, "etc", "carlos", "apps"), 0o755}, |
| 457 | {unitDir, 0o755}, |
| 458 | // The journal cap's drop-in dir. systemd creates it on demand for a |
| 459 | // hand-placed file, but nothing creates it on a fresh box, and the |
| 460 | // write below would fail ENOENT if this line were missing. |
| 461 | {filepath.Join(root, "etc", "systemd", "journald.conf.d"), 0o755}, |
| 462 | {filepath.Join(root, "etc", "tmpfiles.d"), 0o755}, |
| 463 | {filepath.Join(root, "etc", "sysusers.d"), 0o755}, |
| 464 | } |
| 465 | for _, d := range dirs { |
| 466 | if err := bootstrapMakeDir(d.path, d.mode); err != nil { |
| 467 | return fmt.Errorf("mkdir/chmod %s: %v", d.path, err) |
| 468 | } |
| 469 | } |
| 470 | |
| 471 | // The registry file, created here and by nothing else on a fresh box. |
| 472 | // |
| 473 | // It has to exist BEFORE the units below are armed, because four of them |
| 474 | // (carlos-adopt, -stage, -restart, -config-sync) run `carlos adopt` as |
| 475 | // root and open this same file — and on a fresh box adopt's .path and |
| 476 | // .timer can fire before the edge has ever started. Whoever creates the |
| 477 | // file owns it, and an edge running as serviceUser cannot write a |
| 478 | // root-owned registry: that is discussion #8, three region boxes left |
| 479 | // unstartable on 2026-08-31. Creating it here puts it inside the |
| 480 | // descriptor-based ownership walk that privilegedPlan runs, so it is |
| 481 | // serviceUser's before anything can race for it. |
| 482 | // |
| 483 | // An empty file is a valid empty SQLite database — the first opener |
| 484 | // (edge or adopt) runs the migrations into it. Deliberately NOT opened |
| 485 | // as a registry here: on a re-run over a live box this must not touch |
| 486 | // the schema of a database the running edge holds open, and "the file |
| 487 | // exists, owned by the right user" is the whole job. |
| 488 | // |
| 489 | // One constraint this puts on the deploy side: a STAGING run takes this |
| 490 | // branch too. `carlos bootstrap -root <dir>` is how a live box refreshes |
| 491 | // its unit files (pinfra's update-platform-binary.sh stages into |
| 492 | // /tmp/carlos-unitstage), so a staging tree now contains a 0-byte |
| 493 | // data/registry.db. Harmless while that script copies only etc/ out of |
| 494 | // the tree — and destructive the day anything rsyncs the whole tree over |
| 495 | // /, because a 0-byte file would land on top of the live registry and |
| 496 | // empty it. The script lives in the private platform-infrastructure repo; |
| 497 | // whoever widens what it copies has to exclude data/. |
| 498 | regPath := filepath.Join(dataDir, "registry.db") |
| 499 | created, err := bootstrapCreateRegistry(regPath) |
| 500 | if err != nil { |
| 501 | return fmt.Errorf("create %s: %v", regPath, err) |
| 502 | } |
| 503 | if created { |
| 504 | fmt.Fprintf(out, " created %s\n", regPath) |
| 505 | } |
| 506 | |
| 507 | names, err := unitNames() |
| 508 | if err != nil { |
| 509 | return err |
| 510 | } |
| 511 | for _, name := range names { |
| 512 | content, err := unitsFS.ReadFile("units/" + name) |
| 513 | if err != nil { |
| 514 | return fmt.Errorf("read embedded unit %s: %v", name, err) |
| 515 | } |
| 516 | path := filepath.Join(unitDir, name) |
| 517 | if name == claimControlUnit { |
| 518 | if keepControl { |
| 519 | afterControl = beforeControl |
| 520 | fmt.Fprintf(out, " kept %s (host.env cannot be parsed)\n", path) |
| 521 | continue |
| 522 | } |
| 523 | afterControl = claimUnitContent(name, content, claimSettings) |
| 524 | } |
| 525 | if err := os.WriteFile(path, claimUnitContent(name, content, claimSettings), 0o644); err != nil { |
| 526 | return fmt.Errorf("write %s: %v", path, err) |
| 527 | } |
| 528 | fmt.Fprintf(out, " wrote %s\n", path) |
| 529 | } |
| 530 | |
| 531 | // On a live root the privileged plan runs BEFORE the tmpfiles conf is |
| 532 | // written: the conf's /run/carlos line names socketsGroup, and |
| 533 | // systemd-tmpfiles DROPS a line whose group doesn't resolve — so a |
| 534 | // bootstrap interrupted between "conf written" and "groupadd ran" would |
| 535 | // leave a box that looks healthy until its next reboot, when /run/carlos |
| 536 | // simply isn't created and carlos-edge (ReadWritePaths=/run/carlos, no |
| 537 | // "-" prefix) fails its namespace setup forever. Group first, then the |
| 538 | // conf that names it. |
| 539 | if live { |
| 540 | for _, args := range privilegedPlan(consumers) { |
| 541 | if err := runPrivileged(args); err != nil { |
| 542 | return err |
| 543 | } |
| 544 | } |
| 545 | } |
| 546 | |
| 547 | // The sysusers.d snippet closes the same hole for confs that arrive |
| 548 | // WITHOUT a live bootstrap run: a staging root is how a live box |
| 549 | // refreshes its unit files (pinfra's update-platform-binary.sh), and a |
| 550 | // refresh that installs the staged etc/ tree ships the tmpfiles line to |
| 551 | // a box that never ran groupadd. systemd-sysusers.service is ordered |
| 552 | // Before=systemd-tmpfiles-setup.service, so the group exists by the time |
| 553 | // the tmpfiles line is parsed, however the conf got there. |
| 554 | sysusersPath := filepath.Join(root, "etc", "sysusers.d", "carlos.conf") |
| 555 | if err := os.WriteFile(sysusersPath, []byte("g "+socketsGroup+" -\n"), 0o644); err != nil { |
| 556 | return fmt.Errorf("write %s: %v", sysusersPath, err) |
| 557 | } |
| 558 | fmt.Fprintf(out, " wrote %s\n", sysusersPath) |
| 559 | |
| 560 | // The journal cap (discussion #111). Written on every root, live or |
| 561 | // staging, because a staging root is how a live box refreshes its etc/ |
| 562 | // tree — both the self-update apply (systemapply.go's installUnits) and |
| 563 | // pinfra's update-platform-binary.sh stage with `bootstrap -root <dir>` |
| 564 | // and copy named files out of it. The apply path names this one; the |
| 565 | // pinfra script copies only units, sysusers and tmpfiles today, so a box |
| 566 | // updated by hand gets the cap on its next self-update or bootstrap |
| 567 | // rather than on that script's next run. |
| 568 | journaldPath := filepath.Join(root, "etc", "systemd", "journald.conf.d", journaldConfName) |
| 569 | if err := os.WriteFile(journaldPath, []byte(journaldConf), 0o644); err != nil { |
| 570 | return fmt.Errorf("write %s: %v", journaldPath, err) |
| 571 | } |
| 572 | fmt.Fprintf(out, " wrote %s\n", journaldPath) |
| 573 | |
| 574 | tmpfilesPath := filepath.Join(root, "etc", "tmpfiles.d", "carlos.conf") |
| 575 | if err := os.WriteFile(tmpfilesPath, []byte(tmpfilesConf), 0o644); err != nil { |
| 576 | return fmt.Errorf("write %s: %v", tmpfilesPath, err) |
| 577 | } |
| 578 | fmt.Fprintf(out, " wrote %s\n", tmpfilesPath) |
| 579 | |
| 580 | if !live { |
| 581 | if planJSON { |
| 582 | return json.NewEncoder(os.Stdout).Encode(currentBootstrapPlan()) |
| 583 | } |
| 584 | fmt.Printf("bootstrap: staged under %s — on a real box (--root /) it would also:\n", root) |
| 585 | // Printed, never run: a staging root is how a LIVE box refreshes its |
| 586 | // unit files (pinfra's update-platform-binary.sh stages into |
| 587 | // /tmp/carlos-unitstage), so arming anything from here would reach out |
| 588 | // of the temp root and touch the running system from a command whose |
| 589 | // entire contract is "write files, change nothing". |
| 590 | for _, args := range privilegedPlan(consumers) { |
| 591 | fmt.Printf(" %s\n", strings.Join(args, " ")) |
| 592 | } |
| 593 | for _, args := range systemctlPlan(enabledUnits) { |
| 594 | fmt.Printf(" systemctl %s\n", strings.Join(args, " ")) |
| 595 | } |
| 596 | fmt.Printf(" systemctl %s\n", strings.Join(journaldReload, " ")) |
| 597 | return nil |
| 598 | } |
| 599 | |
| 600 | // Repair before arming any root pass that may load these credentials. |
| 601 | if err := repairOffcloudCredentials("/etc/carlos/credentials"); err != nil { |
| 602 | return err |
| 603 | } |
| 604 | for _, args := range systemctlPlan(enabledUnits) { |
| 605 | if err := systemctl(args...); err != nil { |
| 606 | return err |
| 607 | } |
| 608 | } |
| 609 | if err := restartClaimControl(filepath.Join(unitDir, claimControlUnit), beforeControl, afterControl, func(args ...string) (string, error) { return "", systemctl(args...) }); err != nil { |
| 610 | return err |
| 611 | } |
| 612 | // A journald drop-in is inert until journald re-reads its configuration, |
| 613 | // and journald re-reads it only on restart — so a bootstrap that writes |
| 614 | // the cap and stops leaves a box whose journal is capped from its next |
| 615 | // reboot, which on a long-lived box is never. Non-fatal: the cap is a |
| 616 | // safety margin, and refusing to finish bootstrap over it would be a |
| 617 | // worse outcome than an uncapped journal on one box. |
| 618 | if err := systemctl(journaldReload...); err != nil { |
| 619 | fmt.Printf("bootstrap: could not restart systemd-journald (%v) — the journal cap in %s takes effect at the next restart\n", err, journaldConfName) |
| 620 | } |
| 621 | fmt.Printf("bootstrap: units installed, %s enabled (timers and path units armed, not just symlinked)\n", strings.Join(enabledUnits, " + ")) |
| 622 | fmt.Println(" next: copy the carlos binary to /opt/carlos/carlos, stage /etc/carlos/host.env, then: systemctl start carlos-edge") |
| 623 | return nil |
| 624 | } |
| 625 | |
| 626 | // unitNames lists every embedded unit, sorted (embed.FS ReadDir order). |
| 627 | func unitNames() ([]string, error) { |
| 628 | entries, err := unitsFS.ReadDir("units") |
| 629 | if err != nil { |
| 630 | return nil, fmt.Errorf("read embedded units: %v", err) |
| 631 | } |
| 632 | names := make([]string, 0, len(entries)) |
| 633 | for _, e := range entries { |
| 634 | names = append(names, e.Name()) |
| 635 | } |
| 636 | return names, nil |
| 637 | } |
| 638 | |
| 639 | // privilegedPlan is the exact non-systemctl privileged sequence a live |
| 640 | // bootstrap dispatches — and the same sequence a staging run PRINTS instead of |
| 641 | // running (systemctlPlan's discipline, extended to the user/group half when |
| 642 | // issue #103 added the consumers group): the two can never drift into |
| 643 | // describing different behaviour. |
| 644 | // |
| 645 | // The sequence: service user, consumers group, ownership. Every chown |
| 646 | // converges TOWARD the target state, never through an intermediate that |
| 647 | // undoes a previous run's work: /run/carlos is recursively |
| 648 | // serviceUser:socketsGroup. Unsafe entries, including live sockets, are |
| 649 | // logged and skipped; their creating units establish socket ownership. |
| 650 | // The apps/ directory (unit-tenant sockets, no path consumers) is then |
| 651 | // put back to serviceUser:serviceUser. The chmod re-asserts the setgid bit |
| 652 | // the tmpfiles line promises; consumers enroll last, one usermod each |
| 653 | // (`-aG` is append — an existing member is a no-op, so re-runs converge). |
| 654 | // It deliberately does NOT touch group `carlos` memberships: an interim |
| 655 | // grant made there (the tito workaround this replaces) is the operator's to |
| 656 | // revoke, per the deployment notes in the #103 spec. |
| 657 | func privilegedPlan(consumers []string) [][]string { |
| 658 | plan := [][]string{ |
| 659 | {"useradd", "--system", "--user-group", "--no-create-home", "--shell", "/usr/sbin/nologin", serviceUser}, |
| 660 | {"groupadd", "--system", socketsGroup}, |
| 661 | {"chown", "-R", serviceUser + ":" + serviceUser, "/data"}, |
| 662 | {"chown", "-R", serviceUser + ":" + socketsGroup, "/run/carlos"}, |
| 663 | {"chown", "-R", serviceUser + ":" + serviceUser, "/run/carlos/apps"}, |
| 664 | {"chmod", "2750", "/run/carlos"}, |
| 665 | } |
| 666 | for _, u := range consumers { |
| 667 | plan = append(plan, []string{"usermod", "-aG", socketsGroup, u}) |
| 668 | } |
| 669 | return plan |
| 670 | } |
| 671 | |
| 672 | // runPrivileged executes one privilegedPlan step, adding the existence |
| 673 | // guards that keep the raw commands idempotent: useradd and groupadd exit |
| 674 | // non-zero when their subject already exists, and "already provisioned" is |
| 675 | // bootstrap's normal re-run state, not an error. Everything else in the plan |
| 676 | // is idempotent. Ownership and mode steps use descriptor helpers; the |
| 677 | // command text is retained for staging output and is never sent to chown. |
| 678 | func runPrivileged(args []string) error { |
| 679 | if len(args) == 4 && args[0] == "chown" && args[1] == "-R" { |
| 680 | owner := strings.SplitN(args[2], ":", 2) |
| 681 | if len(owner) != 2 { |
| 682 | return fmt.Errorf("invalid ownership %q", args[2]) |
| 683 | } |
| 684 | uid, gid, err := bootstrapOwner(owner[0], owner[1]) |
| 685 | if err != nil { |
| 686 | return err |
| 687 | } |
| 688 | return bootstrapChownTree(args[3], uid, gid) |
| 689 | } |
| 690 | if len(args) == 3 && args[0] == "chmod" && args[1] == "2750" { |
| 691 | return bootstrapMakeDir(args[2], 0o750|os.ModeSetgid) |
| 692 | } |
| 693 | // The subject is derived from the command line itself (both commands |
| 694 | // take it as their final argument), never from a package constant — a |
| 695 | // guard keyed to a constant would silently skip any SECOND useradd or |
| 696 | // groupadd a later plan revision adds. |
| 697 | switch subject := args[len(args)-1]; args[0] { |
| 698 | case "useradd": |
| 699 | if _, err := user.Lookup(subject); err == nil { |
| 700 | return nil |
| 701 | } |
| 702 | case "groupadd": |
| 703 | if _, err := user.LookupGroup(subject); err == nil { |
| 704 | return nil |
| 705 | } |
| 706 | } |
| 707 | out, err := exec.Command(args[0], args[1:]...).CombinedOutput() |
| 708 | if err != nil { |
| 709 | return fmt.Errorf("%s: %v: %s", strings.Join(args, " "), err, strings.TrimSpace(string(out))) |
| 710 | } |
| 711 | return nil |
| 712 | } |
| 713 | |
| 714 | func bootstrapOwner(owner, group string) (int, int, error) { |
| 715 | u, err := user.Lookup(owner) |
| 716 | if err != nil { |
| 717 | return 0, 0, err |
| 718 | } |
| 719 | g, err := user.LookupGroup(group) |
| 720 | if err != nil { |
| 721 | return 0, 0, err |
| 722 | } |
| 723 | uid, err := strconv.Atoi(u.Uid) |
| 724 | if err != nil { |
| 725 | return 0, 0, err |
| 726 | } |
| 727 | gid, err := strconv.Atoi(g.Gid) |
| 728 | return uid, gid, err |
| 729 | } |
| 730 | |
| 731 | func systemctl(args ...string) error { |
| 732 | out, err := exec.Command("systemctl", args...).CombinedOutput() |
| 733 | if err != nil { |
| 734 | return fmt.Errorf("systemctl %s: %v: %s", strings.Join(args, " "), err, strings.TrimSpace(string(out))) |
| 735 | } |
| 736 | return nil |
| 737 | } |
| 738 | |
amadan