| 1 | package main |
| 2 | |
| 3 | import ( |
| 4 | "embed" |
| 5 | "flag" |
| 6 | "fmt" |
| 7 | "io" |
| 8 | "os" |
| 9 | "os/exec" |
| 10 | "os/user" |
| 11 | "path/filepath" |
| 12 | "regexp" |
| 13 | "strconv" |
| 14 | "strings" |
| 15 | "time" |
| 16 | ) |
| 17 | |
| 18 | // The systemd units ship inside the binary (source of truth: deploy/systemd), so |
| 19 | // `lchat bootstrap` can provision a fresh box with no separate files to copy. |
| 20 | // |
| 21 | //go:embed deploy/systemd/lchat-router.service |
| 22 | var unitRouter string |
| 23 | |
| 24 | //go:embed deploy/systemd/lchat@.service |
| 25 | var unitInstance string |
| 26 | |
| 27 | //go:embed deploy/systemd/lchat-bot@.service |
| 28 | var unitBot string |
| 29 | |
| 30 | // The webhook-ingest sidecar (incoming webhooks, docs/webhooks.md). Installs |
| 31 | // everywhere like the bot template; a host only runs one after `add-hooks` |
| 32 | // stages its /data/<host>-hooks.env and enables the unit. |
| 33 | // |
| 34 | //go:embed deploy/systemd/lchat-hooks@.service |
| 35 | var unitHooks string |
| 36 | |
| 37 | // The push relay + home server are optional sibling services (one each per box). |
| 38 | // Their units install alongside the rest but stay inert until enabled with a |
| 39 | // /data/{push,home}.env present, so a box that runs neither is unaffected. |
| 40 | // |
| 41 | //go:embed deploy/systemd/lchat-push.service |
| 42 | var unitPush string |
| 43 | |
| 44 | //go:embed deploy/systemd/lchat-home.service |
| 45 | var unitHome string |
| 46 | |
| 47 | // The connect broker (OAuth relay, docs/integrations.md) is the third sibling |
| 48 | // service; like push/home it installs everywhere and stays inert until an |
| 49 | // operator stages /data/connect.env and enables it. This unit was hand-made on |
| 50 | // the box before it lived here — keep the two in step. |
| 51 | // |
| 52 | //go:embed deploy/systemd/lchat-connect.service |
| 53 | var unitConnect string |
| 54 | |
| 55 | // The links redirector (11ch.at short links, issue #905) is also a sibling |
| 56 | // service, but unlike push/home it's a subcommand of this same binary, not a |
| 57 | // separate one, and holds no secrets or database. The unit installs |
| 58 | // everywhere; only the box declared the short-link apex actually RUNS it |
| 59 | // (`bootstrap --links-apex <host>`, converged from the registry thereafter — |
| 60 | // see syncLinks). Exactly one box in a fleet is the apex, so every other box |
| 61 | // keeps the unit written and disabled. |
| 62 | // |
| 63 | //go:embed deploy/systemd/lchat-links.service |
| 64 | var unitLinks string |
| 65 | |
| 66 | // The control daemon + cloud provisioner (self-serve space creation — see |
| 67 | // docs/cloud-provisioner.md). Like push/home they install everywhere but stay |
| 68 | // inert: controld needs an operator enable (its token file is minted below); |
| 69 | // cloud additionally needs /data/cloud.env. |
| 70 | // |
| 71 | //go:embed deploy/systemd/lchat-control.service |
| 72 | var unitControl string |
| 73 | |
| 74 | //go:embed deploy/systemd/lchat-cloud.service |
| 75 | var unitCloud string |
| 76 | |
| 77 | // The trial-tunnel gateway (`lchat try` — see docs/tunnel-try.md). Installs |
| 78 | // everywhere, stays inert until enabled; it only matters on a box that fronts |
| 79 | // a wildcard domain. |
| 80 | // |
| 81 | //go:embed deploy/systemd/lchat-tunnel.service |
| 82 | var unitTunnel string |
| 83 | |
| 84 | // The public-web fetch service (`lchat fetch run` — see docs/fetch.md). Installs |
| 85 | // everywhere, stays inert until an operator enables it and fronts it with |
| 86 | // `add-service`; instances opt in by pointing LCHAT_FETCH_URL at that route. |
| 87 | // |
| 88 | //go:embed deploy/systemd/lchat-fetch.service |
| 89 | var unitFetch string |
| 90 | |
| 91 | // The auto-updater (updater.go): a root oneshot fired hourly by its timer. |
| 92 | // Enabled by bootstrap — a pass with no reachable release feed is a no-op, so |
| 93 | // it's safe everywhere, and a box provisioned tomorrow auto-updates without a |
| 94 | // runbook step anyone has to remember. |
| 95 | // |
| 96 | //go:embed deploy/systemd/lchat-update.service |
| 97 | var unitUpdate string |
| 98 | |
| 99 | //go:embed deploy/systemd/lchat-update.timer |
| 100 | var unitUpdateTimer string |
| 101 | |
| 102 | // The public status collectors (docs/status.md): two root oneshots, each fired |
| 103 | // by a 5-minute timer — the SSH access ledger and the fleet hardening posture. |
| 104 | // They ride the binary (source: deploy/) for the same reason the units do: |
| 105 | // with the fleet hardened (operator SSH over WireGuard only, root login off, |
| 106 | // /opt/lchat + /etc/systemd/system root-owned) there's no root shell to scp |
| 107 | // them onto a box, so the binary must place them itself. See installCollectors. |
| 108 | // |
| 109 | //go:embed deploy/ssh-ledger-collect.sh |
| 110 | var scriptSSHLedger string |
| 111 | |
| 112 | //go:embed deploy/hardening-posture.sh |
| 113 | var scriptHardening string |
| 114 | |
| 115 | // The main publish collector (fleet + books + the viewer-page uploads) rides |
| 116 | // the binary too, for the same no-root-scp reason — so its per-page upload set |
| 117 | // stays reconciled with main on every `bootstrap --collectors-only`, instead of |
| 118 | // drifting stale on the box (which is how funding/roadmap uploads went missing). |
| 119 | // Only the SCRIPT is embedded; its 5-minute timer is installed separately by the |
| 120 | // script's own `install` verb (docs/status.md), which bootstrap leaves alone. |
| 121 | // |
| 122 | //go:embed deploy/status-publish.sh |
| 123 | var scriptStatusPublish string |
| 124 | |
| 125 | //go:embed deploy/systemd/lchat-ssh-ledger.service |
| 126 | var unitSSHLedger string |
| 127 | |
| 128 | //go:embed deploy/systemd/lchat-ssh-ledger.timer |
| 129 | var unitSSHLedgerTimer string |
| 130 | |
| 131 | //go:embed deploy/systemd/lchat-hardening.service |
| 132 | var unitHardening string |
| 133 | |
| 134 | //go:embed deploy/systemd/lchat-hardening.timer |
| 135 | var unitHardeningTimer string |
| 136 | |
| 137 | // statusPagesFS carries the static status viewer pages the collectors upload — |
| 138 | // the .html, the shared status.css, and the static page scripts: the shared |
| 139 | // masthead+nav builder (status-nav.js — every page mounts it; forgetting it |
| 140 | // here renders the whole family navless, hence the embed test) and the |
| 141 | // funding page's normative waterfall.js. |
| 142 | // The JSON those pages fetch (fleet, carbon, ssh, hardening, contributors, …) |
| 143 | // is collector- and operator-generated and lives mutably on the box, so it is |
| 144 | // deliberately NOT embedded: baking it in would clobber live data with a stale |
| 145 | // snapshot on every deploy. |
| 146 | // |
| 147 | //go:embed web/status/*.html web/status/status.css web/status/status-nav.js web/status/waterfall.js |
| 148 | var statusPagesFS embed.FS |
| 149 | |
| 150 | // serviceUser is the unprivileged account every unit runs as (see the hardened |
| 151 | // units in deploy/systemd). bootstrap creates it and hands it the data + runtime |
| 152 | // dirs; the router keeps CAP_NET_BIND_SERVICE to still bind :80/:443. |
| 153 | const serviceUser = "lchat" |
| 154 | |
| 155 | // tmpfilesConf keeps /run/lchat across reboots (/run is tmpfs); bootstrap also |
| 156 | // creates it now. Owned by the service user so the non-root units can bind their |
| 157 | // sockets there. |
| 158 | const tmpfilesConf = "d /run/lchat 0750 " + serviceUser + " " + serviceUser + " -\n" |
| 159 | |
| 160 | // ensureServiceUser creates the locked, home-less system user the units run as, |
| 161 | // idempotently. |
| 162 | func ensureServiceUser() { |
| 163 | if _, err := user.Lookup(serviceUser); err == nil { |
| 164 | return |
| 165 | } |
| 166 | out, err := exec.Command("useradd", "--system", "--user-group", |
| 167 | "--no-create-home", "--shell", "/usr/sbin/nologin", serviceUser).CombinedOutput() |
| 168 | if err != nil { |
| 169 | die("bootstrap: create user %s: %v: %s", serviceUser, err, strings.TrimSpace(string(out))) |
| 170 | } |
| 171 | } |
| 172 | |
| 173 | // chownToService recursively gives a path to the service user (best-effort exec of |
| 174 | // chown -R, so it also fixes ownership of any data already present). |
| 175 | var chownToService = func(path string) { |
| 176 | out, err := exec.Command("chown", "-R", serviceUser+":"+serviceUser, path).CombinedOutput() |
| 177 | if err != nil { |
| 178 | die("bootstrap: chown %s: %v: %s", path, err, strings.TrimSpace(string(out))) |
| 179 | } |
| 180 | } |
| 181 | |
| 182 | // ensureControlToken mints the control daemon's bearer token into a ROOT-only |
| 183 | // file, idempotently. Root-only is the point: every service shares the lchat |
| 184 | // uid, so this token — not socket permissions — is what separates "may create |
| 185 | // instances" (the cloud unit, via LoadCredential) from everything else. |
| 186 | func ensureControlToken() error { |
| 187 | if _, err := os.Stat(controlTokenPath); err == nil { |
| 188 | return nil |
| 189 | } |
| 190 | tok, err := randToken() |
| 191 | if err != nil { |
| 192 | return err |
| 193 | } |
| 194 | if err := os.MkdirAll(filepath.Dir(controlTokenPath), 0o700); err != nil { |
| 195 | return err |
| 196 | } |
| 197 | if err := os.WriteFile(controlTokenPath, []byte(tok+"\n"), 0o600); err != nil { |
| 198 | return err |
| 199 | } |
| 200 | fmt.Printf(" minted %s (root-only)\n", controlTokenPath) |
| 201 | return nil |
| 202 | } |
| 203 | |
| 204 | // writeFile writes content to path at mode, dying on failure. bootstrap is a |
| 205 | // one-shot root command, so a half-written unit/script means a half-provisioned |
| 206 | // box: fail loud rather than limp on. mode is 0644 for units + pages, 0755 for |
| 207 | // the executable collector scripts. |
| 208 | func writeFile(path, content string, mode os.FileMode) { |
| 209 | if err := os.WriteFile(path, []byte(content), mode); err != nil { |
| 210 | die("bootstrap: write %s: %v", path, err) |
| 211 | } |
| 212 | fmt.Printf(" wrote %s\n", path) |
| 213 | } |
| 214 | |
| 215 | // collectorTimers are the status collectors' timer units. They're oneshot + |
| 216 | // timer (not long-running services), so `enable --now` is the right verb — it |
| 217 | // enables the timer and fires one pass now, with no active service to leave |
| 218 | // stale the way `--now` would for the router. |
| 219 | var collectorTimers = []string{"lchat-ssh-ledger.timer", "lchat-hardening.timer"} |
| 220 | |
| 221 | // writeCollectorFiles lays down the box-managed status collectors: the two root |
| 222 | // oneshot scripts (executable) + their four systemd units, the main publish |
| 223 | // script (executable; its timer is installed by the script's own `install` |
| 224 | // verb, not here), and the static viewer pages the collectors upload. It writes |
| 225 | // files ONLY — the caller reloads systemd and enables the timers — so full |
| 226 | // bootstrap (one daemon-reload covering every unit) and the `--collectors-only` |
| 227 | // refresh share exactly this install logic. |
| 228 | // |
| 229 | // appDir is the on-box install root (/opt/lchat): each unit's ExecStart points at |
| 230 | // <app-dir>/<script>, and the collectors upload the pages from <app-dir>/status- |
| 231 | // pages. unitDir is the systemd unit directory. |
| 232 | func writeCollectorFiles(appDir, unitDir, dataDir string) { |
| 233 | // The collectors' state dirs must exist BEFORE their units first start: |
| 234 | // each unit lists its dir in ReadWritePaths, and on the fleet's systemd a |
| 235 | // missing ReadWritePaths target fails namespace setup for the WHOLE unit — |
| 236 | // including the `ExecStartPre=+mkdir` that was supposed to create it (the |
| 237 | // `+` doesn't escape mount-namespace construction there). Found on the |
| 238 | // hardening collector's first real-box run, 2026-07-19. |
| 239 | for _, d := range []string{filepath.Join(dataDir, "ssh-ledger"), filepath.Join(dataDir, "hardening")} { |
| 240 | if err := os.MkdirAll(d, 0o755); err != nil { |
| 241 | die("bootstrap: mkdir %s: %v", d, err) |
| 242 | } |
| 243 | } |
| 244 | |
| 245 | // The scripts are executable — their units fire them via ExecStart. |
| 246 | writeFile(filepath.Join(appDir, "ssh-ledger-collect.sh"), scriptSSHLedger, 0o755) |
| 247 | writeFile(filepath.Join(appDir, "hardening-posture.sh"), scriptHardening, 0o755) |
| 248 | // The main publish script rides the binary too so its upload set (the pages |
| 249 | // it names, one line each) stays current with main; its own timer, installed |
| 250 | // by `status-publish.sh install`, keeps firing this refreshed copy. |
| 251 | writeFile(filepath.Join(appDir, "status-publish.sh"), scriptStatusPublish, 0o755) |
| 252 | |
| 253 | writeFile(filepath.Join(unitDir, "lchat-ssh-ledger.service"), unitSSHLedger, 0o644) |
| 254 | writeFile(filepath.Join(unitDir, "lchat-ssh-ledger.timer"), unitSSHLedgerTimer, 0o644) |
| 255 | writeFile(filepath.Join(unitDir, "lchat-hardening.service"), unitHardening, 0o644) |
| 256 | writeFile(filepath.Join(unitDir, "lchat-hardening.timer"), unitHardeningTimer, 0o644) |
| 257 | |
| 258 | // The collectors upload the viewer pages from here. Only the embedded .html + |
| 259 | // .css land; the JSON the pages fetch is generated on the box (see |
| 260 | // statusPagesFS) and must stay untouched. |
| 261 | pagesDir := filepath.Join(appDir, "status-pages") |
| 262 | if err := os.MkdirAll(pagesDir, 0o755); err != nil { |
| 263 | die("bootstrap: mkdir %s: %v", pagesDir, err) |
| 264 | } |
| 265 | entries, err := statusPagesFS.ReadDir("web/status") |
| 266 | if err != nil { |
| 267 | die("bootstrap: read embedded status pages: %v", err) |
| 268 | } |
| 269 | for _, e := range entries { |
| 270 | content, err := statusPagesFS.ReadFile("web/status/" + e.Name()) |
| 271 | if err != nil { |
| 272 | die("bootstrap: read embedded %s: %v", e.Name(), err) |
| 273 | } |
| 274 | // basename only — strip the web/status/ embed prefix. |
| 275 | writeFile(filepath.Join(pagesDir, e.Name()), string(content), 0o644) |
| 276 | } |
| 277 | } |
| 278 | |
| 279 | // enableCollectorTimers enables + fires the two collector timers. Split from |
| 280 | // writeCollectorFiles so full bootstrap's single daemon-reload can register all |
| 281 | // units at once; the caller must daemon-reload before calling this. |
| 282 | func enableCollectorTimers() { |
| 283 | for _, timer := range collectorTimers { |
| 284 | if err := systemctl("enable", "--now", timer); err != nil { |
| 285 | die("bootstrap: enable %s: %v", timer, err) |
| 286 | } |
| 287 | } |
| 288 | } |
| 289 | |
| 290 | // setUpdateTimer turns the hourly auto-updater on or off, and is deliberately |
| 291 | // two-sided rather than a "skip enabling" flag. |
| 292 | // |
| 293 | // Some boxes must NEVER track a release channel — the dev mesh runs unmerged |
| 294 | // code, so an updater pass would quietly replace it with the current stable |
| 295 | // build. Today that invariant is held only by the timer happening to be |
| 296 | // disabled, which a single full `bootstrap` would silently undo. Skipping the |
| 297 | // enable would not be enough either: a box that was bootstrapped normally once |
| 298 | // already has the timer on, and a later `--no-update-timer` run must fix that |
| 299 | // rather than leave it. So the off path disables explicitly, making the end |
| 300 | // state a property of the flag instead of a property of the box's history. |
| 301 | func setUpdateTimer(on bool) { |
| 302 | if on { |
| 303 | if err := systemctl("enable", "--now", "lchat-update.timer"); err != nil { |
| 304 | die("bootstrap: enable update timer: %v", err) |
| 305 | } |
| 306 | return |
| 307 | } |
| 308 | if err := systemctl("disable", "--now", "lchat-update.timer"); err != nil { |
| 309 | die("bootstrap: disable update timer: %v (this box must not auto-update — refusing to finish in an unknown state)", err) |
| 310 | } |
| 311 | fmt.Println("bootstrap: auto-update timer DISABLED (--no-update-timer) — this box will not track a release channel") |
| 312 | } |
| 313 | |
| 314 | // linksSocket is where the redirector listens — the contract between |
| 315 | // lchat-links.service's ExecStart and the registry route that fronts it. |
| 316 | const linksSocket = "/run/lchat/links.sock" |
| 317 | |
| 318 | // linksRouted reports whether THIS box is the short-link apex, by asking the |
| 319 | // only thing that actually knows: the router's registry. Any route pointing at |
| 320 | // linksSocket means an operator has declared this box the apex. |
| 321 | // |
| 322 | // Matching on the SOCKET rather than a hostname is what keeps this general — |
| 323 | // it recognises 11ch.at in production and dev.11ch.at on the dev mesh without |
| 324 | // either being named in the code. |
| 325 | // |
| 326 | // A registry we can't read is reported as "not the apex": that is the safe |
| 327 | // direction (an idle box rather than a service we couldn't justify starting), |
| 328 | // and on a fresh box the registry legitimately has nothing in it yet. |
| 329 | // |
| 330 | // The Stat guard matters more than it looks. openRegistry CREATES the database |
| 331 | // (sql.Open + migrate's CREATE TABLE IF NOT EXISTS), and this runs on every |
| 332 | // plain bootstrap — including every self-hoster's install.sh. bootstrap is |
| 333 | // root, while the router runs as the lchat user and creates the registry |
| 334 | // itself moments earlier; if this read won that race it would leave a |
| 335 | // ROOT-OWNED registry the router cannot write, breaking all routing on a |
| 336 | // fresh box. A read must never be the thing that creates it. |
| 337 | func linksRouted(registryPath string) bool { |
| 338 | if _, err := os.Stat(registryPath); err != nil { |
| 339 | return false |
| 340 | } |
| 341 | reg, err := openRegistry(registryPath) |
| 342 | if err != nil { |
| 343 | return false |
| 344 | } |
| 345 | routes, err := reg.list() |
| 346 | if err != nil { |
| 347 | return false |
| 348 | } |
| 349 | for _, rt := range routes { |
| 350 | if rt.Socket == linksSocket { |
| 351 | return true |
| 352 | } |
| 353 | } |
| 354 | return false |
| 355 | } |
| 356 | |
| 357 | // syncLinks converges the short-link redirector (`lchat links run`, issue #905) |
| 358 | // to whether this box is the apex. |
| 359 | // |
| 360 | // The unit has always been written by bootstrap, but nothing ever enabled it — |
| 361 | // so the service existed in the repo and ran nowhere, which is why 11ch.at |
| 362 | // served nothing long after the code merged. |
| 363 | // |
| 364 | // The state is keyed on the ROUTE, not on a flag, and that distinction is the |
| 365 | // whole design. Only one box in a fleet is the short-link apex; every other |
| 366 | // box (and every self-hoster's) has no business running the process at all. |
| 367 | // But if a *flag* decided it, an operator who later ran a plain `bootstrap` on |
| 368 | // the apex box would silently turn short links off — the same footgun |
| 369 | // --no-update-timer has, and has to have, because no timer state persists |
| 370 | // anywhere. Here the route persists in the registry, so it can be the source |
| 371 | // of truth: bootstrap simply makes the box match what the registry says, and |
| 372 | // re-running it is always safe. |
| 373 | // |
| 374 | // restart, not `enable --now`, for exactly the reason the router does it: on |
| 375 | // the installer's update path `--now` is a no-op for an already-active unit, |
| 376 | // which would leave the OLD process running against the NEW binary. The unit |
| 377 | // execs /opt/lchat/lchat directly (not a versioned live symlink), so a |
| 378 | // bootstrap re-run is the only thing that picks up a new build. |
| 379 | // |
| 380 | // Failures warn rather than die(): short links are one optional service, and |
| 381 | // must never be the reason a box fails to come up. |
| 382 | func syncLinks(apex bool) { |
| 383 | if !apex { |
| 384 | // Not the apex — make sure it isn't running. Idempotent: bootstrap has |
| 385 | // just written the unit file, and disabling an already-disabled unit is |
| 386 | // a no-op, so this is quiet on every ordinary box. |
| 387 | if err := systemctl("disable", "--now", "lchat-links"); err != nil { |
| 388 | fmt.Printf("bootstrap: warning: disable lchat-links: %v\n", err) |
| 389 | } |
| 390 | return |
| 391 | } |
| 392 | if err := systemctl("enable", "lchat-links"); err != nil { |
| 393 | fmt.Printf("bootstrap: warning: enable lchat-links: %v (short links will not serve on this box)\n", err) |
| 394 | return |
| 395 | } |
| 396 | if err := systemctl("restart", "lchat-links"); err != nil { |
| 397 | fmt.Printf("bootstrap: warning: start lchat-links: %v (short links will not serve on this box)\n", err) |
| 398 | } |
| 399 | } |
| 400 | |
| 401 | // registerLinksApex makes this box the short-link apex for host: start the |
| 402 | // redirector, then register the route that both points the router at it and |
| 403 | // records the decision durably (so later bootstraps converge to it). |
| 404 | // |
| 405 | // Order matters — the route is registered only once the socket answers, the |
| 406 | // same check `add-service` makes, because a route to a dead socket is a |
| 407 | // 502-shaped hole in the router rather than a pending setup step. |
| 408 | // |
| 409 | // KNOWN LIMITATION worth stating where an operator will meet it: the |
| 410 | // redirector is really only OURS to serve. Two values in linkscmd.go are |
| 411 | // Eleven-specific and not configurable — the apex bounces to |
| 412 | // linksMarketingURL (elevenmessenger.com), and the apple-app-site-association |
| 413 | // advertises appleAppID (our iOS app). Pointing your own apex here therefore |
| 414 | // sends your visitors to our marketing site and publishes our app id under |
| 415 | // your domain. Making those configurable is what self-hostable short links |
| 416 | // would take; until then this flag is for our fleet. |
| 417 | func registerLinksApex(host, registryPath string) { |
| 418 | host = strings.ToLower(strings.TrimSpace(host)) |
| 419 | if !validHost(host) { |
| 420 | die("bootstrap: --links-apex %q is not a valid hostname", host) |
| 421 | } |
| 422 | syncLinks(true) |
| 423 | if err := waitReachable(linksSocket, 10*time.Second); err != nil { |
| 424 | die("bootstrap: links redirector not answering on %s: %v (check: journalctl -u lchat-links)", linksSocket, err) |
| 425 | } |
| 426 | reg, err := openRegistry(registryPath) |
| 427 | if err != nil { |
| 428 | die("bootstrap: open registry: %v", err) |
| 429 | } |
| 430 | if existing, err := reg.get(host); err != nil { |
| 431 | die("bootstrap: %v", err) |
| 432 | } else if existing != nil && existing.Socket != linksSocket { |
| 433 | die("bootstrap: %s is already routed to %s (kind %s) — refusing to repoint it at the links redirector", host, existing.Socket, existing.Kind) |
| 434 | } |
| 435 | if err := reg.upsert(Route{Host: host, Kind: "service", Socket: linksSocket, Status: "ready"}); err != nil { |
| 436 | die("bootstrap: register %s: %v", host, err) |
| 437 | } |
| 438 | if err := systemctl("reload", "lchat-router"); err != nil { |
| 439 | die("bootstrap: reload router: %v (route added; run: systemctl reload lchat-router)", err) |
| 440 | } |
| 441 | fmt.Printf("bootstrap: short-link apex %s -> %s\n", host, linksSocket) |
| 442 | } |
| 443 | |
| 444 | // --- root-SSH lockdown ------------------------------------------------------- |
| 445 | |
| 446 | // The drop-in that turns root SSH off for good — the last step of operator |
| 447 | // setup (docs/instance-router.md § Hardening), encoded here because a box that |
| 448 | // misses the manual step has no root shell left to fix it with; the allowlisted |
| 449 | // `sudo -n /opt/lchat/lchat bootstrap --ssh-noroot` is the remaining root- |
| 450 | // capable path. The 10- prefix matters: sshd config is first-match-wins, so |
| 451 | // this must sort before the distro's 50-*.conf (prohibit-password) to win. |
| 452 | const noRootConfPath = "/etc/ssh/sshd_config.d/10-lchat-noroot.conf" |
| 453 | |
| 454 | const noRootConf = `# Installed by lchat bootstrap (--ssh-noroot). Root SSH is bootstrap-only on |
| 455 | # fleet boxes: operators are named accounts over WireGuard whose root capability |
| 456 | # is the sudoers allowlist (deploy/sudoers-operators). Undo via console/rescue |
| 457 | # only — see docs/instance-router.md § Hardening. |
| 458 | PermitRootLogin no |
| 459 | ` |
| 460 | |
| 461 | // operatorsExist reports whether the lchat-operators group has members — the |
| 462 | // "someone other than root can still get in" gate for the lockdown, so a fresh |
| 463 | // box can never shut root out before operator accounts exist. Parsed from the |
| 464 | // group file directly: os/user resolves groups but can't list their members. |
| 465 | func operatorsExist(groupFile string) bool { |
| 466 | b, err := os.ReadFile(groupFile) |
| 467 | if err != nil { |
| 468 | return false |
| 469 | } |
| 470 | for _, line := range strings.Split(string(b), "\n") { |
| 471 | if f := strings.Split(line, ":"); len(f) == 4 && f[0] == "lchat-operators" && strings.TrimSpace(f[3]) != "" { |
| 472 | return true |
| 473 | } |
| 474 | } |
| 475 | return false |
| 476 | } |
| 477 | |
| 478 | // sshdCheck validates the merged sshd config (`sshd -t`); a var so tests can |
| 479 | // stand in for the daemon binary. |
| 480 | var sshdCheck = func() error { |
| 481 | sshd, err := exec.LookPath("sshd") |
| 482 | if err != nil { |
| 483 | sshd = "/usr/sbin/sshd" // sudo's PATH may omit sbin; the fleet's lives here |
| 484 | } |
| 485 | if out, err := exec.Command(sshd, "-t").CombinedOutput(); err != nil { |
| 486 | return fmt.Errorf("sshd -t: %v: %s", err, strings.TrimSpace(string(out))) |
| 487 | } |
| 488 | return nil |
| 489 | } |
| 490 | |
| 491 | // sshdRootLogin reports sshd's EFFECTIVE permitrootlogin (`sshd -T`) — the |
| 492 | // merged, first-match-wins truth, not the drop-in's content; a var so tests |
| 493 | // can stand in for the daemon binary. |
| 494 | var sshdRootLogin = func() (string, error) { |
| 495 | sshd, err := exec.LookPath("sshd") |
| 496 | if err != nil { |
| 497 | sshd = "/usr/sbin/sshd" |
| 498 | } |
| 499 | out, err := exec.Command(sshd, "-T").Output() |
| 500 | if err != nil { |
| 501 | return "", fmt.Errorf("sshd -T: %v", err) |
| 502 | } |
| 503 | for _, line := range strings.Split(string(out), "\n") { |
| 504 | if f := strings.Fields(line); len(f) == 2 && f[0] == "permitrootlogin" { |
| 505 | return f[1], nil |
| 506 | } |
| 507 | } |
| 508 | return "", fmt.Errorf("sshd -T reported no permitrootlogin") |
| 509 | } |
| 510 | |
| 511 | // applyNoRootSSH converges the drop-in: write, validate the merged config, |
| 512 | // reload sshd, then confirm sshd's EFFECTIVE permitrootlogin is "no". The |
| 513 | // confirm runs even when the file was already correct, and it is the step |
| 514 | // that matters: a correct drop-in can still lose sshd's first-match to an |
| 515 | // earlier-sorting sshd_config.d file — kepler's 00-hardening.conf (2026-07-23) |
| 516 | // made this verb report success while root login stayed enabled. A failed |
| 517 | // `sshd -t` restores the prior state so a bad write can never leave config |
| 518 | // sshd would refuse on its next real restart; a failed effective check leaves |
| 519 | // the (inert, harmless) drop-in in place and errors with the cause. Reload |
| 520 | // only re-reads config: existing sessions and named-account logins are |
| 521 | // untouched — only new root logins are refused. |
| 522 | func applyNoRootSSH(confPath string) (changed bool, err error) { |
| 523 | prev, prevErr := os.ReadFile(confPath) |
| 524 | if prevErr != nil || string(prev) != noRootConf { |
| 525 | if err := os.WriteFile(confPath, []byte(noRootConf), 0o644); err != nil { |
| 526 | return false, err |
| 527 | } |
| 528 | if err := sshdCheck(); err != nil { |
| 529 | if prevErr == nil { |
| 530 | os.WriteFile(confPath, prev, 0o644) |
| 531 | } else { |
| 532 | os.Remove(confPath) |
| 533 | } |
| 534 | return false, err |
| 535 | } |
| 536 | // Ubuntu names the unit ssh; other distros sshd. |
| 537 | if err := systemctl("reload", "ssh"); err != nil { |
| 538 | if err2 := systemctl("reload", "sshd"); err2 != nil { |
| 539 | return false, fmt.Errorf("%v; %v", err, err2) |
| 540 | } |
| 541 | } |
| 542 | changed = true |
| 543 | } |
| 544 | got, gerr := sshdRootLogin() |
| 545 | if gerr != nil { |
| 546 | return changed, fmt.Errorf("verify effective config: %v", gerr) |
| 547 | } |
| 548 | if got != "no" { |
| 549 | return changed, fmt.Errorf("drop-in in place but sshd's effective permitrootlogin is %q — sshd_config is first-match-wins, so a file sorting before %s (ls /etc/ssh/sshd_config.d) or a directive above the Include line is overriding it; rename the offender to sort after it and re-run", got, filepath.Base(confPath)) |
| 550 | } |
| 551 | return changed, nil |
| 552 | } |
| 553 | |
| 554 | // --- operator accounts ------------------------------------------------------- |
| 555 | |
| 556 | // The account half of the hardening recipe's operator-setup stage |
| 557 | // (docs/instance-router.md § Hardening), encoded because a box hardened before |
| 558 | // an operator joined it has no root shell left to add them with — kepler |
| 559 | // carried James's WireGuard peer but no account (2026-07-29). The allowlisted |
| 560 | // `sudo -n /opt/lchat/lchat bootstrap --add-operator <name>` over the tunnel |
| 561 | // is the remaining root-capable path; the keys ride stdin so the committed |
| 562 | // sudoers line (`/opt/lchat/lchat *`) already covers it. |
| 563 | |
| 564 | // Conservative unix account names only — this lands in useradd's argv. |
| 565 | var operatorNameRe = regexp.MustCompile(`^[a-z][a-z0-9-]{0,31}$`) |
| 566 | |
| 567 | // countAuthorizedKeys confirms the piped bytes LOOK like an authorized_keys |
| 568 | // file and counts the keys. Installing an empty or wrong file as someone's |
| 569 | // login credentials must fail loudly here — with root SSH off, a clobbered |
| 570 | // key file is a console/rescue trip. |
| 571 | func countAuthorizedKeys(b []byte) (int, error) { |
| 572 | n := 0 |
| 573 | for _, line := range strings.Split(string(b), "\n") { |
| 574 | line = strings.TrimSpace(line) |
| 575 | if line == "" || strings.HasPrefix(line, "#") { |
| 576 | continue |
| 577 | } |
| 578 | f := strings.Fields(line) |
| 579 | if len(f) < 2 || !(strings.HasPrefix(f[0], "ssh-") || strings.HasPrefix(f[0], "ecdsa-") || strings.HasPrefix(f[0], "sk-")) { |
| 580 | return 0, fmt.Errorf("stdin line %q is not a public key", line) |
| 581 | } |
| 582 | n++ |
| 583 | } |
| 584 | if n == 0 { |
| 585 | return 0, fmt.Errorf("no keys on stdin — pipe the operator's authorized_keys content in") |
| 586 | } |
| 587 | return n, nil |
| 588 | } |
| 589 | |
| 590 | // runAccountCmd is the exec seam for the user/group commands — a var so tests |
| 591 | // record instead of running (the chownToService pattern). |
| 592 | var runAccountCmd = func(name string, args ...string) error { |
| 593 | if out, err := exec.Command(name, args...).CombinedOutput(); err != nil { |
| 594 | return fmt.Errorf("%s %s: %v: %s", name, strings.Join(args, " "), err, strings.TrimSpace(string(out))) |
| 595 | } |
| 596 | return nil |
| 597 | } |
| 598 | |
| 599 | // lookupAccount is the user-database seam: home dir + numeric ids, or |
| 600 | // exists=false. A var for the same reason. |
| 601 | var lookupAccount = func(name string) (home string, uid, gid int, exists bool) { |
| 602 | u, err := user.Lookup(name) |
| 603 | if err != nil { |
| 604 | return "", 0, 0, false |
| 605 | } |
| 606 | uid, _ = strconv.Atoi(u.Uid) |
| 607 | gid, _ = strconv.Atoi(u.Gid) |
| 608 | return u.HomeDir, uid, gid, true |
| 609 | } |
| 610 | |
| 611 | // addOperatorAccount converges one named operator: the account, its groups |
| 612 | // (lchat-operators carries the sudo allowlist, systemd-journal the log |
| 613 | // reads), and ~/.ssh/authorized_keys REPLACED by the piped keys — |
| 614 | // replacement, not append, so re-running with a fresh key set is also the |
| 615 | // rotation path. Idempotent like the rest of bootstrap. |
| 616 | func addOperatorAccount(name string, keys io.Reader) error { |
| 617 | if !operatorNameRe.MatchString(name) { |
| 618 | return fmt.Errorf("invalid operator name %q (want lowercase, [a-z][a-z0-9-]{0,31})", name) |
| 619 | } |
| 620 | b, err := io.ReadAll(io.LimitReader(keys, 1<<20)) |
| 621 | if err != nil { |
| 622 | return err |
| 623 | } |
| 624 | nkeys, err := countAuthorizedKeys(b) |
| 625 | if err != nil { |
| 626 | return err |
| 627 | } |
| 628 | // -f converges on a box whose operator-setup stage already made the group. |
| 629 | if err := runAccountCmd("groupadd", "-f", "lchat-operators"); err != nil { |
| 630 | return err |
| 631 | } |
| 632 | _, _, _, existed := lookupAccount(name) |
| 633 | if !existed { |
| 634 | if err := runAccountCmd("useradd", "-m", "-s", "/bin/bash", name); err != nil { |
| 635 | return err |
| 636 | } |
| 637 | } |
| 638 | if err := runAccountCmd("usermod", "-aG", "lchat-operators,systemd-journal", name); err != nil { |
| 639 | return err |
| 640 | } |
| 641 | home, uid, gid, ok := lookupAccount(name) |
| 642 | if !ok { |
| 643 | return fmt.Errorf("account %s missing after create", name) |
| 644 | } |
| 645 | sshDir := filepath.Join(home, ".ssh") |
| 646 | if err := os.MkdirAll(sshDir, 0o700); err != nil { |
| 647 | return err |
| 648 | } |
| 649 | if err := os.Chmod(sshDir, 0o700); err != nil { |
| 650 | return err |
| 651 | } |
| 652 | akPath := filepath.Join(sshDir, "authorized_keys") |
| 653 | if err := os.WriteFile(akPath, b, 0o600); err != nil { |
| 654 | return err |
| 655 | } |
| 656 | for _, p := range []string{sshDir, akPath} { |
| 657 | if err := os.Chown(p, uid, gid); err != nil { |
| 658 | return err |
| 659 | } |
| 660 | } |
| 661 | verb := "converged" |
| 662 | if !existed { |
| 663 | verb = "created" |
| 664 | } |
| 665 | noun := "keys" |
| 666 | if nkeys == 1 { |
| 667 | noun = "key" |
| 668 | } |
| 669 | fmt.Printf("bootstrap: operator %s %s — groups lchat-operators+systemd-journal, %d %s installed\n", name, verb, nkeys, noun) |
| 670 | return nil |
| 671 | } |
| 672 | |
| 673 | // runBootstrap installs the systemd units + runtime dir and starts the router, so |
| 674 | // provisioning a fresh box is one command after the binary is copied to |
| 675 | // /opt/lchat/lchat. Idempotent — safe to re-run. See docs/instance-router.md. |
| 676 | func runBootstrap(args []string) { |
| 677 | fs := flag.NewFlagSet("bootstrap", flag.ExitOnError) |
| 678 | dataDir := fs.String("data-dir", "/data", "dir for per-instance DBs, env files, the registry, and the acme cache") |
| 679 | unitDir := fs.String("unit-dir", "/etc/systemd/system", "systemd unit directory") |
| 680 | noStart := fs.Bool("no-start", false, "install units but don't enable/start the router") |
| 681 | collectorsOnly := fs.Bool("collectors-only", false, "refresh ONLY the status collectors (scripts, units, viewer pages) + enable their timers — no service user, no control token, no router restart") |
| 682 | linksApex := fs.String("links-apex", "", "make this box the short-link apex for this host (starts the redirector and routes the host at it); without it, a box that isn't already routed as the apex doesn't run the redirector at all") |
| 683 | noUpdateTimer := fs.Bool("no-update-timer", false, "ensure the hourly auto-updater is DISABLED instead of enabled (for boxes that must never track a release channel, e.g. the dev mesh)") |
| 684 | sshNoRoot := fs.Bool("ssh-noroot", false, "ONLY converge the root-SSH lockdown (PermitRootLogin no drop-in + sshd reload) — refused until the lchat-operators group has members") |
| 685 | addOperator := fs.String("add-operator", "", "ONLY create/converge the named operator account (groups lchat-operators+systemd-journal; ~/.ssh/authorized_keys REPLACED from stdin) — the allowlist-reachable form of the operator-setup stage") |
| 686 | _ = fs.Parse(args) |
| 687 | |
| 688 | // --add-operator is operator-setup for a box that already shut root out: |
| 689 | // an existing operator adds the next one over the tunnel, keys on stdin |
| 690 | // (`ops provision operator` is the front door). |
| 691 | if *addOperator != "" { |
| 692 | if err := addOperatorAccount(*addOperator, os.Stdin); err != nil { |
| 693 | die("bootstrap: --add-operator: %v", err) |
| 694 | } |
| 695 | return |
| 696 | } |
| 697 | |
| 698 | // --ssh-noroot is the scoped, allowlist-reachable form of the hardening |
| 699 | // recipe's last step: on a box where root SSH is already unreachable (keys |
| 700 | // pulled) but sshd still says prohibit-password, an operator flips the |
| 701 | // config itself over WireGuard — no root shell involved. Refused until |
| 702 | // operator accounts exist so a fresh box can't shut root out first. |
| 703 | if *sshNoRoot { |
| 704 | if !operatorsExist("/etc/group") { |
| 705 | die("bootstrap: refusing to disable root SSH — the lchat-operators group has no members yet; finish operator setup first (docs/instance-router.md § Hardening)") |
| 706 | } |
| 707 | changed, err := applyNoRootSSH(noRootConfPath) |
| 708 | if err != nil { |
| 709 | die("bootstrap: root-SSH lockdown: %v", err) |
| 710 | } |
| 711 | if changed { |
| 712 | fmt.Println("bootstrap: root SSH disabled (PermitRootLogin no; sshd reloaded) — confirm: sshd -T | grep permitrootlogin") |
| 713 | } else { |
| 714 | fmt.Println("bootstrap: root-SSH lockdown already in place — nothing to do") |
| 715 | } |
| 716 | return |
| 717 | } |
| 718 | |
| 719 | // --collectors-only is the no-root-shell refresh path: with the fleet |
| 720 | // hardened there's no root scp to drop new collector scripts/pages, so an |
| 721 | // operator runs `sudo -n /opt/lchat/lchat bootstrap --collectors-only` over |
| 722 | // WireGuard (allowlisted by `/opt/lchat/lchat *` in deploy/sudoers-operators). |
| 723 | // It deliberately skips everything that touches long-running services or box |
| 724 | // identity — the service user, the control token, and the router's units / |
| 725 | // enable / restart — so it can't cause a member-facing blip. optDir (the |
| 726 | // install root, /opt/lchat) is where the collectors' units expect their |
| 727 | // scripts + status-pages dir. |
| 728 | if *collectorsOnly { |
| 729 | writeCollectorFiles(optDir, *unitDir, *dataDir) |
| 730 | if err := systemctl("daemon-reload"); err != nil { |
| 731 | die("bootstrap: %v", err) |
| 732 | } |
| 733 | enableCollectorTimers() |
| 734 | fmt.Println("bootstrap: status collectors refreshed (scripts + units + viewer pages; timers enabled)") |
| 735 | return |
| 736 | } |
| 737 | |
| 738 | ensureServiceUser() |
| 739 | for _, d := range []string{*dataDir, filepath.Join(*dataDir, "acme"), "/run/lchat"} { |
| 740 | if err := os.MkdirAll(d, 0o750); err != nil { |
| 741 | die("bootstrap: mkdir %s: %v", d, err) |
| 742 | } |
| 743 | } |
| 744 | // The version store + per-instance live symlinks (see versions.go). World- |
| 745 | // readable/executable so the sandboxed instance units can exec their binary; |
| 746 | // only root (the operator running deploy/stage) writes here. |
| 747 | for _, d := range []string{versionsDir(), liveDir()} { |
| 748 | if err := os.MkdirAll(d, 0o755); err != nil { |
| 749 | die("bootstrap: mkdir %s: %v", d, err) |
| 750 | } |
| 751 | } |
| 752 | if err := os.Chmod("/run/lchat", 0o750); err != nil { |
| 753 | die("bootstrap: chmod /run/lchat: %v", err) |
| 754 | } |
| 755 | // The units run as serviceUser, so it must own where they read/write. |
| 756 | chownToService(*dataDir) |
| 757 | chownToService("/run/lchat") |
| 758 | |
| 759 | write := func(path, content string) { writeFile(path, content, 0o644) } |
| 760 | write(filepath.Join(*unitDir, "lchat-router.service"), unitRouter) |
| 761 | write(filepath.Join(*unitDir, "lchat@.service"), unitInstance) |
| 762 | write(filepath.Join(*unitDir, "lchat-bot@.service"), unitBot) |
| 763 | write(filepath.Join(*unitDir, "lchat-hooks@.service"), unitHooks) |
| 764 | write(filepath.Join(*unitDir, "lchat-push.service"), unitPush) |
| 765 | write(filepath.Join(*unitDir, "lchat-home.service"), unitHome) |
| 766 | write(filepath.Join(*unitDir, "lchat-connect.service"), unitConnect) |
| 767 | write(filepath.Join(*unitDir, "lchat-links.service"), unitLinks) |
| 768 | write(filepath.Join(*unitDir, "lchat-control.service"), unitControl) |
| 769 | write(filepath.Join(*unitDir, "lchat-cloud.service"), unitCloud) |
| 770 | write(filepath.Join(*unitDir, "lchat-tunnel.service"), unitTunnel) |
| 771 | write(filepath.Join(*unitDir, "lchat-fetch.service"), unitFetch) |
| 772 | write(filepath.Join(*unitDir, "lchat-update.service"), unitUpdate) |
| 773 | write(filepath.Join(*unitDir, "lchat-update.timer"), unitUpdateTimer) |
| 774 | write("/etc/tmpfiles.d/lchat.conf", tmpfilesConf) |
| 775 | // The status collectors ride along here so the single daemon-reload below |
| 776 | // registers their units too; --no-start still lays down the files (like every |
| 777 | // other unit), and the started path enables their timers. |
| 778 | writeCollectorFiles(optDir, *unitDir, *dataDir) |
| 779 | |
| 780 | if err := ensureControlToken(); err != nil { |
| 781 | die("bootstrap: control token: %v", err) |
| 782 | } |
| 783 | |
| 784 | if err := systemctl("daemon-reload"); err != nil { |
| 785 | die("bootstrap: %v", err) |
| 786 | } |
| 787 | if *noStart { |
| 788 | fmt.Println("bootstrap: units installed (router not started; --no-start)") |
| 789 | return |
| 790 | } |
| 791 | if err := systemctl("enable", "lchat-router"); err != nil { |
| 792 | die("bootstrap: enable router: %v", err) |
| 793 | } |
| 794 | // restart, not `enable --now`: on a re-run over a live box (the installer's |
| 795 | // update path) `--now` is a no-op for an already-active unit, silently |
| 796 | // leaving the OLD router process serving the new binary's units. restart |
| 797 | // also starts an inactive unit, so the fresh-box path is unchanged. |
| 798 | if err := systemctl("restart", "lchat-router"); err != nil { |
| 799 | die("bootstrap: start router: %v", err) |
| 800 | } |
| 801 | setUpdateTimer(!*noUpdateTimer) |
| 802 | // Short links: --links-apex declares this box the apex (and records that in |
| 803 | // the registry); otherwise converge to whatever the registry already says, |
| 804 | // so a re-run neither turns a live apex off nor starts the redirector on a |
| 805 | // box that has no use for it. |
| 806 | registryPath := filepath.Join(*dataDir, "registry.db") |
| 807 | if *linksApex != "" { |
| 808 | registerLinksApex(*linksApex, registryPath) |
| 809 | } else { |
| 810 | syncLinks(linksRouted(registryPath)) |
| 811 | } |
| 812 | // The collector units + pages were written above and picked up by the reload; |
| 813 | // enable their 5-minute timers (safe everywhere — a pass with no /data/status.env |
| 814 | // creds is a near no-op). |
| 815 | enableCollectorTimers() |
| 816 | // Converge the root-SSH lockdown on boxes that completed operator setup, so |
| 817 | // a box that missed the manual step (kepler, 2026-07) picks it up on the |
| 818 | // next provision pass. A box with no operator accounts — a fresh bootstrap, |
| 819 | // or a self-hoster who never ran operator setup — is skipped: shutting root |
| 820 | // out there could shut everyone out. |
| 821 | if operatorsExist("/etc/group") { |
| 822 | changed, err := applyNoRootSSH(noRootConfPath) |
| 823 | if err != nil { |
| 824 | die("bootstrap: root-SSH lockdown: %v", err) |
| 825 | } |
| 826 | if changed { |
| 827 | fmt.Println(" root SSH disabled (PermitRootLogin no; sshd reloaded)") |
| 828 | } |
| 829 | } |
| 830 | autoUpdate := "auto-update" |
| 831 | if *noUpdateTimer { |
| 832 | autoUpdate = "NO auto-update" |
| 833 | } |
| 834 | fmt.Printf("bootstrap: router + links installed and started (%s; status-collector timers enabled)\n", autoUpdate) |
| 835 | fmt.Println(" next: lchat add-instance <host>") |
| 836 | } |
| 837 | |
amadan