| 1 | package main |
| 2 | |
| 3 | // The control daemon: the box's localhost-only provisioning surface, anticipated |
| 4 | // in docs/instance-router.md as "the god surface … never exposed on a served |
| 5 | // hostname". It executes the same creation core as `lchat add-instance` |
| 6 | // (addInstance in control.go) on behalf of the unprivileged cloud provisioner |
| 7 | // service (cloud.go), which is the public, policy-enforcing face at |
| 8 | // cloud.elevenmessenger.com. The split exists because creation needs root |
| 9 | // (systemctl, /opt/lchat writes) while everything network-facing runs sandboxed |
| 10 | // as the lchat user. See docs/cloud-provisioner.md. |
| 11 | // |
| 12 | // Auth: every unit on the box shares the lchat uid, so socket permissions can't |
| 13 | // distinguish the cloud service from a compromised instance. The gate is a |
| 14 | // bearer token in a root-only file (/etc/lchat/control.token, 0600) that the |
| 15 | // cloud unit alone receives via systemd LoadCredential. |
| 16 | |
| 17 | import ( |
| 18 | "crypto/subtle" |
| 19 | "encoding/json" |
| 20 | "errors" |
| 21 | "flag" |
| 22 | "fmt" |
| 23 | "io" |
| 24 | "log" |
| 25 | "net" |
| 26 | "net/http" |
| 27 | "os" |
| 28 | "os/user" |
| 29 | "path/filepath" |
| 30 | "strconv" |
| 31 | "strings" |
| 32 | "sync" |
| 33 | "syscall" |
| 34 | "time" |
| 35 | ) |
| 36 | |
| 37 | // controlTokenPath is root-owned (0600): readable by controld and loadable into |
| 38 | // the cloud unit by systemd, but never by anything running as the lchat user. |
| 39 | const controlTokenPath = "/etc/lchat/control.token" |
| 40 | |
| 41 | // controlSocketPath is the control daemon's unix socket: where controld listens |
| 42 | // (runControld's -socket default below) AND where the control self-update dials |
| 43 | // /healthz to confirm a freshly-swapped control binary came up. One source of |
| 44 | // truth for both, and a var so tests can point the health probe at a temp socket. |
| 45 | var controlSocketPath = envOr("LCHAT_CONTROL_SOCKET", "/run/lchat/control.sock") |
| 46 | |
| 47 | // freeDiskBytes reports the free space where instance DBs live. A var so tests |
| 48 | // can simulate a full disk without filling one. |
| 49 | var freeDiskBytes = func(path string) (uint64, error) { |
| 50 | var st syscall.Statfs_t |
| 51 | if err := syscall.Statfs(path, &st); err != nil { |
| 52 | return 0, err |
| 53 | } |
| 54 | return uint64(st.Bavail) * uint64(st.Bsize), nil |
| 55 | } |
| 56 | |
| 57 | // controld is the daemon's state: the registry it counts and creates into, the |
| 58 | // capacity policy, and the token gate. Creation is serialized (mu) so the |
| 59 | // capacity check and the create it guards are atomic. |
| 60 | type controld struct { |
| 61 | reg *Registry |
| 62 | tokenHash string |
| 63 | maxInstances int |
| 64 | minFreeGB int |
| 65 | // hibernateCreates (env LCHAT_CONTROL_HIBERNATE_CREATES=1) makes every |
| 66 | // provisioner-created space born hibernating (#550): registered + routed, |
| 67 | // env file + staged binary in place, but no systemd unit and no resident |
| 68 | // process — the router's activator wakes it on demand. Off by default: the |
| 69 | // flip to hibernate-by-default is this one env var on the box, not a code |
| 70 | // deploy. Direct `lchat add-instance` is unaffected either way (its |
| 71 | // --hibernate flag stays an explicit per-invocation choice). Requires the |
| 72 | // router to run with LCHAT_HIBERNATE_BUCKET, or created spaces never answer. |
| 73 | hibernateCreates bool |
| 74 | // maxRegistered caps TOTAL instance rows (materialized + hibernating) when |
| 75 | // hibernateCreates is on — a sanity ceiling on registry rows/env files, not |
| 76 | // a resource cap (hibernating rows hold no process). <=0 defaults to |
| 77 | // 10x maxInstances. |
| 78 | maxRegistered int |
| 79 | // homeOrigin (env LCHAT_CONTROL_HOME_ORIGIN) is the box's default |
| 80 | // LCHAT_HOME_ORIGIN for provisioner creates that don't name one. The home |
| 81 | // origin is an environment property (which home server fronts this fleet), |
| 82 | // not per-space policy — without this, a non-production box's spaces point |
| 83 | // at the compiled production default, so their backup nudges and the demo |
| 84 | // banner's live status read ask the wrong home. Empty = keep the compiled |
| 85 | // default (production boxes need nothing). |
| 86 | homeOrigin string |
| 87 | mu sync.Mutex |
| 88 | } |
| 89 | |
| 90 | // controlCreateReq mirrors addInstanceOpts across the socket. AdminToken is the |
| 91 | // migration path (import an existing token) — unused by the cloud provisioner, |
| 92 | // kept so the control API can do everything the CLI can. |
| 93 | type controlCreateReq struct { |
| 94 | Host string `json:"host"` |
| 95 | AdminToken string `json:"admin_token,omitempty"` |
| 96 | AppStoreURL string `json:"app_store_url,omitempty"` |
| 97 | HomeOrigin string `json:"home_origin,omitempty"` |
| 98 | PushRelayURL string `json:"push_relay_url,omitempty"` |
| 99 | PushRelayToken string `json:"push_relay_token,omitempty"` |
| 100 | Entitlement string `json:"entitlement,omitempty"` |
| 101 | // Name seeds the space's display name (→ LCHAT_GROUP_NAME; the create flow |
| 102 | // asks for it up front). Sanitized + capped by addInstance; optional. |
| 103 | Name string `json:"name,omitempty"` |
| 104 | // Replication classes the space at birth (registry.go's durability ladder) |
| 105 | // so a free space never spends a single awake window on the streaming rung. |
| 106 | Replication string `json:"replication,omitempty"` |
| 107 | // HomeRegion is the space's placement region (multi-edge Phase 1, |
| 108 | // docs/multi-edge.md): the mesh owns a space on a box whose LCHAT_MESH_REGION |
| 109 | // matches its home_region (mesh.candidatesFor), so relocating a space is just |
| 110 | // a home_region flip. The cloud provisioner stamps its own LCHAT_CLOUD_REGION |
| 111 | // here at create; "" (the default) keeps the space region-agnostic — hashed |
| 112 | // across the whole fleet exactly as before regions existed. |
| 113 | HomeRegion string `json:"home_region,omitempty"` |
| 114 | } |
| 115 | |
| 116 | // controlErr is the wire error shape: a stable machine code plus a human |
| 117 | // message, so callers never parse message strings. |
| 118 | type controlErr struct { |
| 119 | Code string `json:"error"` |
| 120 | Message string `json:"message"` |
| 121 | } |
| 122 | |
| 123 | func writeControlErr(w http.ResponseWriter, status int, code, msg string) { |
| 124 | writeJSON(w, status, controlErr{Code: code, Message: msg}) |
| 125 | } |
| 126 | |
| 127 | func (c *controld) handler() http.Handler { |
| 128 | mux := http.NewServeMux() |
| 129 | // Unauthenticated, state-free liveness+build probe. The box's own control |
| 130 | // self-update (controlSelfUpdate in updater.go) swaps THIS binary in place and |
| 131 | // then dials /healthz to confirm the new build actually came up before it |
| 132 | // commits — rolling the swap back if it can't. It has to answer the instant the |
| 133 | // HTTP listener binds, so it deliberately touches no DB/registry state a |
| 134 | // half-started process might still lack; it reveals only the compiled-in build |
| 135 | // string, and only over the root-only control socket. No token: the swap probe |
| 136 | // runs in the same already-root updater and there's nothing here worth gating. |
| 137 | mux.HandleFunc("GET /healthz", c.healthzHandler) |
| 138 | mux.HandleFunc("POST /instances", c.auth(c.createHandler)) |
| 139 | mux.HandleFunc("DELETE /instances/{host}", c.auth(c.deleteHandler)) |
| 140 | mux.HandleFunc("GET /instances/{host}/stats", c.auth(c.statsHandler)) |
| 141 | mux.HandleFunc("POST /instances/{host}/quota", c.auth(c.quotaHandler)) |
| 142 | mux.HandleFunc("POST /instances/{host}/meter", c.auth(c.meterHandler)) |
| 143 | mux.HandleFunc("POST /instances/{host}/bot", c.auth(c.botHandler)) |
| 144 | mux.HandleFunc("POST /instances/{host}/replication", c.auth(c.replicationHandler)) |
| 145 | mux.HandleFunc("POST /instances/{host}/materialize", c.auth(c.materializeHandler)) |
| 146 | return mux |
| 147 | } |
| 148 | |
| 149 | // healthzHandler answers the control self-update's health probe with the running |
| 150 | // control binary's build id. It touches nothing on c (no registry, no disk) so a |
| 151 | // process that has only just bound its listener still answers — see the route |
| 152 | // registration in handler() for the full why, including why it's unauthenticated. |
| 153 | func (c *controld) healthzHandler(w http.ResponseWriter, _ *http.Request) { |
| 154 | writeJSON(w, http.StatusOK, map[string]any{"build": buildVersion}) |
| 155 | } |
| 156 | |
| 157 | // replicationHandler records one instance's replication class on its registry |
| 158 | // route (the durability ladder — registry.go) and reloads the router so the |
| 159 | // next wake reads it. The class is wake-time config, so nothing is restarted |
| 160 | // and no sleeping space is woken: an awake instance keeps its current sync |
| 161 | // tick until it next sleeps. Idempotent, and deliberately reload-free when the |
| 162 | // class already matches — the cloud sweep re-pushes every pass, and a router |
| 163 | // reload per space per sweep would be pointless churn. |
| 164 | func (c *controld) replicationHandler(w http.ResponseWriter, r *http.Request) { |
| 165 | host := strings.ToLower(strings.TrimSpace(r.PathValue("host"))) |
| 166 | if !validHost(host) { |
| 167 | writeControlErr(w, http.StatusBadRequest, "invalid_host", "not a valid hostname") |
| 168 | return |
| 169 | } |
| 170 | var req struct { |
| 171 | Class string `json:"class"` |
| 172 | } |
| 173 | if err := json.NewDecoder(io.LimitReader(r.Body, 4<<10)).Decode(&req); err != nil { |
| 174 | writeControlErr(w, http.StatusBadRequest, "bad_request", "invalid JSON body") |
| 175 | return |
| 176 | } |
| 177 | class := strings.ToLower(strings.TrimSpace(req.Class)) |
| 178 | if class == "" || !validReplication(class) { |
| 179 | writeControlErr(w, http.StatusBadRequest, "bad_request", |
| 180 | fmt.Sprintf("class must be %s, %s or %s", replicationStream, replicationBatched, replicationDaily)) |
| 181 | return |
| 182 | } |
| 183 | rt, err := c.reg.get(host) |
| 184 | if err != nil { |
| 185 | log.Printf("controld: replication %s: lookup: %v", host, err) |
| 186 | writeControlErr(w, http.StatusInternalServerError, "lookup_failed", "db error") |
| 187 | return |
| 188 | } |
| 189 | if rt == nil || rt.Kind != "instance" { |
| 190 | writeControlErr(w, http.StatusNotFound, "unknown_host", "no such instance") |
| 191 | return |
| 192 | } |
| 193 | if rt.Replication == class { |
| 194 | writeJSON(w, http.StatusOK, map[string]any{"ok": true, "changed": false}) |
| 195 | return |
| 196 | } |
| 197 | if err := c.reg.setReplication(host, class); err != nil { |
| 198 | log.Printf("controld: replication %s: %v", host, err) |
| 199 | writeControlErr(w, http.StatusInternalServerError, "replication_failed", err.Error()) |
| 200 | return |
| 201 | } |
| 202 | // Best-effort: the registry write is durable either way, and the sweep's |
| 203 | // next push finds changed=false and stops — so a failed reload here must be |
| 204 | // loud (it's the only convergence signal left until something else reloads |
| 205 | // the router). |
| 206 | reloaded := true |
| 207 | if err := systemctl("reload", "lchat-router"); err != nil { |
| 208 | reloaded = false |
| 209 | log.Printf("controld: replication %s → %s: router reload FAILED: %v (class is stored but inert until a reload)", host, class, err) |
| 210 | } |
| 211 | log.Printf("controld: replication %s → %s (reloaded=%v)", host, class, reloaded) |
| 212 | writeJSON(w, http.StatusOK, map[string]any{"ok": true, "changed": true, "reloaded": reloaded}) |
| 213 | } |
| 214 | |
| 215 | // materializeHandler stages the running control binary and points host's live |
| 216 | // symlink at it, if not already done. This is the privileged half of a route's |
| 217 | // first wake on a box that never locally created it (control.go's |
| 218 | // materializeInstance normally does this at add-instance time, but a |
| 219 | // mesh-replicated route — meshreplicate.go — only ever inserts a registry row, |
| 220 | // never a symlink, since the receiving box didn't create the space). |
| 221 | // activator.go's s3Backend.start can't do this itself: the router runs |
| 222 | // sandboxed (ProtectSystem=strict, no write access outside /data and |
| 223 | // /run/lchat — see docs/instance-router.md § Hardening), so only this |
| 224 | // always-unsandboxed root daemon can. Idempotent: safe to call even if the |
| 225 | // host is already materialized (pointLive happily repoints at the same or a |
| 226 | // newer version), and safe to call for a host this box didn't create — the |
| 227 | // row already exists (checked below) is the only prerequisite. |
| 228 | func (c *controld) materializeHandler(w http.ResponseWriter, r *http.Request) { |
| 229 | host := strings.ToLower(strings.TrimSpace(r.PathValue("host"))) |
| 230 | if !validHost(host) { |
| 231 | writeControlErr(w, http.StatusBadRequest, "invalid_host", "not a valid hostname") |
| 232 | return |
| 233 | } |
| 234 | if c.reg != nil { |
| 235 | rt, err := c.reg.get(host) |
| 236 | if err != nil { |
| 237 | log.Printf("controld: materialize %s: lookup: %v", host, err) |
| 238 | writeControlErr(w, http.StatusInternalServerError, "lookup_failed", "db error") |
| 239 | return |
| 240 | } |
| 241 | if rt == nil { |
| 242 | writeControlErr(w, http.StatusNotFound, "unknown_host", "no such route") |
| 243 | return |
| 244 | } |
| 245 | } |
| 246 | ver, err := ensureSelfStaged() |
| 247 | if err != nil { |
| 248 | log.Printf("controld: materialize %s: stage: %v", host, err) |
| 249 | writeControlErr(w, http.StatusInternalServerError, "stage_failed", err.Error()) |
| 250 | return |
| 251 | } |
| 252 | if err := pointLive(host, ver); err != nil { |
| 253 | log.Printf("controld: materialize %s: point live: %v", host, err) |
| 254 | writeControlErr(w, http.StatusInternalServerError, "point_live_failed", err.Error()) |
| 255 | return |
| 256 | } |
| 257 | writeJSON(w, http.StatusOK, map[string]any{"ok": true, "version": ver}) |
| 258 | } |
| 259 | |
| 260 | // quotaHandler applies a storage cap to one instance: upsert |
| 261 | // LCHAT_BLOB_QUOTA_BYTES in its env file (0 = back to the compiled default), |
| 262 | // mirror the billing entitlement alongside it when the caller sends one, and |
| 263 | // bounce a unit-managed instance so it applies now. A hibernating instance's |
| 264 | // env is read at spawn, so its cap simply applies on next wake — never wake a |
| 265 | // space to limit it. |
| 266 | func (c *controld) quotaHandler(w http.ResponseWriter, r *http.Request) { |
| 267 | host := strings.ToLower(strings.TrimSpace(r.PathValue("host"))) |
| 268 | if !validHost(host) { |
| 269 | writeControlErr(w, http.StatusBadRequest, "invalid_host", "not a valid hostname") |
| 270 | return |
| 271 | } |
| 272 | var req struct { |
| 273 | Bytes int64 `json:"bytes"` |
| 274 | Entitlement string `json:"entitlement"` |
| 275 | } |
| 276 | if err := json.NewDecoder(io.LimitReader(r.Body, 4<<10)).Decode(&req); err != nil || req.Bytes < 0 { |
| 277 | writeControlErr(w, http.StatusBadRequest, "bad_request", "bytes required (>= 0; 0 restores the default)") |
| 278 | return |
| 279 | } |
| 280 | if req.Entitlement != "" && req.Entitlement != "free" && req.Entitlement != "paid" { |
| 281 | writeControlErr(w, http.StatusBadRequest, "bad_request", `entitlement must be "free" or "paid" (or omitted)`) |
| 282 | return |
| 283 | } |
| 284 | envPath := filepath.Join(instanceDataDir, host+".env") |
| 285 | if err := setInstanceQuota(envPath, req.Bytes, req.Entitlement); err != nil { |
| 286 | if os.IsNotExist(err) { |
| 287 | writeControlErr(w, http.StatusNotFound, "unknown_host", "no such instance") |
| 288 | return |
| 289 | } |
| 290 | log.Printf("controld: quota %s: %v", host, err) |
| 291 | writeControlErr(w, http.StatusInternalServerError, "quota_failed", err.Error()) |
| 292 | return |
| 293 | } |
| 294 | // The router (service user) reads a hibernating instance's env at wake; the |
| 295 | // file was just rewritten as root, so hand it back (same rule as creation). |
| 296 | if err := matchOwner(envPath, instanceDataDir); err != nil { |
| 297 | log.Printf("controld: quota %s: env ownership: %v", host, err) |
| 298 | } |
| 299 | restarted := false |
| 300 | hibernating := false |
| 301 | if c.reg != nil { |
| 302 | if rt, err := c.reg.get(host); err == nil && rt != nil && rt.Hibernate { |
| 303 | hibernating = true |
| 304 | } |
| 305 | } |
| 306 | if !hibernating { |
| 307 | // try-restart: bounce only if it's running (data-safe; seconds). |
| 308 | if err := systemctl("try-restart", "lchat@"+host); err == nil { |
| 309 | restarted = true |
| 310 | } else { |
| 311 | log.Printf("controld: quota %s: restart: %v (cap applies on next start)", host, err) |
| 312 | } |
| 313 | } |
| 314 | log.Printf("controld: quota %s → %d bytes (restarted=%v)", host, req.Bytes, restarted) |
| 315 | writeJSON(w, http.StatusOK, map[string]any{"ok": true, "restarted": restarted}) |
| 316 | } |
| 317 | |
| 318 | // meterHandler hands one instance its usage-pipe credentials (#857): upsert |
| 319 | // LCHAT_METER_URL + LCHAT_METER_TOKEN in the env and bounce the unit so they |
| 320 | // take effect — the same shape as quotaHandler, minus the registry-quota |
| 321 | // bookkeeping. Both empty clears the pair (metering off). |
| 322 | func (c *controld) meterHandler(w http.ResponseWriter, r *http.Request) { |
| 323 | host := strings.ToLower(strings.TrimSpace(r.PathValue("host"))) |
| 324 | if !validHost(host) { |
| 325 | writeControlErr(w, http.StatusBadRequest, "invalid_host", "not a valid hostname") |
| 326 | return |
| 327 | } |
| 328 | var req struct { |
| 329 | URL string `json:"url"` |
| 330 | Token string `json:"token"` |
| 331 | } |
| 332 | if err := json.NewDecoder(io.LimitReader(r.Body, 4<<10)).Decode(&req); err != nil || |
| 333 | (req.URL == "") != (req.Token == "") { |
| 334 | writeControlErr(w, http.StatusBadRequest, "bad_request", "url and token required together (both empty clears)") |
| 335 | return |
| 336 | } |
| 337 | envPath := filepath.Join(instanceDataDir, host+".env") |
| 338 | if err := setInstanceMeter(envPath, req.URL, req.Token); err != nil { |
| 339 | if os.IsNotExist(err) { |
| 340 | writeControlErr(w, http.StatusNotFound, "unknown_host", "no such instance") |
| 341 | return |
| 342 | } |
| 343 | log.Printf("controld: meter %s: %v", host, err) |
| 344 | writeControlErr(w, http.StatusInternalServerError, "meter_failed", err.Error()) |
| 345 | return |
| 346 | } |
| 347 | // Root just rewrote the env; the router (service user) reads it at wake. |
| 348 | if err := matchOwner(envPath, instanceDataDir); err != nil { |
| 349 | log.Printf("controld: meter %s: env ownership: %v", host, err) |
| 350 | } |
| 351 | restarted := false |
| 352 | hibernating := false |
| 353 | if c.reg != nil { |
| 354 | if rt, err := c.reg.get(host); err == nil && rt != nil && rt.Hibernate { |
| 355 | hibernating = true // picks the env up on its next wake |
| 356 | } |
| 357 | } |
| 358 | if !hibernating { |
| 359 | if err := systemctl("try-restart", "lchat@"+host); err == nil { |
| 360 | restarted = true |
| 361 | } else { |
| 362 | log.Printf("controld: meter %s: restart: %v (applies on next start)", host, err) |
| 363 | } |
| 364 | } |
| 365 | log.Printf("controld: meter %s → %v (restarted=%v)", host, req.URL != "", restarted) |
| 366 | writeJSON(w, http.StatusOK, map[string]any{"ok": true, "restarted": restarted}) |
| 367 | } |
| 368 | |
| 369 | // botHandler attaches a bot sidecar to one instance — the provisioner's half |
| 370 | // of the AI opt-in (#858, docs/ai-provisioning.md phase 2). Cloud spaces get |
| 371 | // **Eleven only** (the product voice; admins add further agents through the |
| 372 | // admin console, which the managed sidecar picks up). Idempotent: a host |
| 373 | // whose bot env is already staged answers ok without touching anything, so |
| 374 | // re-opt-ins and admin toggle flips are safe to signal every time. |
| 375 | // |
| 376 | // The model key comes from controld's own environment (ANTHROPIC_API_KEY) — |
| 377 | // the sidecar holds it, never the instance (CLAUDE.md's bot invariant). With |
| 378 | // no key configured the sidecar still attaches deterministic-only and the |
| 379 | // response says so, so the operator gap is visible rather than silent. |
| 380 | func (c *controld) botHandler(w http.ResponseWriter, r *http.Request) { |
| 381 | host := strings.ToLower(strings.TrimSpace(r.PathValue("host"))) |
| 382 | if !validHost(host) { |
| 383 | writeControlErr(w, http.StatusBadRequest, "invalid_host", "not a valid hostname") |
| 384 | return |
| 385 | } |
| 386 | hibernating := false |
| 387 | if c.reg != nil { |
| 388 | rt, err := c.reg.get(host) |
| 389 | if err != nil || rt == nil || rt.Kind != "instance" { |
| 390 | writeControlErr(w, http.StatusNotFound, "unknown_host", "no such instance") |
| 391 | return |
| 392 | } |
| 393 | hibernating = rt.Hibernate |
| 394 | } |
| 395 | apiKey := strings.TrimSpace(os.Getenv("ANTHROPIC_API_KEY")) |
| 396 | msg, err := attachBot(host, hibernating, []botIdentitySpec{elevenIdentity}, apiKey, "", envOr("LCHAT_BOT_UNIT", "lchat-bot@")) |
| 397 | if err != nil { |
| 398 | if errors.Is(err, os.ErrNotExist) { |
| 399 | // No instance env file: a registered-but-never-materialized host. |
| 400 | writeControlErr(w, http.StatusNotFound, "unknown_host", "no such instance") |
| 401 | return |
| 402 | } |
| 403 | log.Printf("controld: attach-bot %s: %v", host, err) |
| 404 | writeControlErr(w, http.StatusInternalServerError, "attach_failed", err.Error()) |
| 405 | return |
| 406 | } |
| 407 | log.Printf("controld: attach-bot %s — %s (ai=%v)", host, msg, apiKey != "") |
| 408 | writeJSON(w, http.StatusOK, map[string]any{"ok": true, "detail": msg, "ai": apiKey != ""}) |
| 409 | } |
| 410 | |
| 411 | // statsHandler reports the reaper's guards for a space — setup_complete, member |
| 412 | // count, and has_content (any message or thread). Read-only against the live |
| 413 | // instance DB. validHost guards the DB path against traversal before it's joined |
| 414 | // with the data dir. |
| 415 | func (c *controld) statsHandler(w http.ResponseWriter, r *http.Request) { |
| 416 | host := strings.ToLower(strings.TrimSpace(r.PathValue("host"))) |
| 417 | if !validHost(host) { |
| 418 | writeControlErr(w, http.StatusBadRequest, "invalid_host", "not a valid hostname") |
| 419 | return |
| 420 | } |
| 421 | members, people, setupComplete, hasContent, err := instanceStatus(host) |
| 422 | if err != nil { |
| 423 | if os.IsNotExist(err) { |
| 424 | // A hibernating space that has never woken has no local DB at all — |
| 425 | // the first wake boots it fresh, and the setup-invite mint seeds the |
| 426 | // admin user. Its stats are therefore DEFINITIONAL, answered from |
| 427 | // the registry row alone: no users, never set up, no content. No |
| 428 | // socket is dialed and no wake is triggered — a reaper sweep over |
| 429 | // thousands of dormant spaces must stay a pure metadata read, never |
| 430 | // a wave of reaper-induced wakes. (A missing DB on a NON-hibernating |
| 431 | // row stays a 404: a systemd-managed instance creates its DB at |
| 432 | // first boot, so "no DB" there means a failed create — the reaper |
| 433 | // skips those, as before.) |
| 434 | if c.reg != nil { |
| 435 | if rt, rerr := c.reg.get(host); rerr == nil && rt != nil && rt.Kind == "instance" && rt.Hibernate { |
| 436 | writeJSON(w, http.StatusOK, map[string]any{"host": host, "members": 0, "people": 0, "setup_complete": false, "has_content": false, "storage_bytes": 0}) |
| 437 | return |
| 438 | } |
| 439 | } |
| 440 | writeControlErr(w, http.StatusNotFound, "unknown_host", "no such instance") |
| 441 | return |
| 442 | } |
| 443 | log.Printf("controld: stats %s: %v", host, err) |
| 444 | writeControlErr(w, http.StatusInternalServerError, "stats_failed", err.Error()) |
| 445 | return |
| 446 | } |
| 447 | writeJSON(w, http.StatusOK, map[string]any{ |
| 448 | "host": host, "members": members, "people": people, "setup_complete": setupComplete, "has_content": hasContent, |
| 449 | // The storage meter's read (meter.go) — pure filesystem metadata, |
| 450 | // measured alongside the DB stats so one endpoint serves both sweeps. |
| 451 | "storage_bytes": instanceStorageBytes(host), |
| 452 | }) |
| 453 | } |
| 454 | |
| 455 | // deleteHandler deprovisions one instance (self-serve space deletion, reached |
| 456 | // via the cloud provisioner which enforces ownership). The instance's data is |
| 457 | // archived to /data/trash, not destroyed — see deleteInstance — unless the |
| 458 | // caller passes ?discard=1, the demo tier's no-backups contract |
| 459 | // (docs/demo-spaces.md): a demo's data dies with it, by disclosure. |
| 460 | func (c *controld) deleteHandler(w http.ResponseWriter, r *http.Request) { |
| 461 | host := r.PathValue("host") |
| 462 | discard := r.URL.Query().Get("discard") == "1" |
| 463 | c.mu.Lock() |
| 464 | defer c.mu.Unlock() |
| 465 | err := deleteInstance(c.reg, host, discard) |
| 466 | switch { |
| 467 | case errors.Is(err, errInvalidHost): |
| 468 | writeControlErr(w, http.StatusBadRequest, "invalid_host", err.Error()) |
| 469 | return |
| 470 | case errors.Is(err, errUnknownHost): |
| 471 | writeControlErr(w, http.StatusNotFound, "unknown_host", err.Error()) |
| 472 | return |
| 473 | case err != nil: |
| 474 | log.Printf("controld: delete %s: %v", host, err) |
| 475 | writeControlErr(w, http.StatusInternalServerError, "delete_failed", err.Error()) |
| 476 | return |
| 477 | } |
| 478 | if discard { |
| 479 | log.Printf("controld: deleted %s (data DISCARDED — demo contract)", host) |
| 480 | } else { |
| 481 | log.Printf("controld: deleted %s (data archived to trash)", host) |
| 482 | } |
| 483 | writeJSON(w, http.StatusOK, map[string]any{"ok": true}) |
| 484 | } |
| 485 | |
| 486 | // auth gates a handler behind the control token (constant-time hash compare, |
| 487 | // same pattern as the push relay's admin gate). |
| 488 | func (c *controld) auth(next http.HandlerFunc) http.HandlerFunc { |
| 489 | return func(w http.ResponseWriter, r *http.Request) { |
| 490 | if !tokenHashEqual(bearerTokenHeader(r), c.tokenHash) { |
| 491 | writeControlErr(w, http.StatusUnauthorized, "unauthorized", "bad or missing control token") |
| 492 | return |
| 493 | } |
| 494 | next(w, r) |
| 495 | } |
| 496 | } |
| 497 | |
| 498 | // atCapacity applies the box capacity policy: an instance-count ceiling and a |
| 499 | // free-disk floor (disk, not RAM, is the binding resource — see |
| 500 | // docs/instance-router.md § Sizing). It returns a human reason when full. |
| 501 | func (c *controld) atCapacity() (string, error) { |
| 502 | routes, err := c.reg.list() |
| 503 | if err != nil { |
| 504 | return "", err |
| 505 | } |
| 506 | total, materialized := 0, 0 |
| 507 | for _, r := range routes { |
| 508 | if r.Kind != "instance" { |
| 509 | continue |
| 510 | } |
| 511 | total++ |
| 512 | if !r.Hibernate { |
| 513 | materialized++ |
| 514 | } |
| 515 | } |
| 516 | if c.hibernateCreates { |
| 517 | // Hibernating rows hold no unit, no process, no RAM — the per-instance |
| 518 | // resource maxInstances guards is what's MATERIALIZED (systemd-managed, |
| 519 | // always-on), so only those count against it. Registrations still get a |
| 520 | // much higher sanity ceiling (rows, env files, staged symlinks are not |
| 521 | // free forever), and disk stays the real floor below. |
| 522 | if materialized >= c.maxInstances { |
| 523 | return fmt.Sprintf("materialized-instance limit reached (%d/%d)", materialized, c.maxInstances), nil |
| 524 | } |
| 525 | maxReg := c.maxRegistered |
| 526 | if maxReg <= 0 { |
| 527 | maxReg = 10 * c.maxInstances |
| 528 | } |
| 529 | if total >= maxReg { |
| 530 | return fmt.Sprintf("registered-space limit reached (%d/%d)", total, maxReg), nil |
| 531 | } |
| 532 | } else if total >= c.maxInstances { |
| 533 | return fmt.Sprintf("instance limit reached (%d/%d)", total, c.maxInstances), nil |
| 534 | } |
| 535 | free, err := freeDiskBytes(instanceDataDir) |
| 536 | if err != nil { |
| 537 | return "", err |
| 538 | } |
| 539 | if free < uint64(c.minFreeGB)<<30 { |
| 540 | return fmt.Sprintf("low disk (%.1f GB free, floor %d GB)", float64(free)/(1<<30), c.minFreeGB), nil |
| 541 | } |
| 542 | return "", nil |
| 543 | } |
| 544 | |
| 545 | func (c *controld) createHandler(w http.ResponseWriter, r *http.Request) { |
| 546 | var req controlCreateReq |
| 547 | if err := json.NewDecoder(io.LimitReader(r.Body, 1<<20)).Decode(&req); err != nil { |
| 548 | writeControlErr(w, http.StatusBadRequest, "bad_request", "invalid JSON body") |
| 549 | return |
| 550 | } |
| 551 | |
| 552 | // One create at a time: the capacity check and the creation it authorizes |
| 553 | // must not interleave with another create. |
| 554 | c.mu.Lock() |
| 555 | defer c.mu.Unlock() |
| 556 | |
| 557 | reason, err := c.atCapacity() |
| 558 | if err != nil { |
| 559 | writeControlErr(w, http.StatusInternalServerError, "create_failed", err.Error()) |
| 560 | return |
| 561 | } |
| 562 | if reason != "" { |
| 563 | log.Printf("controld: refusing %s: %s", req.Host, reason) |
| 564 | writeControlErr(w, http.StatusServiceUnavailable, "at_capacity", reason) |
| 565 | return |
| 566 | } |
| 567 | |
| 568 | // Demos are NEVER activator-managed, even on hibernate-by-default boxes: |
| 569 | // no lease, no litestream replica, no parked S3 prefix — "no backups" |
| 570 | // (docs/demo-spaces.md) is structural. They hold a materialized slot for |
| 571 | // their 24h life instead; the cloud's live-demo cap bounds how many. |
| 572 | hibernate := c.hibernateCreates |
| 573 | if strings.EqualFold(strings.TrimSpace(req.Entitlement), "demo") { |
| 574 | hibernate = false |
| 575 | } |
| 576 | // The box default fills a create that names no home origin (see the |
| 577 | // controld.homeOrigin comment); an explicit request value still wins. |
| 578 | homeOrigin := req.HomeOrigin |
| 579 | if homeOrigin == "" { |
| 580 | homeOrigin = c.homeOrigin |
| 581 | } |
| 582 | res, err := addInstance(c.reg, addInstanceOpts{ |
| 583 | Host: req.Host, |
| 584 | AdminToken: req.AdminToken, |
| 585 | AppStoreURL: req.AppStoreURL, |
| 586 | HomeOrigin: homeOrigin, |
| 587 | PushRelayURL: req.PushRelayURL, |
| 588 | PushRelayToken: req.PushRelayToken, |
| 589 | Entitlement: req.Entitlement, |
| 590 | GroupName: req.Name, |
| 591 | Replication: strings.ToLower(strings.TrimSpace(req.Replication)), |
| 592 | // Placement region (multi-edge Phase 1): recorded on the registry row, |
| 593 | // where the mesh reads it to place the space (control.go registerInstance |
| 594 | // → registry.go). TrimSpace only — region is matched case-sensitively |
| 595 | // against LCHAT_MESH_REGION (mesh.go), never lower-cased. |
| 596 | HomeRegion: strings.TrimSpace(req.HomeRegion), |
| 597 | // Hibernate-by-default (#550): with LCHAT_CONTROL_HIBERNATE_CREATES=1 |
| 598 | // every provisioner-created space is born activator-managed — the |
| 599 | // create's public-readiness probe is its one (first) wake, minting the |
| 600 | // setup link and warming the cert, and the idle window then puts it to |
| 601 | // sleep. A never-claimed space costs zero RAM from that point on. Off |
| 602 | // by default so the flip stays a per-box config decision. |
| 603 | Hibernate: hibernate, |
| 604 | }) |
| 605 | switch { |
| 606 | case errors.Is(err, errInvalidHost): |
| 607 | writeControlErr(w, http.StatusBadRequest, "invalid_host", err.Error()) |
| 608 | return |
| 609 | case errors.Is(err, errHostExists): |
| 610 | writeControlErr(w, http.StatusConflict, "host_exists", err.Error()) |
| 611 | return |
| 612 | case err != nil: |
| 613 | log.Printf("controld: create %s: %v", req.Host, err) |
| 614 | writeControlErr(w, http.StatusInternalServerError, "create_failed", err.Error()) |
| 615 | return |
| 616 | } |
| 617 | log.Printf("controld: created %s", res.Host) |
| 618 | if res.SetupErr != nil { |
| 619 | // The admin link still bootstraps, but a create whose setup link died |
| 620 | // is a degraded first-run — it must be visible in the journal. |
| 621 | log.Printf("controld: %s: setup link mint FAILED: %v", res.Host, res.SetupErr) |
| 622 | } |
| 623 | writeJSON(w, http.StatusCreated, map[string]any{ |
| 624 | "host": res.Host, |
| 625 | "url": "https://" + res.Host, |
| 626 | "setup_link": res.SetupLink, // "" if the mint failed; admin_link still bootstraps |
| 627 | "admin_link": "https://" + res.Host + "/admin#token=" + res.AdminToken, |
| 628 | }) |
| 629 | } |
| 630 | |
| 631 | // tokenHashEqual compares a presented token against a stored hash in constant |
| 632 | // time (cf. the push relay's admin gate). |
| 633 | func tokenHashEqual(token, wantHash string) bool { |
| 634 | return token != "" && subtle.ConstantTimeCompare([]byte(hashToken(token)), []byte(wantHash)) == 1 |
| 635 | } |
| 636 | |
| 637 | // bearerTokenHeader pulls the token out of "Authorization: Bearer <t>". |
| 638 | func bearerTokenHeader(r *http.Request) string { |
| 639 | const p = "Bearer " |
| 640 | h := r.Header.Get("Authorization") |
| 641 | if len(h) > len(p) && h[:len(p)] == p { |
| 642 | return h[len(p):] |
| 643 | } |
| 644 | return "" |
| 645 | } |
| 646 | |
| 647 | // runControld: `lchat controld` — serve the control API on a local unix socket. |
| 648 | // Runs as root (the lchat-control.service unit): it's the one process allowed to |
| 649 | // call systemctl and write the version store, and it is never routed publicly. |
| 650 | func runControld(args []string) { |
| 651 | fs := flag.NewFlagSet("controld", flag.ExitOnError) |
| 652 | registryPath := fs.String("registry", envOr("LCHAT_REGISTRY", "/data/registry.db"), "router registry SQLite path") |
| 653 | socketPath := fs.String("socket", controlSocketPath, "unix socket to serve the control API on") |
| 654 | tokenFile := fs.String("token-file", envOr("LCHAT_CONTROL_TOKEN_FILE", controlTokenPath), "file holding the control bearer token") |
| 655 | maxInstances := fs.Int("max-instances", envInt("LCHAT_MAX_INSTANCES", 40), "refuse creation beyond this many instances on the box (materialized only under -hibernate-creates)") |
| 656 | minFreeGB := fs.Int("min-free-gb", envInt("LCHAT_MIN_FREE_GB", 5), "refuse creation when the data disk has less than this free (GB)") |
| 657 | trashKeepDays := fs.Int("trash-keep-days", envInt("LCHAT_TRASH_KEEP_DAYS", 0), "purge archived (deleted) spaces from /data/trash after this many days; 0 keeps them forever") |
| 658 | hibernateCreates := fs.Bool("hibernate-creates", os.Getenv("LCHAT_CONTROL_HIBERNATE_CREATES") == "1", "create provisioner spaces as hibernating (activator-managed, no systemd unit) — requires the router to run with LCHAT_HIBERNATE_BUCKET") |
| 659 | maxRegistered := fs.Int("max-registered", envInt("LCHAT_MAX_REGISTERED", 0), "with -hibernate-creates: refuse registration beyond this many total spaces, hibernating included (0 = 10x -max-instances)") |
| 660 | homeOrigin := fs.String("home-origin", os.Getenv("LCHAT_CONTROL_HOME_ORIGIN"), "default LCHAT_HOME_ORIGIN for provisioner creates that don't name one (empty = the compiled production default)") |
| 661 | _ = fs.Parse(args) |
| 662 | |
| 663 | tok, err := os.ReadFile(*tokenFile) |
| 664 | if err != nil { |
| 665 | die("controld: read token file: %v (run `lchat bootstrap` to mint it)", err) |
| 666 | } |
| 667 | reg, err := openRegistry(*registryPath) |
| 668 | if err != nil { |
| 669 | die("controld: open registry: %v", err) |
| 670 | } |
| 671 | c := &controld{ |
| 672 | reg: reg, |
| 673 | tokenHash: hashToken(string(trimNL(tok))), |
| 674 | maxInstances: *maxInstances, |
| 675 | minFreeGB: *minFreeGB, |
| 676 | hibernateCreates: *hibernateCreates, |
| 677 | maxRegistered: *maxRegistered, |
| 678 | homeOrigin: strings.TrimSpace(*homeOrigin), |
| 679 | } |
| 680 | |
| 681 | if err := os.Remove(*socketPath); err != nil && !os.IsNotExist(err) { |
| 682 | die("controld: remove stale socket: %v", err) |
| 683 | } |
| 684 | ln, err := net.Listen("unix", *socketPath) |
| 685 | if err != nil { |
| 686 | die("controld: listen %s: %v", *socketPath, err) |
| 687 | } |
| 688 | // 0660 + group lchat: the sandboxed services can connect (auth is the token, |
| 689 | // not the socket), nobody else on the box can even dial. |
| 690 | if err := os.Chmod(*socketPath, 0o660); err != nil { |
| 691 | die("controld: chmod socket: %v", err) |
| 692 | } |
| 693 | if g, err := user.LookupGroup(serviceUser); err == nil { |
| 694 | if gid, err := strconv.Atoi(g.Gid); err == nil { |
| 695 | _ = os.Chown(*socketPath, -1, gid) // best-effort; needs root |
| 696 | } |
| 697 | } |
| 698 | |
| 699 | go runTrashPurge(time.Duration(*trashKeepDays) * 24 * time.Hour) |
| 700 | |
| 701 | mode := "" |
| 702 | if c.hibernateCreates { |
| 703 | maxReg := c.maxRegistered |
| 704 | if maxReg <= 0 { |
| 705 | maxReg = 10 * c.maxInstances |
| 706 | } |
| 707 | mode = fmt.Sprintf(", hibernate-creates on (max %d registered)", maxReg) |
| 708 | } |
| 709 | log.Printf("controld: serving on %s (registry %s, max %d instances, %d GB disk floor%s)", |
| 710 | *socketPath, *registryPath, *maxInstances, *minFreeGB, mode) |
| 711 | // Say the mesh posture out loud: on a mesh box this daemon MUST see the |
| 712 | // router's mesh env (the unit reads /data/router.env), or every create's |
| 713 | // route replication silently no-ops and peer-owned hosts are born dark — |
| 714 | // the failure that hid for weeks precisely because nothing logged it. |
| 715 | if hc, hcErr := hibernateConfigFromEnv(); hcErr != nil { |
| 716 | log.Printf("controld: mesh route replication OFF (%v)", hcErr) |
| 717 | } else if mesh, meshErr := meshFromEnv(hc); meshErr != nil { |
| 718 | log.Printf("controld: mesh route replication OFF (%v)", meshErr) |
| 719 | } else if mesh == nil { |
| 720 | log.Printf("controld: mesh route replication off (no LCHAT_MESH_SELF — fine off-mesh; on a mesh box check the unit reads /data/router.env)") |
| 721 | } else { |
| 722 | log.Printf("controld: mesh route replication on (self %s, domains %v)", mesh.self, mesh.domains) |
| 723 | } |
| 724 | srv := &http.Server{Handler: c.handler(), ReadHeaderTimeout: 10 * time.Second} |
| 725 | if err := srv.Serve(ln); err != nil { |
| 726 | log.Fatal(err) |
| 727 | } |
| 728 | } |
| 729 | |
| 730 | // trimNL strips a single trailing newline a hand-edited token file may carry. |
| 731 | func trimNL(b []byte) []byte { |
| 732 | for len(b) > 0 && (b[len(b)-1] == '\n' || b[len(b)-1] == '\r') { |
| 733 | b = b[:len(b)-1] |
| 734 | } |
| 735 | return b |
| 736 | } |
| 737 | |
| 738 | // envInt reads an integer env var, falling back on absent or malformed values. |
| 739 | func envInt(key string, def int) int { |
| 740 | if v := os.Getenv(key); v != "" { |
| 741 | if n, err := strconv.Atoi(v); err == nil { |
| 742 | return n |
| 743 | } |
| 744 | } |
| 745 | return def |
| 746 | } |
| 747 | |
amadan