| 1 | package main |
| 2 | |
| 3 | import ( |
| 4 | "bytes" |
| 5 | "context" |
| 6 | "crypto/sha256" |
| 7 | "database/sql" |
| 8 | "encoding/hex" |
| 9 | "encoding/json" |
| 10 | "flag" |
| 11 | "fmt" |
| 12 | "io" |
| 13 | "log" |
| 14 | "net/http" |
| 15 | "os" |
| 16 | "path/filepath" |
| 17 | "runtime" |
| 18 | "strings" |
| 19 | "syscall" |
| 20 | "time" |
| 21 | ) |
| 22 | |
| 23 | // The box-side auto-updater: `lchat update run`, fired hourly by |
| 24 | // lchat-update.timer (root — it stages binaries and restarts units, like |
| 25 | // lchat-control). One pass reads the signed channel feed (release.go), asks |
| 26 | // each instance what its admin chose (GET /api/update-prefs over the unix |
| 27 | // socket — instances are sandboxed and can't update themselves), then |
| 28 | // downloads/stages/applies per those prefs, reusing the exact deploy dance |
| 29 | // (deployHostVersion: repoint → restart → health-check → auto-rollback). |
| 30 | // It shares the deploy flock with `deploy`/`rollback`, so a timer firing |
| 31 | // mid-operator-deploy waits its turn (fails fast, next tick retries). |
| 32 | // |
| 33 | // Trust + anti-replay: the manifest is Ed25519-verified, binaries are hashed |
| 34 | // against the signed sha256, and the last accepted generated_at is persisted — |
| 35 | // a replayed stale feed is refused, so nobody can downgrade the fleet by |
| 36 | // re-serving an old (validly signed) manifest. See docs/server-releases.md. |
| 37 | |
| 38 | func incomingDir() string { return filepath.Join(optDir, "incoming") } |
| 39 | |
| 40 | // manifestStampPath persists the newest generated_at ever accepted. |
| 41 | func manifestStampPath() string { return filepath.Join(optDir, "manifest-stamp") } |
| 42 | |
| 43 | func readManifestStamp() time.Time { |
| 44 | raw, err := os.ReadFile(manifestStampPath()) |
| 45 | if err != nil { |
| 46 | return time.Time{} |
| 47 | } |
| 48 | t, err := time.Parse(time.RFC3339, strings.TrimSpace(string(raw))) |
| 49 | if err != nil { |
| 50 | return time.Time{} |
| 51 | } |
| 52 | return t |
| 53 | } |
| 54 | |
| 55 | func writeManifestStamp(t time.Time) error { |
| 56 | return os.WriteFile(manifestStampPath(), []byte(t.Format(time.RFC3339)+"\n"), 0o644) |
| 57 | } |
| 58 | |
| 59 | // fetchReleaseManifestFrom downloads and signature-verifies the channel feed |
| 60 | // from one URL. |
| 61 | func fetchReleaseManifestFrom(url string) (*releaseManifest, error) { |
| 62 | pub, err := releasePubKey() |
| 63 | if err != nil { |
| 64 | return nil, err |
| 65 | } |
| 66 | client := &http.Client{Timeout: 2 * time.Minute} |
| 67 | resp, err := client.Get(url) |
| 68 | if err != nil { |
| 69 | return nil, err |
| 70 | } |
| 71 | defer resp.Body.Close() |
| 72 | if resp.StatusCode != http.StatusOK { |
| 73 | return nil, fmt.Errorf("fetch manifest: %s", resp.Status) |
| 74 | } |
| 75 | raw, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) |
| 76 | if err != nil { |
| 77 | return nil, err |
| 78 | } |
| 79 | return verifyReleaseManifest(pub, raw) |
| 80 | } |
| 81 | |
| 82 | // fetchReleaseManifest reads the channel feed, falling back to the S3 mirror |
| 83 | // (LCHAT_RELEASE_MIRROR_URL) when the primary fetch fails — or when it |
| 84 | // verifies but is STALE: GitHub's CDN has served a pre-publish manifest for |
| 85 | // minutes after a clobber, which errors nowhere. Stale here means a |
| 86 | // generated_at strictly older than the newest feed this box ever accepted |
| 87 | // (the anti-replay stamp); a feed equal to the stamp is indistinguishable |
| 88 | // from "no new release", so it never consults the mirror — the hourly pass |
| 89 | // stays a single request in the steady state. Signature verification is |
| 90 | // identical on both paths (the mirror adds availability, never trust), and |
| 91 | // every fallback logs loudly. Shared with the instance-side availability |
| 92 | // checker (updatecheck.go), where no stamp file exists and so only outright |
| 93 | // primary failure falls back. |
| 94 | func fetchReleaseManifest(url string) (*releaseManifest, error) { |
| 95 | m, err := fetchReleaseManifestFrom(url) |
| 96 | base := releaseMirrorURL |
| 97 | if base == "" { |
| 98 | return m, err |
| 99 | } |
| 100 | stamp := readManifestStamp() |
| 101 | stale := err == nil && m.GeneratedAt.Before(stamp) |
| 102 | if err == nil && !stale { |
| 103 | return m, nil |
| 104 | } |
| 105 | murl := mirrorManifestURL(base) |
| 106 | mm, merr := fetchReleaseManifestFrom(murl) |
| 107 | switch { |
| 108 | case err != nil && merr != nil: |
| 109 | log.Printf("update: primary feed failed (%v); mirror %s failed too (%v)", err, murl, merr) |
| 110 | return nil, err |
| 111 | case err != nil: |
| 112 | log.Printf("update: primary feed failed (%v) — using mirror %s (signature verified)", err, murl) |
| 113 | return mm, nil |
| 114 | case merr != nil: |
| 115 | log.Printf("update: primary feed is stale (generated_at %s < last accepted %s) and mirror %s failed (%v) — keeping the primary result", |
| 116 | m.GeneratedAt.Format(time.RFC3339), stamp.Format(time.RFC3339), murl, merr) |
| 117 | return m, nil |
| 118 | case mm.GeneratedAt.After(m.GeneratedAt): |
| 119 | log.Printf("update: primary feed is stale (generated_at %s < last accepted %s) — using fresher mirror %s (generated_at %s)", |
| 120 | m.GeneratedAt.Format(time.RFC3339), stamp.Format(time.RFC3339), murl, mm.GeneratedAt.Format(time.RFC3339)) |
| 121 | return mm, nil |
| 122 | } |
| 123 | // The mirror is no fresher: the stale primary stands, and updatePass's |
| 124 | // replay check refuses it — the mirror never masks a replayed feed. |
| 125 | return m, nil |
| 126 | } |
| 127 | |
| 128 | // updatePrefs is what an instance's admin chose (GET /api/update-prefs). |
| 129 | type updatePrefs struct { |
| 130 | AutoUpdate bool `json:"auto_update"` |
| 131 | AutoDownload bool `json:"auto_download"` |
| 132 | Channel string `json:"channel"` |
| 133 | // Pin, when set, freezes the instance on that exact version — the updater |
| 134 | // skips it entirely (pin beats channel). How a canary declares itself. |
| 135 | Pin string `json:"pin"` |
| 136 | // Major, when set, keeps the instance on that major version: minors and |
| 137 | // patches follow the channel, a head whose semver crosses the major is |
| 138 | // held (majorHold, release.go). |
| 139 | Major string `json:"major"` |
| 140 | } |
| 141 | |
| 142 | // fetchUpdatePrefs reads an instance's prefs over its unix socket. A binary |
| 143 | // too old to have the endpoint 404s — that means the defaults (on/on/stable), |
| 144 | // which is exactly how a pre-feature fleet adopts auto-updating. |
| 145 | func fetchUpdatePrefs(socket string) (updatePrefs, error) { |
| 146 | defaults := updatePrefs{AutoUpdate: true, AutoDownload: true, Channel: "stable"} |
| 147 | client := unixHTTPClient(socket) |
| 148 | resp, err := client.Get("http://lchat/api/update-prefs") |
| 149 | if err != nil { |
| 150 | return defaults, err |
| 151 | } |
| 152 | defer resp.Body.Close() |
| 153 | if resp.StatusCode == http.StatusNotFound { |
| 154 | return defaults, nil |
| 155 | } |
| 156 | if resp.StatusCode != http.StatusOK { |
| 157 | return defaults, fmt.Errorf("update-prefs: %s", resp.Status) |
| 158 | } |
| 159 | var p updatePrefs |
| 160 | if err := json.NewDecoder(io.LimitReader(resp.Body, 1<<16)).Decode(&p); err != nil { |
| 161 | return defaults, err |
| 162 | } |
| 163 | if !validChannel(p.Channel) { |
| 164 | p.Channel = "stable" |
| 165 | } |
| 166 | if !validVersion(p.Pin) { |
| 167 | p.Pin = "" // a garbled pin must not freeze anything silently |
| 168 | } |
| 169 | if !validMajor(p.Major) { |
| 170 | p.Major = "" // same rule as the pin |
| 171 | } |
| 172 | return p, nil |
| 173 | } |
| 174 | |
| 175 | // hibernatingPrefs reads an instance's update prefs straight from its parked |
| 176 | // DB — an activator-managed host can't answer the socket fetch without being |
| 177 | // woken, and waking the fleet hourly would defeat hibernation. Read-only, so |
| 178 | // it can't disturb an awake child's WAL. FAIL-CLOSED: an absent DB (never |
| 179 | // woken here; state parked only in S3) or any read trouble returns an error |
| 180 | // and the caller skips the host — the admin's pin or updates-off may live |
| 181 | // only in the unreadable copy, and "the one host that must not move, moves" |
| 182 | // is the failure that matters. A skipped host converges on the pass after |
| 183 | // its first wake leaves a warm local DB. |
| 184 | func hibernatingPrefs(host string) (updatePrefs, error) { |
| 185 | p := updatePrefs{AutoUpdate: true, AutoDownload: true, Channel: "stable"} |
| 186 | path := filepath.Join(instanceDataDir, host+".db") |
| 187 | if _, err := os.Stat(path); err != nil { |
| 188 | return p, err |
| 189 | } |
| 190 | // Even a mode=ro open CREATES <db>-wal/-shm when they're absent (a cleanly |
| 191 | // slept instance has neither) — and this runs as root, so without repair |
| 192 | // they'd be root-owned and the instance (service user) would EACCES on its |
| 193 | // next wake: space down until a manual chown. Fix ownership on the way out. |
| 194 | defer repairDBSidecarOwnership(path) |
| 195 | conn, err := sql.Open("sqlite", "file:"+path+"?mode=ro&_pragma=busy_timeout(3000)") |
| 196 | if err != nil { |
| 197 | return p, err |
| 198 | } |
| 199 | defer conn.Close() |
| 200 | conn.SetMaxOpenConns(1) |
| 201 | ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) |
| 202 | defer cancel() |
| 203 | get := func(key string) (string, bool, error) { |
| 204 | var v string |
| 205 | err := conn.QueryRowContext(ctx, "SELECT value FROM config WHERE key = ?", key).Scan(&v) |
| 206 | if err == sql.ErrNoRows { |
| 207 | return "", false, nil |
| 208 | } |
| 209 | if err != nil { |
| 210 | return "", false, err |
| 211 | } |
| 212 | return v, true, nil |
| 213 | } |
| 214 | // Same semantics as the instance's own readers: booleans default ON when |
| 215 | // absent, an unrecognised channel falls back to stable, a garbled pin |
| 216 | // must not freeze anything silently. |
| 217 | if v, ok, err := get("update_auto"); err != nil { |
| 218 | return p, err |
| 219 | } else if ok { |
| 220 | p.AutoUpdate = v == "1" |
| 221 | } |
| 222 | if v, ok, err := get("update_download"); err != nil { |
| 223 | return p, err |
| 224 | } else if ok { |
| 225 | p.AutoDownload = v == "1" |
| 226 | } |
| 227 | if v, ok, err := get("update_channel"); err != nil { |
| 228 | return p, err |
| 229 | } else if ok && validChannel(v) { |
| 230 | p.Channel = v |
| 231 | } |
| 232 | if v, ok, err := get("update_pin"); err != nil { |
| 233 | return p, err |
| 234 | } else if ok && validVersion(v) { |
| 235 | p.Pin = v |
| 236 | } |
| 237 | if v, ok, err := get("update_major"); err != nil { |
| 238 | return p, err |
| 239 | } else if ok && validMajor(v) { |
| 240 | p.Major = v |
| 241 | } |
| 242 | return p, nil |
| 243 | } |
| 244 | |
| 245 | // repairDBSidecarOwnership chowns <db>-wal/-shm to the DB file's own uid/gid. |
| 246 | // A root-run read-only open of a checkpointed WAL DB creates those two files |
| 247 | // root-owned (mode=ro guards the DB's content, not the WAL control files), |
| 248 | // and the instance — running as the service user — then can't open its own |
| 249 | // DB: the add-bot root-owned-dir incident as a library. Best-effort: non-root |
| 250 | // callers can't chown and can't create root-owned files either. |
| 251 | func repairDBSidecarOwnership(dbPath string) { |
| 252 | st, err := os.Stat(dbPath) |
| 253 | if err != nil { |
| 254 | return |
| 255 | } |
| 256 | sys, ok := st.Sys().(*syscall.Stat_t) |
| 257 | if !ok { |
| 258 | return |
| 259 | } |
| 260 | for _, p := range []string{dbPath + "-wal", dbPath + "-shm"} { |
| 261 | if _, err := os.Stat(p); err == nil { |
| 262 | _ = os.Chown(p, int(sys.Uid), int(sys.Gid)) |
| 263 | } |
| 264 | } |
| 265 | } |
| 266 | |
| 267 | // updaterArch is the architecture this box runs — a var so tests can exercise |
| 268 | // the arm64 resolution without faking runtime.GOARCH. |
| 269 | var updaterArch = runtime.GOARCH |
| 270 | |
| 271 | // downloadBinaryToTemp streams one URL into a temp file in the incoming dir |
| 272 | // and refuses bytes whose sha256 doesn't match the signed manifest entry. |
| 273 | func downloadBinaryToTemp(url, sha, ver string) (string, error) { |
| 274 | if err := os.MkdirAll(incomingDir(), 0o755); err != nil { |
| 275 | return "", err |
| 276 | } |
| 277 | f, err := os.CreateTemp(incomingDir(), "dl-") |
| 278 | if err != nil { |
| 279 | return "", err |
| 280 | } |
| 281 | defer f.Close() |
| 282 | fail := func(err error) (string, error) { |
| 283 | os.Remove(f.Name()) |
| 284 | return "", err |
| 285 | } |
| 286 | client := &http.Client{Timeout: 10 * time.Minute} |
| 287 | resp, err := client.Get(url) |
| 288 | if err != nil { |
| 289 | return fail(err) |
| 290 | } |
| 291 | defer resp.Body.Close() |
| 292 | if resp.StatusCode != http.StatusOK { |
| 293 | return fail(fmt.Errorf("download %s: %s", ver, resp.Status)) |
| 294 | } |
| 295 | h := sha256.New() |
| 296 | if _, err := io.Copy(io.MultiWriter(f, h), resp.Body); err != nil { |
| 297 | return fail(err) |
| 298 | } |
| 299 | if got := hex.EncodeToString(h.Sum(nil)); got != sha { |
| 300 | return fail(fmt.Errorf("download %s: sha256 mismatch (refusing to stage)", ver)) |
| 301 | } |
| 302 | return f.Name(), nil |
| 303 | } |
| 304 | |
| 305 | // downloadReleaseBinary fetches this box's arch of a release into the incoming |
| 306 | // dir and refuses anything whose sha256 doesn't match the signed manifest |
| 307 | // entry. A failed primary download (GitHub 404s brand-new assets for minutes |
| 308 | // after a publish) falls back to the S3 mirror, loudly; the sha256 gate is |
| 309 | // identical on both paths, so a mirror serving the wrong bytes only ever |
| 310 | // causes a refusal. Returns the temp path; the caller stages it and removes |
| 311 | // the temp. |
| 312 | func downloadReleaseBinary(e releaseEntry) (string, error) { |
| 313 | url, sha, ok := e.forArch(updaterArch) |
| 314 | if !ok { |
| 315 | // An arm64 box facing a pre-arm64 release: skip, never install the |
| 316 | // wrong arch. The next release publishes both and the box catches up. |
| 317 | return "", fmt.Errorf("release %s has no %s build — skipping", e.Version, updaterArch) |
| 318 | } |
| 319 | path, err := downloadBinaryToTemp(url, sha, e.Version) |
| 320 | if err == nil { |
| 321 | return path, nil |
| 322 | } |
| 323 | murl := "" |
| 324 | if releaseMirrorURL != "" { |
| 325 | murl = mirrorBinaryURL(releaseMirrorURL, url) |
| 326 | } |
| 327 | if murl == "" { |
| 328 | return "", err |
| 329 | } |
| 330 | log.Printf("update: primary download of %s failed (%v) — trying mirror %s", e.Version, err, murl) |
| 331 | path, merr := downloadBinaryToTemp(murl, sha, e.Version) |
| 332 | if merr != nil { |
| 333 | return "", fmt.Errorf("download %s: primary failed (%v); mirror failed too (%v)", e.Version, err, merr) |
| 334 | } |
| 335 | return path, nil |
| 336 | } |
| 337 | |
| 338 | // ensureVersionStaged makes a manifest entry's binary available in the version |
| 339 | // store, downloading it if needed. Idempotent across hosts in one pass. |
| 340 | func ensureVersionStaged(e releaseEntry) error { |
| 341 | if _, err := os.Stat(versionBinPath(e.Version)); err == nil { |
| 342 | return nil |
| 343 | } |
| 344 | tmp, err := downloadReleaseBinary(e) |
| 345 | if err != nil { |
| 346 | return err |
| 347 | } |
| 348 | defer os.Remove(tmp) |
| 349 | return stageVersion(tmp, e.Version) |
| 350 | } |
| 351 | |
| 352 | // updatePass is one full check-and-apply sweep, split from runUpdateRun so |
| 353 | // tests drive it directly (die()-free: per-host problems log and continue — |
| 354 | // an unreachable instance must not strand the rest of the fleet unchecked). |
| 355 | // ignoreBake is the local, root-only override of the supply-chain bake window |
| 356 | // (bake.go) — deliberately never a publisher-side flag. |
| 357 | func updatePass(reg *Registry, unitTmpl string, dryRun, ignoreBake bool) error { |
| 358 | m, err := fetchReleaseManifest(releaseManifestURL) |
| 359 | if err != nil { |
| 360 | // No feed is a normal state (the releases repo may not exist yet, or |
| 361 | // the box is offline) — never an error exit the timer would flap on. |
| 362 | fmt.Printf("update: no usable manifest (%v) — nothing to do\n", err) |
| 363 | return nil |
| 364 | } |
| 365 | if last := readManifestStamp(); m.GeneratedAt.Before(last) { |
| 366 | return fmt.Errorf("manifest generated_at %s is older than the last accepted %s — refusing a replayed feed", |
| 367 | m.GeneratedAt.Format(time.RFC3339), last.Format(time.RFC3339)) |
| 368 | } |
| 369 | if !dryRun { |
| 370 | if err := writeManifestStamp(m.GeneratedAt); err != nil { |
| 371 | return fmt.Errorf("record manifest stamp: %v", err) |
| 372 | } |
| 373 | } |
| 374 | // Tick the bake clock: record every channel head this box hasn't seen at |
| 375 | // that head before, so the supply-chain window is measured from the box's |
| 376 | // own first sighting, never a feed timestamp (bake.go). Dry runs compute |
| 377 | // eligibility but persist nothing. |
| 378 | bake := loadFirstSeen() |
| 379 | if bake.observe(m, time.Now()) && !dryRun { |
| 380 | if err := bake.save(); err != nil { |
| 381 | return fmt.Errorf("record bake clock: %v", err) |
| 382 | } |
| 383 | } |
| 384 | |
| 385 | routes, err := reg.list() |
| 386 | if err != nil { |
| 387 | return err |
| 388 | } |
| 389 | // A version that failed its health check on one host isn't tried on the |
| 390 | // next this pass — one bad build shouldn't bounce every instance in turn. |
| 391 | poisoned := map[string]bool{} |
| 392 | // Versions instances are pinned to must survive GC even when no channel or |
| 393 | // registry row mentions them (a canary's branch build). |
| 394 | var pinnedVers []string |
| 395 | // Hibernating hosts, deferred to a second phase (see the branch below). |
| 396 | var hibernating []Route |
| 397 | for _, rt := range routes { |
| 398 | if rt.Kind != "instance" { |
| 399 | continue |
| 400 | } |
| 401 | if rt.Status != "ready" { |
| 402 | fmt.Printf(" %s: skipped (status %s)\n", rt.Host, rt.Status) |
| 403 | continue |
| 404 | } |
| 405 | if rt.Hibernate { |
| 406 | // Processed in a second phase below, AFTER every unit-managed host: |
| 407 | // repoints have no health check, so let an awake host discover a |
| 408 | // boot-broken build (and poison it) before any sleeper is moved. |
| 409 | hibernating = append(hibernating, rt) |
| 410 | continue |
| 411 | } |
| 412 | prefs, err := fetchUpdatePrefs(rt.Socket) |
| 413 | if err != nil { |
| 414 | fmt.Printf(" %s: unreachable (%v) — skipped\n", rt.Host, err) |
| 415 | continue |
| 416 | } |
| 417 | if prefs.Pin != "" { |
| 418 | // Pinned: the instance declared its own version — leave it alone |
| 419 | // entirely (this is what keeps a canary from being stomped back to |
| 420 | // the channel head an hour after it's deployed). |
| 421 | pinnedVers = append(pinnedVers, prefs.Pin) |
| 422 | fmt.Printf(" %s: pinned to %s — left alone\n", rt.Host, prefs.Pin) |
| 423 | continue |
| 424 | } |
| 425 | e, ok := m.Channels[prefs.Channel] |
| 426 | if !ok { |
| 427 | fmt.Printf(" %s: channel %s has no release yet\n", rt.Host, prefs.Channel) |
| 428 | continue |
| 429 | } |
| 430 | cur, _ := liveVersion(rt.Host) |
| 431 | if cur == e.Version { |
| 432 | continue // up to date — stay quiet on the hourly tick |
| 433 | } |
| 434 | if hold := majorHold(prefs.Major, e); hold != "" { |
| 435 | fmt.Printf(" %s: %s\n", rt.Host, hold) |
| 436 | continue |
| 437 | } |
| 438 | if hold := bakeHold(bake, prefs.Channel, e.Version, ignoreBake, time.Now()); hold != "" { |
| 439 | fmt.Printf(" %s: %s (%s) is %s — apply now: lchat deploy %s --version %s\n", |
| 440 | rt.Host, e.Version, prefs.Channel, hold, rt.Host, e.Version) |
| 441 | continue |
| 442 | } |
| 443 | if !prefs.AutoDownload && !prefs.AutoUpdate { |
| 444 | fmt.Printf(" %s: %s available on %s (downloads off — admin's call)\n", rt.Host, e.Version, prefs.Channel) |
| 445 | continue |
| 446 | } |
| 447 | if dryRun { |
| 448 | fmt.Printf(" %s: would update %s -> %s (%s)\n", rt.Host, cur, e.Version, prefs.Channel) |
| 449 | continue |
| 450 | } |
| 451 | if err := ensureVersionStaged(e); err != nil { |
| 452 | fmt.Printf(" %s: %v\n", rt.Host, err) |
| 453 | continue |
| 454 | } |
| 455 | if !prefs.AutoUpdate { |
| 456 | fmt.Printf(" %s: %s downloaded and staged (auto-update off — apply with `lchat deploy %s --version %s`)\n", |
| 457 | rt.Host, e.Version, rt.Host, e.Version) |
| 458 | continue |
| 459 | } |
| 460 | if poisoned[e.Version] { |
| 461 | fmt.Printf(" %s: skipping %s (failed health check on another instance this pass)\n", rt.Host, e.Version) |
| 462 | continue |
| 463 | } |
| 464 | rtCopy := rt |
| 465 | changed, err := deployHostVersion(reg, &rtCopy, unitTmpl+rt.Host, e.Version) |
| 466 | if err != nil { |
| 467 | poisoned[e.Version] = true |
| 468 | fmt.Printf(" %s: update to %s failed: %v\n", rt.Host, e.Version, err) |
| 469 | continue |
| 470 | } |
| 471 | if changed { |
| 472 | fmt.Printf(" %s: updated %s -> %s (%s)\n", rt.Host, cur, e.Version, prefs.Channel) |
| 473 | } |
| 474 | } |
| 475 | |
| 476 | // Second phase: hibernating hosts. An activator-managed host has no unit |
| 477 | // to restart and must not be woken just to update (an hourly wake would |
| 478 | // defeat hibernation). Instead: repoint its live symlink — the NEXT wake |
| 479 | // execs the new binary, since the router's spawn resolves |
| 480 | // /opt/lchat/live/<host> at exec time. Prefs come from the parked DB |
| 481 | // (read-only, fail-closed). Two stated trades vs the unit path, both in |
| 482 | // docs/server-releases.md: no health check (nothing runs to probe — a bad |
| 483 | // build surfaces at wake; recover with `lchat rollback <host>`, which |
| 484 | // repoints back), and a currently-AWAKE child keeps its old binary until |
| 485 | // it next sleeps (a running process doesn't follow the symlink). |
| 486 | for _, rt := range hibernating { |
| 487 | prefs, err := hibernatingPrefs(rt.Host) |
| 488 | if err != nil { |
| 489 | // The admin's pin or opt-out may live only in the copy we couldn't |
| 490 | // read — never move a host whose wishes are unknown. It converges |
| 491 | // on the pass after its first wake leaves a warm local DB. |
| 492 | fmt.Printf(" %s: hibernating, parked prefs unreadable (%v) — skipped\n", rt.Host, err) |
| 493 | continue |
| 494 | } |
| 495 | if prefs.Pin != "" { |
| 496 | pinnedVers = append(pinnedVers, prefs.Pin) |
| 497 | fmt.Printf(" %s: pinned to %s — left alone\n", rt.Host, prefs.Pin) |
| 498 | continue |
| 499 | } |
| 500 | e, ok := m.Channels[prefs.Channel] |
| 501 | if !ok { |
| 502 | fmt.Printf(" %s: channel %s has no release yet\n", rt.Host, prefs.Channel) |
| 503 | continue |
| 504 | } |
| 505 | cur, _ := liveVersion(rt.Host) |
| 506 | if cur == e.Version { |
| 507 | continue // up to date — stay quiet on the hourly tick |
| 508 | } |
| 509 | if hold := majorHold(prefs.Major, e); hold != "" { |
| 510 | fmt.Printf(" %s: %s\n", rt.Host, hold) |
| 511 | continue |
| 512 | } |
| 513 | if hold := bakeHold(bake, prefs.Channel, e.Version, ignoreBake, time.Now()); hold != "" { |
| 514 | fmt.Printf(" %s: %s (%s) is %s — apply now: lchat deploy %s --version %s\n", |
| 515 | rt.Host, e.Version, prefs.Channel, hold, rt.Host, e.Version) |
| 516 | continue |
| 517 | } |
| 518 | if !prefs.AutoDownload && !prefs.AutoUpdate { |
| 519 | fmt.Printf(" %s: %s available on %s (downloads off — admin's call)\n", rt.Host, e.Version, prefs.Channel) |
| 520 | continue |
| 521 | } |
| 522 | if dryRun { |
| 523 | fmt.Printf(" %s: would repoint %s -> %s (%s; hibernating)\n", rt.Host, cur, e.Version, prefs.Channel) |
| 524 | continue |
| 525 | } |
| 526 | if err := ensureVersionStaged(e); err != nil { |
| 527 | fmt.Printf(" %s: %v\n", rt.Host, err) |
| 528 | continue |
| 529 | } |
| 530 | if !prefs.AutoUpdate { |
| 531 | fmt.Printf(" %s: %s downloaded and staged (auto-update off — apply with `lchat deploy %s --version %s`)\n", |
| 532 | rt.Host, e.Version, rt.Host, e.Version) |
| 533 | continue |
| 534 | } |
| 535 | if poisoned[e.Version] { |
| 536 | fmt.Printf(" %s: skipping %s (failed health check on another instance this pass)\n", rt.Host, e.Version) |
| 537 | continue |
| 538 | } |
| 539 | if err := pointLive(rt.Host, e.Version); err != nil { |
| 540 | fmt.Printf(" %s: repoint failed: %v\n", rt.Host, err) |
| 541 | continue |
| 542 | } |
| 543 | if err := reg.setVersions(rt.Host, e.Version, cur); err != nil { |
| 544 | fmt.Printf(" %s: repointed but recording the version failed: %v\n", rt.Host, err) |
| 545 | continue |
| 546 | } |
| 547 | fmt.Printf(" %s: repointed %s -> %s (%s; hibernating — runs it on next wake)\n", rt.Host, cur, e.Version, prefs.Channel) |
| 548 | } |
| 549 | |
| 550 | if !dryRun { |
| 551 | gcVersions(reg, m, pinnedVers) |
| 552 | } |
| 553 | // Last, the box's own trust plane: the control binary rides the same |
| 554 | // signed feed (see controlSelfUpdate) — after instances, so a build that |
| 555 | // poisons a health check this pass never also becomes the control plane. |
| 556 | // A self-update failure is already logged loudly inside (it rolls itself |
| 557 | // back); surface the summary here but never fail the whole pass on it — |
| 558 | // the instances above already converged. Passing `poisoned` makes the |
| 559 | // "a build that poisons a health check this pass never also becomes the |
| 560 | // control plane" promise real: a head that just failed on an instance is |
| 561 | // skipped for the control binary too. |
| 562 | if err := controlSelfUpdate(m, bake, poisoned, dryRun, ignoreBake); err != nil { |
| 563 | fmt.Printf("update: control self-update: %v\n", err) |
| 564 | } |
| 565 | // The sibling services (connect/push/home) converge from the same signed |
| 566 | // entry, after the control binary — see updaterservices.go. |
| 567 | serviceBinariesUpdate(m, bake, poisoned, dryRun, ignoreBake) |
| 568 | return nil |
| 569 | } |
| 570 | |
| 571 | // controlSelfUpdate keeps the CONTROL binary — /opt/lchat/lchat, the path |
| 572 | // every privileged operation execs through (router, controld, cloud, this |
| 573 | // very updater) — on its own release channel, from the same signed feed the |
| 574 | // instance pass just verified. Before the operator-allowlist hardening this |
| 575 | // was a manual root `cp`; with root SSH gone (docs/instance-router.md § Box |
| 576 | // access), the already-root update pass is the one place the swap can live — |
| 577 | // and it adds no trust the box didn't have: same signature, same sha, same |
| 578 | // bake window as every instance binary. |
| 579 | // |
| 580 | // LCHAT_CONTROL_CHANNEL (in the unit's /data/update.env) picks the channel; |
| 581 | // "stable" is the default — the trust plane should be the LAST thing on a |
| 582 | // box to take a new build — and "off" disables self-update entirely. The |
| 583 | // swap is an atomic same-filesystem rename over the existing path (running |
| 584 | // services keep their old inode until restarted; sudo never sees a missing |
| 585 | // path), with the outgoing build kept beside it as .prev-<build>. |
| 586 | // |
| 587 | // The swap is NOT committed blind: after bouncing lchat-control/lchat-cloud |
| 588 | // onto the new binary we confirm the control plane reports the new build. The |
| 589 | // strong gate is the control daemon's live /healthz (waitControlHealthy); when |
| 590 | // lchat-control isn't running to answer it, we fall back to the weaker but |
| 591 | // controld-independent `<self> version` integrity check (controlBinaryReports) |
| 592 | // rather than roll back a binary the probe simply couldn't reach. If neither |
| 593 | // gate passes we restore .prev and confirm the box recovered — a boot-broken |
| 594 | // control plane rolling itself back is the whole point of this function, since |
| 595 | // with root SSH gone an operator can't reach in to fix a bricked box by hand. |
| 596 | // lchat-control and lchat-cloud are the only units bounced (backend sockets, |
| 597 | // invisible to members); the ROUTER is deliberately NOT restarted here — its |
| 598 | // front door should never blip on a timer's schedule, and it has no /healthz to |
| 599 | // probe, so it runs the old build until an operator restarts it, noted below. |
| 600 | // |
| 601 | // A rolled-back head is QUARANTINED (writeFailedControlVersion, persisted under |
| 602 | // optDir) so the hourly pass doesn't reinstall + roll it back forever; the |
| 603 | // quarantine clears when a good build commits or the channel moves past it. A |
| 604 | // head that poisoned an instance's health check THIS pass is skipped up front. |
| 605 | // |
| 606 | // Returns an error only when something actually went wrong (a pre-swap failure, |
| 607 | // or a swap that was rolled back); every no-op path returns nil. updatePass |
| 608 | // logs the summary but never fails the pass on it — the instances already |
| 609 | // converged before this runs. |
| 610 | // |
| 611 | // RESIDUAL RISK (deliberately not addressed here): the swap → health-check |
| 612 | // window is not crash-atomic. A reboot or kill in that window can leave the |
| 613 | // loose /opt/lchat/lchat as the unproven NEW build with no process left to run |
| 614 | // the rollback — so nothing auto-recovers until a human renames .prev back. |
| 615 | // Closing it means a boot-time recovery (an ExecStartPre on lchat-router / |
| 616 | // lchat-control that verifies the loose binary and reverts to .prev on doubt); |
| 617 | // that's deferred because it touches the ingress start path, which we don't |
| 618 | // want a timer-adjacent change to destabilise. |
| 619 | func controlSelfUpdate(m *releaseManifest, bake *firstSeenState, poisoned map[string]bool, dryRun, ignoreBake bool) error { |
| 620 | if buildVersion == "dev" { |
| 621 | // An unstamped binary (local build, test binary) has no place in the |
| 622 | // channel ordering — self-updating it would clobber whatever the |
| 623 | // operator is hand-running. Only release-stamped builds converge. |
| 624 | return nil |
| 625 | } |
| 626 | channel := envOr("LCHAT_CONTROL_CHANNEL", "stable") |
| 627 | if channel == "off" { |
| 628 | return nil |
| 629 | } |
| 630 | e, ok := m.Channels[channel] |
| 631 | if !ok || e.Version == buildVersion { |
| 632 | return nil |
| 633 | } |
| 634 | // The channel moved past a previously-quarantined head — forget the |
| 635 | // quarantine and let the new head be tried. |
| 636 | if failed := readFailedControlVersion(); failed != "" && failed != e.Version { |
| 637 | clearFailedControlVersion() |
| 638 | } |
| 639 | // A head we already swapped in, health-checked, and rolled back stays |
| 640 | // quarantined until a human pins/patches the channel or it moves on — never |
| 641 | // reinstalled + rolled back every hour. |
| 642 | if e.Version == readFailedControlVersion() { |
| 643 | fmt.Printf("update: control build %s is quarantined (failed its health check; pin or patch the channel to move)\n", e.Version) |
| 644 | return nil |
| 645 | } |
| 646 | // A build that failed a health check on an instance THIS pass must never |
| 647 | // then become the control plane (updatePass runs us after the instances). |
| 648 | if poisoned[e.Version] { |
| 649 | fmt.Printf("update: skipping control update to %s — it failed a health check on an instance this pass\n", e.Version) |
| 650 | return nil |
| 651 | } |
| 652 | if hold := bakeHold(bake, channel, e.Version, ignoreBake, time.Now()); hold != "" { |
| 653 | fmt.Printf("update: control binary %s -> %s: %s\n", buildVersion, e.Version, hold) |
| 654 | return nil |
| 655 | } |
| 656 | if dryRun { |
| 657 | fmt.Printf("update: control binary would update %s -> %s (%s; dry run)\n", buildVersion, e.Version, channel) |
| 658 | return nil |
| 659 | } |
| 660 | self, err := selfPath() |
| 661 | if err != nil { |
| 662 | return err |
| 663 | } |
| 664 | tmp, err := downloadReleaseBinary(e) |
| 665 | if err != nil { |
| 666 | return err |
| 667 | } |
| 668 | defer os.Remove(tmp) |
| 669 | if err := os.Chmod(tmp, 0o755); err != nil { |
| 670 | return fmt.Errorf("chmod: %v", err) |
| 671 | } |
| 672 | // Stage the verified build into the version store BEFORE swapping it live. |
| 673 | // gcVersions keeps every channel head, so this protects the good target from |
| 674 | // the pruner and makes it reachable to `lchat rollback`/re-deploy — the swap |
| 675 | // alone would leave the new build only as the loose /opt/lchat/lchat, a file |
| 676 | // the version store has never heard of. |
| 677 | if err := stageVersion(tmp, e.Version); err != nil { |
| 678 | return fmt.Errorf("stage %s: %v", e.Version, err) |
| 679 | } |
| 680 | // One .prev at a time: the previous previous build is already two behind |
| 681 | // and reachable from the version store anyway. |
| 682 | if old, _ := filepath.Glob(self + ".prev-*"); len(old) > 0 { |
| 683 | for _, p := range old { |
| 684 | os.Remove(p) |
| 685 | } |
| 686 | } |
| 687 | // .prev is written HERE, from the still-running (known-good) binary, and only |
| 688 | // ever just before a swap — so it always points at the last build that passed |
| 689 | // its health check. A rollback below restores it and returns immediately, |
| 690 | // never advancing .prev past an unproven build; the invariant is that the |
| 691 | // loose /opt/lchat/lchat is always either health-confirmed or being actively |
| 692 | // rolled back to one. |
| 693 | prev := self + ".prev-" + buildVersion |
| 694 | if err := copyFileAtomic(self, prev, 0o755); err != nil { |
| 695 | return fmt.Errorf("keep rollback copy: %v", err) |
| 696 | } |
| 697 | // downloadReleaseBinary lands in incomingDir() — same filesystem as the |
| 698 | // control binary, so this rename atomically replaces the path. |
| 699 | if err := os.Rename(tmp, self); err != nil { |
| 700 | return fmt.Errorf("install: %v", err) |
| 701 | } |
| 702 | // The binary is swapped; prove it comes up before walking away. Bounce the |
| 703 | // backend units onto it, then confirm the control plane reports the new build. |
| 704 | restartControlUnits() |
| 705 | if herr := waitControlHealthy(e.Version, deployHealthTimeout); herr != nil { |
| 706 | // The live probe failed — but it's only AUTHORITATIVE when lchat-control |
| 707 | // is actually running to answer it. try-restart is a no-op on a box where |
| 708 | // the unit is inactive (never enabled, or intentionally stopped), so a |
| 709 | // failed probe there means "nobody answered", not "the binary is bad". |
| 710 | if isActive(controlUnit) { |
| 711 | return rollbackControl(self, prev, e.Version, herr) // authoritative failure |
| 712 | } |
| 713 | // controld isn't up to probe. Fall back to the weaker, controld-independent |
| 714 | // gate: run `<self> version` and require it names the target build. That |
| 715 | // proves the new binary loads and self-reports the right build/arch (a |
| 716 | // corrupt or wrong-arch binary fails), NOT that the service boots — hence |
| 717 | // "weaker", used only because the strong gate is unavailable. controld |
| 718 | // starts fresh from this binary when next needed. |
| 719 | if verr := controlBinaryReports(self, e.Version); verr != nil { |
| 720 | return rollbackControl(self, prev, e.Version, fmt.Errorf("lchat-control inactive and the new binary failed its integrity check: %v", verr)) |
| 721 | } |
| 722 | log.Printf("update: lchat-control inactive — committing control update to %s on the `version` integrity check (weaker gate; live probe unavailable)", e.Version) |
| 723 | // fall through to commit |
| 724 | } |
| 725 | // Committed. Any stale quarantine no longer applies. |
| 726 | clearFailedControlVersion() |
| 727 | // F5: cloud rode the same swap+restart but isn't the health gate (the router |
| 728 | // and chat are unaffected by it) — so don't roll back on it, just flag it. |
| 729 | if !isActive(cloudUnit) { |
| 730 | log.Printf("update: WARN lchat-cloud not active after control update to %s — check the provisioner", e.Version) |
| 731 | } |
| 732 | fmt.Printf("update: control binary %s -> %s (%s; healthy; rollback at %s)\n", buildVersion, e.Version, channel, prev) |
| 733 | fmt.Printf("update: lchat-router still runs %s — restart it when convenient to converge\n", buildVersion) |
| 734 | return nil |
| 735 | } |
| 736 | |
| 737 | // rollbackControl restores the last-known-good control binary (.prev) after a |
| 738 | // swapped-in build failed its health check, confirms the box came back, and |
| 739 | // quarantines the bad build so the hourly pass doesn't reinstall + roll it back |
| 740 | // forever. It returns a non-nil error describing the rollback — or a screaming, |
| 741 | // non-looping CONTROL PLANE BRICKED error if recovery itself fails. |
| 742 | func rollbackControl(self, prev, newVer string, healthErr error) error { |
| 743 | log.Printf("update: control binary %s -> %s is UNHEALTHY (%v) — rolling back to %s", buildVersion, newVer, healthErr, buildVersion) |
| 744 | if rberr := os.Rename(prev, self); rberr != nil { |
| 745 | return fmt.Errorf("CONTROL PLANE BRICKED: %s failed its health check AND restoring %s failed: %v (health error: %v)", newVer, prev, rberr, healthErr) |
| 746 | } |
| 747 | restartControlUnits() |
| 748 | // Confirm the restored binary with the same strong/weak gate: the live probe |
| 749 | // when controld runs, else the `version` integrity check. |
| 750 | var recerr error |
| 751 | if isActive(controlUnit) { |
| 752 | recerr = waitControlHealthy(buildVersion, deployHealthTimeout) |
| 753 | } else { |
| 754 | recerr = controlBinaryReports(self, buildVersion) |
| 755 | } |
| 756 | if recerr != nil { |
| 757 | // Genuine brick: neither the new build nor the restored old one is |
| 758 | // healthy. Scream, but do NOT loop — a human has to look now. |
| 759 | log.Printf("update: CONTROL PLANE BRICKED — rolled back to %s but IT is unhealthy too (%v); box needs a human NOW", buildVersion, recerr) |
| 760 | return fmt.Errorf("CONTROL PLANE BRICKED: %s unhealthy, rolled back to %s but it is unhealthy too (%v) — box needs a human NOW (original health error: %v)", newVer, buildVersion, recerr, healthErr) |
| 761 | } |
| 762 | // Persist the quarantine so the next hourly pass short-circuits this head |
| 763 | // instead of reinstalling + rolling it back again. |
| 764 | if err := writeFailedControlVersion(newVer); err != nil { |
| 765 | log.Printf("update: warning: couldn't persist control quarantine for %s: %v", newVer, err) |
| 766 | } |
| 767 | log.Printf("update: rolled back to %s and it is healthy again — %s is quarantined until a good build ships or you pin/patch the channel", buildVersion, newVer) |
| 768 | return fmt.Errorf("control binary %s failed its health check and was rolled back to %s: %v", newVer, buildVersion, healthErr) |
| 769 | } |
| 770 | |
| 771 | // controlUnit / cloudUnit / linksUnit are the systemd units that exec the |
| 772 | // control binary (/opt/lchat/lchat) directly, rather than a per-instance |
| 773 | // versioned live symlink — so they keep running the OLD inode until something |
| 774 | // restarts them. |
| 775 | const ( |
| 776 | controlUnit = "lchat-control.service" |
| 777 | cloudUnit = "lchat-cloud.service" |
| 778 | linksUnit = "lchat-links.service" |
| 779 | ) |
| 780 | |
| 781 | // controlBinaryUnits is the one list of them, so the updater and the tests that |
| 782 | // assert "one restart each, no rollback round" can't disagree about how many |
| 783 | // that is. Adding a unit here is all it takes to put it on the update path. |
| 784 | var controlBinaryUnits = []string{controlUnit, cloudUnit, linksUnit} |
| 785 | |
| 786 | // restartControlUnits bounces the backend units that exec the control binary — |
| 787 | // lchat-control, lchat-cloud, and lchat-links — onto whatever /opt/lchat/lchat |
| 788 | // now is. The ROUTER is deliberately excluded: it's the fleet's only ingress, |
| 789 | // has no /healthz to probe, and must never blip on the updater's schedule (it |
| 790 | // keeps its old inode until an operator restarts it). Restart failures only |
| 791 | // log — the health probe that follows is the real gate on whether the new |
| 792 | // build is up. |
| 793 | // |
| 794 | // links belongs here for the same reason cloud does, and is the cheapest of |
| 795 | // the three to bounce: it holds no database and no connections, just an |
| 796 | // in-memory verification cache that re-populates lazily. Without it the |
| 797 | // redirector would be the one service that never picks up a new build at all — |
| 798 | // try-restart is a no-op on a box where it isn't running, so boxes that route |
| 799 | // no short links are unaffected. |
| 800 | func restartControlUnits() { |
| 801 | for _, unit := range controlBinaryUnits { |
| 802 | if err := systemctl("try-restart", unit); err != nil { |
| 803 | fmt.Printf("update: %s restart: %v (new build applies on its next start)\n", unit, err) |
| 804 | } |
| 805 | } |
| 806 | } |
| 807 | |
| 808 | // isActive reports whether a systemd unit is currently active (`systemctl |
| 809 | // is-active --quiet` exits 0 only when active). A var so tests can fake the |
| 810 | // box's unit state — it decides whether the control /healthz probe is even |
| 811 | // answerable, so a failed probe on a box that simply isn't running controld |
| 812 | // doesn't roll back a perfectly good binary. |
| 813 | var isActive = func(unit string) bool { |
| 814 | return systemctlQuery("is-active", "--quiet", unit) == nil |
| 815 | } |
| 816 | |
| 817 | // controlBinaryReports is the controld-independent integrity gate: run |
| 818 | // `<path> version` and require its output names wantBuild. It proves the binary |
| 819 | // execs and self-reports the expected build/arch, which is weaker than a live |
| 820 | // /healthz (it does NOT prove the service boots) but is all we have when |
| 821 | // lchat-control isn't running to answer a probe. A var so tests can fake the |
| 822 | // exec without a real binary on disk. |
| 823 | var controlBinaryReports = func(path, wantBuild string) error { |
| 824 | got, err := readBinaryVersion(path) |
| 825 | if err != nil { |
| 826 | return fmt.Errorf("run %s version: %v", path, err) |
| 827 | } |
| 828 | if !strings.Contains(got, wantBuild) { |
| 829 | return fmt.Errorf("%s reported build %q, want %q", path, got, wantBuild) |
| 830 | } |
| 831 | return nil |
| 832 | } |
| 833 | |
| 834 | // controlFailedVersionPath persists the last control head that failed its health |
| 835 | // check and was rolled back (mirrors manifestStampPath). It lives under optDir |
| 836 | // so it survives the updater process and is read on the next hourly pass. |
| 837 | func controlFailedVersionPath() string { return filepath.Join(optDir, ".control-failed-version") } |
| 838 | |
| 839 | // readFailedControlVersion returns the quarantined control build, or "" if none. |
| 840 | func readFailedControlVersion() string { |
| 841 | raw, err := os.ReadFile(controlFailedVersionPath()) |
| 842 | if err != nil { |
| 843 | return "" |
| 844 | } |
| 845 | return strings.TrimSpace(string(raw)) |
| 846 | } |
| 847 | |
| 848 | // writeFailedControlVersion records ver as quarantined. |
| 849 | func writeFailedControlVersion(ver string) error { |
| 850 | return os.WriteFile(controlFailedVersionPath(), []byte(ver+"\n"), 0o644) |
| 851 | } |
| 852 | |
| 853 | // clearFailedControlVersion lifts the quarantine (a good build committed, or the |
| 854 | // channel moved on). Absent file is fine — clearing "nothing" is a no-op. |
| 855 | func clearFailedControlVersion() { |
| 856 | if err := os.Remove(controlFailedVersionPath()); err != nil && !os.IsNotExist(err) { |
| 857 | log.Printf("update: warning: couldn't clear control quarantine file: %v", err) |
| 858 | } |
| 859 | } |
| 860 | |
| 861 | // waitControlHealthy polls the control daemon's /healthz until it reports |
| 862 | // wantBuild, or timeout elapses — the gate on the box's self-update. After the |
| 863 | // control binary is swapped and lchat-control restarted, this confirms the NEW |
| 864 | // build came up (wantBuild = the new version); on the rollback path it confirms |
| 865 | // the OLD build came back (wantBuild = buildVersion, this updater process's own |
| 866 | // still-running id). |
| 867 | // |
| 868 | // lchat-control runs Restart=always (RestartSec=1), so a crash-looping build |
| 869 | // flaps between activating and active: the socket comes and goes and dials fail |
| 870 | // with "connection refused" in between. A transient refusal is therefore |
| 871 | // neither success nor failure — keep retrying across it until either the build |
| 872 | // matches (healthy, return early) or the whole timeout expires (unhealthy). It |
| 873 | // mirrors waitHealthyVersion, but against the control socket and /healthz. |
| 874 | func waitControlHealthy(wantBuild string, timeout time.Duration) error { |
| 875 | deadline := time.Now().Add(timeout) |
| 876 | client := unixHTTPClient(controlSocketPath) |
| 877 | var last error |
| 878 | for time.Now().Before(deadline) { |
| 879 | resp, err := client.Get("http://lchat/healthz") |
| 880 | if err == nil { |
| 881 | var body struct { |
| 882 | Build string `json:"build"` |
| 883 | } |
| 884 | derr := json.NewDecoder(io.LimitReader(resp.Body, 1<<16)).Decode(&body) |
| 885 | resp.Body.Close() |
| 886 | if derr == nil && body.Build == wantBuild { |
| 887 | return nil |
| 888 | } |
| 889 | if derr != nil { |
| 890 | last = fmt.Errorf("read /healthz: %v", derr) |
| 891 | } else { |
| 892 | last = fmt.Errorf("control build %q != expected %q", body.Build, wantBuild) |
| 893 | } |
| 894 | } else { |
| 895 | last = err // often "connection refused" mid-restart — retried, not fatal |
| 896 | } |
| 897 | time.Sleep(150 * time.Millisecond) |
| 898 | } |
| 899 | return fmt.Errorf("control daemon didn't report build %q within %s: %v", wantBuild, timeout, last) |
| 900 | } |
| 901 | |
| 902 | // gcVersions prunes version-store dirs nothing references anymore: kept are |
| 903 | // every route's current + previous version, every channel head, the control |
| 904 | // binary's own build, every pinned version (keepExtra), and whatever any live |
| 905 | // symlink points at (belt and braces — the symlinks are what units actually |
| 906 | // exec). Without this, hourly edge updates would grow /opt/lchat/versions |
| 907 | // without bound on a box that enforces a free-disk floor. Leftover crashed |
| 908 | // downloads are swept too. |
| 909 | func gcVersions(reg *Registry, m *releaseManifest, keepExtra []string) { |
| 910 | keep := map[string]bool{buildVersion: true} |
| 911 | for _, v := range keepExtra { |
| 912 | keep[v] = true |
| 913 | } |
| 914 | routes, err := reg.list() |
| 915 | if err != nil { |
| 916 | return |
| 917 | } |
| 918 | for _, rt := range routes { |
| 919 | keep[rt.Version], keep[rt.PrevVersion] = true, true |
| 920 | } |
| 921 | if m != nil { |
| 922 | for _, e := range m.Channels { |
| 923 | keep[e.Version] = true |
| 924 | } |
| 925 | } |
| 926 | if links, err := os.ReadDir(liveDir()); err == nil { |
| 927 | for _, l := range links { |
| 928 | if v, err := liveVersion(l.Name()); err == nil { |
| 929 | keep[v] = true |
| 930 | } |
| 931 | } |
| 932 | } |
| 933 | entries, err := os.ReadDir(versionsDir()) |
| 934 | if err != nil { |
| 935 | return |
| 936 | } |
| 937 | for _, en := range entries { |
| 938 | if !en.IsDir() || keep[en.Name()] { |
| 939 | continue |
| 940 | } |
| 941 | if err := os.RemoveAll(filepath.Join(versionsDir(), en.Name())); err == nil { |
| 942 | fmt.Printf(" pruned unused version %s\n", en.Name()) |
| 943 | } |
| 944 | } |
| 945 | // Our downloads are temp-then-removed; a surviving dl-* file is a crashed |
| 946 | // pass. (Operator-scp'd files in incoming/ are left alone.) |
| 947 | if files, err := os.ReadDir(incomingDir()); err == nil { |
| 948 | for _, f := range files { |
| 949 | if strings.HasPrefix(f.Name(), "dl-") { |
| 950 | _ = os.Remove(filepath.Join(incomingDir(), f.Name())) |
| 951 | } |
| 952 | } |
| 953 | } |
| 954 | } |
| 955 | |
| 956 | // runUpdate: `lchat update run|status|pin|unpin|major` — the auto-updater CLI. |
| 957 | func runUpdate(args []string) { |
| 958 | if len(args) == 0 { |
| 959 | die("update: subcommand required: run | status | pin | unpin | major") |
| 960 | } |
| 961 | switch args[0] { |
| 962 | case "run": |
| 963 | runUpdateRun(args[1:]) |
| 964 | case "status": |
| 965 | runUpdateStatus(args[1:]) |
| 966 | case "pin": |
| 967 | runUpdatePin(args[1:], true) |
| 968 | case "unpin": |
| 969 | runUpdatePin(args[1:], false) |
| 970 | case "major": |
| 971 | runUpdateMajor(args[1:]) |
| 972 | default: |
| 973 | die("update: unknown subcommand %q (run | status | pin | unpin | major)", args[0]) |
| 974 | } |
| 975 | } |
| 976 | |
| 977 | // pinHost writes update_pin into the instance's own settings via its admin API |
| 978 | // over the unix socket — the pin is instance state (the same value the admin |
| 979 | // panel shows and /api/update-prefs reports), not a registry side-channel. The |
| 980 | // admin token comes from the instance's EnvironmentFile, like `admin-link`. |
| 981 | // The write is read back before reporting success: a pin the operator |
| 982 | // believes is set but isn't is exactly how the next hourly pass stomps a |
| 983 | // canary. |
| 984 | func pinHost(rt *Route, pin string) error { |
| 985 | if err := writeHostSetting(rt, "update_pin", pin); err != nil { |
| 986 | return err |
| 987 | } |
| 988 | return confirmHostSetting(rt, "pin", pin, func(p updatePrefs) string { return p.Pin }) |
| 989 | } |
| 990 | |
| 991 | // confirmHostSetting re-reads /api/update-prefs after a settings write and |
| 992 | // errors when the value didn't stick. An instance binary that predates a |
| 993 | // setting ACCEPTS the POST and silently ignores the unknown field — found |
| 994 | // live: `update major` against a pre-0.11.0 build reported success while |
| 995 | // storing nothing. |
| 996 | func confirmHostSetting(rt *Route, what, want string, read func(updatePrefs) string) error { |
| 997 | p, err := fetchUpdatePrefs(rt.Socket) |
| 998 | if err != nil { |
| 999 | return fmt.Errorf("wrote the %s but couldn't read it back (%v) — verify with `lchat update status`", what, err) |
| 1000 | } |
| 1001 | if got := read(p); got != want { |
| 1002 | return fmt.Errorf("%s accepted the %s write but reports %q — its binary likely predates the setting; update the instance first, then retry", rt.Host, what, got) |
| 1003 | } |
| 1004 | return nil |
| 1005 | } |
| 1006 | |
| 1007 | // writeHostSetting posts one settings key into the instance's own admin API — |
| 1008 | // the shared plumbing behind `update pin`/`update major`. |
| 1009 | func writeHostSetting(rt *Route, key, value string) error { |
| 1010 | env, err := readEnvFile(filepath.Join(instanceDataDir, rt.Host+".env")) |
| 1011 | if err != nil { |
| 1012 | return err |
| 1013 | } |
| 1014 | tok := env["LCHAT_ADMIN_TOKEN"] |
| 1015 | if tok == "" { |
| 1016 | return fmt.Errorf("no LCHAT_ADMIN_TOKEN in %s/%s.env", instanceDataDir, rt.Host) |
| 1017 | } |
| 1018 | body, err := json.Marshal(map[string]string{key: value}) |
| 1019 | if err != nil { |
| 1020 | return err |
| 1021 | } |
| 1022 | req, err := http.NewRequest("POST", "http://lchat/api/admin/settings", bytes.NewReader(body)) |
| 1023 | if err != nil { |
| 1024 | return err |
| 1025 | } |
| 1026 | req.Header.Set("Authorization", "Bearer "+tok) |
| 1027 | resp, err := unixHTTPClient(rt.Socket).Do(req) |
| 1028 | if err != nil { |
| 1029 | return err |
| 1030 | } |
| 1031 | defer resp.Body.Close() |
| 1032 | if resp.StatusCode != http.StatusOK { |
| 1033 | return fmt.Errorf("settings write: %s", resp.Status) |
| 1034 | } |
| 1035 | return nil |
| 1036 | } |
| 1037 | |
| 1038 | // runUpdatePin: `lchat update pin <host> [<ver>]` freezes an instance on a |
| 1039 | // version (default: whatever it's running now); `unpin` resumes its channel. |
| 1040 | func runUpdatePin(args []string, pin bool) { |
| 1041 | cmd := "unpin" |
| 1042 | if pin { |
| 1043 | cmd = "pin" |
| 1044 | } |
| 1045 | fs := flag.NewFlagSet("update "+cmd, flag.ExitOnError) |
| 1046 | registryPath := fs.String("registry", envOr("LCHAT_REGISTRY", "/data/registry.db"), "router registry SQLite path") |
| 1047 | _ = fs.Parse(reorderFlagsFirst(fs, args)) |
| 1048 | |
| 1049 | host := strings.ToLower(strings.TrimSpace(fs.Arg(0))) |
| 1050 | if !validHost(host) { |
| 1051 | die("update %s: invalid host %q", cmd, host) |
| 1052 | } |
| 1053 | reg, err := openRegistry(*registryPath) |
| 1054 | if err != nil { |
| 1055 | die("update %s: open registry: %v", cmd, err) |
| 1056 | } |
| 1057 | rt, err := reg.get(host) |
| 1058 | if err != nil || rt == nil || rt.Kind != "instance" { |
| 1059 | die("update %s: %s is not a managed instance", cmd, host) |
| 1060 | } |
| 1061 | |
| 1062 | ver := "" |
| 1063 | if pin { |
| 1064 | ver = strings.TrimSpace(fs.Arg(1)) |
| 1065 | if ver == "" { |
| 1066 | // Default to what the instance is running — "freeze me here". |
| 1067 | if ver, _ = liveVersion(host); ver == "" { |
| 1068 | die("update pin: %s has no live version — pass one explicitly", host) |
| 1069 | } |
| 1070 | } |
| 1071 | if !validVersion(ver) { |
| 1072 | die("update pin: invalid version %q", ver) |
| 1073 | } |
| 1074 | } |
| 1075 | if err := pinHost(rt, ver); err != nil { |
| 1076 | die("update %s %s: %v", cmd, host, err) |
| 1077 | } |
| 1078 | if pin { |
| 1079 | fmt.Printf("pinned %s to %s (auto-updates paused; `lchat update unpin %s` or pick a channel in its admin panel to resume)\n", host, ver, host) |
| 1080 | } else { |
| 1081 | fmt.Printf("unpinned %s (back on its channel from the next update pass)\n", host) |
| 1082 | } |
| 1083 | } |
| 1084 | |
| 1085 | // runUpdateMajor: `lchat update major <host> <n>|clear` keeps an instance on |
| 1086 | // one major version — minors/patches keep flowing, a major bump holds until |
| 1087 | // the admin raises or clears it. Unlike `pin` there is no default: staying on |
| 1088 | // a major is a statement about the future, so the operator says which one. |
| 1089 | func runUpdateMajor(args []string) { |
| 1090 | fs := flag.NewFlagSet("update major", flag.ExitOnError) |
| 1091 | registryPath := fs.String("registry", envOr("LCHAT_REGISTRY", "/data/registry.db"), "router registry SQLite path") |
| 1092 | _ = fs.Parse(reorderFlagsFirst(fs, args)) |
| 1093 | |
| 1094 | host := strings.ToLower(strings.TrimSpace(fs.Arg(0))) |
| 1095 | if !validHost(host) { |
| 1096 | die("update major: invalid host %q", host) |
| 1097 | } |
| 1098 | major := strings.TrimSpace(fs.Arg(1)) |
| 1099 | if major == "clear" { |
| 1100 | major = "" |
| 1101 | } else if major == "" || !validMajor(major) { |
| 1102 | die("update major: pass a major version number, or `clear` to follow everything") |
| 1103 | } |
| 1104 | reg, err := openRegistry(*registryPath) |
| 1105 | if err != nil { |
| 1106 | die("update major: open registry: %v", err) |
| 1107 | } |
| 1108 | rt, err := reg.get(host) |
| 1109 | if err != nil || rt == nil || rt.Kind != "instance" { |
| 1110 | die("update major: %s is not a managed instance", host) |
| 1111 | } |
| 1112 | if err := writeHostSetting(rt, "update_major", major); err != nil { |
| 1113 | die("update major %s: %v", host, err) |
| 1114 | } |
| 1115 | if err := confirmHostSetting(rt, "major pin", major, func(p updatePrefs) string { return p.Major }); err != nil { |
| 1116 | die("update major: %v", err) |
| 1117 | } |
| 1118 | if major == "" { |
| 1119 | fmt.Printf("cleared %s's major pin (follows its channel fully from the next update pass)\n", host) |
| 1120 | } else { |
| 1121 | fmt.Printf("%s stays on major %s (minors and patches keep flowing; a release crossing major %s is held until this is raised or cleared)\n", host, major, major) |
| 1122 | } |
| 1123 | } |
| 1124 | |
| 1125 | func runUpdateRun(args []string) { |
| 1126 | fs := flag.NewFlagSet("update run", flag.ExitOnError) |
| 1127 | registryPath := fs.String("registry", envOr("LCHAT_REGISTRY", "/data/registry.db"), "router registry SQLite path") |
| 1128 | unitTmpl := fs.String("unit", envOr("LCHAT_UNIT", "lchat@"), "systemd instance unit template prefix") |
| 1129 | dryRun := fs.Bool("dry-run", false, "report what would change without staging or restarting anything") |
| 1130 | ignoreBake := fs.Bool("ignore-bake", false, "install channel heads still inside their supply-chain bake window (the deliberate local override)") |
| 1131 | _ = fs.Parse(reorderFlagsFirst(fs, args)) |
| 1132 | |
| 1133 | unlock, err := acquireDeployLock() |
| 1134 | if err != nil { |
| 1135 | die("update: %v", err) |
| 1136 | } |
| 1137 | defer unlock() |
| 1138 | |
| 1139 | reg, err := openRegistry(*registryPath) |
| 1140 | if err != nil { |
| 1141 | die("update: open registry: %v", err) |
| 1142 | } |
| 1143 | if err := updatePass(reg, *unitTmpl, *dryRun, *ignoreBake); err != nil { |
| 1144 | die("update: %v", err) |
| 1145 | } |
| 1146 | } |
| 1147 | |
| 1148 | // statusPrefs resolves the prefs a status row shows: the live socket when the |
| 1149 | // host answers; a hibernating host's parked DB (the same fail-closed read the |
| 1150 | // update pass trusts) when it doesn't. The error keeps the caller's honest |
| 1151 | // "?" for a host unreadable either way — status must never guess what an |
| 1152 | // admin chose. |
| 1153 | func statusPrefs(rt Route) (updatePrefs, error) { |
| 1154 | p, err := fetchUpdatePrefs(rt.Socket) |
| 1155 | if err != nil && rt.Hibernate { |
| 1156 | return hibernatingPrefs(rt.Host) |
| 1157 | } |
| 1158 | return p, err |
| 1159 | } |
| 1160 | |
| 1161 | // runUpdateStatus prints where every instance stands relative to its channel. |
| 1162 | func runUpdateStatus(args []string) { |
| 1163 | fs := flag.NewFlagSet("update status", flag.ExitOnError) |
| 1164 | registryPath := fs.String("registry", envOr("LCHAT_REGISTRY", "/data/registry.db"), "router registry SQLite path") |
| 1165 | probe := fs.Bool("probe", false, "also read each instance's socket /api/version — the build it's ACTUALLY serving, which an awake hibernating instance can lag behind its symlink (docs/awake-instance-convergence.md)") |
| 1166 | _ = fs.Parse(reorderFlagsFirst(fs, args)) |
| 1167 | |
| 1168 | reg, err := openRegistry(*registryPath) |
| 1169 | if err != nil { |
| 1170 | die("update: open registry: %v", err) |
| 1171 | } |
| 1172 | routes, err := reg.list() |
| 1173 | if err != nil { |
| 1174 | die("update: %v", err) |
| 1175 | } |
| 1176 | m, merr := fetchReleaseManifest(releaseManifestURL) |
| 1177 | bake := loadFirstSeen() |
| 1178 | |
| 1179 | if *probe { |
| 1180 | fmt.Printf("%-34s %-8s %-8s %-12s %-12s %s\n", "HOST", "CHANNEL", "AUTO", "SYMLINK", "SERVING", "AVAILABLE") |
| 1181 | } else { |
| 1182 | fmt.Printf("%-34s %-8s %-8s %-12s %s\n", "HOST", "CHANNEL", "AUTO", "CURRENT", "AVAILABLE") |
| 1183 | } |
| 1184 | staleServing := false |
| 1185 | for _, rt := range routes { |
| 1186 | if rt.Kind != "instance" { |
| 1187 | continue |
| 1188 | } |
| 1189 | channel, auto, major := "?", "?", "" |
| 1190 | if prefs, err := statusPrefs(rt); err == nil { |
| 1191 | channel, major = prefs.Channel, prefs.Major |
| 1192 | switch { |
| 1193 | case prefs.Pin != "": |
| 1194 | auto = "pinned" // pin beats channel; the updater skips this host |
| 1195 | case prefs.AutoDownload && prefs.AutoUpdate: |
| 1196 | auto = "dl+upd" |
| 1197 | case prefs.AutoDownload: |
| 1198 | auto = "dl" |
| 1199 | case prefs.AutoUpdate: |
| 1200 | auto = "upd" |
| 1201 | default: |
| 1202 | auto = "off" |
| 1203 | } |
| 1204 | } |
| 1205 | cur, _ := liveVersion(rt.Host) |
| 1206 | if cur == "" { |
| 1207 | cur = "-" |
| 1208 | } |
| 1209 | avail := "-" |
| 1210 | if m != nil { |
| 1211 | if e, ok := m.Channels[channel]; ok { |
| 1212 | avail = e.Version |
| 1213 | switch { |
| 1214 | case e.Version == cur: |
| 1215 | avail += " (current)" |
| 1216 | case majorHold(major, e) != "": |
| 1217 | avail += " (held by major pin " + major + ")" |
| 1218 | case bakeDelays[channel] > 0: |
| 1219 | // Status reads the bake clock without ticking it — only an |
| 1220 | // update pass observes. An unticked head shows as held too. |
| 1221 | if at, ok := bake.bakeEligibleAt(channel, e.Version); !ok { |
| 1222 | avail += " (held — bake starts on the next update pass)" |
| 1223 | } else if time.Now().Before(at) { |
| 1224 | avail += " (held until " + at.UTC().Format("2006-01-02 15:04 MST") + ")" |
| 1225 | } |
| 1226 | } |
| 1227 | } |
| 1228 | } |
| 1229 | if *probe { |
| 1230 | // The build the process is actually serving (vs. the symlink `cur`). |
| 1231 | serving, ok := probeVersion(rt.Socket) |
| 1232 | if !ok { |
| 1233 | serving = "asleep" // socket didn't answer — hibernating, converges on wake |
| 1234 | } else if serving != cur { |
| 1235 | serving += " ⚠" // awake but stale: symlink moved, process didn't |
| 1236 | staleServing = true |
| 1237 | } |
| 1238 | fmt.Printf("%-34s %-8s %-8s %-12s %-12s %s\n", rt.Host, channel, auto, cur, serving, avail) |
| 1239 | } else { |
| 1240 | fmt.Printf("%-34s %-8s %-8s %-12s %s\n", rt.Host, channel, auto, cur, avail) |
| 1241 | } |
| 1242 | } |
| 1243 | if staleServing { |
| 1244 | fmt.Println("\n⚠ = awake but serving an older build than its symlink names (a running process doesn't follow the") |
| 1245 | fmt.Println(" symlink). Converges on next sleep/wake, or cycle now with: lchat hibernate <host> --off && … --on") |
| 1246 | fmt.Println(" (docs/awake-instance-convergence.md).") |
| 1247 | } |
| 1248 | if merr != nil { |
| 1249 | fmt.Printf("(no manifest: %v)\n", merr) |
| 1250 | } |
| 1251 | } |
| 1252 | |
amadan