eleven / ocho Public

Clone
git clone https://amadan.net/eleven/ocho

Plain git — no account needed to clone.

Download

Download this file

1package main
2
3import (
4 "context"
5 "encoding/json"
6 "flag"
7 "fmt"
8 "io"
9 "net"
10 "net/http"
11 "os"
12 "os/exec"
13 "path/filepath"
14 "regexp"
15 "strings"
16 "time"
17)
18
19// runCmd runs a command and returns its stdout (trimmed by callers). Used to read a
20// built binary's `version` output.
21func runCmd(name string, args ...string) (string, error) {
22 out, err := exec.Command(name, args...).Output()
23 if err != nil {
24 return "", fmt.Errorf("%s %s: %v", name, strings.Join(args, " "), err)
25 }
26 return string(out), nil
27}
28
29// Versioned, side-by-side deploys.
30//
31// The box runs one `lchat` binary per instance, but they need not be the SAME
32// binary: each instance's systemd unit execs /opt/lchat/live/<host>, a symlink
33// into a content store at /opt/lchat/versions/<ver>/lchat. Swapping one link (and
34// restarting one unit) moves one instance to a new build while every other
35// instance stays put — so a new build can be canaried on `test` before it's
36// promoted fleet-wide, and rolled back per instance. The registry records each
37// instance's current + previous version (source of truth for `lchat instances`
38// and `rollback`). See docs/instance-router.md § Versioned deploys.
39
40// buildVersion is the binary's build id, compiled in via
41// -ldflags "-X main.buildVersion=$(git rev-parse --short HEAD)" (see
42// deploy/build-linux.sh). "dev" means an un-stamped local build. `lchat version`
43// prints it; the running instance reports it at /api/version.
44var buildVersion = "dev"
45
46// optDir is the on-box install root: the version store and the per-instance
47// "live" symlinks live under it. A var with an env override so tests can point it
48// at a temp dir instead of the real /opt/lchat.
49var optDir = envOr("LCHAT_OPT_DIR", "/opt/lchat")
50
51func versionsDir() string { return filepath.Join(optDir, "versions") }
52func liveDir() string { return filepath.Join(optDir, "live") }
53func versionBinPath(ver string) string { return filepath.Join(versionsDir(), ver, "lchat") }
54
55// liveLink is the per-instance pointer the systemd unit execs
56// (/opt/lchat/live/<host>) — a symlink to a versionBinPath. Swapping it atomically
57// is how one instance moves between versions independently of the others.
58func liveLink(host string) string { return filepath.Join(liveDir(), host) }
59
60// versionRe keeps a version usable as a directory name (no slashes, no traversal).
61var versionRe = regexp.MustCompile(`^[a-zA-Z0-9._-]+$`)
62
63func validVersion(v string) bool {
64 return v != "" && len(v) <= 64 && v != "." && v != ".." && versionRe.MatchString(v)
65}
66
67// readBinaryVersion runs `<path> version` to read a built binary's compiled-in id,
68// so `stage` can key the store by it without the operator retyping the sha.
69func readBinaryVersion(path string) (string, error) {
70 out, err := runCmd(path, "version")
71 if err != nil {
72 return "", err
73 }
74 return strings.TrimSpace(out), nil
75}
76
77// copyFileAtomic copies src to dst via a temp file + rename, so a reader never sees
78// a half-written binary (and a crashed copy leaves the old one intact).
79func copyFileAtomic(src, dst string, mode os.FileMode) error {
80 in, err := os.Open(src)
81 if err != nil {
82 return err
83 }
84 defer in.Close()
85 tmp := dst + ".tmp"
86 out, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode)
87 if err != nil {
88 return err
89 }
90 if _, err := io.Copy(out, in); err != nil {
91 out.Close()
92 os.Remove(tmp)
93 return err
94 }
95 if err := out.Close(); err != nil {
96 os.Remove(tmp)
97 return err
98 }
99 // O_CREATE's mode is masked by umask; set it explicitly so the binary is +x.
100 if err := os.Chmod(tmp, mode); err != nil {
101 os.Remove(tmp)
102 return err
103 }
104 if err := os.Rename(tmp, dst); err != nil {
105 os.Remove(tmp)
106 return err
107 }
108 return nil
109}
110
111// stageVersion copies a built binary into the store under ver, so instances can be
112// pointed at it. Idempotent: re-staging the same ver overwrites its binary.
113func stageVersion(srcPath, ver string) error {
114 if !validVersion(ver) {
115 return fmt.Errorf("invalid version %q (allowed: letters, digits, . _ -)", ver)
116 }
117 dir := filepath.Join(versionsDir(), ver)
118 if err := os.MkdirAll(dir, 0o755); err != nil {
119 return err
120 }
121 // MkdirAll's mode is masked by the caller's umask, so a tight operator
122 // umask (e.g. 077 from a hardened shell) leaves the version dir 0700 —
123 // the sandboxed instance user then can't traverse in to exec its binary
124 // (203/EXEC "Permission denied", found staging a canary). Force it, like
125 // copyFileAtomic already forces the binary's +x.
126 if err := os.Chmod(dir, 0o755); err != nil {
127 return err
128 }
129 return copyFileAtomic(srcPath, versionBinPath(ver), 0o755)
130}
131
132// pointLive atomically repoints an instance's live symlink at ver's binary. It
133// verifies ver is staged first, so a typo can't leave a dangling link that would
134// fail the unit's ExecStart on the next restart.
135func pointLive(host, ver string) error {
136 target := versionBinPath(ver)
137 if _, err := os.Stat(target); err != nil {
138 return fmt.Errorf("version %q is not staged (%s): run `lchat stage --file <binary>` first", ver, target)
139 }
140 if err := os.MkdirAll(liveDir(), 0o755); err != nil {
141 return err
142 }
143 link := liveLink(host)
144 tmp := link + ".tmp"
145 _ = os.Remove(tmp) // a leftover temp link from a crashed swap
146 if err := os.Symlink(target, tmp); err != nil {
147 return err
148 }
149 if err := os.Rename(tmp, link); err != nil {
150 os.Remove(tmp)
151 return err
152 }
153 return nil
154}
155
156// liveVersion reports the version an instance's live symlink currently points at,
157// or "" (no error) if the instance hasn't been put under version management yet.
158func liveVersion(host string) (string, error) {
159 target, err := os.Readlink(liveLink(host))
160 if err != nil {
161 if os.IsNotExist(err) {
162 return "", nil
163 }
164 return "", err
165 }
166 // target = /opt/lchat/versions/<ver>/lchat → the parent dir's name is <ver>.
167 return filepath.Base(filepath.Dir(target)), nil
168}
169
170// selfPath is the running control binary — what add-instance / init-versions stage
171// so a fresh instance runs the same build as the operator's `lchat`. A var for tests.
172var selfPath = func() (string, error) { return os.Executable() }
173
174// ensureSelfStaged stages the running control binary under its build id and returns
175// that id, so add-instance can point a fresh instance at a real staged version
176// before the unit (ExecStart=/opt/lchat/live/<host>) first starts.
177func ensureSelfStaged() (string, error) {
178 self, err := selfPath()
179 if err != nil {
180 return "", err
181 }
182 if err := stageVersion(self, buildVersion); err != nil {
183 return "", err
184 }
185 return buildVersion, nil
186}
187
188// runStage: `lchat stage --file <binary>` — register a built binary in the store
189// under its build id (or --version), ready for `lchat deploy --version`.
190func runStage(args []string) {
191 fs := flag.NewFlagSet("stage", flag.ExitOnError)
192 file := fs.String("file", "", "path to the built lchat binary to register (required)")
193 verFlag := fs.String("version", "", "version id to store it under (default: the binary's own `lchat version`)")
194 _ = fs.Parse(reorderFlagsFirst(fs, args))
195
196 if *file == "" {
197 die("stage: --file <path> is required")
198 }
199 ver := strings.TrimSpace(*verFlag)
200 if ver == "" {
201 v, err := readBinaryVersion(*file)
202 if err != nil {
203 die("stage: can't read the build id from %s: %v\n build it with -ldflags \"-X main.buildVersion=$(git rev-parse --short HEAD)\" (see deploy/build-linux.sh), or pass --version", *file, err)
204 }
205 ver = v
206 }
207 if ver == "" || ver == "dev" {
208 die("stage: refusing to stage build id %q — build with -ldflags \"-X main.buildVersion=$(git rev-parse --short HEAD)\" or pass --version <id>", ver)
209 }
210 if !validVersion(ver) {
211 die("stage: invalid version %q (allowed: letters, digits, . _ -)", ver)
212 }
213 if err := stageVersion(*file, ver); err != nil {
214 die("stage: %v", err)
215 }
216 fmt.Printf("staged %s -> %s\n", ver, versionBinPath(ver))
217}
218
219// runInitVersions: `lchat init-versions` — one-time adoption. Stage the running
220// control binary and point every instance's live symlink at it, so the versioned
221// unit (ExecStart=/opt/lchat/live/<host>) resolves for instances that predate it.
222// Idempotent: it only creates a live link that's missing, never repoints a managed
223// instance (use `deploy --version` for that).
224func runInitVersions(args []string) {
225 fs := flag.NewFlagSet("init-versions", flag.ExitOnError)
226 registryPath := fs.String("registry", envOr("LCHAT_REGISTRY", "/data/registry.db"), "router registry SQLite path")
227 from := fs.String("from", "", "binary to seed the store with (default: the running control binary)")
228 verFlag := fs.String("version", "", "version id for the seed binary (default: its own build id)")
229 _ = fs.Parse(reorderFlagsFirst(fs, args))
230
231 src := strings.TrimSpace(*from)
232 ver := strings.TrimSpace(*verFlag)
233 if src == "" {
234 self, err := selfPath()
235 if err != nil {
236 die("init-versions: locate self: %v", err)
237 }
238 src = self
239 if ver == "" {
240 ver = buildVersion
241 }
242 } else if ver == "" {
243 v, err := readBinaryVersion(src)
244 if err != nil {
245 die("init-versions: read build id from %s: %v (or pass --version)", src, err)
246 }
247 ver = v
248 }
249 if !validVersion(ver) {
250 die("init-versions: invalid seed version %q (build with -ldflags, or pass --version)", ver)
251 }
252 if err := stageVersion(src, ver); err != nil {
253 die("init-versions: stage: %v", err)
254 }
255 fmt.Printf("seeded %s -> %s\n", ver, versionBinPath(ver))
256
257 reg, err := openRegistry(*registryPath)
258 if err != nil {
259 die("init-versions: open registry: %v", err)
260 }
261 routes, err := reg.list()
262 if err != nil {
263 die("init-versions: %v", err)
264 }
265 for _, rt := range routes {
266 if rt.Kind != "instance" {
267 continue
268 }
269 cur, err := liveVersion(rt.Host)
270 if err != nil {
271 die("init-versions: %s: read live link: %v", rt.Host, err)
272 }
273 if cur != "" {
274 fmt.Printf(" %s already on %s (left as-is)\n", rt.Host, cur)
275 continue
276 }
277 if err := pointLive(rt.Host, ver); err != nil {
278 die("init-versions: %s: %v", rt.Host, err)
279 }
280 rt.Version = ver
281 if err := reg.upsert(rt); err != nil {
282 die("init-versions: %s: record version: %v", rt.Host, err)
283 }
284 fmt.Printf(" %s -> %s\n", rt.Host, ver)
285 }
286 fmt.Println("init-versions: done (restart instances onto the versioned unit with `lchat deploy --all`)")
287}
288
289// runRollback: `lchat rollback <host>` (or --all) — repoint the instance(s) at the
290// previous version recorded at the last `deploy --version`, restart, health-check.
291// It swaps current<->previous so a second rollback returns to where you were.
292func runRollback(args []string) {
293 fs := flag.NewFlagSet("rollback", flag.ExitOnError)
294 registryPath := fs.String("registry", envOr("LCHAT_REGISTRY", "/data/registry.db"), "router registry SQLite path")
295 unitTmpl := fs.String("unit", envOr("LCHAT_UNIT", "lchat@"), "systemd instance unit template prefix")
296 all := fs.Bool("all", false, "roll back every instance that has a previous version")
297 _ = fs.Parse(reorderFlagsFirst(fs, args))
298
299 unlock, err := acquireDeployLock()
300 if err != nil {
301 die("rollback: %v", err)
302 }
303 defer unlock()
304
305 reg, err := openRegistry(*registryPath)
306 if err != nil {
307 die("rollback: open registry: %v", err)
308 }
309 hosts, err := deployTargets(reg, *all, fs.Arg(0), "rollback")
310 if err != nil {
311 die("%v", err)
312 }
313 for _, host := range hosts {
314 rt, err := reg.get(host)
315 if err != nil || rt == nil {
316 die("rollback %s: unknown host (run add-instance first)", host)
317 }
318 if rt.Kind != "instance" {
319 die("rollback %s: not an instance (kind %q)", host, rt.Kind)
320 }
321 prev := strings.TrimSpace(rt.PrevVersion)
322 if prev == "" {
323 if *all {
324 continue // nothing to roll back to; skip in a fleet rollback
325 }
326 die("rollback %s: no previous version recorded", host)
327 }
328 cur, _ := liveVersion(host)
329 if rt.Hibernate {
330 // No unit to restart — a pure repoint, mirroring the auto-updater's
331 // hibernating phase. This is the rescue path when a repointed build
332 // turns out bad: the next wake execs the previous version.
333 if err := pointLive(host, prev); err != nil {
334 die("rollback %s: %v", host, err)
335 }
336 if err := reg.setVersions(host, prev, cur); err != nil {
337 die("rollback %s: record: %v", host, err)
338 }
339 fmt.Printf("rolled back %s to %s (hibernating — runs it on next wake)\n", host, prev)
340 continue
341 }
342 if err := pointLive(host, prev); err != nil {
343 die("rollback %s: %v", host, err)
344 }
345 if err := systemctl("restart", *unitTmpl+host); err != nil {
346 die("rollback %s: restart: %v", host, err)
347 }
348 if err := waitHealthy(rt.Socket, 20*time.Second); err != nil {
349 die("rollback %s: %v", host, err)
350 }
351 if err := reg.setVersions(host, prev, cur); err != nil {
352 die("rollback %s: record: %v", host, err)
353 }
354 restartBotSidecar(host)
355 fmt.Printf("rolled back %s to %s\n", host, prev)
356 }
357}
358
359// deployTargets resolves the host list shared by deploy/rollback: every instance
360// (--all) or the one validated positional host.
361func deployTargets(reg *Registry, all bool, arg, cmd string) ([]string, error) {
362 if all {
363 routes, err := reg.list()
364 if err != nil {
365 return nil, fmt.Errorf("%s: %v", cmd, err)
366 }
367 var hosts []string
368 for _, r := range routes {
369 if r.Kind == "instance" {
370 hosts = append(hosts, r.Host)
371 }
372 }
373 if len(hosts) == 0 {
374 return nil, fmt.Errorf("%s: no instances", cmd)
375 }
376 return hosts, nil
377 }
378 h := strings.ToLower(strings.TrimSpace(arg))
379 if !validHost(h) {
380 return nil, fmt.Errorf("%s: invalid host %q (or pass --all)", cmd, h)
381 }
382 return []string{h}, nil
383}
384
385// waitHealthyVersion is waitHealthy plus a build check: it waits until the instance
386// not only answers but reports the expected build at /api/version — so a deploy
387// fails loudly if the process is still the old binary (a stale socket, or a restart
388// that didn't take), rather than silently "succeeding".
389func waitHealthyVersion(socket, wantBuild string, timeout time.Duration) error {
390 deadline := time.Now().Add(timeout)
391 client := unixHTTPClient(socket)
392 var last error
393 for time.Now().Before(deadline) {
394 resp, err := client.Get("http://lchat/api/version")
395 if err == nil {
396 var body struct {
397 Build string `json:"build"`
398 }
399 derr := json.NewDecoder(io.LimitReader(resp.Body, 1<<16)).Decode(&body)
400 resp.Body.Close()
401 if derr == nil && body.Build == wantBuild {
402 return nil
403 }
404 if derr != nil {
405 last = fmt.Errorf("read /api/version: %v", derr)
406 } else {
407 last = fmt.Errorf("live build %q != expected %q", body.Build, wantBuild)
408 }
409 } else {
410 last = err
411 }
412 time.Sleep(150 * time.Millisecond)
413 }
414 return fmt.Errorf("instance didn't report build %q within %s: %v", wantBuild, timeout, last)
415}
416
417// probeVersion reads the build the process behind a socket is ACTUALLY serving
418// (its /api/version), not the symlink target. This is how `update status
419// --probe` catches an awake hibernating instance that repointed its symlink but
420// still runs the old binary (docs/awake-instance-convergence.md). "" + ok=false
421// when the socket doesn't answer (asleep, or no such instance).
422func probeVersion(socket string) (build string, ok bool) {
423 resp, err := unixHTTPClient(socket).Get("http://lchat/api/version")
424 if err != nil {
425 return "", false
426 }
427 defer resp.Body.Close()
428 var body struct {
429 Build string `json:"build"`
430 }
431 if json.NewDecoder(io.LimitReader(resp.Body, 1<<16)).Decode(&body) != nil {
432 return "", false
433 }
434 return body.Build, body.Build != ""
435}
436
437// unixHTTPClient dials a unix socket over HTTP — the router↔instance transport,
438// reused by the health checks.
439func unixHTTPClient(socket string) *http.Client {
440 return &http.Client{
441 Timeout: 3 * time.Second,
442 Transport: &http.Transport{
443 DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
444 return (&net.Dialer{}).DialContext(ctx, "unix", socket)
445 },
446 },
447 }
448}
449
450// ---- sibling installs -------------------------------------------------------
451
452// siblingBinaries names the box's non-instance service binaries that `lchat
453// install` may replace, each with the unit that runs it. These are the fixed
454// ExecStart targets of their units — the door the SSH hardening deliberately
455// closed for ad-hoc root shells, reopened as one audited, allowlisted verb
456// (operators run it via `sudo -n /opt/lchat/lchat install …`). Extending the
457// set is a reviewed PR, never a box edit.
458var siblingBinaries = map[string]string{
459 "home": "lchat-home",
460 "push": "lchat-push",
461 "connect": "lchat-connect",
462}
463
464// installSibling swaps one sibling service's binary to a staged version:
465// back up the current binary (cp to <name>.prev-<date>, the convention the
466// box already uses), then atomically copy the staged build over the unit's
467// ExecStart path. It deliberately does NOT restart the unit — the operator
468// does that visibly via lchat-svc, so an install can be staged ahead of a
469// maintenance window. Returns the human summary to print.
470func installSibling(name, ver string) (string, error) {
471 unit, ok := siblingBinaries[name]
472 if !ok {
473 return "", fmt.Errorf("unknown sibling %q (know: home, push, connect)", name)
474 }
475 src := versionBinPath(ver)
476 if _, err := os.Stat(src); err != nil {
477 return "", fmt.Errorf("version %q is not staged — run `lchat stage --file <binary>` first", ver)
478 }
479 target := filepath.Join(optDir, name)
480 prev := ""
481 if _, err := os.Stat(target); err == nil {
482 prev = target + ".prev-" + time.Now().Format("20060102-150405")
483 if err := copyFileAtomic(target, prev, 0o755); err != nil {
484 return "", fmt.Errorf("back up current binary: %v", err)
485 }
486 }
487 if err := copyFileAtomic(src, target, 0o755); err != nil {
488 return "", fmt.Errorf("install: %v", err)
489 }
490 msg := fmt.Sprintf("installed %s %s -> %s", name, ver, target)
491 if prev != "" {
492 msg += fmt.Sprintf(" (previous at %s)", prev)
493 }
494 msg += fmt.Sprintf("\nrestart to pick it up: sudo -n /usr/local/sbin/lchat-svc restart %s", unit)
495 return msg, nil
496}
497
498// runInstall: `lchat install <name> --version <ver>` — put a staged build onto
499// a sibling service's fixed binary path. The staged version store doesn't care
500// what a binary IS (stage --file records any build under its own version), so
501// sibling binaries ride the same store the messenger versions do.
502func runInstall(args []string) {
503 fs := flag.NewFlagSet("install", flag.ExitOnError)
504 verFlag := fs.String("version", "", "staged version to install (see `lchat stage`)")
505 _ = fs.Parse(reorderFlagsFirst(fs, args))
506 name := strings.TrimSpace(fs.Arg(0))
507 ver := strings.TrimSpace(*verFlag)
508 if name == "" || ver == "" {
509 die("install: usage: lchat install <home|push|connect> --version <ver>")
510 }
511 if !validVersion(ver) {
512 die("install: invalid version %q", ver)
513 }
514 msg, err := installSibling(name, ver)
515 if err != nil {
516 die("install: %v", err)
517 }
518 fmt.Println(msg)
519}
520