eleven / messenger Public

server: versioned, side-by-side per-instance deploys (#189)

The router box ran one shared /opt/lchat/lchat, so a new build couldn't be
canaried on `test` without every instance picking it up on its next restart.
Make each instance's version independent so canary-then-promote is first-class.

- Store: /opt/lchat/versions/<ver>/lchat, keyed by build id (git short sha
  stamped via -ldflags -X main.buildVersion; deploy/build-linux.sh).
- Pointer: the instance unit now execs /opt/lchat/live/<host> (a symlink into
  the store); the registry records each host's current + previous version.
- CLI: stage, deploy --version (atomic repoint + version-verified health check
  + auto-rollback), rollback, init-versions, version. `instances` gains a
  VERSION column; add-instance stages + points the live link before first start.
- /api/version now also reports the binary `build`, so a --version deploy
  confirms the intended binary is actually serving (not just "answers 200").

Existing registries get prev_version via an additive ALTER; bootstrap creates
the store dirs. init-versions is the one-time adoption step for the live box.
Docs: docs/instance-router.md § Versioned, side-by-side deploys.
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Paul Campbell cf63e2834f986928726f44ce081e4be5ac8735dd parent 44effc7
10 files changed, +767 −72
  • bootstrap.go +8 −0
  • bootstrap_test.go +3 −2
  • control.go +93 −34
  • deploy/build-linux.sh +21 −0
  • deploy/systemd/lchat@.service +6 −1
  • docs/instance-router.md +63 −23
  • main.go +20 −2
  • registry.go +26 −10
  • versions.go +400 −0
  • versions_test.go +127 −0
diff --git a/bootstrap.go b/bootstrap.go
index e155f5b..54078ea 100644
--- a/bootstrap.go
+++ b/bootstrap.go
@@ -81,6 +81,14 @@ func runBootstrap(args []string) {
die("bootstrap: mkdir %s: %v", d, err)
}
}
+ // The version store + per-instance live symlinks (see versions.go). World-
+ // readable/executable so the sandboxed instance units can exec their binary;
+ // only root (the operator running deploy/stage) writes here.
+ for _, d := range []string{versionsDir(), liveDir()} {
+ if err := os.MkdirAll(d, 0o755); err != nil {
+ die("bootstrap: mkdir %s: %v", d, err)
+ }
+ }
if err := os.Chmod("/run/lchat", 0o750); err != nil {
die("bootstrap: chmod /run/lchat: %v", err)
}
diff --git a/bootstrap_test.go b/bootstrap_test.go
index 054d65d..6e1fad8 100644
--- a/bootstrap_test.go
+++ b/bootstrap_test.go
@@ -14,8 +14,9 @@ func TestEmbeddedUnits(t *testing.T) {
if !strings.Contains(unitInstance, "EnvironmentFile=-/data/%i.env") {
t.Error("instance unit missing EnvironmentFile directive")
}
- if !strings.Contains(unitInstance, "ExecStart=/opt/lchat/lchat --socket /run/lchat/%i.sock --db /data/%i.db") {
- t.Error("instance unit missing its ExecStart line")
+ // Execs the per-instance live symlink (versioned deploys), not one shared binary.
+ if !strings.Contains(unitInstance, "ExecStart=/opt/lchat/live/%i --socket /run/lchat/%i.sock --db /data/%i.db") {
+ t.Error("instance unit missing its versioned ExecStart line")
}
if !strings.Contains(unitBot, "ExecStart=/usr/bin/node /opt/lchat/agent/agent.mjs") {
t.Error("bot unit missing its node ExecStart line")
diff --git a/control.go b/control.go
index 8a650df..cad2289 100644
--- a/control.go
+++ b/control.go
@@ -159,6 +159,18 @@ func runAddInstance(args []string) {
die("add-instance: register: %v", err)
}
+ // The instance's unit execs /opt/lchat/live/<host>; stage the running control
+ // binary and point that symlink at it, so ExecStart resolves on first start.
+ // A fresh instance therefore runs whatever build the operator's `lchat` is.
+ ver, err := ensureSelfStaged()
+ if err != nil {
+ die("add-instance: stage binary: %v", err)
+ }
+ if err := pointLive(host, ver); err != nil {
+ die("add-instance: point live symlink: %v", err)
+ }
+ route.Version = ver
+
unit := *unitTmpl + host
if err := systemctl("enable", "--now", unit); err != nil {
markInstanceFailed(reg, host, unit, "start: "+err.Error())
@@ -290,9 +302,13 @@ func runInstances(args []string) {
fmt.Println("(no routes)")
return
}
- fmt.Printf("%-34s %-9s %-9s %s\n", "HOST", "KIND", "STATUS", "SOCKET")
+ fmt.Printf("%-34s %-9s %-9s %-12s %s\n", "HOST", "KIND", "STATUS", "VERSION", "SOCKET")
for _, r := range routes {
- fmt.Printf("%-34s %-9s %-9s %s\n", r.Host, r.Kind, r.Status, r.Socket)
+ ver := r.Version
+ if ver == "" {
+ ver = "-"
+ }
+ fmt.Printf("%-34s %-9s %-9s %-12s %s\n", r.Host, r.Kind, r.Status, ver, r.Socket)
if r.LastError != "" {
fmt.Printf(" last error: %s\n", r.LastError)
}
@@ -320,43 +336,43 @@ func runAdminLink(args []string) {
fmt.Printf("https://%s/admin#token=%s\n", host, tok)
}
-// runDeploy: `lchat deploy <host>` (or --all) — restart the instance on the new
-// on-disk binary. SIGTERM drains it gracefully and the router retries dials across
-// the brief gap, so the swap drops no requests. Ship the new binary first (scp +
-// atomic mv over /opt/lchat/lchat); deploy then rolls it in. See
-// docs/instance-router.md.
+// runDeploy: `lchat deploy <host>` (or --all) — restart the instance(s) so they
+// pick up a new binary. SIGTERM drains gracefully and the router retries dials
+// across the brief gap, so the swap drops no requests.
+//
+// With --version <ver> it first repoints the instance's /opt/lchat/live/<host>
+// symlink at that staged version (see `lchat stage`), then health-checks that the
+// instance actually reports that build — auto-rolling-back to the prior version if
+// it doesn't. That's the side-by-side path: `deploy test --version <ver>` moves one
+// instance while the rest stay put; `deploy --all --version <ver>` promotes.
+//
+// Without --version it just restarts on the instance's current version — the
+// classic "I've already swapped the binary" path. See docs/instance-router.md.
func runDeploy(args []string) {
fs := flag.NewFlagSet("deploy", flag.ExitOnError)
registryPath := fs.String("registry", envOr("LCHAT_REGISTRY", "/data/registry.db"), "router registry SQLite path")
unitTmpl := fs.String("unit", envOr("LCHAT_UNIT", "lchat@"), "systemd instance unit template prefix")
all := fs.Bool("all", false, "deploy every instance, one at a time")
+ verFlag := fs.String("version", "", "point the instance(s) at this staged version before restarting (see `lchat stage`)")
_ = fs.Parse(reorderFlagsFirst(fs, args))
+ ver := strings.TrimSpace(*verFlag)
+ if ver != "" {
+ if !validVersion(ver) {
+ die("deploy: invalid version %q", ver)
+ }
+ if _, err := os.Stat(versionBinPath(ver)); err != nil {
+ die("deploy: version %q is not staged — run `lchat stage --file <binary>` first", ver)
+ }
+ }
+
reg, err := openRegistry(*registryPath)
if err != nil {
die("deploy: open registry: %v", err)
}
-
- var hosts []string
- if *all {
- routes, err := reg.list()
- if err != nil {
- die("deploy: %v", err)
- }
- for _, r := range routes {
- if r.Kind == "instance" {
- hosts = append(hosts, r.Host)
- }
- }
- } else {
- h := strings.ToLower(strings.TrimSpace(fs.Arg(0)))
- if !validHost(h) {
- die("deploy: invalid host %q (or pass --all)", h)
- }
- hosts = []string{h}
- }
- if len(hosts) == 0 {
- die("deploy: no instances to deploy")
+ hosts, err := deployTargets(reg, *all, fs.Arg(0), "deploy")
+ if err != nil {
+ die("%v", err)
}
for _, host := range hosts {
@@ -370,16 +386,59 @@ func runDeploy(args []string) {
if rt.Kind != "instance" {
die("deploy %s: not an instance (kind %q) — only messenger instances are deployed", host, rt.Kind)
}
- if err := systemctl("restart", *unitTmpl+host); err != nil {
- die("deploy %s: restart: %v", host, err)
+ unit := *unitTmpl + host
+
+ if ver == "" {
+ // Restart on the current version (binary already in place).
+ if err := systemctl("restart", unit); err != nil {
+ die("deploy %s: restart: %v", host, err)
+ }
+ if err := waitHealthy(rt.Socket, 20*time.Second); err != nil {
+ die("deploy %s: %v", host, err)
+ }
+ fmt.Printf("deployed %s\n", host)
+ continue
}
- // rt.Socket is what the router actually dials (stored at add-instance time),
- // so health-check exactly that rather than recomputing a path.
- if err := waitHealthy(rt.Socket, 20*time.Second); err != nil {
+
+ // --version: repoint, restart, verify the build took, roll back if not.
+ prev, _ := liveVersion(host) // "" if this instance isn't versioned yet
+ if prev == ver {
+ fmt.Printf("deployed %s (already on %s)\n", host, ver)
+ continue
+ }
+ if err := pointLive(host, ver); err != nil {
die("deploy %s: %v", host, err)
}
- fmt.Printf("deployed %s\n", host)
+ if err := systemctl("restart", unit); err != nil {
+ rollbackLive(host, prev, unit, rt.Socket)
+ die("deploy %s: restart: %v (rolled back)", host, err)
+ }
+ if err := waitHealthyVersion(rt.Socket, ver, 20*time.Second); err != nil {
+ fmt.Printf("deploy %s: %v — rolling back to %q\n", host, err, prev)
+ rollbackLive(host, prev, unit, rt.Socket)
+ die("deploy %s: failed health check, rolled back to %q", host, prev)
+ }
+ rt.PrevVersion, rt.Version = prev, ver
+ if err := reg.upsert(*rt); err != nil {
+ die("deploy %s: record version: %v", host, err)
+ }
+ fmt.Printf("deployed %s @ %s\n", host, ver)
+ }
+}
+
+// rollbackLive best-effort restores an instance to prev after a failed
+// --version deploy: repoint the symlink and restart. If there is no prev (the
+// instance had no version yet) it leaves the failed link in place — the operator
+// must intervene either way, and the die() that follows says so.
+func rollbackLive(host, prev, unit, socket string) {
+ if prev == "" {
+ return
+ }
+ if err := pointLive(host, prev); err != nil {
+ return
}
+ _ = systemctl("restart", unit)
+ _ = waitHealthy(socket, 20*time.Second)
}
// waitHealthy blocks until the instance answers an HTTP request on its socket, or
diff --git a/deploy/build-linux.sh b/deploy/build-linux.sh
new file mode 100755
index 0000000..7d0f4ff
--- /dev/null
+++ b/deploy/build-linux.sh
@@ -0,0 +1,21 @@
+#!/usr/bin/env sh
+# Build the linux/amd64 lchat binary for the router box, stamped with a build id
+# (the git short sha, or $1) so `lchat stage`/`deploy --version` can identify and
+# health-check it. Prints the build id and the output path.
+#
+# ./deploy/build-linux.sh # id = git short sha
+# ./deploy/build-linux.sh test-scroll # id = test-scroll (e.g. a canary label)
+# OUT=/tmp/lchat ./deploy/build-linux.sh
+set -eu
+
+VER="${1:-$(git rev-parse --short HEAD)}"
+if [ -z "$VER" ]; then
+ echo "build-linux: could not determine a build id (pass one as \$1)" >&2
+ exit 1
+fi
+OUT="${OUT:-lchat-linux-amd64}"
+
+GOOS=linux GOARCH=amd64 CGO_ENABLED=0 \
+ go build -ldflags "-X main.buildVersion=${VER}" -o "$OUT" .
+
+echo "built ${VER} -> ${OUT}"
diff --git a/deploy/systemd/lchat@.service b/deploy/systemd/lchat@.service
index 87135d9..57347cc 100644
--- a/deploy/systemd/lchat@.service
+++ b/deploy/systemd/lchat@.service
@@ -7,7 +7,12 @@ Wants=network-online.target
# add-instance writes /data/<host>.env (0600) with LCHAT_ADMIN_TOKEN + per-instance
# config. The leading '-' makes it optional so a self-provisioning instance still boots.
EnvironmentFile=-/data/%i.env
-ExecStart=/opt/lchat/lchat --socket /run/lchat/%i.sock --db /data/%i.db
+# Execs a per-instance symlink (/opt/lchat/live/<host> → /opt/lchat/versions/<ver>/
+# lchat) rather than one shared binary, so each instance runs its own version and a
+# build can be canaried on one instance before it's promoted. add-instance /
+# init-versions create the link; `lchat deploy --version` repoints it. See
+# docs/instance-router.md § Versioned deploys.
+ExecStart=/opt/lchat/live/%i --socket /run/lchat/%i.sock --db /data/%i.db
Restart=always
RestartSec=1
diff --git a/docs/instance-router.md b/docs/instance-router.md
index 6959b7a..f02f84a 100644
--- a/docs/instance-router.md
+++ b/docs/instance-router.md
@@ -113,9 +113,11 @@ many hosts a backend serves. That leaves two ways to slice processes:
**One backend per instance (the default).** Each instance is its own process,
socket, and DB — full crash and resource isolation, and per-version deploys come
-*for free*: point one instance's unit at a new build and leave the rest on the old
-one (a `trunk` instance next to `stable` ones) with no coordination. Canary is
-just "which binary does this unit run."
+*for free*: each unit execs a per-instance `live` symlink into a version store, so
+moving one instance to a new build is one `lchat deploy <host> --version <ver>`
+while the rest stay put (a `trunk` instance next to `stable` ones), no coordination.
+Canary is just "which binary does this unit run." See *Versioned, side-by-side
+deploys* below.
**One backend per version (an option, later).** Alternatively a single process
serves *many* hosts at one version (an in-process host map, one DB per host), and
@@ -296,34 +298,65 @@ retrying state machine are refinements the schema already allows.
`lchat admin-link <host>` recovers the admin link from the EnvironmentFile (the DB
holds only the token's hash).
-## Zero-downtime deploy (per instance)
+## Versioned, side-by-side deploys
-`lchat deploy <host>` (or `--all`) rolls an instance onto the new on-disk binary
-with no dropped requests — via **graceful drain + a retrying router**, rather than
-a dual-socket blue/green. (Simpler, and enough: clients already auto-reconnect, so
-the marginal gain of true zero-latency didn't justify alternating dual units and
-two processes sharing one SQLite DB.)
+Per-version deploys are the point of this box, so they're first-class rather than
+"point a unit at a different file by hand." Two pieces make one instance's version
+independent of every other's:
-1. Ship the new binary (`scp` + atomic `mv` over `/opt/lchat/lchat`).
-2. `systemctl restart lchat@<host>`. On **SIGTERM** the instance drains in-flight
+- **A content store**: built binaries live at `/opt/lchat/versions/<ver>/lchat`,
+ keyed by build id (the git short sha, stamped in at build via
+ `-ldflags "-X main.buildVersion=<sha>"` — see `deploy/build-linux.sh`).
+- **A per-instance pointer**: each instance's unit execs `/opt/lchat/live/<host>`,
+ a symlink into the store. The registry records each host's current + previous
+ version (`lchat instances` shows it).
+
+```
+ /opt/lchat/live/test.luan.chat ─▶ versions/<canary>/lchat (one instance moved)
+ /opt/lchat/live/family.luan.chat ─▶ versions/<stable>/lchat
+ /opt/lchat/live/paulca… ─▶ versions/<stable>/lchat
+```
+
+**Flow (canary → promote):**
+
+```
+./deploy/build-linux.sh # build linux/amd64, stamped with the git sha
+scp lchat-linux-amd64 root@box:/opt/lchat/incoming
+ssh box lchat stage --file /opt/lchat/incoming # register it → prints <ver>
+ssh box lchat deploy test.luan.chat --version <ver> # canary ONE instance
+# …verify test…
+ssh box lchat deploy --all --version <ver> # promote the rest
+ssh box lchat rollback test.luan.chat # or step back to its previous version
+```
+
+`stage` copies a binary into the store (reading its own build id via `lchat
+version`). `deploy --version <ver>` atomically repoints `live/<host>`, restarts,
+and health-checks — see below. `deploy <host>` with **no** `--version` just
+restarts on the current version (the binary's already in place). `rollback`
+repoints to the recorded previous version and restarts. `init-versions` is the
+one-time adoption step (stage the running binary, create a `live/<host>` link for
+every existing instance) when moving a box onto this scheme.
+
+**The restart is a graceful drain + a retrying router** (rather than dual-socket
+blue/green — simpler, and enough since clients auto-reconnect):
+
+1. `systemctl restart lchat@<host>`. On **SIGTERM** the instance drains in-flight
HTTP (`http.Server.Shutdown`) and closes WebSockets (so browsers reconnect),
then exits; systemd starts the new process, which rebinds the socket.
-3. Across the brief gap (old exited, new not yet bound → dials fail
+2. Across the brief gap (old exited, new not yet bound → dials fail
`ENOENT`/`ECONNREFUSED`) the **router retries** the dial for a short window, so
requests queue a beat instead of failing.
-4. `deploy` waits for the new process to answer an HTTP health check (retrying —
- catches a stale socket or a broken binary) before reporting success; `--all`
- stops on the first failure.
+3. For a `--version` deploy, `deploy` polls `/api/version` until the instance
+ reports the **expected `build`** (not merely "answers 200") — so a stale socket
+ or a restart that didn't take fails loudly. On failure it **auto-rolls-back**:
+ repoint the symlink to the previous version, restart, and error out. `--all`
+ stops on the first failure (already-promoted hosts stay on the new version).
Blast radius is **one instance**: its WebSocket clients reconnect once; HTTP
requests see at most a little added latency. Canary = deploy one instance alone.
Verified in practice: 120 requests against an instance during a live deploy all
returned 200.
-The router itself changes rarely; when it does, its own front listener would get
-zero-downtime via `SO_REUSEPORT` / systemd socket activation (not yet wired — a
-router deploy is currently a ~1s blip for all hosts).
-
The router itself changes rarely; when it does, its own front listener gets
zero-downtime via `SO_REUSEPORT` / systemd socket activation.
@@ -453,12 +486,19 @@ touching anything real.
- `lchat bootstrap` — one-time box setup: write systemd units, make dirs, start
the router (no toolchain needed — just the copied binary).
- `lchat add-instance <host>` — create + start + warm cert (async; `instances`
- shows status).
-- `lchat deploy <host> [--version …]` — zero-downtime socket swap.
-- `lchat deploy --all` — roll the fleet, one instance at a time.
+ shows status). Stages the running binary + points the host's `live` link first.
+- `lchat stage --file <binary>` — register a built binary in the version store.
+- `lchat deploy <host> [--version <ver>]` — zero-downtime restart; with `--version`,
+ repoint the host at that staged version and health-check the `build` (auto-rollback
+ on mismatch).
+- `lchat deploy --all [--version <ver>]` — roll the fleet, one instance at a time.
+- `lchat rollback <host> [--all]` — step back to the recorded previous version.
+- `lchat init-versions` — one-time adoption: seed the store + create `live` links
+ for existing instances.
+- `lchat version` — print the binary's build id (also at `/api/version`).
- `lchat admin-link <host>` — recover the admin link from the instance DB (no
platform env lookup needed).
-- `lchat instances` — the status table.
+- `lchat instances` — the status table (now with a VERSION column).
Operator specifics — box addresses, DNS/record ids, the concrete migration
runbook — live in `CLAUDE.local.md`, not here.
diff --git a/main.go b/main.go
index 06d6891..f0636d8 100644
--- a/main.go
+++ b/main.go
@@ -110,6 +110,21 @@ func main() {
case "admin-link":
runAdminLink(os.Args[2:])
return
+ case "version":
+ // The build id compiled in via -ldflags "-X main.buildVersion=<id>".
+ // `stage` reads it from a built binary; the running instance reports it
+ // at /api/version so a deploy can confirm the swap actually took.
+ fmt.Println(buildVersion)
+ return
+ case "stage":
+ runStage(os.Args[2:])
+ return
+ case "rollback":
+ runRollback(os.Args[2:])
+ return
+ case "init-versions":
+ runInitVersions(os.Args[2:])
+ return
}
}
@@ -466,9 +481,12 @@ func frontendVersion() string {
return fmt.Sprintf("%x", h.Sum(nil))[:12]
}
-// getVersion returns the current frontend build hash.
+// getVersion returns the current frontend build hash (clients poll it to
+// auto-reload) plus the binary build id. `version` drives the frontend reload;
+// `build` is the compiled-in id `lchat deploy --version` health-checks against, so
+// a deploy can confirm the intended binary is actually the one now serving.
func (s *Server) getVersion(w http.ResponseWriter, r *http.Request) {
- writeJSON(w, http.StatusOK, map[string]any{"version": s.version})
+ writeJSON(w, http.StatusOK, map[string]any{"version": s.version, "build": buildVersion})
}
// ChangelogEntry is one embedded release note (see the changelogFS comment).
diff --git a/registry.go b/registry.go
index 9339d86..b969f68 100644
--- a/registry.go
+++ b/registry.go
@@ -2,6 +2,7 @@ package main
import (
"database/sql"
+ "strings"
"time"
)
@@ -22,6 +23,7 @@ type Route struct {
LastError string
DBPath string
Version string
+ PrevVersion string
AppStoreURL string
HomeOrigin string
PushRelayURL string
@@ -55,7 +57,7 @@ func openRegistry(path string) (*Registry, error) {
// migrate is additive (CREATE TABLE IF NOT EXISTS), like the instance schema, so
// deploying new code over an existing registry is safe.
func (r *Registry) migrate() error {
- _, err := r.sql.Exec(`
+ if _, err := r.sql.Exec(`
CREATE TABLE IF NOT EXISTS routes (
host TEXT PRIMARY KEY,
kind TEXT NOT NULL,
@@ -64,19 +66,32 @@ CREATE TABLE IF NOT EXISTS routes (
last_error TEXT NOT NULL DEFAULT '',
db_path TEXT NOT NULL DEFAULT '',
version TEXT NOT NULL DEFAULT '',
+ prev_version TEXT NOT NULL DEFAULT '',
app_store_url TEXT NOT NULL DEFAULT '',
home_origin TEXT NOT NULL DEFAULT '',
push_relay_url TEXT NOT NULL DEFAULT '',
push_relay_token TEXT NOT NULL DEFAULT '',
created_at INTEGER NOT NULL
-);`)
- return err
+);`); err != nil {
+ return err
+ }
+ // Additive column migrations for registries created before a column existed —
+ // CREATE TABLE IF NOT EXISTS won't add columns to an existing table. Ignore the
+ // "duplicate column" error so this stays idempotent.
+ for _, stmt := range []string{
+ `ALTER TABLE routes ADD COLUMN prev_version TEXT NOT NULL DEFAULT ''`,
+ } {
+ if _, err := r.sql.Exec(stmt); err != nil && !strings.Contains(err.Error(), "duplicate column") {
+ return err
+ }
+ }
+ return nil
}
// list returns every route, ordered by host.
func (r *Registry) list() ([]Route, error) {
rows, err := r.sql.Query(`
-SELECT host, kind, socket, status, last_error, db_path, version,
+SELECT host, kind, socket, status, last_error, db_path, version, prev_version,
app_store_url, home_origin, push_relay_url, push_relay_token, created_at
FROM routes ORDER BY host`)
if err != nil {
@@ -87,7 +102,7 @@ FROM routes ORDER BY host`)
for rows.Next() {
var t Route
if err := rows.Scan(&t.Host, &t.Kind, &t.Socket, &t.Status, &t.LastError,
- &t.DBPath, &t.Version, &t.AppStoreURL, &t.HomeOrigin,
+ &t.DBPath, &t.Version, &t.PrevVersion, &t.AppStoreURL, &t.HomeOrigin,
&t.PushRelayURL, &t.PushRelayToken, &t.CreatedAt); err != nil {
return nil, err
}
@@ -100,10 +115,10 @@ FROM routes ORDER BY host`)
func (r *Registry) get(host string) (*Route, error) {
var t Route
err := r.sql.QueryRow(`
-SELECT host, kind, socket, status, last_error, db_path, version,
+SELECT host, kind, socket, status, last_error, db_path, version, prev_version,
app_store_url, home_origin, push_relay_url, push_relay_token, created_at
FROM routes WHERE host = ?`, host).Scan(&t.Host, &t.Kind, &t.Socket, &t.Status,
- &t.LastError, &t.DBPath, &t.Version, &t.AppStoreURL, &t.HomeOrigin,
+ &t.LastError, &t.DBPath, &t.Version, &t.PrevVersion, &t.AppStoreURL, &t.HomeOrigin,
&t.PushRelayURL, &t.PushRelayToken, &t.CreatedAt)
if err == sql.ErrNoRows {
return nil, nil
@@ -135,15 +150,16 @@ func (r *Registry) upsert(t Route) error {
t.Kind = "instance"
}
_, err := r.sql.Exec(`
-INSERT INTO routes (host, kind, socket, status, last_error, db_path, version,
+INSERT INTO routes (host, kind, socket, status, last_error, db_path, version, prev_version,
app_store_url, home_origin, push_relay_url, push_relay_token, created_at)
-VALUES (?,?,?,?,?,?,?,?,?,?,?,?)
+VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)
ON CONFLICT(host) DO UPDATE SET
kind=excluded.kind, socket=excluded.socket, status=excluded.status,
last_error=excluded.last_error, db_path=excluded.db_path, version=excluded.version,
+ prev_version=excluded.prev_version,
app_store_url=excluded.app_store_url, home_origin=excluded.home_origin,
push_relay_url=excluded.push_relay_url, push_relay_token=excluded.push_relay_token`,
- t.Host, t.Kind, t.Socket, t.Status, t.LastError, t.DBPath, t.Version,
+ t.Host, t.Kind, t.Socket, t.Status, t.LastError, t.DBPath, t.Version, t.PrevVersion,
t.AppStoreURL, t.HomeOrigin, t.PushRelayURL, t.PushRelayToken, t.CreatedAt)
return err
}
diff --git a/versions.go b/versions.go
new file mode 100644
index 0000000..4a41fb8
--- /dev/null
+++ b/versions.go
@@ -0,0 +1,400 @@
+package main
+
+import (
+ "context"
+ "encoding/json"
+ "flag"
+ "fmt"
+ "io"
+ "net"
+ "net/http"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "regexp"
+ "strings"
+ "time"
+)
+
+// runCmd runs a command and returns its stdout (trimmed by callers). Used to read a
+// built binary's `version` output.
+func runCmd(name string, args ...string) (string, error) {
+ out, err := exec.Command(name, args...).Output()
+ if err != nil {
+ return "", fmt.Errorf("%s %s: %v", name, strings.Join(args, " "), err)
+ }
+ return string(out), nil
+}
+
+// Versioned, side-by-side deploys.
+//
+// The box runs one `lchat` binary per instance, but they need not be the SAME
+// binary: each instance's systemd unit execs /opt/lchat/live/<host>, a symlink
+// into a content store at /opt/lchat/versions/<ver>/lchat. Swapping one link (and
+// restarting one unit) moves one instance to a new build while every other
+// instance stays put — so a new build can be canaried on `test` before it's
+// promoted fleet-wide, and rolled back per instance. The registry records each
+// instance's current + previous version (source of truth for `lchat instances`
+// and `rollback`). See docs/instance-router.md § Versioned deploys.
+
+// buildVersion is the binary's build id, compiled in via
+// -ldflags "-X main.buildVersion=$(git rev-parse --short HEAD)" (see
+// deploy/build-linux.sh). "dev" means an un-stamped local build. `lchat version`
+// prints it; the running instance reports it at /api/version.
+var buildVersion = "dev"
+
+// optDir is the on-box install root: the version store and the per-instance
+// "live" symlinks live under it. A var with an env override so tests can point it
+// at a temp dir instead of the real /opt/lchat.
+var optDir = envOr("LCHAT_OPT_DIR", "/opt/lchat")
+
+func versionsDir() string { return filepath.Join(optDir, "versions") }
+func liveDir() string { return filepath.Join(optDir, "live") }
+func versionBinPath(ver string) string { return filepath.Join(versionsDir(), ver, "lchat") }
+
+// liveLink is the per-instance pointer the systemd unit execs
+// (/opt/lchat/live/<host>) — a symlink to a versionBinPath. Swapping it atomically
+// is how one instance moves between versions independently of the others.
+func liveLink(host string) string { return filepath.Join(liveDir(), host) }
+
+// versionRe keeps a version usable as a directory name (no slashes, no traversal).
+var versionRe = regexp.MustCompile(`^[a-zA-Z0-9._-]+$`)
+
+func validVersion(v string) bool {
+ return v != "" && len(v) <= 64 && v != "." && v != ".." && versionRe.MatchString(v)
+}
+
+// readBinaryVersion runs `<path> version` to read a built binary's compiled-in id,
+// so `stage` can key the store by it without the operator retyping the sha.
+func readBinaryVersion(path string) (string, error) {
+ out, err := runCmd(path, "version")
+ if err != nil {
+ return "", err
+ }
+ return strings.TrimSpace(out), nil
+}
+
+// copyFileAtomic copies src to dst via a temp file + rename, so a reader never sees
+// a half-written binary (and a crashed copy leaves the old one intact).
+func copyFileAtomic(src, dst string, mode os.FileMode) error {
+ in, err := os.Open(src)
+ if err != nil {
+ return err
+ }
+ defer in.Close()
+ tmp := dst + ".tmp"
+ out, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode)
+ if err != nil {
+ return err
+ }
+ if _, err := io.Copy(out, in); err != nil {
+ out.Close()
+ os.Remove(tmp)
+ return err
+ }
+ if err := out.Close(); err != nil {
+ os.Remove(tmp)
+ return err
+ }
+ // O_CREATE's mode is masked by umask; set it explicitly so the binary is +x.
+ if err := os.Chmod(tmp, mode); err != nil {
+ os.Remove(tmp)
+ return err
+ }
+ if err := os.Rename(tmp, dst); err != nil {
+ os.Remove(tmp)
+ return err
+ }
+ return nil
+}
+
+// stageVersion copies a built binary into the store under ver, so instances can be
+// pointed at it. Idempotent: re-staging the same ver overwrites its binary.
+func stageVersion(srcPath, ver string) error {
+ if !validVersion(ver) {
+ return fmt.Errorf("invalid version %q (allowed: letters, digits, . _ -)", ver)
+ }
+ if err := os.MkdirAll(filepath.Join(versionsDir(), ver), 0o755); err != nil {
+ return err
+ }
+ return copyFileAtomic(srcPath, versionBinPath(ver), 0o755)
+}
+
+// pointLive atomically repoints an instance's live symlink at ver's binary. It
+// verifies ver is staged first, so a typo can't leave a dangling link that would
+// fail the unit's ExecStart on the next restart.
+func pointLive(host, ver string) error {
+ target := versionBinPath(ver)
+ if _, err := os.Stat(target); err != nil {
+ return fmt.Errorf("version %q is not staged (%s): run `lchat stage --file <binary>` first", ver, target)
+ }
+ if err := os.MkdirAll(liveDir(), 0o755); err != nil {
+ return err
+ }
+ link := liveLink(host)
+ tmp := link + ".tmp"
+ _ = os.Remove(tmp) // a leftover temp link from a crashed swap
+ if err := os.Symlink(target, tmp); err != nil {
+ return err
+ }
+ if err := os.Rename(tmp, link); err != nil {
+ os.Remove(tmp)
+ return err
+ }
+ return nil
+}
+
+// liveVersion reports the version an instance's live symlink currently points at,
+// or "" (no error) if the instance hasn't been put under version management yet.
+func liveVersion(host string) (string, error) {
+ target, err := os.Readlink(liveLink(host))
+ if err != nil {
+ if os.IsNotExist(err) {
+ return "", nil
+ }
+ return "", err
+ }
+ // target = /opt/lchat/versions/<ver>/lchat → the parent dir's name is <ver>.
+ return filepath.Base(filepath.Dir(target)), nil
+}
+
+// selfPath is the running control binary — what add-instance / init-versions stage
+// so a fresh instance runs the same build as the operator's `lchat`. A var for tests.
+var selfPath = func() (string, error) { return os.Executable() }
+
+// ensureSelfStaged stages the running control binary under its build id and returns
+// that id, so add-instance can point a fresh instance at a real staged version
+// before the unit (ExecStart=/opt/lchat/live/<host>) first starts.
+func ensureSelfStaged() (string, error) {
+ self, err := selfPath()
+ if err != nil {
+ return "", err
+ }
+ if err := stageVersion(self, buildVersion); err != nil {
+ return "", err
+ }
+ return buildVersion, nil
+}
+
+// runStage: `lchat stage --file <binary>` — register a built binary in the store
+// under its build id (or --version), ready for `lchat deploy --version`.
+func runStage(args []string) {
+ fs := flag.NewFlagSet("stage", flag.ExitOnError)
+ file := fs.String("file", "", "path to the built lchat binary to register (required)")
+ verFlag := fs.String("version", "", "version id to store it under (default: the binary's own `lchat version`)")
+ _ = fs.Parse(reorderFlagsFirst(fs, args))
+
+ if *file == "" {
+ die("stage: --file <path> is required")
+ }
+ ver := strings.TrimSpace(*verFlag)
+ if ver == "" {
+ v, err := readBinaryVersion(*file)
+ if err != nil {
+ die("stage: can't read the build id from %s: %v\n build it with -ldflags \"-X main.buildVersion=$(git rev-parse --short HEAD)\" (see deploy/build-linux.sh), or pass --version", *file, err)
+ }
+ ver = v
+ }
+ if ver == "" || ver == "dev" {
+ die("stage: refusing to stage build id %q — build with -ldflags \"-X main.buildVersion=$(git rev-parse --short HEAD)\" or pass --version <id>", ver)
+ }
+ if !validVersion(ver) {
+ die("stage: invalid version %q (allowed: letters, digits, . _ -)", ver)
+ }
+ if err := stageVersion(*file, ver); err != nil {
+ die("stage: %v", err)
+ }
+ fmt.Printf("staged %s -> %s\n", ver, versionBinPath(ver))
+}
+
+// runInitVersions: `lchat init-versions` — one-time adoption. Stage the running
+// control binary and point every instance's live symlink at it, so the versioned
+// unit (ExecStart=/opt/lchat/live/<host>) resolves for instances that predate it.
+// Idempotent: it only creates a live link that's missing, never repoints a managed
+// instance (use `deploy --version` for that).
+func runInitVersions(args []string) {
+ fs := flag.NewFlagSet("init-versions", flag.ExitOnError)
+ registryPath := fs.String("registry", envOr("LCHAT_REGISTRY", "/data/registry.db"), "router registry SQLite path")
+ from := fs.String("from", "", "binary to seed the store with (default: the running control binary)")
+ verFlag := fs.String("version", "", "version id for the seed binary (default: its own build id)")
+ _ = fs.Parse(reorderFlagsFirst(fs, args))
+
+ src := strings.TrimSpace(*from)
+ ver := strings.TrimSpace(*verFlag)
+ if src == "" {
+ self, err := selfPath()
+ if err != nil {
+ die("init-versions: locate self: %v", err)
+ }
+ src = self
+ if ver == "" {
+ ver = buildVersion
+ }
+ } else if ver == "" {
+ v, err := readBinaryVersion(src)
+ if err != nil {
+ die("init-versions: read build id from %s: %v (or pass --version)", src, err)
+ }
+ ver = v
+ }
+ if !validVersion(ver) {
+ die("init-versions: invalid seed version %q (build with -ldflags, or pass --version)", ver)
+ }
+ if err := stageVersion(src, ver); err != nil {
+ die("init-versions: stage: %v", err)
+ }
+ fmt.Printf("seeded %s -> %s\n", ver, versionBinPath(ver))
+
+ reg, err := openRegistry(*registryPath)
+ if err != nil {
+ die("init-versions: open registry: %v", err)
+ }
+ routes, err := reg.list()
+ if err != nil {
+ die("init-versions: %v", err)
+ }
+ for _, rt := range routes {
+ if rt.Kind != "instance" {
+ continue
+ }
+ cur, err := liveVersion(rt.Host)
+ if err != nil {
+ die("init-versions: %s: read live link: %v", rt.Host, err)
+ }
+ if cur != "" {
+ fmt.Printf(" %s already on %s (left as-is)\n", rt.Host, cur)
+ continue
+ }
+ if err := pointLive(rt.Host, ver); err != nil {
+ die("init-versions: %s: %v", rt.Host, err)
+ }
+ rt.Version = ver
+ if err := reg.upsert(rt); err != nil {
+ die("init-versions: %s: record version: %v", rt.Host, err)
+ }
+ fmt.Printf(" %s -> %s\n", rt.Host, ver)
+ }
+ fmt.Println("init-versions: done (restart instances onto the versioned unit with `lchat deploy --all`)")
+}
+
+// runRollback: `lchat rollback <host>` (or --all) — repoint the instance(s) at the
+// previous version recorded at the last `deploy --version`, restart, health-check.
+// It swaps current<->previous so a second rollback returns to where you were.
+func runRollback(args []string) {
+ fs := flag.NewFlagSet("rollback", flag.ExitOnError)
+ registryPath := fs.String("registry", envOr("LCHAT_REGISTRY", "/data/registry.db"), "router registry SQLite path")
+ unitTmpl := fs.String("unit", envOr("LCHAT_UNIT", "lchat@"), "systemd instance unit template prefix")
+ all := fs.Bool("all", false, "roll back every instance that has a previous version")
+ _ = fs.Parse(reorderFlagsFirst(fs, args))
+
+ reg, err := openRegistry(*registryPath)
+ if err != nil {
+ die("rollback: open registry: %v", err)
+ }
+ hosts, err := deployTargets(reg, *all, fs.Arg(0), "rollback")
+ if err != nil {
+ die("%v", err)
+ }
+ for _, host := range hosts {
+ rt, err := reg.get(host)
+ if err != nil || rt == nil {
+ die("rollback %s: unknown host (run add-instance first)", host)
+ }
+ if rt.Kind != "instance" {
+ die("rollback %s: not an instance (kind %q)", host, rt.Kind)
+ }
+ prev := strings.TrimSpace(rt.PrevVersion)
+ if prev == "" {
+ if *all {
+ continue // nothing to roll back to; skip in a fleet rollback
+ }
+ die("rollback %s: no previous version recorded", host)
+ }
+ cur, _ := liveVersion(host)
+ if err := pointLive(host, prev); err != nil {
+ die("rollback %s: %v", host, err)
+ }
+ if err := systemctl("restart", *unitTmpl+host); err != nil {
+ die("rollback %s: restart: %v", host, err)
+ }
+ if err := waitHealthy(rt.Socket, 20*time.Second); err != nil {
+ die("rollback %s: %v", host, err)
+ }
+ rt.Version, rt.PrevVersion = prev, cur
+ if err := reg.upsert(*rt); err != nil {
+ die("rollback %s: record: %v", host, err)
+ }
+ fmt.Printf("rolled back %s to %s\n", host, prev)
+ }
+}
+
+// deployTargets resolves the host list shared by deploy/rollback: every instance
+// (--all) or the one validated positional host.
+func deployTargets(reg *Registry, all bool, arg, cmd string) ([]string, error) {
+ if all {
+ routes, err := reg.list()
+ if err != nil {
+ return nil, fmt.Errorf("%s: %v", cmd, err)
+ }
+ var hosts []string
+ for _, r := range routes {
+ if r.Kind == "instance" {
+ hosts = append(hosts, r.Host)
+ }
+ }
+ if len(hosts) == 0 {
+ return nil, fmt.Errorf("%s: no instances", cmd)
+ }
+ return hosts, nil
+ }
+ h := strings.ToLower(strings.TrimSpace(arg))
+ if !validHost(h) {
+ return nil, fmt.Errorf("%s: invalid host %q (or pass --all)", cmd, h)
+ }
+ return []string{h}, nil
+}
+
+// waitHealthyVersion is waitHealthy plus a build check: it waits until the instance
+// not only answers but reports the expected build at /api/version — so a deploy
+// fails loudly if the process is still the old binary (a stale socket, or a restart
+// that didn't take), rather than silently "succeeding".
+func waitHealthyVersion(socket, wantBuild string, timeout time.Duration) error {
+ deadline := time.Now().Add(timeout)
+ client := unixHTTPClient(socket)
+ var last error
+ for time.Now().Before(deadline) {
+ resp, err := client.Get("http://lchat/api/version")
+ if err == nil {
+ var body struct {
+ Build string `json:"build"`
+ }
+ derr := json.NewDecoder(io.LimitReader(resp.Body, 1<<16)).Decode(&body)
+ resp.Body.Close()
+ if derr == nil && body.Build == wantBuild {
+ return nil
+ }
+ if derr != nil {
+ last = fmt.Errorf("read /api/version: %v", derr)
+ } else {
+ last = fmt.Errorf("live build %q != expected %q", body.Build, wantBuild)
+ }
+ } else {
+ last = err
+ }
+ time.Sleep(150 * time.Millisecond)
+ }
+ return fmt.Errorf("instance didn't report build %q within %s: %v", wantBuild, timeout, last)
+}
+
+// unixHTTPClient dials a unix socket over HTTP — the router↔instance transport,
+// reused by the health checks.
+func unixHTTPClient(socket string) *http.Client {
+ return &http.Client{
+ Timeout: 3 * time.Second,
+ Transport: &http.Transport{
+ DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
+ return (&net.Dialer{}).DialContext(ctx, "unix", socket)
+ },
+ },
+ }
+}
diff --git a/versions_test.go b/versions_test.go
new file mode 100644
index 0000000..dede662
--- /dev/null
+++ b/versions_test.go
@@ -0,0 +1,127 @@
+package main
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+)
+
+// withTempOpt points optDir at a fresh temp dir for a test and restores it after,
+// so the version-store helpers never touch the real /opt/lchat.
+func withTempOpt(t *testing.T) string {
+ t.Helper()
+ dir := t.TempDir()
+ prev := optDir
+ optDir = dir
+ t.Cleanup(func() { optDir = prev })
+ return dir
+}
+
+// writeFakeBinary drops an executable file to stand in for a built binary.
+func writeFakeBinary(t *testing.T, path, content string) {
+ t.Helper()
+ if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(path, []byte(content), 0o755); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestValidVersion(t *testing.T) {
+ ok := []string{"abc123", "v1.2.3", "test-scroll", "dev", "a_b.c-d"}
+ bad := []string{"", ".", "..", "a/b", "with space", "semi;colon", "../etc"}
+ for _, v := range ok {
+ if !validVersion(v) {
+ t.Errorf("validVersion(%q) = false, want true", v)
+ }
+ }
+ for _, v := range bad {
+ if validVersion(v) {
+ t.Errorf("validVersion(%q) = true, want false", v)
+ }
+ }
+}
+
+func TestStageAndPointLive(t *testing.T) {
+ withTempOpt(t)
+ src := filepath.Join(t.TempDir(), "lchat-build")
+ writeFakeBinary(t, src, "#!/bin/sh\necho hi\n")
+
+ if err := stageVersion(src, "abc123"); err != nil {
+ t.Fatalf("stageVersion: %v", err)
+ }
+ // The staged binary exists and is executable.
+ info, err := os.Stat(versionBinPath("abc123"))
+ if err != nil {
+ t.Fatalf("staged binary missing: %v", err)
+ }
+ if info.Mode().Perm()&0o100 == 0 {
+ t.Errorf("staged binary not executable: %v", info.Mode())
+ }
+
+ // Pointing an instance's live link resolves to the staged binary and reports
+ // the right version.
+ if err := pointLive("test.luan.chat", "abc123"); err != nil {
+ t.Fatalf("pointLive: %v", err)
+ }
+ got, err := liveVersion("test.luan.chat")
+ if err != nil {
+ t.Fatalf("liveVersion: %v", err)
+ }
+ if got != "abc123" {
+ t.Errorf("liveVersion = %q, want abc123", got)
+ }
+ // The symlink actually points at the version binary.
+ resolved, err := filepath.EvalSymlinks(liveLink("test.luan.chat"))
+ if err != nil {
+ t.Fatalf("EvalSymlinks: %v", err)
+ }
+ if want, _ := filepath.EvalSymlinks(versionBinPath("abc123")); resolved != want {
+ t.Errorf("live link resolves to %q, want %q", resolved, want)
+ }
+}
+
+func TestPointLiveUnstagedFails(t *testing.T) {
+ withTempOpt(t)
+ if err := pointLive("test.luan.chat", "nope"); err == nil {
+ t.Error("pointLive to an unstaged version should fail, but succeeded")
+ }
+}
+
+func TestPointLiveAtomicSwap(t *testing.T) {
+ withTempOpt(t)
+ src := filepath.Join(t.TempDir(), "lchat-build")
+ writeFakeBinary(t, src, "bin")
+ for _, v := range []string{"v1", "v2"} {
+ if err := stageVersion(src, v); err != nil {
+ t.Fatalf("stageVersion %s: %v", v, err)
+ }
+ }
+ if err := pointLive("h", "v1"); err != nil {
+ t.Fatal(err)
+ }
+ // Repointing to a new version replaces the link in place (no leftover .tmp).
+ if err := pointLive("h", "v2"); err != nil {
+ t.Fatal(err)
+ }
+ if got, _ := liveVersion("h"); got != "v2" {
+ t.Errorf("after repoint liveVersion = %q, want v2", got)
+ }
+ if _, err := os.Lstat(liveLink("h") + ".tmp"); !os.IsNotExist(err) {
+ t.Errorf("leftover temp symlink after swap: %v", err)
+ }
+}
+
+// liveVersion on an instance that was never versioned returns ("", nil), not an
+// error — init-versions/deploy rely on that to detect an unmanaged instance.
+func TestLiveVersionMissing(t *testing.T) {
+ withTempOpt(t)
+ got, err := liveVersion("never.seen")
+ if err != nil {
+ t.Errorf("liveVersion(missing) err = %v, want nil", err)
+ }
+ if got != "" {
+ t.Errorf("liveVersion(missing) = %q, want empty", got)
+ }
+}