amadan
server: box-side auto-updater — lchat update run/status + hourly timer (#383)
The piece that acts on the release channels: a root oneshot (fired hourly by lchat-update.timer, installed + enabled by bootstrap) that fetches the signed manifest, reads each ready instance's admin prefs over its unix socket (404 = defaults: on/on/stable), downloads + sha256-verifies + stages the channel's binary, and applies it via the shared deploy dance. - control.go: the per-host apply block extracted from runDeploy into deployHostVersion (die-free, returned errors) — one implementation of repoint/restart/health-check/rollback for deploy and the updater - a box-wide flock (/opt/lchat/.deploy.lock) serialises deploy, rollback and update run, so a timer pass can't interleave an operator's deploy - anti-replay: newest accepted generated_at persisted to /opt/lchat/manifest-stamp; older signed manifests are refused - a failed health check poisons that version for the rest of the pass; per-host errors log and continue (an unreachable instance never strands the fleet) - version-store GC after each pass (keep: route current/prev, channel heads, live symlink targets, the control binary's own build) + dl-* sweep - lchat update status: HOST/CHANNEL/AUTO/CURRENT/AVAILABLE fleet table - validateManifest now allows plain-http URLs on loopback only (local feeds and tests); remote stays https-only
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
12 files changed,
+941
−45
bootstrap.go+17 −1control.go+67 −19deploy/systemd/lchat-update.service+31 −0deploy/systemd/lchat-update.timer+12 −0docs/instance-router.md+37 −0docs/server-releases.md+2 −2main.go+3 −0release.go+14 −2updatecheck.go+3 −21updater.go+387 −0updater_test.go+362 −0versions.go+6 −0
diff --git a/bootstrap.go b/bootstrap.go| index 573aa04..3064b5f 100644 |
| --- a/bootstrap.go |
| +++ b/bootstrap.go |
| @@ -44,6 +44,17 @@ var unitControl string |
| //go:embed deploy/systemd/lchat-cloud.service |
| var unitCloud string |
| +// The auto-updater (updater.go): a root oneshot fired hourly by its timer. |
| +// Enabled by bootstrap — a pass with no reachable release feed is a no-op, so |
| +// it's safe everywhere, and a box provisioned tomorrow auto-updates without a |
| +// runbook step anyone has to remember. |
| +// |
| +//go:embed deploy/systemd/lchat-update.service |
| +var unitUpdate string |
| + |
| +//go:embed deploy/systemd/lchat-update.timer |
| +var unitUpdateTimer string |
| + |
| // serviceUser is the unprivileged account every unit runs as (see the hardened |
| // units in deploy/systemd). bootstrap creates it and hands it the data + runtime |
| // dirs; the router keeps CAP_NET_BIND_SERVICE to still bind :80/:443. |
| @@ -142,6 +153,8 @@ func runBootstrap(args []string) { |
| write(filepath.Join(*unitDir, "lchat-home.service"), unitHome) |
| write(filepath.Join(*unitDir, "lchat-control.service"), unitControl) |
| write(filepath.Join(*unitDir, "lchat-cloud.service"), unitCloud) |
| + write(filepath.Join(*unitDir, "lchat-update.service"), unitUpdate) |
| + write(filepath.Join(*unitDir, "lchat-update.timer"), unitUpdateTimer) |
| write("/etc/tmpfiles.d/lchat.conf", tmpfilesConf) |
| if err := ensureControlToken(); err != nil { |
| @@ -158,6 +171,9 @@ func runBootstrap(args []string) { |
| if err := systemctl("enable", "--now", "lchat-router"); err != nil { |
| die("bootstrap: start router: %v", err) |
| } |
| - fmt.Println("bootstrap: router installed and started") |
| + if err := systemctl("enable", "--now", "lchat-update.timer"); err != nil { |
| + die("bootstrap: enable update timer: %v", err) |
| + } |
| + fmt.Println("bootstrap: router installed and started (auto-update timer enabled)") |
| fmt.Println(" next: lchat add-instance <host>") |
| } |
diff --git a/control.go b/control.go| index 180dfb7..a786c5c 100644 |
| --- a/control.go |
| +++ b/control.go |
| @@ -14,6 +14,7 @@ import ( |
| "regexp" |
| "sort" |
| "strings" |
| + "syscall" |
| "time" |
| ) |
| @@ -509,6 +510,12 @@ func runDeploy(args []string) { |
| } |
| } |
| + unlock, err := acquireDeployLock() |
| + if err != nil { |
| + die("deploy: %v", err) |
| + } |
| + defer unlock() |
| + |
| reg, err := openRegistry(*registryPath) |
| if err != nil { |
| die("deploy: open registry: %v", err) |
| @@ -544,31 +551,72 @@ func runDeploy(args []string) { |
| } |
| // --version: repoint, restart, verify the build took, roll back if not. |
| - prev, _ := liveVersion(host) // "" if this instance isn't versioned yet |
| - if prev == ver { |
| - fmt.Printf("deployed %s (already on %s)\n", host, ver) |
| - continue |
| - } |
| - if err := pointLive(host, ver); err != nil { |
| + changed, err := deployHostVersion(reg, rt, unit, ver) |
| + if err != nil { |
| die("deploy %s: %v", host, err) |
| } |
| - if err := systemctl("restart", unit); err != nil { |
| - rollbackLive(host, prev, unit, rt.Socket) |
| - die("deploy %s: restart: %v (rolled back)", host, err) |
| - } |
| - if err := waitHealthyVersion(rt.Socket, ver, 20*time.Second); err != nil { |
| - fmt.Printf("deploy %s: %v — rolling back to %q\n", host, err, prev) |
| - rollbackLive(host, prev, unit, rt.Socket) |
| - die("deploy %s: failed health check, rolled back to %q", host, prev) |
| - } |
| - rt.PrevVersion, rt.Version = prev, ver |
| - if err := reg.upsert(*rt); err != nil { |
| - die("deploy %s: record version: %v", host, err) |
| + if changed { |
| + fmt.Printf("deployed %s @ %s\n", host, ver) |
| + } else { |
| + fmt.Printf("deployed %s (already on %s)\n", host, ver) |
| } |
| - fmt.Printf("deployed %s @ %s\n", host, ver) |
| } |
| } |
| +// deployHostVersion is the one per-host apply dance, shared by `deploy |
| +// --version` and the auto-updater (updater.go): repoint the live symlink at a |
| +// staged version, restart, verify the instance actually reports that build, |
| +// roll back on failure, and record current+previous in the registry. Returns |
| +// changed=false when the host was already on ver. |
| +// |
| +// deployHealthTimeout is a var so tests can shrink the failure path's wait. |
| +var deployHealthTimeout = 20 * time.Second |
| + |
| +func deployHostVersion(reg *Registry, rt *Route, unit, ver string) (changed bool, err error) { |
| + host := rt.Host |
| + prev, _ := liveVersion(host) // "" if this instance isn't versioned yet |
| + if prev == ver { |
| + return false, nil |
| + } |
| + if err := pointLive(host, ver); err != nil { |
| + return false, err |
| + } |
| + if err := systemctl("restart", unit); err != nil { |
| + rollbackLive(host, prev, unit, rt.Socket) |
| + return false, fmt.Errorf("restart: %v (rolled back to %q)", err, prev) |
| + } |
| + if err := waitHealthyVersion(rt.Socket, ver, deployHealthTimeout); err != nil { |
| + rollbackLive(host, prev, unit, rt.Socket) |
| + return false, fmt.Errorf("failed health check (%v), rolled back to %q", err, prev) |
| + } |
| + rt.PrevVersion, rt.Version = prev, ver |
| + if err := reg.upsert(*rt); err != nil { |
| + return true, fmt.Errorf("record version: %v", err) |
| + } |
| + return true, nil |
| +} |
| + |
| +// acquireDeployLock serialises everything that runs the repoint/restart/ |
| +// health-check dance — deploy, rollback, and the timer-fired `update run` — |
| +// across processes, so an auto-update can never interleave with an operator's |
| +// deploy mid-swap. Non-blocking: the second caller gets a clear error rather |
| +// than silently queueing behind an unknown amount of work. |
| +func acquireDeployLock() (func(), error) { |
| + path := filepath.Join(optDir, ".deploy.lock") |
| + f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o644) |
| + if err != nil { |
| + return nil, err |
| + } |
| + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { |
| + f.Close() |
| + return nil, fmt.Errorf("another deploy/rollback/update is running (%s is held)", path) |
| + } |
| + return func() { |
| + _ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN) |
| + f.Close() |
| + }, nil |
| +} |
| + |
| // rollbackLive best-effort restores an instance to prev after a failed |
| // --version deploy: repoint the symlink and restart. If there is no prev (the |
| // instance had no version yet) it leaves the failed link in place — the operator |
diff --git a/deploy/systemd/lchat-update.service b/deploy/systemd/lchat-update.service| new file mode 100644 |
| index 0000000..3599b1f |
| --- /dev/null |
| +++ b/deploy/systemd/lchat-update.service |
| @@ -0,0 +1,31 @@ |
| +[Unit] |
| +Description=lchat auto-updater (one release-channel update pass) |
| +After=network-online.target |
| +Wants=network-online.target |
| + |
| +[Service] |
| +Type=oneshot |
| +# Optional override: LCHAT_UPDATE_MANIFEST_URL (point a box at another feed). |
| +EnvironmentFile=-/data/update.env |
| +ExecStart=/opt/lchat/lchat update run |
| + |
| +# --- hardening (see docs/instance-router.md § Hardening) --- |
| +# Runs as ROOT deliberately, like lchat-control: an update pass writes the |
| +# version store (/opt/lchat) and restarts instance units via systemctl. It is |
| +# fired by lchat-update.timer, never routed or listening. NoNewPrivileges and |
| +# the syscall filter are omitted for the systemctl child; the rest applies. |
| +ProtectHome=yes |
| +PrivateTmp=yes |
| +PrivateDevices=yes |
| +ProtectClock=yes |
| +ProtectHostname=yes |
| +ProtectKernelTunables=yes |
| +ProtectKernelModules=yes |
| +ProtectKernelLogs=yes |
| +RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX |
| +RestrictNamespaces=yes |
| +RestrictRealtime=yes |
| +LockPersonality=yes |
| +MemoryDenyWriteExecute=yes |
| +MemoryMax=256M |
| +TasksMax=64 |
diff --git a/deploy/systemd/lchat-update.timer b/deploy/systemd/lchat-update.timer| new file mode 100644 |
| index 0000000..81dc1fc |
| --- /dev/null |
| +++ b/deploy/systemd/lchat-update.timer |
| @@ -0,0 +1,12 @@ |
| +[Unit] |
| +Description=hourly lchat release-channel update check |
| + |
| +[Timer] |
| +# A short boot delay so instances are up before the first pass; the random |
| +# spread keeps a fleet of boxes from hitting the feed in the same second. |
| +OnBootSec=5min |
| +OnUnitActiveSec=1h |
| +RandomizedDelaySec=10min |
| + |
| +[Install] |
| +WantedBy=timers.target |
diff --git a/docs/instance-router.md b/docs/instance-router.md| index 1f2e1d0..2004cc9 100644 |
| --- a/docs/instance-router.md |
| +++ b/docs/instance-router.md |
| @@ -402,6 +402,43 @@ returned 200. |
| The router itself changes rarely; when it does, its own front listener gets |
| zero-downtime via `SO_REUSEPORT` / systemd socket activation. |
| +## 🤖 Auto-updating (release channels) |
| + |
| +Instances can keep themselves current without an operator driving `deploy` — |
| +built on the versioned-deploy machinery above plus the signed release feed |
| +described in `docs/server-releases.md`: |
| + |
| +- **`lchat update run`** (fired hourly by `lchat-update.timer`, installed + |
| + enabled by `bootstrap`; root, like `lchat-control`) fetches and |
| + Ed25519-verifies the channel manifest, asks each *ready* instance what its |
| + admin chose (`GET /api/update-prefs` over its unix socket — a pre-feature |
| + binary 404s, which reads as the defaults: auto-update on, stable channel), |
| + then per instance: downloads + sha256-checks + stages the channel's binary |
| + and applies it with the exact `deploy --version` dance (repoint → restart → |
| + health-check → auto-rollback). A version that fails its health check on one |
| + host is not tried on the rest that pass ("poisoned" for the run); other |
| + per-host failures log and continue, so one unreachable instance never |
| + strands the fleet. |
| +- **Prefs are per-instance admin settings** (`update_auto`, `update_download`, |
| + `update_channel` in the instance's own config table, set from the admin |
| + consoles). Download-only means "staged, apply manually with |
| + `lchat deploy <host> --version <ver>`". **A pinned canary gets stomped like |
| + `deploy --all` would** — flip that instance's auto-update off first. |
| +- **A deploy flock** (`/opt/lchat/.deploy.lock`) serialises `deploy`, |
| + `rollback`, and `update run` across processes; the loser errors immediately |
| + ("another deploy/rollback/update is running") instead of interleaving the |
| + swap dance. A timer pass that loses simply retries next tick. |
| +- **Anti-replay**: the newest accepted `generated_at` persists in |
| + `/opt/lchat/manifest-stamp`; older (validly signed but stale) manifests are |
| + refused, so the feed can be pulled backward only by *re-signing* — a |
| + deliberate act with the key. |
| +- **GC**: after a pass, version-store dirs nothing references (no route's |
| + current/previous, no channel head, no live symlink, not the control binary's |
| + own build) are pruned, and crashed `dl-*` downloads in `incoming/` are |
| + swept — hourly edge updates stay disk-bounded. |
| +- **`lchat update status`** prints HOST / CHANNEL / AUTO / CURRENT / AVAILABLE |
| + across the fleet; `lchat update run --dry-run` shows what a pass would do. |
| + |
| ## Supervision (systemd) |
| - `lchat-router.service` — `Restart=always`, socket-activated (its own |
diff --git a/docs/server-releases.md b/docs/server-releases.md| index 75c4278..6c5b1c5 100644 |
| --- a/docs/server-releases.md |
| +++ b/docs/server-releases.md |
| @@ -4,8 +4,8 @@ How server builds get released, how the three channels work, and the trust |
| model behind instance auto-updating. The publishing tool is |
| `deploy/release-server.sh`; the format/signing code is `release.go` (shared by |
| the publisher and the box updater so they can't drift). The box-side updater |
| -that consumes this feed is described in `docs/instance-router.md` (§ auto-update, |
| -once it lands). |
| +that consumes this feed is described in `docs/instance-router.md` |
| +(§ Auto-updating). |
| ## The shape |
diff --git a/main.go b/main.go| index 57912fb..7a215ee 100644 |
| --- a/main.go |
| +++ b/main.go |
| @@ -159,6 +159,9 @@ func main() { |
| case "release": |
| runRelease(os.Args[2:]) |
| return |
| + case "update": |
| + runUpdate(os.Args[2:]) |
| + return |
| case "controld": |
| runControld(os.Args[2:]) |
| return |
diff --git a/release.go b/release.go| index 20be70e..208627d 100644 |
| --- a/release.go |
| +++ b/release.go |
| @@ -9,6 +9,7 @@ import ( |
| "encoding/json" |
| "flag" |
| "fmt" |
| + "net/url" |
| "os" |
| "strings" |
| "time" |
| @@ -52,7 +53,7 @@ func validChannel(c string) bool { |
| // releasePubKeyB64 is the Ed25519 public key server releases are signed with |
| // (the private half lives in 1Password, never in the repo). A var so tests can |
| -// swap in their own trust root via setReleasePubKey. |
| +// swap in their own trust root, and `release verify --pubkey` can override it. |
| var releasePubKeyB64 = "6pL2QKDCfXqcYGR6Vr5uUZZKrpV6YVwg9rV0Okb8LY8=" |
| // releaseManifestURL is where the box updater and instances look for the feed. |
| @@ -107,7 +108,7 @@ func validateManifest(m *releaseManifest) error { |
| if !validVersion(e.Version) || e.Version == "dev" { |
| return fmt.Errorf("channel %s: invalid version %q", ch, e.Version) |
| } |
| - if !strings.HasPrefix(e.URL, "https://") { |
| + if !strings.HasPrefix(e.URL, "https://") && !isLoopbackURL(e.URL) { |
| return fmt.Errorf("channel %s: binary URL must be https (%q)", ch, e.URL) |
| } |
| if raw, err := hex.DecodeString(e.SHA256); err != nil || len(raw) != sha256.Size { |
| @@ -117,6 +118,17 @@ func validateManifest(m *releaseManifest) error { |
| return nil |
| } |
| +// isLoopbackURL allows plain-http binary URLs only on loopback — local feeds |
| +// and tests; anything remote must be https. |
| +func isLoopbackURL(u string) bool { |
| + p, err := url.Parse(u) |
| + if err != nil || p.Scheme != "http" { |
| + return false |
| + } |
| + h := p.Hostname() |
| + return h == "localhost" || h == "127.0.0.1" || h == "::1" |
| +} |
| + |
| // signReleaseManifest stamps generated_at (now — this is what makes replayed |
| // old feeds detectable) and wraps the payload in the signed wire format. |
| func signReleaseManifest(priv ed25519.PrivateKey, m *releaseManifest) ([]byte, error) { |
diff --git a/updatecheck.go b/updatecheck.go| index 75d88d1..3b61e7b 100644 |
| --- a/updatecheck.go |
| +++ b/updatecheck.go |
| @@ -1,8 +1,6 @@ |
| package main |
| import ( |
| - "fmt" |
| - "io" |
| "net/http" |
| "sync" |
| "time" |
| @@ -50,28 +48,12 @@ func (s *Server) runUpdateChecker() { |
| } |
| } |
| -// checkForUpdate fetches and verifies the release manifest, keeping the whole |
| +// checkForUpdate fetches and verifies the release manifest (the same fetch |
| +// the box updater runs — fetchReleaseManifest, updater.go), keeping the whole |
| // channel map — so an admin switching channels sees that channel's version |
| // immediately, without waiting for the next fetch. |
| func (s *Server) checkForUpdate() error { |
| - pub, err := releasePubKey() |
| - if err != nil { |
| - return err |
| - } |
| - client := &http.Client{Timeout: 30 * time.Second} |
| - resp, err := client.Get(releaseManifestURL) |
| - if err != nil { |
| - return err |
| - } |
| - defer resp.Body.Close() |
| - if resp.StatusCode != http.StatusOK { |
| - return fmt.Errorf("fetch manifest: %s", resp.Status) |
| - } |
| - raw, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) |
| - if err != nil { |
| - return err |
| - } |
| - m, err := verifyReleaseManifest(pub, raw) |
| + m, err := fetchReleaseManifest(releaseManifestURL) |
| if err != nil { |
| return err |
| } |
diff --git a/updater.go b/updater.go| new file mode 100644 |
| index 0000000..20ce644 |
| --- /dev/null |
| +++ b/updater.go |
| @@ -0,0 +1,387 @@ |
| +package main |
| + |
| +import ( |
| + "crypto/sha256" |
| + "encoding/hex" |
| + "encoding/json" |
| + "flag" |
| + "fmt" |
| + "io" |
| + "net/http" |
| + "os" |
| + "path/filepath" |
| + "strings" |
| + "time" |
| +) |
| + |
| +// The box-side auto-updater: `lchat update run`, fired hourly by |
| +// lchat-update.timer (root — it stages binaries and restarts units, like |
| +// lchat-control). One pass reads the signed channel feed (release.go), asks |
| +// each instance what its admin chose (GET /api/update-prefs over the unix |
| +// socket — instances are sandboxed and can't update themselves), then |
| +// downloads/stages/applies per those prefs, reusing the exact deploy dance |
| +// (deployHostVersion: repoint → restart → health-check → auto-rollback). |
| +// It shares the deploy flock with `deploy`/`rollback`, so a timer firing |
| +// mid-operator-deploy waits its turn (fails fast, next tick retries). |
| +// |
| +// Trust + anti-replay: the manifest is Ed25519-verified, binaries are hashed |
| +// against the signed sha256, and the last accepted generated_at is persisted — |
| +// a replayed stale feed is refused, so nobody can downgrade the fleet by |
| +// re-serving an old (validly signed) manifest. See docs/server-releases.md. |
| + |
| +func incomingDir() string { return filepath.Join(optDir, "incoming") } |
| + |
| +// manifestStampPath persists the newest generated_at ever accepted. |
| +func manifestStampPath() string { return filepath.Join(optDir, "manifest-stamp") } |
| + |
| +func readManifestStamp() time.Time { |
| + raw, err := os.ReadFile(manifestStampPath()) |
| + if err != nil { |
| + return time.Time{} |
| + } |
| + t, err := time.Parse(time.RFC3339, strings.TrimSpace(string(raw))) |
| + if err != nil { |
| + return time.Time{} |
| + } |
| + return t |
| +} |
| + |
| +func writeManifestStamp(t time.Time) error { |
| + return os.WriteFile(manifestStampPath(), []byte(t.Format(time.RFC3339)+"\n"), 0o644) |
| +} |
| + |
| +// fetchReleaseManifest downloads and signature-verifies the channel feed. |
| +// Shared with the instance-side availability checker (updatecheck.go). |
| +func fetchReleaseManifest(url string) (*releaseManifest, error) { |
| + pub, err := releasePubKey() |
| + if err != nil { |
| + return nil, err |
| + } |
| + client := &http.Client{Timeout: 2 * time.Minute} |
| + resp, err := client.Get(url) |
| + if err != nil { |
| + return nil, err |
| + } |
| + defer resp.Body.Close() |
| + if resp.StatusCode != http.StatusOK { |
| + return nil, fmt.Errorf("fetch manifest: %s", resp.Status) |
| + } |
| + raw, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) |
| + if err != nil { |
| + return nil, err |
| + } |
| + return verifyReleaseManifest(pub, raw) |
| +} |
| + |
| +// updatePrefs is what an instance's admin chose (GET /api/update-prefs). |
| +type updatePrefs struct { |
| + AutoUpdate bool `json:"auto_update"` |
| + AutoDownload bool `json:"auto_download"` |
| + Channel string `json:"channel"` |
| +} |
| + |
| +// fetchUpdatePrefs reads an instance's prefs over its unix socket. A binary |
| +// too old to have the endpoint 404s — that means the defaults (on/on/stable), |
| +// which is exactly how a pre-feature fleet adopts auto-updating. |
| +func fetchUpdatePrefs(socket string) (updatePrefs, error) { |
| + defaults := updatePrefs{AutoUpdate: true, AutoDownload: true, Channel: "stable"} |
| + client := unixHTTPClient(socket) |
| + resp, err := client.Get("http://lchat/api/update-prefs") |
| + if err != nil { |
| + return defaults, err |
| + } |
| + defer resp.Body.Close() |
| + if resp.StatusCode == http.StatusNotFound { |
| + return defaults, nil |
| + } |
| + if resp.StatusCode != http.StatusOK { |
| + return defaults, fmt.Errorf("update-prefs: %s", resp.Status) |
| + } |
| + var p updatePrefs |
| + if err := json.NewDecoder(io.LimitReader(resp.Body, 1<<16)).Decode(&p); err != nil { |
| + return defaults, err |
| + } |
| + if !validChannel(p.Channel) { |
| + p.Channel = "stable" |
| + } |
| + return p, nil |
| +} |
| + |
| +// downloadReleaseBinary fetches a release into the incoming dir and refuses |
| +// anything whose sha256 doesn't match the signed manifest entry. Returns the |
| +// temp path; the caller stages it and removes the temp. |
| +func downloadReleaseBinary(e releaseEntry) (string, error) { |
| + if err := os.MkdirAll(incomingDir(), 0o755); err != nil { |
| + return "", err |
| + } |
| + f, err := os.CreateTemp(incomingDir(), "dl-") |
| + if err != nil { |
| + return "", err |
| + } |
| + defer f.Close() |
| + fail := func(err error) (string, error) { |
| + os.Remove(f.Name()) |
| + return "", err |
| + } |
| + client := &http.Client{Timeout: 10 * time.Minute} |
| + resp, err := client.Get(e.URL) |
| + if err != nil { |
| + return fail(err) |
| + } |
| + defer resp.Body.Close() |
| + if resp.StatusCode != http.StatusOK { |
| + return fail(fmt.Errorf("download %s: %s", e.Version, resp.Status)) |
| + } |
| + h := sha256.New() |
| + if _, err := io.Copy(io.MultiWriter(f, h), resp.Body); err != nil { |
| + return fail(err) |
| + } |
| + if got := hex.EncodeToString(h.Sum(nil)); got != e.SHA256 { |
| + return fail(fmt.Errorf("download %s: sha256 mismatch (refusing to stage)", e.Version)) |
| + } |
| + return f.Name(), nil |
| +} |
| + |
| +// ensureVersionStaged makes a manifest entry's binary available in the version |
| +// store, downloading it if needed. Idempotent across hosts in one pass. |
| +func ensureVersionStaged(e releaseEntry) error { |
| + if _, err := os.Stat(versionBinPath(e.Version)); err == nil { |
| + return nil |
| + } |
| + tmp, err := downloadReleaseBinary(e) |
| + if err != nil { |
| + return err |
| + } |
| + defer os.Remove(tmp) |
| + return stageVersion(tmp, e.Version) |
| +} |
| + |
| +// updatePass is one full check-and-apply sweep, split from runUpdateRun so |
| +// tests drive it directly (die()-free: per-host problems log and continue — |
| +// an unreachable instance must not strand the rest of the fleet unchecked). |
| +func updatePass(reg *Registry, unitTmpl string, dryRun bool) error { |
| + m, err := fetchReleaseManifest(releaseManifestURL) |
| + if err != nil { |
| + // No feed is a normal state (the releases repo may not exist yet, or |
| + // the box is offline) — never an error exit the timer would flap on. |
| + fmt.Printf("update: no usable manifest (%v) — nothing to do\n", err) |
| + return nil |
| + } |
| + if last := readManifestStamp(); m.GeneratedAt.Before(last) { |
| + return fmt.Errorf("manifest generated_at %s is older than the last accepted %s — refusing a replayed feed", |
| + m.GeneratedAt.Format(time.RFC3339), last.Format(time.RFC3339)) |
| + } |
| + if !dryRun { |
| + if err := writeManifestStamp(m.GeneratedAt); err != nil { |
| + return fmt.Errorf("record manifest stamp: %v", err) |
| + } |
| + } |
| + |
| + routes, err := reg.list() |
| + if err != nil { |
| + return err |
| + } |
| + // A version that failed its health check on one host isn't tried on the |
| + // next this pass — one bad build shouldn't bounce every instance in turn. |
| + poisoned := map[string]bool{} |
| + for _, rt := range routes { |
| + if rt.Kind != "instance" { |
| + continue |
| + } |
| + if rt.Status != "ready" { |
| + fmt.Printf(" %s: skipped (status %s)\n", rt.Host, rt.Status) |
| + continue |
| + } |
| + prefs, err := fetchUpdatePrefs(rt.Socket) |
| + if err != nil { |
| + fmt.Printf(" %s: unreachable (%v) — skipped\n", rt.Host, err) |
| + continue |
| + } |
| + e, ok := m.Channels[prefs.Channel] |
| + if !ok { |
| + fmt.Printf(" %s: channel %s has no release yet\n", rt.Host, prefs.Channel) |
| + continue |
| + } |
| + cur, _ := liveVersion(rt.Host) |
| + if cur == e.Version { |
| + continue // up to date — stay quiet on the hourly tick |
| + } |
| + if !prefs.AutoDownload && !prefs.AutoUpdate { |
| + fmt.Printf(" %s: %s available on %s (downloads off — admin's call)\n", rt.Host, e.Version, prefs.Channel) |
| + continue |
| + } |
| + if dryRun { |
| + fmt.Printf(" %s: would update %s -> %s (%s)\n", rt.Host, cur, e.Version, prefs.Channel) |
| + continue |
| + } |
| + if err := ensureVersionStaged(e); err != nil { |
| + fmt.Printf(" %s: %v\n", rt.Host, err) |
| + continue |
| + } |
| + if !prefs.AutoUpdate { |
| + fmt.Printf(" %s: %s downloaded and staged (auto-update off — apply with `lchat deploy %s --version %s`)\n", |
| + rt.Host, e.Version, rt.Host, e.Version) |
| + continue |
| + } |
| + if poisoned[e.Version] { |
| + fmt.Printf(" %s: skipping %s (failed health check on another instance this pass)\n", rt.Host, e.Version) |
| + continue |
| + } |
| + rtCopy := rt |
| + changed, err := deployHostVersion(reg, &rtCopy, unitTmpl+rt.Host, e.Version) |
| + if err != nil { |
| + poisoned[e.Version] = true |
| + fmt.Printf(" %s: update to %s failed: %v\n", rt.Host, e.Version, err) |
| + continue |
| + } |
| + if changed { |
| + fmt.Printf(" %s: updated %s -> %s (%s)\n", rt.Host, cur, e.Version, prefs.Channel) |
| + } |
| + } |
| + if !dryRun { |
| + gcVersions(reg, m) |
| + } |
| + return nil |
| +} |
| + |
| +// gcVersions prunes version-store dirs nothing references anymore: kept are |
| +// every route's current + previous version, every channel head, the control |
| +// binary's own build, and whatever any live symlink points at (belt and |
| +// braces — the symlinks are what units actually exec). Without this, hourly |
| +// edge updates would grow /opt/lchat/versions without bound on a box that |
| +// enforces a free-disk floor. Leftover crashed downloads are swept too. |
| +func gcVersions(reg *Registry, m *releaseManifest) { |
| + keep := map[string]bool{buildVersion: true} |
| + routes, err := reg.list() |
| + if err != nil { |
| + return |
| + } |
| + for _, rt := range routes { |
| + keep[rt.Version], keep[rt.PrevVersion] = true, true |
| + } |
| + if m != nil { |
| + for _, e := range m.Channels { |
| + keep[e.Version] = true |
| + } |
| + } |
| + if links, err := os.ReadDir(liveDir()); err == nil { |
| + for _, l := range links { |
| + if v, err := liveVersion(l.Name()); err == nil { |
| + keep[v] = true |
| + } |
| + } |
| + } |
| + entries, err := os.ReadDir(versionsDir()) |
| + if err != nil { |
| + return |
| + } |
| + for _, en := range entries { |
| + if !en.IsDir() || keep[en.Name()] { |
| + continue |
| + } |
| + if err := os.RemoveAll(filepath.Join(versionsDir(), en.Name())); err == nil { |
| + fmt.Printf(" pruned unused version %s\n", en.Name()) |
| + } |
| + } |
| + // Our downloads are temp-then-removed; a surviving dl-* file is a crashed |
| + // pass. (Operator-scp'd files in incoming/ are left alone.) |
| + if files, err := os.ReadDir(incomingDir()); err == nil { |
| + for _, f := range files { |
| + if strings.HasPrefix(f.Name(), "dl-") { |
| + _ = os.Remove(filepath.Join(incomingDir(), f.Name())) |
| + } |
| + } |
| + } |
| +} |
| + |
| +// runUpdate: `lchat update run|status` — the auto-updater CLI. |
| +func runUpdate(args []string) { |
| + if len(args) == 0 { |
| + die("update: subcommand required: run | status") |
| + } |
| + switch args[0] { |
| + case "run": |
| + runUpdateRun(args[1:]) |
| + case "status": |
| + runUpdateStatus(args[1:]) |
| + default: |
| + die("update: unknown subcommand %q (run | status)", args[0]) |
| + } |
| +} |
| + |
| +func runUpdateRun(args []string) { |
| + fs := flag.NewFlagSet("update run", flag.ExitOnError) |
| + registryPath := fs.String("registry", envOr("LCHAT_REGISTRY", "/data/registry.db"), "router registry SQLite path") |
| + unitTmpl := fs.String("unit", envOr("LCHAT_UNIT", "lchat@"), "systemd instance unit template prefix") |
| + dryRun := fs.Bool("dry-run", false, "report what would change without staging or restarting anything") |
| + _ = fs.Parse(reorderFlagsFirst(fs, args)) |
| + |
| + unlock, err := acquireDeployLock() |
| + if err != nil { |
| + die("update: %v", err) |
| + } |
| + defer unlock() |
| + |
| + reg, err := openRegistry(*registryPath) |
| + if err != nil { |
| + die("update: open registry: %v", err) |
| + } |
| + if err := updatePass(reg, *unitTmpl, *dryRun); err != nil { |
| + die("update: %v", err) |
| + } |
| +} |
| + |
| +// runUpdateStatus prints where every instance stands relative to its channel. |
| +func runUpdateStatus(args []string) { |
| + fs := flag.NewFlagSet("update status", flag.ExitOnError) |
| + registryPath := fs.String("registry", envOr("LCHAT_REGISTRY", "/data/registry.db"), "router registry SQLite path") |
| + _ = fs.Parse(reorderFlagsFirst(fs, args)) |
| + |
| + reg, err := openRegistry(*registryPath) |
| + if err != nil { |
| + die("update: open registry: %v", err) |
| + } |
| + routes, err := reg.list() |
| + if err != nil { |
| + die("update: %v", err) |
| + } |
| + m, merr := fetchReleaseManifest(releaseManifestURL) |
| + |
| + fmt.Printf("%-34s %-8s %-8s %-12s %s\n", "HOST", "CHANNEL", "AUTO", "CURRENT", "AVAILABLE") |
| + for _, rt := range routes { |
| + if rt.Kind != "instance" { |
| + continue |
| + } |
| + channel, auto := "?", "?" |
| + if prefs, err := fetchUpdatePrefs(rt.Socket); err == nil { |
| + channel = prefs.Channel |
| + switch { |
| + case prefs.AutoDownload && prefs.AutoUpdate: |
| + auto = "dl+upd" |
| + case prefs.AutoDownload: |
| + auto = "dl" |
| + case prefs.AutoUpdate: |
| + auto = "upd" |
| + default: |
| + auto = "off" |
| + } |
| + } |
| + cur, _ := liveVersion(rt.Host) |
| + if cur == "" { |
| + cur = "-" |
| + } |
| + avail := "-" |
| + if m != nil { |
| + if e, ok := m.Channels[channel]; ok { |
| + avail = e.Version |
| + if e.Version == cur { |
| + avail += " (current)" |
| + } |
| + } |
| + } |
| + fmt.Printf("%-34s %-8s %-8s %-12s %s\n", rt.Host, channel, auto, cur, avail) |
| + } |
| + if merr != nil { |
| + fmt.Printf("(no manifest: %v)\n", merr) |
| + } |
| +} |
diff --git a/updater_test.go b/updater_test.go| new file mode 100644 |
| index 0000000..0ec1c16 |
| --- /dev/null |
| +++ b/updater_test.go |
| @@ -0,0 +1,362 @@ |
| +package main |
| + |
| +import ( |
| + "crypto/ed25519" |
| + "crypto/rand" |
| + "crypto/sha256" |
| + "encoding/base64" |
| + "encoding/hex" |
| + "net" |
| + "net/http" |
| + "net/http/httptest" |
| + "os" |
| + "path/filepath" |
| + "strings" |
| + "sync" |
| + "testing" |
| + "time" |
| +) |
| + |
| +// startFakeInstance serves /api/update-prefs + /api/version on a real unix |
| +// socket. Its reported build follows the host's live symlink — so pointLive + |
| +// a stubbed systemctl restart looks exactly like a real swap to |
| +// waitHealthyVersion. prefsCode 404 simulates a pre-feature binary; brokenVer |
| +// (if set) makes that version report a wrong build, forcing the rollback path. |
| +func startFakeInstance(t *testing.T, host string, prefsCode int, prefs updatePrefs, brokenVer string) string { |
| + t.Helper() |
| + // Short base dir: unix socket paths have a ~104-char limit on darwin. |
| + dir, err := os.MkdirTemp("", "lch") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + t.Cleanup(func() { os.RemoveAll(dir) }) |
| + socket := filepath.Join(dir, host+".sock") |
| + |
| + mux := http.NewServeMux() |
| + mux.HandleFunc("/api/update-prefs", func(w http.ResponseWriter, r *http.Request) { |
| + if prefsCode != http.StatusOK { |
| + http.Error(w, "no such endpoint", prefsCode) |
| + return |
| + } |
| + writeJSON(w, http.StatusOK, map[string]any{ |
| + "auto_update": prefs.AutoUpdate, "auto_download": prefs.AutoDownload, "channel": prefs.Channel, |
| + }) |
| + }) |
| + mux.HandleFunc("/api/version", func(w http.ResponseWriter, r *http.Request) { |
| + v, _ := liveVersion(host) |
| + if brokenVer != "" && v == brokenVer { |
| + v = "borked" |
| + } |
| + writeJSON(w, http.StatusOK, map[string]any{"build": v}) |
| + }) |
| + |
| + ln, err := net.Listen("unix", socket) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + srv := &http.Server{Handler: mux} |
| + go srv.Serve(ln) |
| + t.Cleanup(func() { srv.Close() }) |
| + return socket |
| +} |
| + |
| +// startFeed signs entries into a manifest and serves it (plus each version's |
| +// binary bytes) from a loopback server, swapping in a test trust root. |
| +func startFeed(t *testing.T, binaries map[string]string) (*httptest.Server, func(channels map[string]string)) { |
| + t.Helper() |
| + pub, priv, err := ed25519.GenerateKey(rand.Reader) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + |
| + var mu sync.Mutex |
| + var wrapper []byte |
| + mux := http.NewServeMux() |
| + mux.HandleFunc("/manifest.json", func(w http.ResponseWriter, r *http.Request) { |
| + mu.Lock() |
| + defer mu.Unlock() |
| + w.Write(wrapper) |
| + }) |
| + mux.HandleFunc("/bin/", func(w http.ResponseWriter, r *http.Request) { |
| + ver := strings.TrimPrefix(r.URL.Path, "/bin/") |
| + w.Write([]byte(binaries[ver])) |
| + }) |
| + ts := httptest.NewServer(mux) |
| + t.Cleanup(ts.Close) |
| + |
| + oldKey, oldURL := releasePubKeyB64, releaseManifestURL |
| + releasePubKeyB64 = base64.StdEncoding.EncodeToString(pub) |
| + releaseManifestURL = ts.URL + "/manifest.json" |
| + t.Cleanup(func() { releasePubKeyB64, releaseManifestURL = oldKey, oldURL }) |
| + |
| + setChannels := func(channels map[string]string) { |
| + m := &releaseManifest{Channels: map[string]releaseEntry{}} |
| + for ch, ver := range channels { |
| + sum := sha256.Sum256([]byte(binaries[ver])) |
| + m.Channels[ch] = releaseEntry{ |
| + Version: ver, |
| + URL: ts.URL + "/bin/" + ver, |
| + SHA256: hex.EncodeToString(sum[:]), |
| + } |
| + } |
| + w, err := signReleaseManifest(priv, m) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + mu.Lock() |
| + wrapper = w |
| + mu.Unlock() |
| + } |
| + return ts, setChannels |
| +} |
| + |
| +// stubSystemctl records unit restarts and leaves the "restart" to the fake |
| +// instance's symlink-following /api/version. |
| +func stubSystemctl(t *testing.T) *[]string { |
| + t.Helper() |
| + var calls []string |
| + prev := systemctl |
| + systemctl = func(args ...string) error { |
| + calls = append(calls, strings.Join(args, " ")) |
| + return nil |
| + } |
| + t.Cleanup(func() { systemctl = prev }) |
| + return &calls |
| +} |
| + |
| +func seedInstance(t *testing.T, reg *Registry, host, socket, ver string) { |
| + t.Helper() |
| + src := filepath.Join(t.TempDir(), "old") |
| + writeFakeBinary(t, src, "old-binary-"+ver) |
| + if err := stageVersion(src, ver); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := pointLive(host, ver); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := reg.upsert(Route{Host: host, Kind: "instance", Status: "ready", Socket: socket, Version: ver}); err != nil { |
| + t.Fatal(err) |
| + } |
| +} |
| + |
| +func TestUpdatePassAppliesPerPrefs(t *testing.T) { |
| + withTempOpt(t) |
| + calls := stubSystemctl(t) |
| + shrink := deployHealthTimeout |
| + deployHealthTimeout = 2 * time.Second |
| + t.Cleanup(func() { deployHealthTimeout = shrink }) |
| + |
| + _, setChannels := startFeed(t, map[string]string{ |
| + "new1234": "new-binary-content", |
| + "beta5555": "beta-binary-content", |
| + }) |
| + setChannels(map[string]string{"stable": "new1234", "beta": "beta5555"}) |
| + |
| + reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db")) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + |
| + // h1: defaults-by-404 (old binary) — updates to stable. |
| + // h2: auto-update off — downloaded + staged, never restarted. |
| + // h3: everything off, beta channel — beta's binary must not even download. |
| + // h4: explicit on/on/stable — updates. |
| + s1 := startFakeInstance(t, "h1.example.com", http.StatusNotFound, updatePrefs{}, "") |
| + s2 := startFakeInstance(t, "h2.example.com", http.StatusOK, updatePrefs{AutoUpdate: false, AutoDownload: true, Channel: "stable"}, "") |
| + s3 := startFakeInstance(t, "h3.example.com", http.StatusOK, updatePrefs{AutoUpdate: false, AutoDownload: false, Channel: "beta"}, "") |
| + s4 := startFakeInstance(t, "h4.example.com", http.StatusOK, updatePrefs{AutoUpdate: true, AutoDownload: true, Channel: "stable"}, "") |
| + seedInstance(t, reg, "h1.example.com", s1, "old1111") |
| + seedInstance(t, reg, "h2.example.com", s2, "old1111") |
| + seedInstance(t, reg, "h3.example.com", s3, "old1111") |
| + seedInstance(t, reg, "h4.example.com", s4, "old1111") |
| + |
| + if err := updatePass(reg, "lchat@", false); err != nil { |
| + t.Fatalf("updatePass: %v", err) |
| + } |
| + |
| + assertLive := func(host, want string) { |
| + t.Helper() |
| + if v, _ := liveVersion(host); v != want { |
| + t.Errorf("%s live version = %q, want %q", host, v, want) |
| + } |
| + } |
| + assertLive("h1.example.com", "new1234") |
| + assertLive("h2.example.com", "old1111") // staged only |
| + assertLive("h3.example.com", "old1111") |
| + assertLive("h4.example.com", "new1234") |
| + |
| + // h2's chosen build is staged and byte-exact; h3's channel never downloaded. |
| + staged, err := os.ReadFile(versionBinPath("new1234")) |
| + if err != nil || string(staged) != "new-binary-content" { |
| + t.Fatalf("staged binary = %q, %v", staged, err) |
| + } |
| + if _, err := os.Stat(versionBinPath("beta5555")); !os.IsNotExist(err) { |
| + t.Fatal("beta5555 was downloaded despite downloads off") |
| + } |
| + |
| + // Registry recorded the moves (and only the moves). |
| + rt, _ := reg.get("h1.example.com") |
| + if rt.Version != "new1234" || rt.PrevVersion != "old1111" { |
| + t.Fatalf("h1 registry = %s/%s", rt.Version, rt.PrevVersion) |
| + } |
| + rt, _ = reg.get("h2.example.com") |
| + if rt.Version != "old1111" { |
| + t.Fatalf("h2 registry = %s", rt.Version) |
| + } |
| + |
| + // Exactly two restarts: h1 and h4. |
| + got := strings.Join(*calls, "; ") |
| + if !strings.Contains(got, "restart lchat@h1.example.com") || !strings.Contains(got, "restart lchat@h4.example.com") || len(*calls) != 2 { |
| + t.Fatalf("systemctl calls = %v", *calls) |
| + } |
| + |
| + // The pass recorded the manifest stamp for anti-replay. |
| + if readManifestStamp().IsZero() { |
| + t.Fatal("manifest stamp not written") |
| + } |
| +} |
| + |
| +func TestUpdatePassRollsBackAndPoisons(t *testing.T) { |
| + withTempOpt(t) |
| + calls := stubSystemctl(t) |
| + shrink := deployHealthTimeout |
| + deployHealthTimeout = 500 * time.Millisecond |
| + t.Cleanup(func() { deployHealthTimeout = shrink }) |
| + |
| + _, setChannels := startFeed(t, map[string]string{"bad9999": "does-not-boot"}) |
| + setChannels(map[string]string{"stable": "bad9999"}) |
| + |
| + reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db")) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + // Both instances report "borked" while on bad9999 → health check fails. |
| + s1 := startFakeInstance(t, "h1.example.com", http.StatusNotFound, updatePrefs{}, "bad9999") |
| + s2 := startFakeInstance(t, "h2.example.com", http.StatusNotFound, updatePrefs{}, "bad9999") |
| + seedInstance(t, reg, "h1.example.com", s1, "old1111") |
| + seedInstance(t, reg, "h2.example.com", s2, "old1111") |
| + |
| + if err := updatePass(reg, "lchat@", false); err != nil { |
| + t.Fatalf("updatePass: %v", err) |
| + } |
| + |
| + // h1 tried, failed, rolled back; h2 was spared (the version is poisoned). |
| + for _, host := range []string{"h1.example.com", "h2.example.com"} { |
| + if v, _ := liveVersion(host); v != "old1111" { |
| + t.Errorf("%s live version = %q, want rollback to old1111", host, v) |
| + } |
| + rt, _ := reg.get(host) |
| + if rt.Version != "old1111" { |
| + t.Errorf("%s registry version = %q", host, rt.Version) |
| + } |
| + } |
| + // h1: restart onto bad + rollback restart = 2 calls; h2: none. |
| + if len(*calls) != 2 || strings.Contains(strings.Join(*calls, ";"), "h2.example.com") { |
| + t.Fatalf("systemctl calls = %v", *calls) |
| + } |
| +} |
| + |
| +func TestUpdatePassRefusesReplayedManifest(t *testing.T) { |
| + withTempOpt(t) |
| + stubSystemctl(t) |
| + |
| + _, setChannels := startFeed(t, map[string]string{"new1234": "x"}) |
| + setChannels(map[string]string{"stable": "new1234"}) |
| + |
| + // Pretend we've already accepted a manifest from the future. |
| + if err := writeManifestStamp(time.Now().Add(time.Hour)); err != nil { |
| + t.Fatal(err) |
| + } |
| + reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db")) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := updatePass(reg, "lchat@", false); err == nil || !strings.Contains(err.Error(), "replayed") { |
| + t.Fatalf("want replay refusal, got %v", err) |
| + } |
| +} |
| + |
| +func TestUpdatePassDryRunTouchesNothing(t *testing.T) { |
| + withTempOpt(t) |
| + calls := stubSystemctl(t) |
| + |
| + _, setChannels := startFeed(t, map[string]string{"new1234": "content"}) |
| + setChannels(map[string]string{"stable": "new1234"}) |
| + |
| + reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db")) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + s1 := startFakeInstance(t, "h1.example.com", http.StatusNotFound, updatePrefs{}, "") |
| + seedInstance(t, reg, "h1.example.com", s1, "old1111") |
| + |
| + if err := updatePass(reg, "lchat@", true); err != nil { |
| + t.Fatal(err) |
| + } |
| + if v, _ := liveVersion("h1.example.com"); v != "old1111" { |
| + t.Fatalf("dry-run moved the symlink to %q", v) |
| + } |
| + if _, err := os.Stat(versionBinPath("new1234")); !os.IsNotExist(err) { |
| + t.Fatal("dry-run downloaded/staged a binary") |
| + } |
| + if len(*calls) != 0 { |
| + t.Fatalf("dry-run ran systemctl: %v", *calls) |
| + } |
| + if !readManifestStamp().IsZero() { |
| + t.Fatal("dry-run wrote the manifest stamp") |
| + } |
| +} |
| + |
| +func TestGCVersionsKeepsEverythingReferenced(t *testing.T) { |
| + withTempOpt(t) |
| + |
| + reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db")) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + bin := filepath.Join(t.TempDir(), "b") |
| + writeFakeBinary(t, bin, "x") |
| + for _, v := range []string{"cur1", "prev1", "head1", "orphan1", "orphan2", "linked1"} { |
| + if err := stageVersion(bin, v); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + reg.upsert(Route{Host: "a.example.com", Kind: "instance", Status: "ready", Version: "cur1", PrevVersion: "prev1"}) |
| + // A live symlink nothing in the registry mentions must still survive. |
| + if err := pointLive("b.example.com", "linked1"); err != nil { |
| + t.Fatal(err) |
| + } |
| + m := &releaseManifest{Channels: map[string]releaseEntry{"stable": {Version: "head1"}}} |
| + |
| + gcVersions(reg, m) |
| + |
| + for _, keep := range []string{"cur1", "prev1", "head1", "linked1"} { |
| + if _, err := os.Stat(versionBinPath(keep)); err != nil { |
| + t.Errorf("gc pruned %s (should be kept)", keep) |
| + } |
| + } |
| + for _, gone := range []string{"orphan1", "orphan2"} { |
| + if _, err := os.Stat(versionBinPath(gone)); !os.IsNotExist(err) { |
| + t.Errorf("gc kept %s (should be pruned)", gone) |
| + } |
| + } |
| + // buildVersion ("dev" under test) is always kept — the control binary's own. |
| + // (No dir staged for it here; just assert gc didn't error on its absence.) |
| +} |
| + |
| +func TestAcquireDeployLockExcludes(t *testing.T) { |
| + withTempOpt(t) |
| + unlock, err := acquireDeployLock() |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := acquireDeployLock(); err == nil { |
| + t.Fatal("second acquire succeeded — deploy and update could interleave") |
| + } |
| + unlock() |
| + unlock2, err := acquireDeployLock() |
| + if err != nil { |
| + t.Fatalf("re-acquire after release: %v", err) |
| + } |
| + unlock2() |
| +} |
diff --git a/versions.go b/versions.go| index 4a41fb8..d3981fc 100644 |
| --- a/versions.go |
| +++ b/versions.go |
| @@ -287,6 +287,12 @@ func runRollback(args []string) { |
| all := fs.Bool("all", false, "roll back every instance that has a previous version") |
| _ = fs.Parse(reorderFlagsFirst(fs, args)) |
| + unlock, err := acquireDeployLock() |
| + if err != nil { |
| + die("rollback: %v", err) |
| + } |
| + defer unlock() |
| + |
| reg, err := openRegistry(*registryPath) |
| if err != nil { |
| die("rollback: open registry: %v", err) |