amadan
server: auto-update hibernating instances by repointing live/<host> — no wake (#778)
* server: auto-update hibernating instances by repointing live/<host> — no wake
The updater skipped hibernate-flagged hosts entirely ("flip off to
update"), which stopped mattering little and started mattering completely
when the fleet went hibernate-by-default: a publish reached nobody — the
whole fleet stayed on the old build indefinitely (the recorded
hibernated-instances-dont-converge-on-publish gap).
The fix rides what wake already does: the router's spawn execs
/opt/lchat/live/<host> at wake time, so updating a sleeping instance is
just repointing that symlink. Prefs (channel/pin/auto flags) are read from
the parked DB read-only — asking over the socket would wake it — with an
absent DB reading as the same on/on/stable defaults the socket path uses.
Pins hold (canaries stay canaries), the registry records version +
prev-version, staged binaries and GC behave as for unit-managed hosts.
Stated trades, in the code and docs/server-releases.md: no health check
(nothing runs to probe — a bad build surfaces at wake; recover by
repointing back), and a currently-awake child keeps its old binary until
it next sleeps — so a sidecar-bearing space that never sleeps still needs
the flip-off flow.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
* server: review fixes — root-owned WAL sidecars, narrow version writes, fail-closed prefs, rescue paths
Adversarial review findings, all fixed:
- CRITICAL: a root-run mode=ro open of a cleanly-slept (checkpointed, no
-wal/-shm) DB CREATES those files root-owned; the instance (service user)
then EACCESes opening its own DB on next wake — space down until a manual
chown, fleet-wide, hourly. repairDBSidecarOwnership() chowns them back to
the DB's owner after every parked-prefs read.
- Registry writes for versions are now a narrow UPDATE (setVersions), not a
full upsert: updatePass/deploy hold their route reads across long
downloads, and the upsert wrote back next_due/hibernate as of READ time —
clobbering the activator's mid-pass setNextDue (a silently missed
scheduled wake) or an operator's hibernate flip. deployHostVersion and
rollback switched too.
- hibernatingPrefs is fail-closed: an absent or unreadable parked DB skips
the host (its pin/opt-out may live only in the copy we can't read) — it
converges on the pass after its first wake leaves a warm DB.
- Hibernating hosts process in a second phase AFTER every unit-managed host,
so a boot-broken build poisoned by an awake health check never reaches a
sleeper in the same pass.
- Rescue paths exist now: lchat rollback and lchat deploy --version handle
hibernating hosts as pure repoints (docs updated to name them) instead of
refusing.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---------
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
6 files changed,
+334
−22
control.go+36 −6docs/server-releases.md+14 −0registry.go+11 −0updater.go+162 −5updater_test.go+97 −0versions.go+14 −11
diff --git a/control.go b/control.go| index 39375ac..64d06c4 100644 |
| --- a/control.go |
| +++ b/control.go |
| @@ -889,13 +889,40 @@ func runDeploy(args []string) { |
| } |
| if rt.Hibernate { |
| // No unit to restart — and starting one would fight the activator's |
| - // child for the socket. Skip quietly in a fleet promote; refuse |
| - // loudly when the operator named this host. |
| - if *all { |
| - fmt.Printf("skipped %s (hibernating — flip it back to deploy: lchat hibernate %s --off)\n", host, host) |
| + // child for the socket. A --version deploy is still meaningful as a |
| + // pure repoint (the next wake execs it — same dance as the |
| + // auto-updater's hibernating phase, and the rescue path for a bad |
| + // repointed build). A plain restart-deploy has nothing to restart. |
| + if ver == "" { |
| + if *all { |
| + fmt.Printf("skipped %s (hibernating — nothing to restart; use --version to repoint)\n", host) |
| + continue |
| + } |
| + die("deploy %s: instance is hibernating (no unit to restart) — use --version to repoint it for its next wake", host) |
| + } |
| + cur, _ := liveVersion(host) |
| + if cur == ver { |
| + fmt.Printf("deployed %s (already on %s)\n", host, ver) |
| continue |
| } |
| - die("deploy %s: instance is hibernating (activator-managed, no unit) — flip it back first: lchat hibernate %s --off", host, host) |
| + if err := pointLive(host, ver); err != nil { |
| + die("deploy %s: %v", host, err) |
| + } |
| + if err := reg.setVersions(host, ver, cur); err != nil { |
| + die("deploy %s: repointed but recording the version failed: %v", host, err) |
| + } |
| + fmt.Printf("deployed %s @ %s (hibernating — no health check; runs it on next wake)\n", host, ver) |
| + if *pinFlag { |
| + // The pin lives in the instance's DB, written over its socket — |
| + // which wakes it, onto the build just repointed. Deliberate: a |
| + // pinned deploy is a canary, and its operator is about to poke |
| + // it anyway. |
| + if err := pinHost(rt, ver); err != nil { |
| + die("deploy %s: deployed @ %s but PINNING FAILED (%v) — the next update pass may move it; retry with `lchat update pin %s %s`", host, ver, err, host, ver) |
| + } |
| + fmt.Printf("pinned %s to %s (auto-updates paused; the pin write woke it)\n", host, ver) |
| + } |
| + continue |
| } |
| unit := *unitTmpl + host |
| @@ -963,7 +990,10 @@ func deployHostVersion(reg *Registry, rt *Route, unit, ver string) (changed bool |
| return false, fmt.Errorf("failed health check (%v), rolled back to %q", err, prev) |
| } |
| rt.PrevVersion, rt.Version = prev, ver |
| - if err := reg.upsert(*rt); err != nil { |
| + // Narrow UPDATE, not a full upsert: rt was read before a possibly-long |
| + // download, and writing every column back would clobber whatever the |
| + // activator or an operator changed meanwhile (next_due, hibernate). |
| + if err := reg.setVersions(host, ver, prev); err != nil { |
| return true, fmt.Errorf("record version: %v", err) |
| } |
| restartBotSidecar(host) |
diff --git a/docs/server-releases.md b/docs/server-releases.md| index b521b53..2aeda4c 100644 |
| --- a/docs/server-releases.md |
| +++ b/docs/server-releases.md |
| @@ -156,6 +156,20 @@ manifest — and each pass restarts updated instances' bot sidecars in |
| lock-step. Pinned instances (canaries) are reported and left alone. With the |
| whole fleet riding `edge`, `publish … && ship` is a complete manual deploy. |
| +**Hibernating hosts** (docs/planet-scale.md) have no unit to restart and are |
| +never woken just to update. The pass instead reads their prefs from the |
| +parked DB (read-only, fail-closed — a host whose prefs can't be read is |
| +skipped, never moved) and **repoints the live symlink** — the next wake |
| +execs the new binary. They're processed *after* every unit-managed host, so |
| +a boot-broken build that fails a health check on an awake instance poisons |
| +the version before any sleeper is moved. Two stated trades vs the unit |
| +path: there is no per-host health check (a bad build surfaces at wake — |
| +recover with `lchat rollback <host>`, which repoints back without waking; |
| +`lchat deploy <host> --version <ver>` likewise repoints), and a |
| +currently-awake child keeps its old binary until it next sleeps. A |
| +sidecar-bearing space that never sleeps therefore still needs the flip-off |
| +flow (`lchat hibernate <host> --off`, update, `--on`). |
| + |
| Same rule as deploys and TestFlight: **publish from landed `main`**, never a |
| feature branch (the script refuses a dirty tree). Publishing is manual for now; |
| a CI job cutting `edge` on every push to `main` is the natural next step. |
diff --git a/registry.go b/registry.go| index a0624b9..52582a7 100644 |
| --- a/registry.go |
| +++ b/registry.go |
| @@ -207,6 +207,17 @@ func (r *Registry) setHibernate(host string, on bool) error { |
| return err |
| } |
| +// setVersions records a host's current + previous version, same narrow-UPDATE |
| +// contract as setHibernate/setNextDue. Deploys and update passes hold their |
| +// route reads for a while (minutes, when a download runs) — a full upsert |
| +// would write back next_due/hibernate as they were at READ time, clobbering |
| +// the activator's mid-pass setNextDue (a silently missed scheduled wake) or |
| +// an operator's hibernate flip. |
| +func (r *Registry) setVersions(host, ver, prev string) error { |
| + _, err := r.sql.Exec(`UPDATE routes SET version = ?, prev_version = ? WHERE host = ?`, ver, prev, host) |
| + return err |
| +} |
| + |
| // ---- due-time index (#550 Part 4 v2 follow-up) -------------------------------- |
| // |
| // The activator's persistence for scheduled deliveries on hibernating hosts: |
diff --git a/updater.go b/updater.go| index 49b5210..64851df 100644 |
| --- a/updater.go |
| +++ b/updater.go |
| @@ -2,7 +2,9 @@ package main |
| import ( |
| "bytes" |
| + "context" |
| "crypto/sha256" |
| + "database/sql" |
| "encoding/hex" |
| "encoding/json" |
| "flag" |
| @@ -14,6 +16,7 @@ import ( |
| "path/filepath" |
| "runtime" |
| "strings" |
| + "syscall" |
| "time" |
| ) |
| @@ -162,6 +165,93 @@ func fetchUpdatePrefs(socket string) (updatePrefs, error) { |
| return p, nil |
| } |
| +// hibernatingPrefs reads an instance's update prefs straight from its parked |
| +// DB — an activator-managed host can't answer the socket fetch without being |
| +// woken, and waking the fleet hourly would defeat hibernation. Read-only, so |
| +// it can't disturb an awake child's WAL. FAIL-CLOSED: an absent DB (never |
| +// woken here; state parked only in S3) or any read trouble returns an error |
| +// and the caller skips the host — the admin's pin or updates-off may live |
| +// only in the unreadable copy, and "the one host that must not move, moves" |
| +// is the failure that matters. A skipped host converges on the pass after |
| +// its first wake leaves a warm local DB. |
| +func hibernatingPrefs(host string) (updatePrefs, error) { |
| + p := updatePrefs{AutoUpdate: true, AutoDownload: true, Channel: "stable"} |
| + path := filepath.Join(instanceDataDir, host+".db") |
| + if _, err := os.Stat(path); err != nil { |
| + return p, err |
| + } |
| + // Even a mode=ro open CREATES <db>-wal/-shm when they're absent (a cleanly |
| + // slept instance has neither) — and this runs as root, so without repair |
| + // they'd be root-owned and the instance (service user) would EACCES on its |
| + // next wake: space down until a manual chown. Fix ownership on the way out. |
| + defer repairDBSidecarOwnership(path) |
| + conn, err := sql.Open("sqlite", "file:"+path+"?mode=ro&_pragma=busy_timeout(3000)") |
| + if err != nil { |
| + return p, err |
| + } |
| + defer conn.Close() |
| + conn.SetMaxOpenConns(1) |
| + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) |
| + defer cancel() |
| + get := func(key string) (string, bool, error) { |
| + var v string |
| + err := conn.QueryRowContext(ctx, "SELECT value FROM config WHERE key = ?", key).Scan(&v) |
| + if err == sql.ErrNoRows { |
| + return "", false, nil |
| + } |
| + if err != nil { |
| + return "", false, err |
| + } |
| + return v, true, nil |
| + } |
| + // Same semantics as the instance's own readers: booleans default ON when |
| + // absent, an unrecognised channel falls back to stable, a garbled pin |
| + // must not freeze anything silently. |
| + if v, ok, err := get("update_auto"); err != nil { |
| + return p, err |
| + } else if ok { |
| + p.AutoUpdate = v == "1" |
| + } |
| + if v, ok, err := get("update_download"); err != nil { |
| + return p, err |
| + } else if ok { |
| + p.AutoDownload = v == "1" |
| + } |
| + if v, ok, err := get("update_channel"); err != nil { |
| + return p, err |
| + } else if ok && validChannel(v) { |
| + p.Channel = v |
| + } |
| + if v, ok, err := get("update_pin"); err != nil { |
| + return p, err |
| + } else if ok && validVersion(v) { |
| + p.Pin = v |
| + } |
| + return p, nil |
| +} |
| + |
| +// repairDBSidecarOwnership chowns <db>-wal/-shm to the DB file's own uid/gid. |
| +// A root-run read-only open of a checkpointed WAL DB creates those two files |
| +// root-owned (mode=ro guards the DB's content, not the WAL control files), |
| +// and the instance — running as the service user — then can't open its own |
| +// DB: the add-bot root-owned-dir incident as a library. Best-effort: non-root |
| +// callers can't chown and can't create root-owned files either. |
| +func repairDBSidecarOwnership(dbPath string) { |
| + st, err := os.Stat(dbPath) |
| + if err != nil { |
| + return |
| + } |
| + sys, ok := st.Sys().(*syscall.Stat_t) |
| + if !ok { |
| + return |
| + } |
| + for _, p := range []string{dbPath + "-wal", dbPath + "-shm"} { |
| + if _, err := os.Stat(p); err == nil { |
| + _ = os.Chown(p, int(sys.Uid), int(sys.Gid)) |
| + } |
| + } |
| +} |
| + |
| // updaterArch is the architecture this box runs — a var so tests can exercise |
| // the arm64 resolution without faking runtime.GOARCH. |
| var updaterArch = runtime.GOARCH |
| @@ -278,6 +368,8 @@ func updatePass(reg *Registry, unitTmpl string, dryRun bool) error { |
| // Versions instances are pinned to must survive GC even when no channel or |
| // registry row mentions them (a canary's branch build). |
| var pinnedVers []string |
| + // Hibernating hosts, deferred to a second phase (see the branch below). |
| + var hibernating []Route |
| for _, rt := range routes { |
| if rt.Kind != "instance" { |
| continue |
| @@ -287,11 +379,10 @@ func updatePass(reg *Registry, unitTmpl string, dryRun bool) error { |
| continue |
| } |
| if rt.Hibernate { |
| - // An activator-managed host has no unit for deployHostVersion to |
| - // restart — starting one would fight the router's child for the |
| - // socket (and an asleep host can't answer the prefs fetch anyway). |
| - // Its updates ride the flip-off flow for now. |
| - fmt.Printf(" %s: hibernating — left alone (no unit; flip off to update: lchat hibernate %s --off)\n", rt.Host, rt.Host) |
| + // Processed in a second phase below, AFTER every unit-managed host: |
| + // repoints have no health check, so let an awake host discover a |
| + // boot-broken build (and poison it) before any sleeper is moved. |
| + hibernating = append(hibernating, rt) |
| continue |
| } |
| prefs, err := fetchUpdatePrefs(rt.Socket) |
| @@ -348,6 +439,72 @@ func updatePass(reg *Registry, unitTmpl string, dryRun bool) error { |
| fmt.Printf(" %s: updated %s -> %s (%s)\n", rt.Host, cur, e.Version, prefs.Channel) |
| } |
| } |
| + |
| + // Second phase: hibernating hosts. An activator-managed host has no unit |
| + // to restart and must not be woken just to update (an hourly wake would |
| + // defeat hibernation). Instead: repoint its live symlink — the NEXT wake |
| + // execs the new binary, since the router's spawn resolves |
| + // /opt/lchat/live/<host> at exec time. Prefs come from the parked DB |
| + // (read-only, fail-closed). Two stated trades vs the unit path, both in |
| + // docs/server-releases.md: no health check (nothing runs to probe — a bad |
| + // build surfaces at wake; recover with `lchat rollback <host>`, which |
| + // repoints back), and a currently-AWAKE child keeps its old binary until |
| + // it next sleeps (a running process doesn't follow the symlink). |
| + for _, rt := range hibernating { |
| + prefs, err := hibernatingPrefs(rt.Host) |
| + if err != nil { |
| + // The admin's pin or opt-out may live only in the copy we couldn't |
| + // read — never move a host whose wishes are unknown. It converges |
| + // on the pass after its first wake leaves a warm local DB. |
| + fmt.Printf(" %s: hibernating, parked prefs unreadable (%v) — skipped\n", rt.Host, err) |
| + continue |
| + } |
| + if prefs.Pin != "" { |
| + pinnedVers = append(pinnedVers, prefs.Pin) |
| + fmt.Printf(" %s: pinned to %s — left alone\n", rt.Host, prefs.Pin) |
| + continue |
| + } |
| + e, ok := m.Channels[prefs.Channel] |
| + if !ok { |
| + fmt.Printf(" %s: channel %s has no release yet\n", rt.Host, prefs.Channel) |
| + continue |
| + } |
| + cur, _ := liveVersion(rt.Host) |
| + if cur == e.Version { |
| + continue // up to date — stay quiet on the hourly tick |
| + } |
| + if !prefs.AutoDownload && !prefs.AutoUpdate { |
| + fmt.Printf(" %s: %s available on %s (downloads off — admin's call)\n", rt.Host, e.Version, prefs.Channel) |
| + continue |
| + } |
| + if dryRun { |
| + fmt.Printf(" %s: would repoint %s -> %s (%s; hibernating)\n", rt.Host, cur, e.Version, prefs.Channel) |
| + continue |
| + } |
| + if err := ensureVersionStaged(e); err != nil { |
| + fmt.Printf(" %s: %v\n", rt.Host, err) |
| + continue |
| + } |
| + if !prefs.AutoUpdate { |
| + fmt.Printf(" %s: %s downloaded and staged (auto-update off — apply with `lchat deploy %s --version %s`)\n", |
| + rt.Host, e.Version, rt.Host, e.Version) |
| + continue |
| + } |
| + if poisoned[e.Version] { |
| + fmt.Printf(" %s: skipping %s (failed health check on another instance this pass)\n", rt.Host, e.Version) |
| + continue |
| + } |
| + if err := pointLive(rt.Host, e.Version); err != nil { |
| + fmt.Printf(" %s: repoint failed: %v\n", rt.Host, err) |
| + continue |
| + } |
| + if err := reg.setVersions(rt.Host, e.Version, cur); err != nil { |
| + fmt.Printf(" %s: repointed but recording the version failed: %v\n", rt.Host, err) |
| + continue |
| + } |
| + fmt.Printf(" %s: repointed %s -> %s (%s; hibernating — runs it on next wake)\n", rt.Host, cur, e.Version, prefs.Channel) |
| + } |
| + |
| if !dryRun { |
| gcVersions(reg, m, pinnedVers) |
| } |
diff --git a/updater_test.go b/updater_test.go| index 2e521ba..5897281 100644 |
| --- a/updater_test.go |
| +++ b/updater_test.go |
| @@ -831,3 +831,100 @@ func TestAcquireDeployLockExcludes(t *testing.T) { |
| } |
| unlock2() |
| } |
| + |
| +// Hibernating hosts (#550): no unit to restart and never woken — the pass |
| +// repoints their live symlink instead, so the NEXT wake execs the new binary. |
| +// Prefs come from the parked DB; no DB reads as the defaults; pins hold. |
| +func TestUpdatePassRepointsHibernating(t *testing.T) { |
| + withTempOpt(t) |
| + calls := stubSystemctl(t) |
| + |
| + oldData := instanceDataDir |
| + instanceDataDir = t.TempDir() |
| + t.Cleanup(func() { instanceDataDir = oldData }) |
| + |
| + _, setChannels := startFeed(t, map[string]string{ |
| + "new1234": "new-binary-content", |
| + "edge777": "edge-binary-content", |
| + }) |
| + setChannels(map[string]string{"stable": "new1234", "edge": "edge777"}) |
| + |
| + reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db")) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + seed := func(host string) { |
| + t.Helper() |
| + src := filepath.Join(t.TempDir(), "old") |
| + writeFakeBinary(t, src, "old-binary") |
| + if err := stageVersion(src, "old1111"); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := pointLive(host, "old1111"); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := reg.upsert(Route{Host: host, Kind: "instance", Status: "ready", |
| + Socket: filepath.Join(t.TempDir(), "no.sock"), Hibernate: true, Version: "old1111"}); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + seedDB := func(host string, kv map[string]string) { |
| + t.Helper() |
| + db, err := openDB(filepath.Join(instanceDataDir, host+".db")) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + for k, v := range kv { |
| + if err := db.setConfig(k, v); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + db.sql.Close() |
| + } |
| + |
| + // h1: parked DB with no update keys — defaults (stable/on/on) → repointed. |
| + // h2: parked DB says edge → repointed to the edge head. |
| + // h3: pinned in the parked DB → untouched. |
| + // h4: downloads+updates off → untouched. |
| + // h5: NO parked DB — prefs unknowable, fail-closed → untouched. |
| + for _, h := range []string{"h1.hib", "h2.hib", "h3.hib", "h4.hib", "h5.hib"} { |
| + seed(h) |
| + } |
| + seedDB("h1.hib", map[string]string{}) |
| + seedDB("h2.hib", map[string]string{"update_channel": "edge"}) |
| + seedDB("h3.hib", map[string]string{"update_pin": "old1111"}) |
| + seedDB("h4.hib", map[string]string{"update_auto": "0", "update_download": "0"}) |
| + |
| + if err := updatePass(reg, "lchat@", false); err != nil { |
| + t.Fatalf("updatePass: %v", err) |
| + } |
| + |
| + assertLive := func(host, want string) { |
| + t.Helper() |
| + if v, _ := liveVersion(host); v != want { |
| + t.Errorf("%s live version = %q, want %q", host, v, want) |
| + } |
| + } |
| + assertLive("h1.hib", "new1234") |
| + assertLive("h2.hib", "edge777") |
| + assertLive("h3.hib", "old1111") |
| + assertLive("h4.hib", "old1111") |
| + assertLive("h5.hib", "old1111") |
| + |
| + // The registry records the move (and the previous version, for rollback). |
| + rt, _ := reg.get("h1.hib") |
| + if rt.Version != "new1234" || rt.PrevVersion != "old1111" { |
| + t.Errorf("h1 registry = %q prev %q", rt.Version, rt.PrevVersion) |
| + } |
| + |
| + // The whole point: no unit was restarted, nothing was woken. |
| + if len(*calls) != 0 { |
| + t.Fatalf("systemctl touched for hibernating hosts: %v", *calls) |
| + } |
| + |
| + // A second pass is quiet + idempotent. |
| + if err := updatePass(reg, "lchat@", false); err != nil { |
| + t.Fatalf("second pass: %v", err) |
| + } |
| + assertLive("h1.hib", "new1234") |
| +} |
diff --git a/versions.go b/versions.go| index 7e48d8b..49818e3 100644 |
| --- a/versions.go |
| +++ b/versions.go |
| @@ -309,15 +309,6 @@ func runRollback(args []string) { |
| if rt.Kind != "instance" { |
| die("rollback %s: not an instance (kind %q)", host, rt.Kind) |
| } |
| - if rt.Hibernate { |
| - // Same guard as deploy: no unit to restart, and starting one would |
| - // fight the activator's child for the socket. |
| - if *all { |
| - fmt.Printf("skipped %s (hibernating — flip it back to roll back: lchat hibernate %s --off)\n", host, host) |
| - continue |
| - } |
| - die("rollback %s: instance is hibernating (activator-managed, no unit) — flip it back first: lchat hibernate %s --off", host, host) |
| - } |
| prev := strings.TrimSpace(rt.PrevVersion) |
| if prev == "" { |
| if *all { |
| @@ -326,6 +317,19 @@ func runRollback(args []string) { |
| die("rollback %s: no previous version recorded", host) |
| } |
| cur, _ := liveVersion(host) |
| + if rt.Hibernate { |
| + // No unit to restart — a pure repoint, mirroring the auto-updater's |
| + // hibernating phase. This is the rescue path when a repointed build |
| + // turns out bad: the next wake execs the previous version. |
| + if err := pointLive(host, prev); err != nil { |
| + die("rollback %s: %v", host, err) |
| + } |
| + if err := reg.setVersions(host, prev, cur); err != nil { |
| + die("rollback %s: record: %v", host, err) |
| + } |
| + fmt.Printf("rolled back %s to %s (hibernating — runs it on next wake)\n", host, prev) |
| + continue |
| + } |
| if err := pointLive(host, prev); err != nil { |
| die("rollback %s: %v", host, err) |
| } |
| @@ -335,8 +339,7 @@ func runRollback(args []string) { |
| if err := waitHealthy(rt.Socket, 20*time.Second); err != nil { |
| die("rollback %s: %v", host, err) |
| } |
| - rt.Version, rt.PrevVersion = prev, cur |
| - if err := reg.upsert(*rt); err != nil { |
| + if err := reg.setVersions(host, prev, cur); err != nil { |
| die("rollback %s: record: %v", host, err) |
| } |
| restartBotSidecar(host) |