amadan

eleven / messenger Public

server: hibernate --on re-run finishes the handover it promised (#722)

The litestream guard's two-step leaves flag-set-but-unit-running; the
re-run early-exited on the flag ('already hibernating') with the unit
still holding the socket — hit live on the first bulk flip. The
already-hibernating short-circuit now also requires the unit to be
stopped (new read-only systemctlQuery seam so state queries don't
pollute call-order test assertions).
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Paul Campbell 23c3b37f2407d4a06e8dc0bec44a550b6fb9cbb3 parent 8e150cb
2 files changed, +48 −1
  • control.go +20 −1
  • control_test.go +28 −0
diff --git a/control.go b/control.go
index 95ae102..40eceb6 100644
--- a/control.go
+++ b/control.go
@@ -619,10 +619,15 @@ func runHibernate(args []string) {
die("hibernate: %s is %s, not ready — fix the instance first", host, rt.Status)
}
if *on {
- if rt.Hibernate {
+ if rt.Hibernate && !unitActive(*unitTmpl+host) {
fmt.Printf("%s is already hibernating\n", host)
return
}
+ // Flag set but the unit still runs = a re-run after the litestream
+ // guard's two-step (the error message PROMISES this re-run works —
+ // hit live on the first bulk flip, where the early-exit above
+ // stranded hosts flagged-but-materialized). Fall through and finish
+ // the handover; hibernateOn is idempotent over an already-set flag.
if err := hibernateOn(reg, rt, *unitTmpl, *botUnit, *routerUnit); err != nil {
die("hibernate %s --on: %v", host, err)
}
@@ -667,6 +672,13 @@ func runHibernate(args []string) {
// Nothing is checkpointed here and no S3 is touched: the DB is already local,
// so the first wake's restore sees it as the warm cache and skips; replication
// under the fresh lease is what backfills S3.
+// unitActive reports whether the systemd unit is running — the discriminator
+// between "already fully hibernating" and "flag set, handover unfinished"
+// (the litestream guard's two-step leaves exactly that state behind).
+func unitActive(unit string) bool {
+ return systemctlQuery("is-active", "--quiet", unit) == nil
+}
+
func hibernateOn(reg *Registry, rt *Route, unitTmpl, botUnit, routerUnit string) error {
// Existing instances' env files were written for the systemd path (root
// reads them); the router's activator reads them as the service user, so
@@ -1110,6 +1122,13 @@ func waitReachable(socket string, timeout time.Duration) error {
// systemctl is a var so tests can stand in for systemd (e.g. bind the instance
// socket themselves when they see "enable --now"); everything else in the
// creation path — env files, registry, symlinks, health checks — runs for real.
+// systemctlQuery is the read-only twin of systemctl (is-active etc.) — its own
+// seam so tests asserting exact mutating-call sequences aren't polluted by
+// state queries.
+var systemctlQuery = func(args ...string) error {
+ return exec.Command("systemctl", args...).Run()
+}
+
var systemctl = func(args ...string) error {
out, err := exec.Command("systemctl", args...).CombinedOutput()
if err != nil {
diff --git a/control_test.go b/control_test.go
index ced6dc3..a1b996e 100644
--- a/control_test.go
+++ b/control_test.go
@@ -796,3 +796,31 @@ func TestHibernateOnRefusesWhileBoxLitestreamTracks(t *testing.T) {
t.Fatalf("re-run flip sequence = %v", calls)
}
}
+
+// The litestream guard's two-step leaves flag-set-but-unit-running behind; the
+// promised re-run must FINISH the handover, not early-exit on the flag (hit
+// live on the first bulk flip: "already hibernating" with the unit still up).
+func TestHibernateOnRerunFinishesHandover(t *testing.T) {
+ reg := controlTestEnv(t)
+ const host = "flip-rerun.example.com"
+ if _, err := addInstance(reg, addInstanceOpts{Host: host}); err != nil {
+ t.Fatal(err)
+ }
+ if err := reg.setHibernate(host, true); err != nil { // the guard's leftover state
+ t.Fatal(err)
+ }
+ oldQ := systemctlQuery
+ systemctlQuery = func(args ...string) error { return nil } // unit reads active
+ defer func() { systemctlQuery = oldQ }()
+ var calls []string
+ systemctl = func(args ...string) error { calls = append(calls, strings.Join(args, " ")); return nil }
+ confirmPublicReady = func(string, string) { calls = append(calls, "confirm") }
+
+ rt, _ := reg.get(host)
+ if err := hibernateOn(reg, rt, "lchat@", "lchat-bot@", "lchat-router"); err != nil {
+ t.Fatalf("re-run: %v", err)
+ }
+ if len(calls) == 0 || calls[0] != "disable --now lchat@"+host {
+ t.Fatalf("re-run must stop the unit first, got %v", calls)
+ }
+}