amadan
server: hibernation hardening — the two lessons from the first real flip (#681)
1. hibernateOn refuses while the box-wide litestream config still tracks the host: two litestream processes share one derived meta dir and corrupt each other's LTX state (hit live flipping friends — a stuck error loop and failed checkpoints). The flag is set before the guard on purpose: the operator's fix (re-run fleet-backup.sh, which excludes hibernate=1 hosts) depends on it, and the flag is inert until the reload the guard prevents. 2. LeasedReplicator.Close releases the lease under its own 5s budget: a wedged final sync exhausts the caller's ctx, and a dead-ctx release left the lease held until TTL — the next wake 503'd for 30s over nothing.
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
3 files changed,
+97
−1
control.go+34 −0control_test.go+55 −0internal/activator/replica.go+8 −1
diff --git a/control.go b/control.go| index 843fe14..95ae102 100644 |
| --- a/control.go |
| +++ b/control.go |
| @@ -677,6 +677,18 @@ func hibernateOn(reg *Registry, rt *Route, unitTmpl, botUnit, routerUnit string) |
| if err := reg.setHibernate(rt.Host, true); err != nil { |
| return fmt.Errorf("flag: %v", err) |
| } |
| + // Two litestream processes on one DB share the same derived meta dir and |
| + // corrupt each other's LTX state (hit live on the first real flip: the |
| + // box-wide backup replicator still tracked the host when the activator's |
| + // embedded one started, and a config regen yanked files out from under |
| + // it — a stuck error loop and failed checkpoints). The flag is set above |
| + // ON PURPOSE before this guard: the operator's fix — re-running the backup |
| + // script, which excludes hibernate=1 hosts — only works because the flag |
| + // already reads true, and the flag is inert until the router reload that |
| + // happens only after this guard passes. |
| + if litestreamConfigTracks(rt.Host) { |
| + return fmt.Errorf("%s is still replicated by the box-wide litestream (%s): re-run deploy/fleet-backup.sh — it now excludes this host, the flag is already set — then run this flip again", rt.Host, litestreamConfigPath()) |
| + } |
| if err := systemctl("disable", "--now", unitTmpl+rt.Host); err != nil { |
| return fmt.Errorf("stop unit: %v (flag is set but the unit still runs — the host serves as before; re-run, or undo with `lchat hibernate %s --off`)", err, rt.Host) |
| } |
| @@ -1110,6 +1122,28 @@ var systemctl = func(args ...string) error { |
| // (the box hardening), so this is "give the file to whoever runs the router" |
| // without parsing systemd units for a user name. A no-op when both are |
| // already the same (the single-user dev case). |
| +// litestreamConfigPath is where the box-wide backup replicator's config lives |
| +// (deploy/fleet-backup.sh writes it); overridable for tests and unusual boxes. |
| +func litestreamConfigPath() string { |
| + if p := os.Getenv("LCHAT_LITESTREAM_CONFIG"); p != "" { |
| + return p |
| + } |
| + return "/etc/litestream.yml" |
| +} |
| + |
| +// litestreamConfigTracks reports whether the box-wide litestream config still |
| +// lists the host's DB. A missing config means no box-wide replicator (dev |
| +// machines, self-hosters) — not tracked. The check is a plain line match |
| +// against the exact "path: /data/<host>.db" the generator emits; parsing YAML |
| +// for one substring would be ceremony. |
| +func litestreamConfigTracks(host string) bool { |
| + b, err := os.ReadFile(litestreamConfigPath()) |
| + if err != nil { |
| + return false |
| + } |
| + return strings.Contains(string(b), "path: "+filepath.Join(instanceDataDir, host+".db")+"\n") |
| +} |
| + |
| func matchOwner(path, dir string) error { |
| st, err := os.Stat(dir) |
| if err != nil { |
diff --git a/control_test.go b/control_test.go| index 38d48a5..ced6dc3 100644 |
| --- a/control_test.go |
| +++ b/control_test.go |
| @@ -741,3 +741,58 @@ func TestMatchOwner(t *testing.T) { |
| t.Fatalf("owner mismatch after matchOwner") |
| } |
| } |
| + |
| +// The flip-window guard: while the box-wide litestream config still tracks a |
| +// host's DB, flipping it on must stop before touching the unit — two |
| +// replicators share one meta dir and corrupt it (hit live on the first real |
| +// flip). The flag must already be SET when the guard fires, because the |
| +// operator's fix (re-running the backup script, which excludes hibernate=1 |
| +// hosts) depends on it — and the flag is inert until the reload the guard |
| +// prevents. |
| +func TestHibernateOnRefusesWhileBoxLitestreamTracks(t *testing.T) { |
| + reg := controlTestEnv(t) |
| + const host = "flip-guard.example.com" |
| + if _, err := addInstance(reg, addInstanceOpts{Host: host}); err != nil { |
| + t.Fatalf("addInstance: %v", err) |
| + } |
| + |
| + conf := filepath.Join(t.TempDir(), "litestream.yml") |
| + body := "dbs:\n - path: " + filepath.Join(instanceDataDir, host+".db") + "\n" |
| + if err := os.WriteFile(conf, []byte(body), 0o644); err != nil { |
| + t.Fatal(err) |
| + } |
| + t.Setenv("LCHAT_LITESTREAM_CONFIG", conf) |
| + |
| + var calls []string |
| + systemctl = func(args ...string) error { |
| + calls = append(calls, strings.Join(args, " ")) |
| + return nil |
| + } |
| + confirmPublicReady = func(string, string) { t.Fatal("must not warm while guarded") } |
| + |
| + rt, _ := reg.get(host) |
| + err := hibernateOn(reg, rt, "lchat@", "lchat-bot@", "lchat-router") |
| + if err == nil || !strings.Contains(err.Error(), "fleet-backup.sh") { |
| + t.Fatalf("want the re-run-fleet-backup refusal, got %v", err) |
| + } |
| + if len(calls) != 0 { |
| + t.Fatalf("guarded flip touched systemd: %v", calls) |
| + } |
| + if got, _ := reg.get(host); !got.Hibernate { |
| + t.Fatalf("flag must be set when the guard fires (the operator's fix depends on it)") |
| + } |
| + |
| + // The exact two-step flow the error prescribes: the config regen drops the |
| + // host (the flag now reads true), and the re-run flip proceeds. |
| + if err := os.WriteFile(conf, []byte("dbs: []\n"), 0o644); err != nil { |
| + t.Fatal(err) |
| + } |
| + confirmPublicReady = func(string, string) {} |
| + rt, _ = reg.get(host) |
| + if err := hibernateOn(reg, rt, "lchat@", "lchat-bot@", "lchat-router"); err != nil { |
| + t.Fatalf("re-run flip after exclusion: %v", err) |
| + } |
| + if len(calls) == 0 || calls[0] != "disable --now lchat@"+host { |
| + t.Fatalf("re-run flip sequence = %v", calls) |
| + } |
| +} |
diff --git a/internal/activator/replica.go b/internal/activator/replica.go| index 177fbe0..cf4a1d8 100644 |
| --- a/internal/activator/replica.go |
| +++ b/internal/activator/replica.go |
| @@ -241,7 +241,14 @@ func (r *LeasedReplicator) Close(ctx context.Context) error { |
| lease := r.rn.stop() |
| if lease != nil { |
| - if e := r.leaser.ReleaseLease(ctx, lease); e != nil && err == nil { |
| + // The release gets its OWN budget, never the caller's: a wedged final |
| + // sync exhausts ctx, and releasing under a dead context leaves the |
| + // lease held until TTL — the next wake then 503s for up to 30s over |
| + // nothing (hit live on the first real instance flip). The release is |
| + // one small S3 DELETE; five seconds is generous. |
| + relCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) |
| + defer cancel() |
| + if e := r.leaser.ReleaseLease(relCtx, lease); e != nil && err == nil { |
| err = e |
| } |
| } |