amadan
server: converge awake hibernating instances onto shipped builds (+ update status --probe) (#834)
* server: update status --probe + awake-convergence design doc The convergence issue (awake hibernate-managed instances keep serving the old binary after a ship — the box repoints the symlink but a running process doesn't follow it) split into a safe now-piece + a design proposal. Now (safe, read-only, shipped here): - `lchat update status --probe` reads each instance's socket /api/version — the build it's ACTUALLY serving — alongside the symlink target, flags awake- but-stale instances (⚠), and prints how to converge them. Stops the false 'current' reading that made a shipped release look fully rolled out when the live processes were still on older builds. probeVersion helper + test. Proposal (docs/awake-instance-convergence.md, NOT implemented): - The real fix — cycle an awake child onto the new symlink via the existing graceful stop, gated on inflight==0 so no live WebSocket is dropped — needs a new per-host router control channel (router.go flags it as a 'later step') and changes a deliberate policy. It touches the live-serving path that fronts every member space, so it's written up for human review before any code lands, with the open decisions (gate policy, control-channel surface, urgency) called out. No urgency: the 72h bake window already spaces releases, and the lazy next-wake path eventually converges most spaces. go test green. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * server: converge awake hibernating instances onto shipped builds A running activator child never re-reads its live/<host> symlink, so an awake hibernate-managed instance kept serving its old binary after a ship repointed the symlink. Idle-hibernate converges sleeping spaces on their next wake, but an always-awake space (every bot-sidecar host) never idles, so it served the stale build indefinitely — and every status surface reads the symlink, so the fleet looked converged while it wasn't. Close the gap in the sweep the activator already runs: the backend's start() reads liveVersion(host) immediately before cmd.Start() and returns it, so the entry records the build the kernel actually execed (reading it back in wake() after the up-to-30s health wait would let a concurrent ship record the new target while the child runs the old binary — the sweep would then never converge it). A fourth staleness case, alongside lost-lease / dead-child / idle: when the live symlink has diverged from spawnVersion for longer than a grace window (90s), gracefully cycle the child via the same beginStopLocked path — SIGTERM drain, final S3 checkpoint, lease release — leaving it asleep so a member request (or the due-time index for a scheduled charter) wakes it onto the new binary. The grace/reset logic is a pure convergeDecision helper (unit-tested directly): first drift starts the clock, a rollback to spawnVersion clears it, a transient read error is 'unknown' and neither cycles nor resets. Rate-limited to one cycle per sweep tick so a fleet-wide ship staggers instead of blipping every space at once. Unlike idle-hibernate this can fire with a request in flight (the whole point — the target spaces never reach inflight==0); the stop drains HTTP just as systemctl restart does. No new router control channel and no repoint-site call: the updater/deploy still just pointLive, and the sweep picks up the drift within one tick (<=30s). Pinned canaries fall out automatically (their symlink is never repointed). liveVer added as a sweep-read seam and execVerFn on the fake backend models the exec-time read; tests cover the cycle, the grace, no-thrash, the per-tick rate limit, the in-flight-request path, and the full convergeDecision state machine. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> ---------
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
6 files changed,
+550
−16
activator.go+113 −12activator_test.go+181 −2docs/awake-instance-convergence.md+177 −0updater.go+25 −2versions.go+20 −0versions_test.go+34 −0
diff --git a/activator.go b/activator.go| index 13f4411..4552ede 100644 |
| --- a/activator.go |
| +++ b/activator.go |
| @@ -133,8 +133,12 @@ type actBackend interface { |
| // restore materializes the local DB from S3 when it's absent (a no-op when |
| // the warm-cache copy is still on disk). |
| restore(ctx context.Context, host, dbPath string) error |
| - // start spawns the instance process and waits until its socket accepts. |
| - start(ctx context.Context, rt Route) (actProc, error) |
| + // start spawns the instance process and waits until its socket accepts. It |
| + // also returns the build it execed (the live/<host> symlink target read at |
| + // spawn), so the sweep can later spot a repoint under a still-running child |
| + // and converge it. Empty when unknown — the sweep treats that as "don't |
| + // cycle" (fail-safe). |
| + start(ctx context.Context, rt Route) (actProc, string, error) |
| // nextDue asks a running instance for its earliest pending scheduled |
| // delivery (unix seconds; 0 = nothing scheduled). Queried at sleep time, |
| // while the socket still answers, to feed the due-time index. |
| @@ -221,6 +225,16 @@ type actEntry struct { |
| rep actRep |
| proc actProc |
| + // Convergence: the `live/<host>` symlink target this child was spawned |
| + // against. A running process never re-reads the symlink, so once a ship |
| + // (or `deploy --version`) repoints it, an awake child keeps serving the old |
| + // build until it's cycled. spawnVersion lets the sweep notice the drift; |
| + // staleSince is when it first did, so a brief transient (a symlink moved |
| + // then rolled straight back) doesn't trigger a needless cycle. Both empty/ |
| + // zero until the wake resolves. |
| + spawnVersion string |
| + staleSince time.Time |
| + |
| // Bot sidecar co-spawn: the sidecar exists exactly while its instance is |
| // awake. All sidecar fields are written under a.mu in the awake state only |
| // (wake's attach, the sweeper's restart); a stop flips the state under the |
| @@ -247,6 +261,47 @@ const ( |
| // behind this timeout is insurance. |
| const sidecarStopTimeout = 5 * time.Second |
| +// Build convergence: how long an awake child may serve a symlink-stale binary |
| +// before the sweep gracefully cycles it, and how many to cycle per tick. |
| +// - convergeGrace comfortably exceeds one sweep interval (≤30s) and the |
| +// sidecar's ~32s reconnect, so a genuinely-persistent drift is required — |
| +// a symlink that moved then rolled straight back never trips it, and a |
| +// briefly-busy space gets a chance to idle-hibernate (which converges for |
| +// free) first. |
| +// - one per tick staggers a fleet-wide ship across sweeps instead of blipping |
| +// every space's connections at once. |
| +// |
| +// Convergence is a latency/observability fix, not urgent (the supply-chain bake |
| +// window already holds releases 72h), so a leisurely, gentle cadence is right. |
| +const ( |
| + convergeGrace = 90 * time.Second |
| + maxConvergeCyclesPerSweep = 1 |
| +) |
| + |
| +// convergeDecision is the pure state machine behind build convergence: given the |
| +// symlink target read this tick (cur/readErr), the build the child execed |
| +// (spawnVer), and when the drift was first seen (staleSince), it says whether to |
| +// cycle now and returns the staleSince to carry forward. Split out so the grace/ |
| +// reset logic is unit-testable without a live activator. Three cases: |
| +// - read error: unknown — neither cycle nor clear the grace; retry next tick. |
| +// - converged / rolled-back / nothing to compare (cur=="" or spawnVer=="" or |
| +// cur==spawnVer): clear the grace clock, don't cycle. |
| +// - stale (symlink moved since spawn): start the grace clock on first sight, |
| +// cycle once the drift has persisted past grace. |
| +func convergeDecision(cur string, readErr error, spawnVer string, staleSince, now time.Time, grace time.Duration) (cycle bool, nextStale time.Time) { |
| + switch { |
| + case readErr != nil: |
| + return false, staleSince |
| + case cur == "" || spawnVer == "" || cur == spawnVer: |
| + return false, time.Time{} |
| + default: |
| + if staleSince.IsZero() { |
| + staleSince = now |
| + } |
| + return now.Sub(staleSince) >= grace, staleSince |
| + } |
| +} |
| + |
| // wakeWaitBound caps how long a request queues on someone else's wake or stop |
| // before giving up with 503 — a wedged wake must not pile up connections |
| // forever. A var so tests can shrink it. |
| @@ -270,7 +325,8 @@ type activatorSvc struct { |
| cfg hibernateConfig |
| backend actBackend |
| due actDueStore |
| - now func() time.Time // a seam for the unit tests' clock |
| + now func() time.Time // a seam for the unit tests' clock |
| + liveVer func(string) (string, error) // live/<host> symlink target; a seam for tests |
| // mesh, when non-nil, is the multi-box placement layer (mesh.go): requests |
| // for hosts whose HRW owner is another live box are proxied there, and the |
| @@ -292,6 +348,7 @@ func newActivator(cfg hibernateConfig, backend actBackend, due actDueStore) *act |
| backend: backend, |
| due: due, |
| now: time.Now, |
| + liveVer: liveVersion, |
| entries: make(map[string]*actEntry), |
| } |
| } |
| @@ -502,6 +559,13 @@ func (a *activatorSvc) wake(e *actEntry) { |
| e.sidecarBackoff = 0 |
| e.sidecarRetryAt = time.Time{} // a wake-time spawn failure retries on the next sweep |
| e.lastActive = a.now() |
| + // The build this child actually execed, captured at Start() inside the |
| + // backend (not re-read here — a ship during the wake window would make a |
| + // re-read name the new target while the process runs the old binary, and |
| + // the sweep would never converge it). Empty = unknown → the sweep won't |
| + // cycle. Fresh wake, so it can't be stale yet. |
| + e.spawnVersion = w.spawnVersion |
| + e.staleSince = time.Time{} |
| } |
| a.mu.Unlock() |
| close(e.ready) |
| @@ -516,6 +580,7 @@ type wokenInstance struct { |
| proc actProc |
| sidecar actProc |
| sidecarWanted bool |
| + spawnVersion string // live/<host> target read at exec — for build convergence |
| } |
| func (a *activatorSvc) doWake(ctx context.Context, rt Route) (*wokenInstance, error) { |
| @@ -551,7 +616,7 @@ func (a *activatorSvc) doWake(ctx context.Context, rt Route) (*wokenInstance, er |
| // on boot, so waiting for the socket guarantees the file exists for |
| // litestream to open. Writes that land in the gap are picked up by the |
| // replica's first sync. |
| - proc, err := a.backend.start(ctx, rt) |
| + proc, spawnVer, err := a.backend.start(ctx, rt) |
| if err != nil { |
| _ = lease.release(ctx) |
| return nil, fmt.Errorf("start %s: %w", host, err) |
| @@ -572,7 +637,7 @@ func (a *activatorSvc) doWake(ctx context.Context, rt Route) (*wokenInstance, er |
| // (its marked requests pass through the router while we're awake). A |
| // spawn failure is loud but never fails the wake: members come before |
| // bots, and the sweeper retries with backoff. |
| - w := &wokenInstance{rep: rep, proc: proc} |
| + w := &wokenInstance{rep: rep, proc: proc, spawnVersion: spawnVer} |
| if a.backend.hasSidecar(host) { |
| w.sidecarWanted = true |
| if sc, err := a.backend.startSidecar(ctx, rt); err != nil { |
| @@ -705,6 +770,7 @@ func (a *activatorSvc) sweepOnce() { |
| log.Printf("activator: due-time sweep: %v", err) |
| } |
| a.mu.Lock() |
| + cycled := 0 // build-convergence cycles started this tick (rate-limited) |
| for _, e := range a.entries { |
| if e.state != actAwake { |
| continue |
| @@ -724,6 +790,31 @@ func (a *activatorSvc) sweepOnce() { |
| a.beginStopLocked(e) |
| continue |
| } |
| + // Converge onto a shipped build. A running child never re-reads |
| + // live/<host>, so once a ship or `deploy --version` repoints the symlink |
| + // this awake child is serving a stale binary. Idle spaces converge for |
| + // free on their next wake; this cycle is for the always-awake ones — |
| + // notably every bot-sidecar host, which never sleeps — that otherwise |
| + // serve the old build indefinitely. Gracefully cycle: beginStopLocked |
| + // drains via SIGTERM, takes the final checkpoint, releases the lease, and |
| + // leaves the host asleep; a member request (or the due-time index for |
| + // scheduled charters) then wakes it onto the new binary — the same clean |
| + // cycle `systemctl restart` gives a unit-managed host. Unlike idle |
| + // hibernation this can fire with a request in flight; the stop still |
| + // drains HTTP gracefully and reconnecting clients coalesce onto the fresh |
| + // wake, so a long upload past the ~10s drain is cut exactly as a restart |
| + // would cut it. A pinned canary never trips this: the updater leaves its |
| + // symlink alone, so spawnVersion still matches. See |
| + // docs/awake-instance-convergence.md. |
| + cur, readErr := a.liveVer(e.host) |
| + cycle, next := convergeDecision(cur, readErr, e.spawnVersion, e.staleSince, now, convergeGrace) |
| + e.staleSince = next |
| + if cycle && cycled < maxConvergeCyclesPerSweep { |
| + log.Printf("activator: %s serving %s but live symlink is now %s — cycling onto the new build", e.host, e.spawnVersion, cur) |
| + a.beginStopLocked(e) |
| + cycled++ |
| + continue |
| + } |
| // Sidecar supervision — Restart=always, sweep-driven, no supervisor |
| // goroutine: a sidecar that died (or failed to spawn at wake) while |
| // its instance stays awake is respawned with doubling, capped backoff; |
| @@ -1051,7 +1142,7 @@ func (b *s3Backend) nextDue(ctx context.Context, rt Route) (int64, error) { |
| // socket and answer. A var so tests can shrink the failure path. |
| var spawnHealthTimeout = 30 * time.Second |
| -func (b *s3Backend) start(ctx context.Context, rt Route) (actProc, error) { |
| +func (b *s3Backend) start(ctx context.Context, rt Route) (actProc, string, error) { |
| host := rt.Host |
| dbPath := rt.DBPath |
| if dbPath == "" { |
| @@ -1067,17 +1158,17 @@ func (b *s3Backend) start(ctx context.Context, rt Route) (actProc, error) { |
| if _, err := os.Stat(bin); err != nil { |
| ver, err := ensureSelfStaged() |
| if err != nil { |
| - return nil, fmt.Errorf("stage binary: %w", err) |
| + return nil, "", fmt.Errorf("stage binary: %w", err) |
| } |
| if err := pointLive(host, ver); err != nil { |
| - return nil, fmt.Errorf("point live symlink: %w", err) |
| + return nil, "", fmt.Errorf("point live symlink: %w", err) |
| } |
| } |
| // The env file is optional, like the unit's "-" prefix. |
| fileEnv, err := readEnvFile(filepath.Join(instanceDataDir, host+".env")) |
| if err != nil && !os.IsNotExist(err) { |
| - return nil, fmt.Errorf("read env file: %w", err) |
| + return nil, "", fmt.Errorf("read env file: %w", err) |
| } |
| cmd := exec.Command(bin, "--socket", rt.Socket, "--db", dbPath) |
| @@ -1086,8 +1177,18 @@ func (b *s3Backend) start(ctx context.Context, rt Route) (actProc, error) { |
| // systemd); there's no per-instance unit to own a log stream anymore. |
| cmd.Stdout = os.Stdout |
| cmd.Stderr = os.Stderr |
| + // Record the build we're about to exec — read the symlink immediately before |
| + // Start() so it names the version the kernel will resolve. Reading it here, |
| + // rather than back in wake() after the health wait, closes a real window: a |
| + // concurrent ship's pointLive during the up-to-30s wait would otherwise make |
| + // us record the NEW target while this child runs the OLD binary, and the |
| + // sweep would then think it's already converged and never cycle it. The |
| + // residual sub-exec race (a repoint between this read and Start) is in the |
| + // safe direction — we'd record the older version, so the sweep cycles once |
| + // too often rather than never. Empty on a read error: the sweep won't cycle. |
| + spawnVer, _ := liveVersion(host) |
| if err := cmd.Start(); err != nil { |
| - return nil, err |
| + return nil, "", err |
| } |
| p := &childProc{cmd: cmd, done: make(chan struct{})} |
| @@ -1100,9 +1201,9 @@ func (b *s3Backend) start(ctx context.Context, rt Route) (actProc, error) { |
| stopCtx, cancel := context.WithTimeout(context.Background(), actStopTimeout) |
| defer cancel() |
| _ = p.stop(stopCtx) |
| - return nil, err |
| + return nil, "", err |
| } |
| - return p, nil |
| + return p, spawnVer, nil |
| } |
| // childEnv approximates a systemd unit's environment from inside the router: |
diff --git a/activator_test.go b/activator_test.go| index a24796b..fd70fb5 100644 |
| --- a/activator_test.go |
| +++ b/activator_test.go |
| @@ -108,6 +108,13 @@ type fakeBackend struct { |
| dueVal int64 // what the instance reports at the sleep-time next-due probe |
| dueErr error // when non-nil, the probe fails (dead socket, missing token, …) |
| + // execVerFn models the live/<host> target the child execs at spawn — start() |
| + // returns it as the spawnVersion, exactly as s3Backend reads the symlink just |
| + // before Start(). Nil → "" (unknown), which disables convergence, so tests |
| + // that don't care are unaffected. Point it at the same source as the sweep's |
| + // liveVer seam to model a symlink faithfully (both read the current target). |
| + execVerFn func() string |
| + |
| // Bot sidecar co-spawn. sidecarStaged is hasSidecar's answer (false by |
| // default, so every pre-co-spawn test doubles as the "host without a |
| // sidecar spawns none" case); sidecarErr injects spawn failures. |
| @@ -153,18 +160,23 @@ func (b *fakeBackend) restore(context.Context, string, string) error { |
| return nil |
| } |
| -func (b *fakeBackend) start(context.Context, Route) (actProc, error) { |
| +func (b *fakeBackend) start(context.Context, Route) (actProc, string, error) { |
| b.mu.Lock() |
| b.starts++ |
| b.order = append(b.order, "start:instance") |
| gate := b.startGate |
| + fn := b.execVerFn |
| proc := &fakeProc{label: "instance", rec: b} |
| b.procs = append(b.procs, proc) |
| b.mu.Unlock() |
| if gate != nil { |
| <-gate |
| } |
| - return proc, nil |
| + ver := "" |
| + if fn != nil { |
| + ver = fn() |
| + } |
| + return proc, ver, nil |
| } |
| func (b *fakeBackend) nextDue(context.Context, Route) (int64, error) { |
| @@ -393,6 +405,173 @@ func TestActivatorInflightKeepsAwake(t *testing.T) { |
| waitFor(t, "hibernation", func() bool { return a.entryCount() == 0 }) |
| } |
| +// A ship repoints live/<host> under an awake child. A running process never |
| +// re-reads its symlink, so idle-hibernate is the only thing that converges an |
| +// instance — and an always-awake space (every bot-sidecar host) never idles. |
| +// The sweep closes that gap: it notices the drift and, after a grace window and |
| +// rate-limited to one per tick, gracefully cycles the child so its next wake |
| +// execs the new binary. See docs/awake-instance-convergence.md. |
| +func TestActivatorConvergesOnSymlinkRepoint(t *testing.T) { |
| + a, b, clk, _ := testActivator(t, time.Hour) // idle far away — isolate the convergence path |
| + var ver atomic.Value // the live/<host> symlink target; swapped to model a ship |
| + ver.Store("v1") |
| + // Model one symlink: the child execs whatever it points at now (execVerFn, |
| + // frozen into spawnVersion at wake), and the sweep reads the current target |
| + // (liveVer). A "ship" swaps ver; the running child keeps its frozen version. |
| + a.liveVer = func(string) (string, error) { return ver.Load().(string), nil } |
| + b.execVerFn = func() string { return ver.Load().(string) } |
| + |
| + release, err := a.beginRequest(context.Background(), testRoute("one.example.com")) |
| + if err != nil { |
| + t.Fatalf("beginRequest: %v", err) |
| + } |
| + release() |
| + |
| + // Symlink unchanged: never cycles, however long it stays awake. |
| + clk.advance(3 * convergeGrace) |
| + a.sweepOnce() |
| + if a.entryCount() != 1 { |
| + t.Fatal("cycled an instance whose symlink never moved") |
| + } |
| + |
| + // Ship: repoint the symlink. The first sweep after only starts the grace |
| + // clock — it must not cycle yet, so a move-then-rollback never drops a live |
| + // connection. |
| + ver.Store("v2") |
| + a.sweepOnce() |
| + if a.entryCount() != 1 { |
| + t.Fatal("cycled inside the grace window — a transient repoint would drop a live connection") |
| + } |
| + |
| + // Past the grace: gracefully cycle (stop → final checkpoint → lease release → |
| + // asleep). The reconnecting client/sidecar would wake it onto v2. |
| + clk.advance(convergeGrace + time.Second) |
| + a.sweepOnce() |
| + waitFor(t, "convergence cycle", func() bool { return a.entryCount() == 0 }) |
| + if !b.lastProc().isStopped() { |
| + t.Fatal("stale instance's process was not stopped") |
| + } |
| + if !b.lastRep().isClosed() { |
| + t.Fatal("convergence cycle skipped the final checkpoint / lease release") |
| + } |
| + |
| + // Re-wake against the now-current symlink: spawnVersion tracks v2, so it |
| + // never cycles again — no thrash. |
| + release, err = a.beginRequest(context.Background(), testRoute("one.example.com")) |
| + if err != nil { |
| + t.Fatalf("re-wake after convergence: %v", err) |
| + } |
| + release() |
| + clk.advance(3 * convergeGrace) |
| + a.sweepOnce() |
| + if a.entryCount() != 1 { |
| + t.Fatal("cycled again after already converging onto the current build") |
| + } |
| +} |
| + |
| +// Two awake children go stale at once (a fleet-wide ship). One sweep cycles at |
| +// most one — a staggered convergence, never a simultaneous blip of every space. |
| +func TestActivatorConvergenceRateLimitsPerSweep(t *testing.T) { |
| + a, b, clk, _ := testActivator(t, time.Hour) |
| + var ver atomic.Value |
| + ver.Store("v1") |
| + a.liveVer = func(string) (string, error) { return ver.Load().(string), nil } |
| + b.execVerFn = func() string { return ver.Load().(string) } |
| + |
| + for _, h := range []string{"one.example.com", "two.example.com"} { |
| + release, err := a.beginRequest(context.Background(), testRoute(h)) |
| + if err != nil { |
| + t.Fatalf("beginRequest %s: %v", h, err) |
| + } |
| + release() |
| + } |
| + if a.entryCount() != 2 { |
| + t.Fatalf("want 2 awake instances, got %d", a.entryCount()) |
| + } |
| + |
| + ver.Store("v2") // ship under both |
| + a.sweepOnce() // starts the grace clock for both |
| + clk.advance(convergeGrace + time.Second) |
| + |
| + a.sweepOnce() // both are past grace, but only one may cycle this tick |
| + waitFor(t, "first convergence cycle", func() bool { return a.entryCount() == 1 }) |
| + // A moment later, the sweep takes the second. |
| + a.sweepOnce() |
| + waitFor(t, "second convergence cycle", func() bool { return a.entryCount() == 0 }) |
| +} |
| + |
| +// The feature's whole premise: converge a space that NEVER reaches inflight==0 |
| +// (an open WebSocket, or a bot sidecar's persistent connection), which |
| +// idle-hibernate can never touch. Unlike idle-hibernate, convergence fires with |
| +// a request in flight — the graceful stop drains it exactly as a restart would. |
| +func TestActivatorConvergesWithRequestInFlight(t *testing.T) { |
| + a, b, clk, _ := testActivator(t, time.Hour) |
| + var ver atomic.Value |
| + ver.Store("v1") |
| + a.liveVer = func(string) (string, error) { return ver.Load().(string), nil } |
| + b.execVerFn = func() string { return ver.Load().(string) } |
| + |
| + // Hold the request open for the whole test — inflight stays 1, so the idle |
| + // path (which needs inflight==0) can never converge this host. |
| + release, err := a.beginRequest(context.Background(), testRoute("one.example.com")) |
| + if err != nil { |
| + t.Fatalf("beginRequest: %v", err) |
| + } |
| + defer release() |
| + |
| + ver.Store("v2") // ship under the busy space |
| + a.sweepOnce() // grace starts; must not cycle a busy space mid-grace |
| + if a.entryCount() != 1 { |
| + t.Fatal("cycled inside the grace window") |
| + } |
| + clk.advance(convergeGrace + time.Second) |
| + a.sweepOnce() |
| + // Cycles despite the in-flight request — idle-hibernate never would. |
| + waitFor(t, "busy-space convergence", func() bool { return a.entryCount() == 0 }) |
| + if !b.lastProc().isStopped() { |
| + t.Fatal("busy stale instance's process was not stopped") |
| + } |
| +} |
| + |
| +// convergeDecision's state machine, exhaustively: the grace clock starts on |
| +// first drift, survives a rollback correctly, and a read error never clears |
| +// accrued grace (it means "unknown", not "converged"). |
| +func TestConvergeDecision(t *testing.T) { |
| + base := time.Unix(1_700_000_000, 0) |
| + set := base.Add(-40 * time.Second) // a grace clock that started 40s ago |
| + errBoom := errors.New("readlink boom") |
| + cases := []struct { |
| + name string |
| + cur string |
| + readErr error |
| + spawnVer string |
| + staleSince time.Time |
| + wantCycle bool |
| + wantStale time.Time |
| + }{ |
| + {"matches — clears grace", "v1", nil, "v1", set, false, time.Time{}}, |
| + {"empty cur — no data, clears", "", nil, "v1", set, false, time.Time{}}, |
| + {"empty spawnVer — unknown, clears", "v2", nil, "", set, false, time.Time{}}, |
| + {"first drift — starts grace, no cycle", "v2", nil, "v1", time.Time{}, false, base}, |
| + {"drift within grace — holds, no cycle", "v2", nil, "v1", base.Add(-30 * time.Second), false, base.Add(-30 * time.Second)}, |
| + {"drift past grace — cycles", "v2", nil, "v1", base.Add(-convergeGrace - time.Second), true, base.Add(-convergeGrace - time.Second)}, |
| + {"read error — preserves accrued grace", "", errBoom, "v1", set, false, set}, |
| + {"read error before any drift — stays zero", "", errBoom, "v1", time.Time{}, false, time.Time{}}, |
| + {"rollback to spawn — clears grace", "v1", nil, "v1", base.Add(-convergeGrace - time.Second), false, time.Time{}}, |
| + } |
| + for _, c := range cases { |
| + t.Run(c.name, func(t *testing.T) { |
| + cycle, next := convergeDecision(c.cur, c.readErr, c.spawnVer, c.staleSince, base, convergeGrace) |
| + if cycle != c.wantCycle { |
| + t.Errorf("cycle = %v, want %v", cycle, c.wantCycle) |
| + } |
| + if !next.Equal(c.wantStale) { |
| + t.Errorf("nextStale = %v, want %v", next, c.wantStale) |
| + } |
| + }) |
| + } |
| +} |
| + |
| // A lost lease (failed renewal) stands the instance down immediately — no |
| // waiting for idle: someone else may own the S3 prefix now. |
| func TestActivatorLeaseLostStandsDown(t *testing.T) { |
diff --git a/docs/awake-instance-convergence.md b/docs/awake-instance-convergence.md| new file mode 100644 |
| index 0000000..91918df |
| --- /dev/null |
| +++ b/docs/awake-instance-convergence.md |
| @@ -0,0 +1,177 @@ |
| +# 🤖 Awake-instance convergence: diagnosis + fix |
| + |
| +**Status: IMPLEMENTED (sweep-driven).** This documents a production gap — awake |
| +hibernate-managed instances kept serving their old binary after a release ships |
| +— and the fix that closed it. The fix landed as a fourth staleness case in the |
| +activator's existing idle sweep (`activator.go`, `sweepOnce`); it needed **no |
| +new router control channel**, no updater changes, and no change to the deploy |
| +path. The sections below keep the original diagnosis, then describe what was |
| +built and why it differs from the first proposal. |
| + |
| +## The symptom |
| + |
| +After `release-server.sh ship` (or the hourly `lchat update run`) repoints an |
| +instance's `live/<host>` symlink to a new binary: |
| + |
| +- `lchat instances` / `lchat update status` show the **new** version. |
| +- But `/api/version` over the instance's socket returns the **old** build. |
| + |
| +Hibernating instances converge on their next wake; **awake** ones (anything |
| +with an open WebSocket or recent traffic — notably every bot-sidecar-bearing |
| +space, which never sleeps) serve the stale binary indefinitely. The reporting |
| +surfaces all read the symlink target, so the fleet *looks* converged while it |
| +isn't. (Confirmed live 2026-07-13: after promoting a release, `family`, |
| +`test`, and `team` each served a different older build than the symlink named.) |
| + |
| +## Why — the exact mechanism |
| + |
| +An awake hibernate-flagged instance is a **child process of the router**, not a |
| +`systemd` unit (`activator.go`: `activatorSvc.entries[host].proc`, an |
| +`*exec.Cmd` via `childProc`). It's spawned once by `s3Backend.start`, which |
| +execs `liveLink(host)` = `/opt/lchat/live/<host>` → `versions/<ver>/lchat`. **A |
| +running child never re-reads the symlink** — that's the root of the gap. |
| + |
| +The update path (`updater.go updatePass`) has two branches: |
| + |
| +- **Unit-managed hosts** repoint the symlink *and* `systemctl restart` the unit |
| + (`deployHostVersion` in `control.go`) — so they converge immediately. |
| +- **Hibernating hosts** (`updater.go` phase 2) do **only** `pointLive` — a pure |
| + symlink repoint, no signal to the running process. The code comment is |
| + explicit: *"a currently-AWAKE child keeps its old binary until it next |
| + sleeps (a running process doesn't follow the symlink)."* |
| + |
| +`update status` (`runUpdateStatus`) and `lchat instances` both build their |
| +"current" column from `liveVersion(host)` (`os.Readlink`), never from the |
| +child's `/api/version` — hence the false "converged" reading. |
| + |
| +### What already exists (and what doesn't) |
| + |
| +- **#778** added the whole hibernating-repoint phase, `hibernatingPrefs` |
| + (parked-DB read), WAL-ownership repair, and the narrow `setVersions`. |
| +- **#774** added parked-DB pref reads to `update status`. |
| +- The "compare symlinks, never restart awake children" behaviour is |
| + `updater.go`'s symlink comparison + `deploy`'s hibernating branch — not a |
| + separate function. |
| + |
| +**What does NOT exist anywhere:** a primitive that cycles an *awake* activator |
| +child onto the new symlink. `lchat deploy <host> --version` also only repoints |
| +for hibernating hosts (`control.go`, the `if rt.Hibernate` branch is a pure |
| +`pointLive`). The only graceful cycle today is the heavyweight manual |
| +`hibernateOff` → update → `hibernateOn` flip documented in |
| +`docs/server-releases.md`. |
| + |
| +## Why awake children aren't restarted (the design tension) |
| + |
| +This is deliberate, for two real reasons: |
| + |
| +1. **Dropped connections.** An instance is kept awake *by* in-flight work, |
| + including open WebSockets (idle = zero in-flight AND elapsed idle window, |
| + `activator.go`). Cycling an awake child cuts live WS/requests. |
| +2. **The single-writer S3 lease.** Each wake acquires a per-host Litestream |
| + lease; a naive external `pkill` races the lease and can 502 within the |
| + router's dial-retry window (the sweep only reaps every `Idle/4`, ≤30s). |
| + |
| +So "never converges" is the current cost of "never drop a live connection." |
| + |
| +## The safe primitive that already exists |
| + |
| +The activator's own stop **is** graceful: `childProc.stop` sends `SIGTERM` |
| +(the instance's 10s drain), `SIGKILL` only on overstay; `beginStopLocked` → |
| +`stop` takes the final S3 checkpoint, releases the lease, stops the sidecar |
| +first, and deletes the entry — leaving the host **asleep**. The next request or |
| +due-wake re-execs `liveLink(host)`, which now resolves to the **new** binary. |
| +So a respawn-onto-new-symlink path exists; it's just **lazy (next request), not |
| +triggered by a ship.** |
| + |
| +## The fix as built (smallest safe change) |
| + |
| +The convergence check is a **fourth staleness case in the sweep** the activator |
| +already runs (`sweepOnce`, every `Idle/4`, capped 30s). The sweep already reaps |
| +awake children for three reasons — lost lease, dead child, idle — each via the |
| +graceful `beginStopLocked`. Convergence is the fourth: |
| + |
| +1. **Record the spawned build.** The backend's `start` reads `liveVersion(host)` |
| + **immediately before `cmd.Start()`** and returns it, so `spawnVersion` names |
| + the build the kernel actually resolved the symlink to — not a value re-read |
| + later in `wake()`, which a ship landing during the up-to-30s health wait |
| + could make disagree with the running binary. A read error leaves it empty, |
| + which disables the check for that entry (fail-safe: never cycle on a version |
| + we couldn't read). |
| +2. **Notice the drift, then wait out a grace.** Each sweep compares the live |
| + symlink against `spawnVersion` (the pure `convergeDecision` helper holds this |
| + state machine, so it's unit-tested directly). On first divergence it stamps |
| + `staleSince`; only once the drift has persisted `convergeGrace` (90s) does it |
| + cycle. The grace means a symlink that moved then rolled straight back (a bad |
| + promote reverted) never drops a connection, and a briefly-busy space gets a |
| + chance to idle-hibernate — which converges for free — first. A transient |
| + symlink-read error is treated as "unknown" — it neither cycles nor resets the |
| + accrued grace. |
| +3. **Cycle gracefully, one per tick.** Past the grace it calls the same |
| + `beginStopLocked` the other three cases use: sidecar stopped first, SIGTERM |
| + drain, final S3 checkpoint, lease released, entry deleted → **asleep**. A |
| + subsequent **member request** — or the **due-time index** for a space with a |
| + scheduled charter — then wakes it, re-execing `liveLink(host)` = the new |
| + binary. (The sidecar itself is SIGKILLed during the stop and can't self-wake; |
| + a bot space with no members present and no scheduled work simply stays asleep |
| + until someone arrives, which is correct hibernation.) At most |
| + `maxConvergeCyclesPerSweep` (1) per tick, so a fleet-wide ship staggers across |
| + sweeps instead of blipping every space at once. |
| + |
| +Unlike idle-hibernation (which only fires at `inflight == 0`), this cycle can |
| +fire with a request in flight — that's deliberate, since the always-awake spaces |
| +it targets never reach `inflight == 0`. The graceful stop drains in-flight HTTP |
| +just as `systemctl restart` does, so the only thing cut is a request still |
| +running past the instance's ~10s drain (a large upload) — identical to a unit |
| +restart landing mid-request, not a new failure mode. |
| + |
| +No repoint-site call, no control channel: the updater/deploy still just |
| +`pointLive`, and the router's own sweep picks up the drift within ≤30s. This |
| +**reuses** the graceful stop and the lazy wake-re-execs-the-symlink path — it |
| +extends the existing convergence machinery rather than adding a new one. |
| + |
| +### Exclusions (all automatic — no special-casing) |
| + |
| +- **Pinned canaries.** The updater never repoints a pinned host's symlink, so |
| + `spawnVersion` keeps matching and the check never fires. Honouring the pin |
| + needs no code here — it falls out of "cycle only when the symlink moved." |
| +- **Unit-managed (non-hibernating) hosts.** Not activator children — they have |
| + no `entries[host]`, so the sweep never sees them; they converge via |
| + `systemctl restart` as before. |
| +- **Bot-sidecar spaces are the PRIMARY target.** They never idle, so the idle |
| + case never converges them; this cycle does, and the sidecar is stopped-first |
| + and re-spawned on wake, so it's sidecar-safe. This automates exactly what the |
| + manual `hibernate --off`/`--on` flip did by hand. |
| + |
| +## Why this shape, not the first proposal |
| + |
| +The original proposal (kept in git history) added a per-host `cycle` verb to a |
| +new router control socket and gated on `inflight == 0`. Both were dropped: |
| + |
| +- **No control channel.** A push from the repoint site would have added the |
| + first per-host *command* into the live router — new attack surface on the |
| + process fronting every space. The sweep already wakes hosts on a timer, so a |
| + ≤30s lazy pull needs none of that. The convergence latency (seconds to a |
| + couple of minutes) is irrelevant given the 72h supply-chain bake window. |
| +- **Grace window, not `inflight == 0`.** An `inflight == 0` gate would leave the |
| + *primary* target — always-awake bot-sidecar spaces, whose persistent |
| + WebSocket keeps `inflight > 0` forever — stale indefinitely, defeating the |
| + point. Cycling them requires a brief connection drop, which the graceful |
| + SIGTERM drain + the router's `actStopping` connection-shed makes clean: |
| + clients reconnect onto the new binary, exactly as they do across a normal |
| + unit `systemctl restart`. The grace window (plus one-per-tick) is what keeps |
| + it gentle — a space that goes quiet converges via idle-hibernate first, and |
| + only a persistently-awake space is force-cycled, once, per ship. |
| + |
| +This is a *latency/observability* fix, not a correctness emergency: the bake |
| +window holds releases 72h and the lazy next-wake path already converged |
| +sleeping spaces. The gap was the always-awake spaces and the dishonest `status` |
| +column — both now closed (see below). |
| + |
| +## Companion: honest `status` (already shipped) |
| + |
| +`lchat update status --probe` reads each instance's `/api/version` over its |
| +socket and flags an awake-stale instance (⚠) where the old symlink-only column |
| +read "current". That shipped first as a small, safe, independent step; this |
| +sweep fix removes the underlying staleness the probe surfaces. The manual |
| +`lchat hibernate <host> --off` → update → `--on` flip still works for forcing an |
| +immediate converge, but is no longer needed for routine ships. |
diff --git a/updater.go b/updater.go| index ff04903..4bf2ee7 100644 |
| --- a/updater.go |
| +++ b/updater.go |
| @@ -816,6 +816,7 @@ func statusPrefs(rt Route) (updatePrefs, error) { |
| func runUpdateStatus(args []string) { |
| fs := flag.NewFlagSet("update status", flag.ExitOnError) |
| registryPath := fs.String("registry", envOr("LCHAT_REGISTRY", "/data/registry.db"), "router registry SQLite path") |
| + probe := fs.Bool("probe", false, "also read each instance's socket /api/version — the build it's ACTUALLY serving, which an awake hibernating instance can lag behind its symlink (docs/awake-instance-convergence.md)") |
| _ = fs.Parse(reorderFlagsFirst(fs, args)) |
| reg, err := openRegistry(*registryPath) |
| @@ -829,7 +830,12 @@ func runUpdateStatus(args []string) { |
| m, merr := fetchReleaseManifest(releaseManifestURL) |
| bake := loadFirstSeen() |
| - fmt.Printf("%-34s %-8s %-8s %-12s %s\n", "HOST", "CHANNEL", "AUTO", "CURRENT", "AVAILABLE") |
| + if *probe { |
| + fmt.Printf("%-34s %-8s %-8s %-12s %-12s %s\n", "HOST", "CHANNEL", "AUTO", "SYMLINK", "SERVING", "AVAILABLE") |
| + } else { |
| + fmt.Printf("%-34s %-8s %-8s %-12s %s\n", "HOST", "CHANNEL", "AUTO", "CURRENT", "AVAILABLE") |
| + } |
| + staleServing := false |
| for _, rt := range routes { |
| if rt.Kind != "instance" { |
| continue |
| @@ -874,7 +880,24 @@ func runUpdateStatus(args []string) { |
| } |
| } |
| } |
| - fmt.Printf("%-34s %-8s %-8s %-12s %s\n", rt.Host, channel, auto, cur, avail) |
| + if *probe { |
| + // The build the process is actually serving (vs. the symlink `cur`). |
| + serving, ok := probeVersion(rt.Socket) |
| + if !ok { |
| + serving = "asleep" // socket didn't answer — hibernating, converges on wake |
| + } else if serving != cur { |
| + serving += " ⚠" // awake but stale: symlink moved, process didn't |
| + staleServing = true |
| + } |
| + fmt.Printf("%-34s %-8s %-8s %-12s %-12s %s\n", rt.Host, channel, auto, cur, serving, avail) |
| + } else { |
| + fmt.Printf("%-34s %-8s %-8s %-12s %s\n", rt.Host, channel, auto, cur, avail) |
| + } |
| + } |
| + if staleServing { |
| + fmt.Println("\n⚠ = awake but serving an older build than its symlink names (a running process doesn't follow the") |
| + fmt.Println(" symlink). Converges on next sleep/wake, or cycle now with: lchat hibernate <host> --off && … --on") |
| + fmt.Println(" (docs/awake-instance-convergence.md).") |
| } |
| if merr != nil { |
| fmt.Printf("(no manifest: %v)\n", merr) |
diff --git a/versions.go b/versions.go| index 49818e3..22201ef 100644 |
| --- a/versions.go |
| +++ b/versions.go |
| @@ -405,6 +405,26 @@ func waitHealthyVersion(socket, wantBuild string, timeout time.Duration) error { |
| return fmt.Errorf("instance didn't report build %q within %s: %v", wantBuild, timeout, last) |
| } |
| +// probeVersion reads the build the process behind a socket is ACTUALLY serving |
| +// (its /api/version), not the symlink target. This is how `update status |
| +// --probe` catches an awake hibernating instance that repointed its symlink but |
| +// still runs the old binary (docs/awake-instance-convergence.md). "" + ok=false |
| +// when the socket doesn't answer (asleep, or no such instance). |
| +func probeVersion(socket string) (build string, ok bool) { |
| + resp, err := unixHTTPClient(socket).Get("http://lchat/api/version") |
| + if err != nil { |
| + return "", false |
| + } |
| + defer resp.Body.Close() |
| + var body struct { |
| + Build string `json:"build"` |
| + } |
| + if json.NewDecoder(io.LimitReader(resp.Body, 1<<16)).Decode(&body) != nil { |
| + return "", false |
| + } |
| + return body.Build, body.Build != "" |
| +} |
| + |
| // unixHTTPClient dials a unix socket over HTTP — the router↔instance transport, |
| // reused by the health checks. |
| func unixHTTPClient(socket string) *http.Client { |
diff --git a/versions_test.go b/versions_test.go| index a78242a..93005e5 100644 |
| --- a/versions_test.go |
| +++ b/versions_test.go |
| @@ -1,6 +1,9 @@ |
| package main |
| import ( |
| + "io" |
| + "net" |
| + "net/http" |
| "os" |
| "path/filepath" |
| "testing" |
| @@ -125,3 +128,34 @@ func TestLiveVersionMissing(t *testing.T) { |
| t.Errorf("liveVersion(missing) = %q, want empty", got) |
| } |
| } |
| + |
| +// probeVersion reads the build a socket actually serves — the honest half of |
| +// `update status --probe` (docs/awake-instance-convergence.md). A tiny HTTP |
| +// server over a real unix socket stands in for a running instance. |
| +func TestProbeVersion(t *testing.T) { |
| + dir := t.TempDir() |
| + sock := filepath.Join(dir, "inst.sock") |
| + ln, err := net.Listen("unix", sock) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| + if r.URL.Path == "/api/version" { |
| + io.WriteString(w, `{"build":"abc1234","version":"deadbeef"}`) |
| + return |
| + } |
| + http.NotFound(w, r) |
| + })} |
| + go srv.Serve(ln) |
| + defer srv.Close() |
| + |
| + build, ok := probeVersion(sock) |
| + if !ok || build != "abc1234" { |
| + t.Fatalf("probeVersion served socket: build=%q ok=%v, want abc1234/true", build, ok) |
| + } |
| + |
| + // A socket that doesn't answer → ok=false (asleep / no instance), never a panic. |
| + if _, ok := probeVersion(filepath.Join(dir, "nope.sock")); ok { |
| + t.Fatal("probeVersion of a dead socket returned ok=true") |
| + } |
| +} |