amadan
server: per-instance hibernation opt-in — routes.hibernate gates the activator per host (#659)
LCHAT_HIBERNATE_BUCKET used to flip EVERY instance to activator management at once, which made a production canary impossible. Hibernation is now a per-host flag in the registry: - registry: routes gains hibernate INTEGER NOT NULL DEFAULT 0 (guarded additive ALTER, like next_due); Route.Hibernate plumbed through list/get/upsert, plus a narrow setHibernate so flips can't clobber concurrent next_due writes. dueInstances only returns hibernating hosts — the due index must never spawn a child against a systemd unit. - router: an upstream hibernates only when act != nil && kind=instance && Hibernate — so with the env set and no route flagged the router behaves exactly as today (binary + env ship first, flags flip per instance). reload() now also reconciles the activator (retireExcept): a host un-flagged or removed mid-flight gets its child stopped, checkpointed, and its lease released. - add-instance --hibernate: registers the route flagged and skips only the systemd half (enable --now + health check). Everything the activator's spawn needs still materializes: env file (admin token), staged binary + live symlink, ready row + router reload; the public-readiness probe is the first wake and the setup-invite mint rides the woken socket. - lchat hibernate <host> --on/--off flips an existing instance, ordered so the host is never unserveable (flag → stop unit → reload → warm wake; flag → reload/release → wait for the socket to go quiet → enable unit). lchat instances grows a HIBERNATE column. - deploy/rollback/auto-update refuse or skip hibernating hosts (no unit to restart; starting one would fight the child for the socket). - controld: provisioner-created spaces stay Hibernate=false for now.
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
11 files changed,
+837
−43
activator.go+56 −4activator_live_test.go+3 −1activator_test.go+147 −2control.go+227 −13control_test.go+321 −3controld.go+3 −0main.go+3 −0registry.go+33 −9router.go+27 −11updater.go+8 −0versions.go+9 −0
diff --git a/activator.go b/activator.go| index f91e509..79e23df 100644 |
| --- a/activator.go |
| +++ b/activator.go |
| @@ -15,10 +15,15 @@ package main |
| // Scope (this phase): ONE box. No gossip/HRW placement, no peer proxying — |
| // membership of one, hash picks self, leases uncontested except against a |
| // previous incarnation of ourselves. Explicit non-goals held for follow-ups: |
| -// - No bot sidecar co-spawn: a hibernating instance's sidecar isn't managed. |
| -// - No migration of existing fleet instances: setting the env on a box whose |
| -// instances run under systemd is unsupported (the units would fight the |
| -// activator over sockets and DBs). |
| +// - No bot sidecar co-spawn: a hibernating instance's sidecar isn't managed |
| +// (a running sidecar's WebSocket also holds the instance awake forever). |
| +// |
| +// Opt-in is PER HOST (routes.hibernate, registry.go): with the bucket env set, |
| +// only flagged instances are activator-managed; the rest stay on their systemd |
| +// units and are proxied straight. That's what makes a production canary safe — |
| +// the env ships fleet-wide as a no-op, then `lchat hibernate <host> --on` |
| +// (control.go) migrates exactly one instance: its unit stops, the flag flips, |
| +// and the first request wakes it here. |
| // |
| // The due-time index (a former non-goal, now in): scheduled future-messages/ |
| // reminders fire on time even while their instance sleeps. At sleep the stop |
| @@ -534,6 +539,53 @@ func (a *activatorSvc) sweepOnce() { |
| a.mu.Unlock() |
| } |
| +// retireExcept stops (final checkpoint + lease release) every live entry whose |
| +// host is not in keep — the reload-time reconcile behind `lchat hibernate |
| +// <host> --off` (and, as a bonus, route removal while awake). The flip clears |
| +// the route's flag and SIGHUPs the router; the map swap stops routing requests |
| +// through the activator, and this puts the child down and gives the S3 lease |
| +// back so the host's systemd unit can bind the socket. It reuses the ordinary |
| +// stop path — the narrowest hook that exists — so the release still records |
| +// next_due and checkpoints to S3 like any sleep (the flip-off's caller waits |
| +// for the socket to go quiet, not for the checkpoint; see hibernateOff). |
| +// In-flight requests on a retired child are cut off as a deploy restart would |
| +// cut them — the flip is a deliberate operator action. |
| +func (a *activatorSvc) retireExcept(keep map[string]bool) { |
| + a.mu.Lock() |
| + var waking []*actEntry |
| + for _, e := range a.entries { |
| + if keep[e.host] { |
| + continue |
| + } |
| + switch e.state { |
| + case actAwake: |
| + log.Printf("activator: %s is no longer hibernate-managed — releasing", e.host) |
| + a.beginStopLocked(e) |
| + case actWaking: |
| + // A wake can't be stopped mid-flight; finish the retire once it |
| + // resolves (below, off-lock). |
| + waking = append(waking, e) |
| + case actStopping: |
| + // Already on its way out. |
| + } |
| + } |
| + a.mu.Unlock() |
| + for _, e := range waking { |
| + go func(e *actEntry) { |
| + <-e.ready |
| + a.mu.Lock() |
| + defer a.mu.Unlock() |
| + // Only if THIS entry is still the live one and its wake succeeded — |
| + // a failed wake already removed itself, and a fresh entry for the |
| + // same host means the route was re-flagged in the meantime. |
| + if a.entries[e.host] == e && e.state == actAwake { |
| + log.Printf("activator: %s is no longer hibernate-managed — releasing (wake finished first)", e.host) |
| + a.beginStopLocked(e) |
| + } |
| + }(e) |
| + } |
| +} |
| + |
| // shutdown checkpoints and stops every child — the router going down takes its |
| // instances with it (they're its children; leaving them orphaned would strand |
| // the leases until TTL expiry and leave the sockets bound against the next |
diff --git a/activator_live_test.go b/activator_live_test.go| index d7efbd0..e723359 100644 |
| --- a/activator_live_test.go |
| +++ b/activator_live_test.go |
| @@ -95,7 +95,9 @@ func TestActivatorLiveHibernation(t *testing.T) { |
| } |
| route := Route{Host: host, Kind: "instance", |
| Socket: filepath.Join(sockDir, host+".sock"), |
| - DBPath: filepath.Join(dataDir, host+".db")} |
| + DBPath: filepath.Join(dataDir, host+".db"), |
| + // Per-host opt-in: only hibernate-flagged routes are activator-managed. |
| + Hibernate: true} |
| if err := reg.upsert(route); err != nil { |
| t.Fatalf("upsert: %v", err) |
| } |
diff --git a/activator_test.go b/activator_test.go| index 3973c79..8735d89 100644 |
| --- a/activator_test.go |
| +++ b/activator_test.go |
| @@ -199,8 +199,11 @@ func testActivator(t *testing.T, idle time.Duration) (*activatorSvc, *fakeBacken |
| } |
| func testRoute(host string) Route { |
| + // Hibernate is set because these routes model activator-managed hosts — |
| + // the per-host gate (dueInstances, the router's upstream flag) only acts |
| + // on flagged instances. |
| return Route{Host: host, Kind: "instance", Socket: "/tmp/nope.sock", |
| - DBPath: filepath.Join("/tmp", host+".db")} |
| + DBPath: filepath.Join("/tmp", host+".db"), Hibernate: true} |
| } |
| // (waitFor — poll until a condition holds — is shared from bot_run_test.go; |
| @@ -432,7 +435,7 @@ func TestActivatorLeaseContention503(t *testing.T) { |
| if err != nil { |
| t.Fatalf("open registry: %v", err) |
| } |
| - if err := reg.upsert(Route{Host: "held.example.com", Kind: "instance", Socket: "/tmp/held.sock"}); err != nil { |
| + if err := reg.upsert(Route{Host: "held.example.com", Kind: "instance", Socket: "/tmp/held.sock", Hibernate: true}); err != nil { |
| t.Fatalf("upsert: %v", err) |
| } |
| rt, err := newRouter(reg, a) |
| @@ -460,6 +463,148 @@ func TestActivatorLeaseContention503(t *testing.T) { |
| } |
| } |
| +// Per-host gating (#550 Part 4 v2 opt-in): with the activator CONFIGURED, only |
| +// a hibernate-flagged instance goes through the wake path; an unflagged one is |
| +// proxied straight, exactly as if there were no activator at all — the |
| +// deploy-safety property that lets the bucket env ship fleet-wide before any |
| +// host is flagged. |
| +func TestRouterPerHostHibernateGating(t *testing.T) { |
| + a, b, _, reg := testActivator(t, time.Minute) |
| + |
| + // Two live backends playing the instances. The flagged host's listener |
| + // stands in for the child a wake would have spawned (the fake backend's |
| + // start() spawns nothing). |
| + bind := func(name string) string { |
| + sock := tmpSock(t) |
| + ln, err := net.Listen("unix", sock) |
| + if err != nil { |
| + t.Fatalf("listen unix: %v", err) |
| + } |
| + srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| + io.WriteString(w, name) |
| + })} |
| + go srv.Serve(ln) |
| + t.Cleanup(func() { srv.Close() }) |
| + return sock |
| + } |
| + if err := reg.upsert(Route{Host: "hib.example.com", Kind: "instance", Socket: bind("hib"), Hibernate: true}); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := reg.upsert(Route{Host: "plain.example.com", Kind: "instance", Socket: bind("plain")}); err != nil { |
| + t.Fatal(err) |
| + } |
| + |
| + rt, err := newRouter(reg, a) |
| + if err != nil { |
| + t.Fatalf("new router: %v", err) |
| + } |
| + front := httptest.NewServer(rt) |
| + t.Cleanup(front.Close) |
| + |
| + // The unflagged host: proxied straight, the activator never touched. |
| + if body, code := proxyGet(t, front.URL, "plain.example.com"); code != 200 || body != "plain" { |
| + t.Fatalf("unflagged host: got %d %q, want 200 plain", code, body) |
| + } |
| + if acquires, _, starts := b.counts(); acquires != 0 || starts != 0 { |
| + t.Fatalf("unflagged instance touched the activator (acquires=%d starts=%d)", acquires, starts) |
| + } |
| + |
| + // The flagged host: served through a full wake — lease, restore, spawn. |
| + if body, code := proxyGet(t, front.URL, "hib.example.com"); code != 200 || body != "hib" { |
| + t.Fatalf("flagged host: got %d %q, want 200 hib", code, body) |
| + } |
| + if acquires, restores, starts := b.counts(); acquires != 1 || restores != 1 || starts != 1 { |
| + t.Fatalf("flagged host skipped the wake path: acquires=%d restores=%d starts=%d (want 1 each)", acquires, restores, starts) |
| + } |
| +} |
| + |
| +// A reload that finds an awake host no longer hibernate-flagged retires its |
| +// child — stopped, final-checkpointed (replication closed), entry forgotten — |
| +// the router-process half of `lchat hibernate --off`. And once the flag is |
| +// off, a recorded next_due no longer wakes it: the due index only acts on |
| +// hibernating hosts, so a systemd-managed instance can never be spawned |
| +// against its own unit. |
| +func TestActivatorReloadRetiresUnflaggedHost(t *testing.T) { |
| + a, b, clk, reg := testActivator(t, time.Hour) |
| + route := testRoute("flip.example.com") |
| + if err := reg.upsert(route); err != nil { |
| + t.Fatal(err) |
| + } |
| + rt, err := newRouter(reg, a) |
| + if err != nil { |
| + t.Fatalf("new router: %v", err) |
| + } |
| + |
| + release, err := a.beginRequest(context.Background(), route) |
| + if err != nil { |
| + t.Fatalf("beginRequest: %v", err) |
| + } |
| + release() |
| + // The retire's sleep re-records next_due AFTER the flag is cleared; an |
| + // already-past value must stay inert under the flag filter. |
| + b.setDue(clk.now().Add(-5 * time.Second).Unix()) |
| + |
| + if err := reg.setHibernate("flip.example.com", false); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := rt.reload(); err != nil { |
| + t.Fatalf("reload: %v", err) |
| + } |
| + waitFor(t, "retire", func() bool { return a.entryCount() == 0 }) |
| + if !b.lastProc().isStopped() { |
| + t.Fatal("retired child was not stopped") |
| + } |
| + if !b.lastRep().isClosed() { |
| + t.Fatal("retired child's replication was not closed (no final checkpoint/lease release)") |
| + } |
| + |
| + // The due sweeper must not resurrect a host that's back under systemd. |
| + a.sweepOnce() |
| + if _, _, starts := b.counts(); starts != 1 { |
| + t.Fatalf("starts=%d after retire, want 1 (no due wake for a systemd-managed host)", starts) |
| + } |
| +} |
| + |
| +// A retire that lands mid-wake can't stop the child yet — it finishes the job |
| +// once the wake resolves, so a flip-off during the small wake window still |
| +// releases cleanly. |
| +func TestActivatorRetireDuringWake(t *testing.T) { |
| + a, b, clk, reg := testActivator(t, time.Hour) |
| + route := testRoute("mid.example.com") |
| + if err := reg.upsert(route); err != nil { |
| + t.Fatal(err) |
| + } |
| + rt, err := newRouter(reg, a) |
| + if err != nil { |
| + t.Fatalf("new router: %v", err) |
| + } |
| + |
| + // Start a request-less (due-time) wake and hold it in the waking state. |
| + b.startGate = make(chan struct{}) |
| + if err := reg.setNextDue("mid.example.com", clk.now().Add(-time.Second).Unix()); err != nil { |
| + t.Fatal(err) |
| + } |
| + a.sweepOnce() |
| + waitFor(t, "the wake to begin", func() bool { _, _, s := b.counts(); return s == 1 }) |
| + |
| + // Flip off mid-wake: the reload's retire must defer, not deadlock. |
| + if err := reg.setHibernate("mid.example.com", false); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := rt.reload(); err != nil { |
| + t.Fatalf("reload: %v", err) |
| + } |
| + close(b.startGate) |
| + |
| + waitFor(t, "post-wake retire", func() bool { return a.entryCount() == 0 }) |
| + if !b.lastProc().isStopped() { |
| + t.Fatal("child woken mid-retire was not stopped") |
| + } |
| + if !b.lastRep().isClosed() { |
| + t.Fatal("child woken mid-retire left replication open (lease never released)") |
| + } |
| +} |
| + |
| // With hibernation on, sibling services (kind "service") are untouched: the |
| // router proxies them directly, no wake, no activity tracking. |
| func TestActivatorServiceRoutesBypass(t *testing.T) { |
diff --git a/control.go b/control.go| index 2084331..01a75eb 100644 |
| --- a/control.go |
| +++ b/control.go |
| @@ -117,8 +117,12 @@ type addInstanceOpts struct { |
| PushRelayToken string |
| HomeRegion string // geo seam (#550 Part 7): "" = the single implicit region |
| Provider string // co-op seam (#550 Part 10): "" = the single implicit provider |
| - UnitTmpl string // default "lchat@" |
| - RouterUnit string // default "lchat-router" |
| + // Hibernate registers the instance as activator-managed (#550 Part 4 v2): |
| + // no systemd unit is created — the router wakes it on first request. Needs |
| + // the router running with LCHAT_HIBERNATE_BUCKET, or the host never answers. |
| + Hibernate bool |
| + UnitTmpl string // default "lchat@" |
| + RouterUnit string // default "lchat-router" |
| } |
| // addInstanceResult is what a successful creation yields. SetupLink may be empty |
| @@ -183,7 +187,7 @@ func registerInstance(reg *Registry, opts addInstanceOpts) (Route, string, error |
| route := Route{Host: host, Kind: "instance", Socket: socket, |
| DBPath: filepath.Join(instanceDataDir, host+".db"), Status: "creating", |
| AppStoreURL: opts.AppStoreURL, HomeOrigin: opts.HomeOrigin, PushRelayURL: opts.PushRelayURL, |
| - HomeRegion: opts.HomeRegion, Provider: opts.Provider} |
| + HomeRegion: opts.HomeRegion, Provider: opts.Provider, Hibernate: opts.Hibernate} |
| if err := reg.upsert(route); err != nil { |
| return Route{}, "", fmt.Errorf("register: %v", err) |
| } |
| @@ -239,14 +243,32 @@ func materializeInstance(reg *Registry, route Route, token string, opts addInsta |
| } |
| route.Version = ver |
| - unit := unitTmpl + host |
| - if err := systemctl("enable", "--now", unit); err != nil { |
| - markInstanceFailed(reg, host, unit, "start: "+err.Error()) |
| - return nil, fmt.Errorf("start %s: %v", unit, err) |
| - } |
| - if err := waitHealthy(socket, 15*time.Second); err != nil { |
| - markInstanceFailed(reg, host, unit, "health: "+err.Error()) |
| - return nil, err |
| + // The systemd half — the ONLY part a hibernating instance skips. The split |
| + // is derived from what the activator's spawn (s3Backend.start) needs at |
| + // first wake: the env file above (the spawn layers /data/<host>.env over a |
| + // clean env, and the admin token's plaintext lives nowhere else — without |
| + // it the woken instance would mint its own token and every saved admin |
| + // link would be dead); the staged binary + live symlink above (the spawn |
| + // execs /opt/lchat/live/<host>, and staging here keeps the instance under |
| + // normal version management instead of self-healing onto the router's own |
| + // build); and the ready row + router reload below (or no request can ever |
| + // reach the host to wake it). What a hibernating instance must NOT have is |
| + // a unit holding the socket — the instance refuses to clobber a live |
| + // listener (listen() in main.go), so a running unit would turn every wake |
| + // into "socket already in use". No DB is created either: the first wake |
| + // finds no replica in S3 and boots fresh. And there's no health check here |
| + // because nothing runs yet — the public-readiness probe below is the first |
| + // request, and the wake it triggers is the health check. |
| + if !route.Hibernate { |
| + unit := unitTmpl + host |
| + if err := systemctl("enable", "--now", unit); err != nil { |
| + markInstanceFailed(reg, host, unit, "start: "+err.Error()) |
| + return nil, fmt.Errorf("start %s: %v", unit, err) |
| + } |
| + if err := waitHealthy(socket, 15*time.Second); err != nil { |
| + markInstanceFailed(reg, host, unit, "health: "+err.Error()) |
| + return nil, err |
| + } |
| } |
| route.Status = "ready" |
| @@ -264,6 +286,13 @@ func materializeInstance(reg *Registry, route Route, token string, opts addInsta |
| // first visitor doesn't race on-demand issuance. Best-effort: on failure we |
| // still return (the instance passed its own health check); the public path is |
| // almost certainly up a beat later. |
| + // |
| + // For a hibernating instance this probe is also its FIRST WAKE: the request |
| + // makes the router's activator spawn the child, which binds the socket the |
| + // setup-invite mint below dials. If the wake never lands (e.g. the router |
| + // isn't running with LCHAT_HIBERNATE_BUCKET), the mint fails and we degrade |
| + // to the usual "setup link unavailable" path — the route and admin link |
| + // still stand. |
| confirmPublicReady(host, routerHTTPSAddr) |
| // The first-user setup link: claiming it makes you the instance's admin and |
| @@ -290,6 +319,7 @@ func runAddInstance(args []string) { |
| homeOrigin := fs.String("home-origin", "", "per-instance LCHAT_HOME_ORIGIN") |
| pushRelayURL := fs.String("push-relay-url", "", "per-instance LCHAT_PUSH_RELAY_URL") |
| pushRelayToken := fs.String("push-relay-token", "", "per-instance LCHAT_PUSH_RELAY_TOKEN") |
| + hibernate := fs.Bool("hibernate", false, "register as activator-managed (no systemd unit; the router wakes it on demand — requires the router to run with LCHAT_HIBERNATE_BUCKET)") |
| _ = fs.Parse(reorderFlagsFirst(fs, args)) |
| reg, err := openRegistry(*registryPath) |
| @@ -303,6 +333,7 @@ func runAddInstance(args []string) { |
| HomeOrigin: *homeOrigin, |
| PushRelayURL: *pushRelayURL, |
| PushRelayToken: *pushRelayToken, |
| + Hibernate: *hibernate, |
| UnitTmpl: *unitTmpl, |
| RouterUnit: *routerUnit, |
| }) |
| @@ -311,6 +342,9 @@ func runAddInstance(args []string) { |
| } |
| fmt.Printf("added %s\n", res.Host) |
| + if *hibernate { |
| + fmt.Printf(" hibernating: no systemd unit — the router wakes it per request and sleeps it when idle\n") |
| + } |
| if res.SetupErr != nil { |
| fmt.Printf(" (setup link unavailable: %v — mint later: lchat setup-invite %s)\n", res.SetupErr, res.Host) |
| } else { |
| @@ -481,19 +515,189 @@ func runInstances(args []string) { |
| fmt.Println("(no routes)") |
| return |
| } |
| - fmt.Printf("%-34s %-9s %-9s %-12s %s\n", "HOST", "KIND", "STATUS", "VERSION", "SOCKET") |
| + fmt.Printf("%-34s %-9s %-9s %-12s %-9s %s\n", "HOST", "KIND", "STATUS", "VERSION", "HIBERNATE", "SOCKET") |
| for _, r := range routes { |
| ver := r.Version |
| if ver == "" { |
| ver = "-" |
| } |
| - fmt.Printf("%-34s %-9s %-9s %-12s %s\n", r.Host, r.Kind, r.Status, ver, r.Socket) |
| + // "-" for the common (systemd-managed) case keeps the table quiet; |
| + // hibernating hosts stand out at a glance. |
| + hib := "-" |
| + if r.Hibernate { |
| + hib = "yes" |
| + } |
| + fmt.Printf("%-34s %-9s %-9s %-12s %-9s %s\n", r.Host, r.Kind, r.Status, ver, hib, r.Socket) |
| if r.LastError != "" { |
| fmt.Printf(" last error: %s\n", r.LastError) |
| } |
| } |
| } |
| +// runHibernate: `lchat hibernate <host> --on|--off` — flip one instance between |
| +// systemd-managed (always-on) and activator-managed (hibernating). Per-host by |
| +// design: a production canary can hibernate while the rest of the fleet stays |
| +// on its units. --on requires the router to be running with |
| +// LCHAT_HIBERNATE_BUCKET, or the host will sit unwakeable once its unit stops. |
| +func runHibernate(args []string) { |
| + fs := flag.NewFlagSet("hibernate", flag.ExitOnError) |
| + registryPath := fs.String("registry", envOr("LCHAT_REGISTRY", "/data/registry.db"), "router registry SQLite path") |
| + unitTmpl := fs.String("unit", envOr("LCHAT_UNIT", "lchat@"), "systemd instance unit template prefix") |
| + routerUnit := fs.String("router-unit", envOr("LCHAT_ROUTER_UNIT", "lchat-router"), "router systemd unit to reload") |
| + on := fs.Bool("on", false, "hand the instance to the activator: stop+disable its unit; the router wakes it per request") |
| + off := fs.Bool("off", false, "hand the instance back to systemd: the activator releases it, then its unit is enabled+started") |
| + _ = fs.Parse(reorderFlagsFirst(fs, args)) |
| + |
| + if *on == *off { |
| + die("hibernate: pass exactly one of --on / --off") |
| + } |
| + host := strings.ToLower(strings.TrimSpace(fs.Arg(0))) |
| + if !validHost(host) { |
| + die("hibernate: invalid host %q", host) |
| + } |
| + reg, err := openRegistry(*registryPath) |
| + if err != nil { |
| + die("hibernate: open registry: %v", err) |
| + } |
| + rt, err := reg.get(host) |
| + if err != nil { |
| + die("hibernate: %v", err) |
| + } |
| + if rt == nil { |
| + die("hibernate: unknown host %q", host) |
| + } |
| + if rt.Kind != "instance" { |
| + die("hibernate: %q is a %s route, not an instance", host, rt.Kind) |
| + } |
| + if rt.Status != "ready" { |
| + die("hibernate: %s is %s, not ready — fix the instance first", host, rt.Status) |
| + } |
| + if *on { |
| + if rt.Hibernate { |
| + fmt.Printf("%s is already hibernating\n", host) |
| + return |
| + } |
| + if err := hibernateOn(reg, rt, *unitTmpl, *routerUnit); err != nil { |
| + die("hibernate %s --on: %v", host, err) |
| + } |
| + fmt.Printf("%s is now activator-managed: unit stopped+disabled; the router wakes it per request and sleeps it when idle\n", host) |
| + fmt.Printf(" note: a bot sidecar (lchat-bot@%s) keeps a WebSocket open and would hold it awake forever — stop the sidecar if this host should actually sleep\n", host) |
| + return |
| + } |
| + if !rt.Hibernate { |
| + fmt.Printf("%s is not hibernating\n", host) |
| + return |
| + } |
| + if err := hibernateOff(reg, rt, *unitTmpl, *routerUnit); err != nil { |
| + die("hibernate %s --off: %v", host, err) |
| + } |
| + fmt.Printf("%s is back under systemd: unit enabled and healthy; the activator has released it\n", host) |
| +} |
| + |
| +// hibernateOn flips a running, systemd-managed instance to activator-managed. |
| +// Ordering — chosen so the host is never left unserveable: |
| +// |
| +// 1. Flag the route. A registry write alone changes nothing live (the |
| +// router's map only swaps on reload), so a crash right after leaves the |
| +// unit serving exactly as before. |
| +// 2. Stop + disable the unit — BEFORE the reload, because once the route is |
| +// activator-managed the first wake's child must be able to bind the |
| +// socket, and the instance refuses to clobber a live listener (listen() |
| +// in main.go): a still-running unit would turn every wake into "socket |
| +// already in use" 503s. This opens the one unavoidable gap — the socket |
| +// is dead until the wake in step 4 — and requests in it ride the router's |
| +// 13s dial-retry window, the same shape as a deploy restart. Disable (not |
| +// just stop) so a reboot can't resurrect the unit to fight the activator. |
| +// 3. Reload the router: the route becomes activator-managed. |
| +// 4. A warm request through the public path — best-effort, but it both |
| +// closes the gap immediately (the wake binds the socket, so any retrying |
| +// dials from step 2 connect) and proves the wake path works while the |
| +// operator is still watching. |
| +// |
| +// Nothing is checkpointed here and no S3 is touched: the DB is already local, |
| +// so the first wake's restore sees it as the warm cache and skips; replication |
| +// under the fresh lease is what backfills S3. |
| +func hibernateOn(reg *Registry, rt *Route, unitTmpl, routerUnit string) error { |
| + if err := reg.setHibernate(rt.Host, true); err != nil { |
| + return fmt.Errorf("flag: %v", err) |
| + } |
| + if err := systemctl("disable", "--now", unitTmpl+rt.Host); err != nil { |
| + return fmt.Errorf("stop unit: %v (flag is set but the unit still runs — the host serves as before; re-run, or undo with `lchat hibernate %s --off`)", err, rt.Host) |
| + } |
| + if err := systemctl("reload", routerUnit); err != nil { |
| + return fmt.Errorf("reload %s: %v (unit is stopped — reload the router by hand to bring the host back)", routerUnit, err) |
| + } |
| + confirmPublicReady(rt.Host, routerHTTPSAddr) |
| + return nil |
| +} |
| + |
| +// hibernateOff flips an activator-managed instance back under systemd. The |
| +// mirror of hibernateOn, with one extra wait: |
| +// |
| +// 1. Clear the flag (inert until reload). |
| +// 2. Reload the router: the map swap stops routing requests through the |
| +// activator, and the reload-time reconcile (retireExcept, activator.go) |
| +// makes the activator release the host — stop the child if it's awake, |
| +// take the final checkpoint, give the S3 lease back. |
| +// 3. WAIT for the child to actually let go of the socket before starting the |
| +// unit. The release is asynchronous inside the router process and the |
| +// instance refuses to start over a live listener, so enabling the unit too |
| +// early would just crash-loop it against the child. Dial-probing the |
| +// socket is the narrowest thing observable from outside the router: quiet |
| +// means the child's listener is gone (an already-asleep host has none, so |
| +// this returns on the first probe). The final S3 checkpoint may still be |
| +// finishing when the unit starts — that's fine: the replica goes stale by |
| +// design once the flag is off, and a later flip back on wakes from the |
| +// local DB (the warm cache, which is the newer truth). |
| +// 4. Enable + start the unit and health-check it, exactly like add-instance. |
| +// |
| +// Requests between the child's stop and the unit binding ride the router's 13s |
| +// dial-retry window, same as a deploy restart. |
| +func hibernateOff(reg *Registry, rt *Route, unitTmpl, routerUnit string) error { |
| + if err := reg.setHibernate(rt.Host, false); err != nil { |
| + return fmt.Errorf("flag: %v", err) |
| + } |
| + if err := systemctl("reload", routerUnit); err != nil { |
| + return fmt.Errorf("reload %s: %v (flag is cleared; reload by hand, then re-run)", routerUnit, err) |
| + } |
| + if err := waitSocketQuiet(rt.Socket, hibernateReleaseTimeout); err != nil { |
| + return fmt.Errorf("%v — the activator hasn't released %s yet (a long final checkpoint?); re-run once it has", err, rt.Host) |
| + } |
| + unit := unitTmpl + rt.Host |
| + if err := systemctl("enable", "--now", unit); err != nil { |
| + return fmt.Errorf("start %s: %v", unit, err) |
| + } |
| + if err := waitHealthy(rt.Socket, 20*time.Second); err != nil { |
| + return fmt.Errorf("%v (unit started but unhealthy — check journalctl -u %s)", err, unit) |
| + } |
| + return nil |
| +} |
| + |
| +// hibernateReleaseTimeout bounds how long a flip-off waits for the activator to |
| +// release the child: the stop path's own budgets (next-due probe 5s + SIGTERM |
| +// drain 20s + final checkpoint 30s) total under a minute; leave headroom. A var |
| +// so tests can shrink the failure path. |
| +var hibernateReleaseTimeout = 90 * time.Second |
| + |
| +// waitSocketQuiet blocks until nothing accepts on the socket (or the timeout |
| +// elapses) — how the flip-off observes, from outside the router process, that |
| +// the activator's child has released the socket. A dial failure is the "quiet" |
| +// signal whether the socket file lingers (ECONNREFUSED) or not (ENOENT). |
| +func waitSocketQuiet(socket string, timeout time.Duration) error { |
| + deadline := time.Now().Add(timeout) |
| + for { |
| + c, err := net.DialTimeout("unix", socket, 500*time.Millisecond) |
| + if err != nil { |
| + return nil |
| + } |
| + c.Close() |
| + if time.Now().After(deadline) { |
| + return fmt.Errorf("socket %s still has a listener after %s", socket, timeout) |
| + } |
| + time.Sleep(250 * time.Millisecond) |
| + } |
| +} |
| + |
| // runAdminLink: `lchat admin-link <host>` — recover the admin link from the |
| // instance's EnvironmentFile (the DB holds only the token's hash). |
| func runAdminLink(args []string) { |
| @@ -575,6 +779,16 @@ func runDeploy(args []string) { |
| if rt.Kind != "instance" { |
| die("deploy %s: not an instance (kind %q) — only messenger instances are deployed", host, rt.Kind) |
| } |
| + if rt.Hibernate { |
| + // No unit to restart — and starting one would fight the activator's |
| + // child for the socket. Skip quietly in a fleet promote; refuse |
| + // loudly when the operator named this host. |
| + if *all { |
| + fmt.Printf("skipped %s (hibernating — flip it back to deploy: lchat hibernate %s --off)\n", host, host) |
| + continue |
| + } |
| + die("deploy %s: instance is hibernating (activator-managed, no unit) — flip it back first: lchat hibernate %s --off", host, host) |
| + } |
| unit := *unitTmpl + host |
| if ver == "" { |
diff --git a/control_test.go b/control_test.go| index 1408efa..9d5791e 100644 |
| --- a/control_test.go |
| +++ b/control_test.go |
| @@ -3,11 +3,14 @@ package main |
| import ( |
| "database/sql" |
| "flag" |
| + "net" |
| + "net/http" |
| "os" |
| "path/filepath" |
| "reflect" |
| "strings" |
| "testing" |
| + "time" |
| ) |
| func TestFlagsAfterPositional(t *testing.T) { |
| @@ -284,10 +287,11 @@ func TestRegistryDueTimeIndex(t *testing.T) { |
| t.Fatal(err) |
| } |
| seed := []Route{ |
| - {Host: "sleepy.example.com", Kind: "instance", Socket: "/run/a.sock"}, |
| - {Host: "quiet.example.com", Kind: "instance", Socket: "/run/b.sock"}, |
| + {Host: "sleepy.example.com", Kind: "instance", Socket: "/run/a.sock", Hibernate: true}, |
| + {Host: "quiet.example.com", Kind: "instance", Socket: "/run/b.sock", Hibernate: true}, |
| {Host: "svc.example.com", Kind: "service", Socket: "/run/c.sock"}, |
| - {Host: "broken.example.com", Kind: "instance", Socket: "/run/d.sock", Status: "failed"}, |
| + {Host: "broken.example.com", Kind: "instance", Socket: "/run/d.sock", Status: "failed", Hibernate: true}, |
| + {Host: "manual.example.com", Kind: "instance", Socket: "/run/e.sock"}, // systemd-managed (hibernate off) |
| } |
| for _, rt := range seed { |
| if err := reg.upsert(rt); err != nil { |
| @@ -299,6 +303,7 @@ func TestRegistryDueTimeIndex(t *testing.T) { |
| "sleepy.example.com": now - 5, // due |
| "svc.example.com": now - 5, // due but a service — never woken |
| "broken.example.com": now - 5, // due but not ready — never woken |
| + "manual.example.com": now - 5, // due but NOT hibernating — always on, never due-woken |
| // quiet: next_due stays 0 — nothing scheduled, never due-woken |
| } { |
| if err := reg.setNextDue(host, due); err != nil { |
| @@ -334,8 +339,321 @@ func TestRegistryDueTimeIndex(t *testing.T) { |
| t.Fatalf("route rewrite clobbered next_due: %d, want %d", got.NextDue, now+60) |
| } |
| + // The narrow setters stay orthogonal: the rewrite above carried the |
| + // hibernate flag, and setHibernate must not touch next_due. |
| + if got, _ := reg.get("sleepy.example.com"); !got.Hibernate { |
| + t.Fatal("route rewrite dropped the hibernate flag") |
| + } |
| + if err := reg.setHibernate("sleepy.example.com", false); err != nil { |
| + t.Fatalf("setHibernate: %v", err) |
| + } |
| + if got, _ := reg.get("sleepy.example.com"); got.Hibernate || got.NextDue != now+60 { |
| + t.Fatalf("setHibernate clobbered the route: hibernate=%v next_due=%d", got.Hibernate, got.NextDue) |
| + } |
| + |
| // An unknown host is a quiet no-op — it can't be woken anyway. |
| if err := reg.setNextDue("ghost.example.com", now); err != nil { |
| t.Fatalf("setNextDue unknown host: %v", err) |
| } |
| } |
| + |
| +// A registry created before the hibernate column existed must gain it on open, |
| +// with existing rows reading as hibernate=off — the live fleet's registry is |
| +// exactly such a DB, so the migration must default every real instance to the |
| +// classic systemd shape (the whole point of the per-host opt-in). |
| +func TestRegistryMigratesHibernateColumn(t *testing.T) { |
| + path := filepath.Join(t.TempDir(), "reg.db") |
| + |
| + // The schema as of the due-time index — everything current EXCEPT hibernate. |
| + old, err := sql.Open("sqlite", path) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := old.Exec(` |
| +CREATE TABLE routes ( |
| + host TEXT PRIMARY KEY, |
| + kind TEXT NOT NULL, |
| + socket TEXT NOT NULL, |
| + status TEXT NOT NULL DEFAULT 'ready', |
| + last_error TEXT NOT NULL DEFAULT '', |
| + db_path TEXT NOT NULL DEFAULT '', |
| + version TEXT NOT NULL DEFAULT '', |
| + prev_version TEXT NOT NULL DEFAULT '', |
| + app_store_url TEXT NOT NULL DEFAULT '', |
| + home_origin TEXT NOT NULL DEFAULT '', |
| + push_relay_url TEXT NOT NULL DEFAULT '', |
| + push_relay_token TEXT NOT NULL DEFAULT '', |
| + home_region TEXT NOT NULL DEFAULT '', |
| + provider TEXT NOT NULL DEFAULT '', |
| + next_due INTEGER NOT NULL DEFAULT 0, |
| + created_at INTEGER NOT NULL |
| +); |
| +INSERT INTO routes (host, kind, socket, created_at) VALUES ('old.example.com', 'instance', '/run/lchat/old.sock', 1700000000); |
| +`); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := old.Close(); err != nil { |
| + t.Fatal(err) |
| + } |
| + |
| + reg, err := openRegistry(path) |
| + if err != nil { |
| + t.Fatalf("open over pre-hibernate registry: %v", err) |
| + } |
| + rt, err := reg.get("old.example.com") |
| + if err != nil || rt == nil { |
| + t.Fatalf("get pre-existing row: %+v, %v", rt, err) |
| + } |
| + if rt.Hibernate { |
| + t.Error("pre-existing row: hibernate=true, want the systemd-managed default (false)") |
| + } |
| + |
| + // The flag round-trips through both the narrow setter and a full upsert. |
| + if err := reg.setHibernate("old.example.com", true); err != nil { |
| + t.Fatalf("setHibernate: %v", err) |
| + } |
| + if got, _ := reg.get("old.example.com"); got == nil || !got.Hibernate { |
| + t.Fatalf("after setHibernate, route = %+v, want hibernate on", got) |
| + } |
| + rt, _ = reg.get("old.example.com") |
| + rt.Hibernate = false |
| + if err := reg.upsert(*rt); err != nil { |
| + t.Fatalf("upsert with hibernate: %v", err) |
| + } |
| + if got, _ := reg.get("old.example.com"); got == nil || got.Hibernate { |
| + t.Fatalf("after upsert, route = %+v, want hibernate off", got) |
| + } |
| + |
| + // migrate is idempotent — reopening must not trip on the added column. |
| + if _, err := openRegistry(path); err != nil { |
| + t.Errorf("reopen after migration: %v", err) |
| + } |
| +} |
| + |
| +// `add-instance --hibernate` produces a registered, unmaterialized-but- |
| +// spawnable route: everything the activator's spawn needs at first wake (the |
| +// env file with the admin token, a staged binary behind the live symlink, a |
| +// ready+flagged route with the router reloaded) and nothing systemd (no unit |
| +// enabled, no add-time health check, no DB — the first wake boots fresh). The |
| +// public-readiness probe doubles as that first wake — simulated here by |
| +// binding the socket when it fires — and the setup-invite mint rides the |
| +// woken socket, so the space is claimable the moment add-instance returns. |
| +func TestAddInstanceHibernate(t *testing.T) { |
| + reg := controlTestEnv(t) |
| + const host = "hib.example.com" |
| + |
| + // Replace the harness's systemd fake with a pure recorder: the hibernating |
| + // path must never enable a unit, so nothing here binds the socket for it. |
| + var calls []string |
| + systemctl = func(args ...string) error { |
| + calls = append(calls, strings.Join(args, " ")) |
| + return nil |
| + } |
| + // Play the router+activator: the readiness probe is the first request, and |
| + // the wake it triggers binds the instance socket — answering the health |
| + // check and the invite mint like a freshly spawned child would. |
| + confirmPublicReady = func(h, _ string) { |
| + calls = append(calls, "confirm "+h) |
| + ln, err := net.Listen("unix", filepath.Join(instanceSocketDir, h+".sock")) |
| + if err != nil { |
| + t.Errorf("bind woken socket: %v", err) |
| + return |
| + } |
| + mux := http.NewServeMux() |
| + mux.HandleFunc("GET /", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(200) }) |
| + mux.HandleFunc("POST /api/admin/key-invites", func(w http.ResponseWriter, r *http.Request) { |
| + writeJSON(w, 200, map[string]any{"ok": true}) |
| + }) |
| + srv := &http.Server{Handler: mux} |
| + go srv.Serve(ln) |
| + t.Cleanup(func() { srv.Close() }) |
| + } |
| + |
| + res, err := addInstance(reg, addInstanceOpts{Host: host, Hibernate: true}) |
| + if err != nil { |
| + t.Fatalf("addInstance --hibernate: %v", err) |
| + } |
| + |
| + // Registered: ready, flagged, under version management. |
| + rt, err := reg.get(host) |
| + if err != nil || rt == nil { |
| + t.Fatalf("route = %+v, %v", rt, err) |
| + } |
| + if rt.Status != "ready" || !rt.Hibernate || rt.Version == "" { |
| + t.Errorf("route = %+v, want a ready hibernating instance with a version", *rt) |
| + } |
| + |
| + // Spawnable: the env file holds the admin token (the spawn's env layering |
| + // and every saved admin link depend on it)… |
| + env, err := readEnvFile(filepath.Join(instanceDataDir, host+".env")) |
| + if err != nil { |
| + t.Fatalf("read env: %v", err) |
| + } |
| + if env["LCHAT_ADMIN_TOKEN"] != res.AdminToken { |
| + t.Error("env token != result token") |
| + } |
| + // …and the live symlink resolves to a staged binary (the spawn execs it). |
| + if ver, err := liveVersion(host); err != nil || ver == "" { |
| + t.Errorf("live symlink not staged: ver=%q err=%v", ver, err) |
| + } |
| + |
| + // Unmaterialized: no unit, no DB; the router WAS reloaded (the route must |
| + // serve before any request can wake it) and only then probed. |
| + reloadIdx, confirmIdx := -1, -1 |
| + for i, c := range calls { |
| + if strings.HasPrefix(c, "enable ") { |
| + t.Errorf("a systemd unit was touched: %q", c) |
| + } |
| + if c == "reload lchat-router" { |
| + reloadIdx = i |
| + } |
| + if c == "confirm "+host { |
| + confirmIdx = i |
| + } |
| + } |
| + if reloadIdx == -1 || confirmIdx == -1 || confirmIdx < reloadIdx { |
| + t.Errorf("want router reload before the wake probe, got calls %v", calls) |
| + } |
| + if _, err := os.Stat(filepath.Join(instanceDataDir, host+".db")); !os.IsNotExist(err) { |
| + t.Error("a DB was created — the first wake should boot fresh") |
| + } |
| + |
| + // Claimable: the setup link was minted through the woken socket. |
| + if !strings.HasPrefix(res.SetupLink, "https://"+host+"/join/setup#") { |
| + t.Errorf("setup link = %q (err %v)", res.SetupLink, res.SetupErr) |
| + } |
| +} |
| + |
| +// The flip-on ordering (hibernateOn): flag first — it's inert until a reload — |
| +// then the unit stops BEFORE the router reloads (a wake must be able to bind |
| +// the socket), then the warm request closes the gap. A crash at any seam |
| +// leaves the host serveable. |
| +func TestHibernateOnOrdering(t *testing.T) { |
| + reg := controlTestEnv(t) |
| + const host = "flip-on.example.com" |
| + if _, err := addInstance(reg, addInstanceOpts{Host: host}); err != nil { |
| + t.Fatalf("addInstance: %v", err) |
| + } |
| + |
| + var calls []string |
| + flagAtDisable, flagAtReload := false, false |
| + systemctl = func(args ...string) error { |
| + call := strings.Join(args, " ") |
| + calls = append(calls, call) |
| + rt, _ := reg.get(host) |
| + switch call { |
| + case "disable --now lchat@" + host: |
| + flagAtDisable = rt.Hibernate |
| + case "reload lchat-router": |
| + flagAtReload = rt.Hibernate |
| + } |
| + return nil |
| + } |
| + confirmPublicReady = func(h, _ string) { calls = append(calls, "confirm "+h) } |
| + |
| + rt, _ := reg.get(host) |
| + if err := hibernateOn(reg, rt, "lchat@", "lchat-router"); err != nil { |
| + t.Fatalf("hibernateOn: %v", err) |
| + } |
| + |
| + want := []string{"disable --now lchat@" + host, "reload lchat-router", "confirm " + host} |
| + if !reflect.DeepEqual(calls, want) { |
| + t.Errorf("flip-on sequence = %v, want %v", calls, want) |
| + } |
| + if !flagAtDisable || !flagAtReload { |
| + t.Errorf("flag not set before unit stop/reload (disable=%v reload=%v) — a crash mid-flip would strand the ordering", flagAtDisable, flagAtReload) |
| + } |
| + if got, _ := reg.get(host); got == nil || !got.Hibernate { |
| + t.Error("flag not set after flip-on") |
| + } |
| +} |
| + |
| +// The flip-off ordering (hibernateOff): flag cleared, router reloaded (which |
| +// makes the activator release the host), and the unit started ONLY after the |
| +// activator's child has let the socket go — verified by a fake child that |
| +// holds the socket until the reload lands, releasing it a beat later like the |
| +// real (asynchronous) reconcile. |
| +func TestHibernateOffOrdering(t *testing.T) { |
| + reg := controlTestEnv(t) |
| + const host = "flip-off.example.com" |
| + sock := filepath.Join(instanceSocketDir, host+".sock") |
| + if err := reg.upsert(Route{Host: host, Kind: "instance", Socket: sock, Status: "ready", Hibernate: true}); err != nil { |
| + t.Fatal(err) |
| + } |
| + |
| + // The fake activator child, awake and holding the socket. |
| + child, err := net.Listen("unix", sock) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + childSrv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(200) })} |
| + go childSrv.Serve(child) |
| + t.Cleanup(func() { childSrv.Close() }) |
| + |
| + var calls []string |
| + systemctl = func(args ...string) error { |
| + call := strings.Join(args, " ") |
| + calls = append(calls, call) |
| + switch { |
| + case call == "reload lchat-router": |
| + // The reconcile's release is asynchronous inside the router |
| + // process — model that: the child lets go a beat after the reload. |
| + go func() { |
| + time.Sleep(300 * time.Millisecond) |
| + childSrv.Close() |
| + }() |
| + case call == "enable --now lchat@"+host: |
| + // The ordering crux: by unit-start time the child must be gone… |
| + if c, err := net.DialTimeout("unix", sock, 100*time.Millisecond); err == nil { |
| + c.Close() |
| + t.Error("unit started while the activator's child still held the socket") |
| + } |
| + // …and the flag off, so systemd unambiguously owns the host. |
| + if rt, _ := reg.get(host); rt.Hibernate { |
| + t.Error("unit started with the hibernate flag still set") |
| + } |
| + // Bind like a started unit would (clearing the stale socket file |
| + // first, as the instance's listen() does), so waitHealthy passes. |
| + _ = os.Remove(sock) |
| + ln, err := net.Listen("unix", sock) |
| + if err != nil { |
| + return err |
| + } |
| + srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(200) })} |
| + go srv.Serve(ln) |
| + t.Cleanup(func() { srv.Close() }) |
| + } |
| + return nil |
| + } |
| + |
| + rt, _ := reg.get(host) |
| + if err := hibernateOff(reg, rt, "lchat@", "lchat-router"); err != nil { |
| + t.Fatalf("hibernateOff: %v", err) |
| + } |
| + want := []string{"reload lchat-router", "enable --now lchat@" + host} |
| + if !reflect.DeepEqual(calls, want) { |
| + t.Errorf("flip-off sequence = %v, want %v", calls, want) |
| + } |
| + if got, _ := reg.get(host); got == nil || got.Hibernate { |
| + t.Error("flag still set after flip-off") |
| + } |
| +} |
| + |
| +// waitSocketQuiet is the flip-off's release observation: quiet means no |
| +// listener (dial failure), whether or not the socket file lingers. |
| +func TestWaitSocketQuiet(t *testing.T) { |
| + sock := tmpSock(t) |
| + // No listener: quiet immediately. |
| + if err := waitSocketQuiet(sock, time.Second); err != nil { |
| + t.Errorf("no listener: %v, want quiet", err) |
| + } |
| + // A held listener: not quiet — the bounded wait must report it. |
| + ln, err := net.Listen("unix", sock) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + defer ln.Close() |
| + if err := waitSocketQuiet(sock, 400*time.Millisecond); err == nil { |
| + t.Error("live listener reported quiet — the flip-off would start a unit against a live child") |
| + } |
| +} |
diff --git a/controld.go b/controld.go| index 76e7a90..3ccb6bb 100644 |
| --- a/controld.go |
| +++ b/controld.go |
| @@ -205,6 +205,9 @@ func (c *controld) createHandler(w http.ResponseWriter, r *http.Request) { |
| HomeOrigin: req.HomeOrigin, |
| PushRelayURL: req.PushRelayURL, |
| PushRelayToken: req.PushRelayToken, |
| + // Provisioner-created spaces stay systemd-managed (Hibernate=false) for |
| + // now; hibernate-by-default cloud spaces are a later, deliberate flip. |
| + Hibernate: false, |
| }) |
| switch { |
| case errors.Is(err, errInvalidHost): |
diff --git a/main.go b/main.go| index 7029639..98b66ab 100644 |
| --- a/main.go |
| +++ b/main.go |
| @@ -139,6 +139,9 @@ func main() { |
| case "instances": |
| runInstances(os.Args[2:]) |
| return |
| + case "hibernate": |
| + runHibernate(os.Args[2:]) |
| + return |
| case "admin-link": |
| runAdminLink(os.Args[2:]) |
| return |
diff --git a/registry.go b/registry.go| index c2333b3..a0624b9 100644 |
| --- a/registry.go |
| +++ b/registry.go |
| @@ -34,6 +34,14 @@ type Route struct { |
| // them yet; register() just records what a caller sets. |
| HomeRegion string |
| Provider string |
| + // Hibernate is the per-host opt-in to activator management (#550 Part 4 |
| + // v2): with the router's LCHAT_HIBERNATE_BUCKET set, a flagged instance |
| + // runs as an on-demand child — woken per request, slept when idle — instead |
| + // of an always-on systemd unit. Off (the default) keeps the classic systemd |
| + // shape even when the router has the env set, so one production canary can |
| + // hibernate while the rest of the box is untouched. Flipped by |
| + // `lchat hibernate <host> --on/--off` via setHibernate. |
| + Hibernate bool |
| // NextDue is the due-time index (#550 Part 4 v2): the earliest pending |
| // scheduled delivery (unix seconds; 0 = none) recorded by the activator when |
| // the instance hibernates, so the sweeper can wake it in time to fire. In |
| @@ -85,6 +93,7 @@ CREATE TABLE IF NOT EXISTS routes ( |
| push_relay_token TEXT NOT NULL DEFAULT '', |
| home_region TEXT NOT NULL DEFAULT '', |
| provider TEXT NOT NULL DEFAULT '', |
| + hibernate INTEGER NOT NULL DEFAULT 0, |
| next_due INTEGER NOT NULL DEFAULT 0, |
| created_at INTEGER NOT NULL |
| );`); err != nil { |
| @@ -98,6 +107,7 @@ CREATE TABLE IF NOT EXISTS routes ( |
| `ALTER TABLE routes ADD COLUMN home_region TEXT NOT NULL DEFAULT ''`, |
| `ALTER TABLE routes ADD COLUMN provider TEXT NOT NULL DEFAULT ''`, |
| `ALTER TABLE routes ADD COLUMN next_due INTEGER NOT NULL DEFAULT 0`, |
| + `ALTER TABLE routes ADD COLUMN hibernate INTEGER NOT NULL DEFAULT 0`, |
| } { |
| if _, err := r.sql.Exec(stmt); err != nil && !strings.Contains(err.Error(), "duplicate column") { |
| return err |
| @@ -111,7 +121,7 @@ func (r *Registry) list() ([]Route, error) { |
| rows, err := r.sql.Query(` |
| SELECT host, kind, socket, status, last_error, db_path, version, prev_version, |
| app_store_url, home_origin, push_relay_url, push_relay_token, |
| - home_region, provider, next_due, created_at |
| + home_region, provider, hibernate, next_due, created_at |
| FROM routes ORDER BY host`) |
| if err != nil { |
| return nil, err |
| @@ -122,7 +132,7 @@ FROM routes ORDER BY host`) |
| var t Route |
| if err := rows.Scan(&t.Host, &t.Kind, &t.Socket, &t.Status, &t.LastError, |
| &t.DBPath, &t.Version, &t.PrevVersion, &t.AppStoreURL, &t.HomeOrigin, |
| - &t.PushRelayURL, &t.PushRelayToken, &t.HomeRegion, &t.Provider, &t.NextDue, &t.CreatedAt); err != nil { |
| + &t.PushRelayURL, &t.PushRelayToken, &t.HomeRegion, &t.Provider, &t.Hibernate, &t.NextDue, &t.CreatedAt); err != nil { |
| return nil, err |
| } |
| out = append(out, t) |
| @@ -136,10 +146,10 @@ func (r *Registry) get(host string) (*Route, error) { |
| err := r.sql.QueryRow(` |
| SELECT host, kind, socket, status, last_error, db_path, version, prev_version, |
| app_store_url, home_origin, push_relay_url, push_relay_token, |
| - home_region, provider, next_due, created_at |
| + home_region, provider, hibernate, next_due, created_at |
| FROM routes WHERE host = ?`, host).Scan(&t.Host, &t.Kind, &t.Socket, &t.Status, |
| &t.LastError, &t.DBPath, &t.Version, &t.PrevVersion, &t.AppStoreURL, &t.HomeOrigin, |
| - &t.PushRelayURL, &t.PushRelayToken, &t.HomeRegion, &t.Provider, &t.NextDue, &t.CreatedAt) |
| + &t.PushRelayURL, &t.PushRelayToken, &t.HomeRegion, &t.Provider, &t.Hibernate, &t.NextDue, &t.CreatedAt) |
| if err == sql.ErrNoRows { |
| return nil, nil |
| } |
| @@ -172,8 +182,8 @@ func (r *Registry) upsert(t Route) error { |
| _, err := r.sql.Exec(` |
| INSERT INTO routes (host, kind, socket, status, last_error, db_path, version, prev_version, |
| app_store_url, home_origin, push_relay_url, push_relay_token, |
| - home_region, provider, next_due, created_at) |
| -VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) |
| + home_region, provider, hibernate, next_due, created_at) |
| +VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) |
| ON CONFLICT(host) DO UPDATE SET |
| kind=excluded.kind, socket=excluded.socket, status=excluded.status, |
| last_error=excluded.last_error, db_path=excluded.db_path, version=excluded.version, |
| @@ -181,10 +191,19 @@ ON CONFLICT(host) DO UPDATE SET |
| app_store_url=excluded.app_store_url, home_origin=excluded.home_origin, |
| push_relay_url=excluded.push_relay_url, push_relay_token=excluded.push_relay_token, |
| home_region=excluded.home_region, provider=excluded.provider, |
| - next_due=excluded.next_due`, |
| + hibernate=excluded.hibernate, next_due=excluded.next_due`, |
| t.Host, t.Kind, t.Socket, t.Status, t.LastError, t.DBPath, t.Version, t.PrevVersion, |
| t.AppStoreURL, t.HomeOrigin, t.PushRelayURL, t.PushRelayToken, |
| - t.HomeRegion, t.Provider, t.NextDue, t.CreatedAt) |
| + t.HomeRegion, t.Provider, t.Hibernate, t.NextDue, t.CreatedAt) |
| + return err |
| +} |
| + |
| +// setHibernate flips host's activator-managed flag without touching the rest of |
| +// the route — a narrow UPDATE (like setNextDue) so the flip can never clobber a |
| +// concurrent next_due write or a deploy's read-modify-write, and vice versa. An |
| +// unknown host is a quiet no-op, matching setNextDue. |
| +func (r *Registry) setHibernate(host string, on bool) error { |
| + _, err := r.sql.Exec(`UPDATE routes SET hibernate = ? WHERE host = ?`, on, host) |
| return err |
| } |
| @@ -214,7 +233,12 @@ func (r *Registry) dueInstances(now int64) ([]Route, error) { |
| } |
| var due []Route |
| for _, t := range routes { |
| - if t.Kind == "instance" && t.Status == "ready" && t.NextDue > 0 && t.NextDue <= now { |
| + // Only hibernating hosts: the index exists to wake sleeping instances, |
| + // and a systemd-managed host is always on — spawning a child for it |
| + // would fight its unit over the socket. This also makes a leftover |
| + // next_due inert on a host flipped back to systemd (the flip-off's |
| + // release re-records one after the flag is already clear). |
| + if t.Kind == "instance" && t.Status == "ready" && t.Hibernate && t.NextDue > 0 && t.NextDue <= now { |
| due = append(due, t) |
| } |
| } |
diff --git a/router.go b/router.go| index 78e94ae..d09537d 100644 |
| --- a/router.go |
| +++ b/router.go |
| @@ -26,10 +26,11 @@ import ( |
| type router struct { |
| reg *Registry |
| upstreams atomic.Pointer[map[string]upstream] |
| - // act, when non-nil, is the hibernation activator (activator.go): instance |
| - // routes stop being always-on systemd backends and become on-demand child |
| - // processes woken per request. Nil (LCHAT_HIBERNATE_BUCKET unset) keeps |
| - // today's proxy-only behavior. |
| + // act, when non-nil, is the hibernation activator (activator.go): |
| + // hibernate-flagged instance routes stop being always-on systemd backends |
| + // and become on-demand child processes woken per request. Nil |
| + // (LCHAT_HIBERNATE_BUCKET unset) keeps today's proxy-only behavior; with it |
| + // set, unflagged routes are still proxied straight (the flag is per host). |
| act *activatorSvc |
| // fallbackURL, when set, is where requests for unknown hosts are redirected |
| // (e.g. the apex marketing site) instead of a bare 404 — a wildcard DNS |
| @@ -68,21 +69,36 @@ func (rt *router) reload() error { |
| return err |
| } |
| m := make(map[string]upstream, len(routes)) |
| + // The hosts the activator should manage after this reload — feeds both the |
| + // per-upstream gate and the retire pass below. |
| + managed := make(map[string]bool) |
| for _, t := range routes { |
| // Only serve ready routes: a 'creating'/'failed' entry has no working |
| // backend, so proxying it (and minting a cert for it) would just stall → 502. |
| if t.Socket == "" || t.Status != "ready" { |
| continue |
| } |
| - // With hibernation on, every instance is activator-managed; sibling |
| - // services (home, push) are externally run and stay proxy-only. |
| - m[t.Host] = upstream{ |
| - proxy: unixProxy(t.Socket), |
| - route: t, |
| - hibernating: rt.act != nil && t.Kind == "instance", |
| + // Hibernation is per-host: only an instance whose route carries the |
| + // hibernate flag is activator-managed; unflagged instances and sibling |
| + // services (home, push) are proxied straight. With the bucket env set |
| + // but no route flagged, the router therefore behaves exactly as it does |
| + // without the activator — so the binary + env can ship fleet-wide |
| + // first, and the flag flips one instance at a time. |
| + hib := rt.act != nil && t.Kind == "instance" && t.Hibernate |
| + if hib { |
| + managed[t.Host] = true |
| } |
| + m[t.Host] = upstream{proxy: unixProxy(t.Socket), route: t, hibernating: hib} |
| } |
| rt.upstreams.Store(&m) |
| + // Reconcile the activator with the new route set: a host un-flagged (or |
| + // removed) since the last reload may still have a live child holding its |
| + // socket and S3 lease — retire it so a systemd unit can take over cleanly. |
| + // After the map swap above, so no new request re-enters the activator for a |
| + // host being retired. |
| + if rt.act != nil { |
| + rt.act.retireExcept(managed) |
| + } |
| return nil |
| } |
| @@ -217,7 +233,7 @@ func runRouter(args []string) { |
| // for sleeping instances. |
| act = newActivator(*hc, &s3Backend{cfg: *hc}, reg) |
| act.start() |
| - log.Printf("router: hibernation active (bucket %s, prefix %q, idle %s) — instances wake on demand", |
| + log.Printf("router: hibernation active (bucket %s, prefix %q, idle %s) — flagged instances wake on demand", |
| hc.Bucket, hc.Prefix, hc.Idle) |
| } |
diff --git a/updater.go b/updater.go| index c15766b..7fb3802 100644 |
| --- a/updater.go |
| +++ b/updater.go |
| @@ -213,6 +213,14 @@ func updatePass(reg *Registry, unitTmpl string, dryRun bool) error { |
| fmt.Printf(" %s: skipped (status %s)\n", rt.Host, rt.Status) |
| continue |
| } |
| + if rt.Hibernate { |
| + // An activator-managed host has no unit for deployHostVersion to |
| + // restart — starting one would fight the router's child for the |
| + // socket (and an asleep host can't answer the prefs fetch anyway). |
| + // Its updates ride the flip-off flow for now. |
| + fmt.Printf(" %s: hibernating — left alone (no unit; flip off to update: lchat hibernate %s --off)\n", rt.Host, rt.Host) |
| + continue |
| + } |
| prefs, err := fetchUpdatePrefs(rt.Socket) |
| if err != nil { |
| fmt.Printf(" %s: unreachable (%v) — skipped\n", rt.Host, err) |
diff --git a/versions.go b/versions.go| index 6f71cc2..7e48d8b 100644 |
| --- a/versions.go |
| +++ b/versions.go |
| @@ -309,6 +309,15 @@ func runRollback(args []string) { |
| if rt.Kind != "instance" { |
| die("rollback %s: not an instance (kind %q)", host, rt.Kind) |
| } |
| + if rt.Hibernate { |
| + // Same guard as deploy: no unit to restart, and starting one would |
| + // fight the activator's child for the socket. |
| + if *all { |
| + fmt.Printf("skipped %s (hibernating — flip it back to roll back: lchat hibernate %s --off)\n", host, host) |
| + continue |
| + } |
| + die("rollback %s: instance is hibernating (activator-managed, no unit) — flip it back first: lchat hibernate %s --off", host, host) |
| + } |
| prev := strings.TrimSpace(rt.PrevVersion) |
| if prev == "" { |
| if *all { |