eleven / messenger Public

server: hibernate-by-default provisioner creates behind LCHAT_CONTROL_HIBERNATE_CREATES (#550) (#663)

With the env flag on (default OFF), controld creates every provisioner space
with Hibernate=true: route + env + staged binary, no systemd unit, no resident
process — the create's readiness probe is the one first wake (mints the setup
link, warms the cert), and the activator's idle window puts it to sleep. A
never-claimed space then costs zero RAM.

Reaper interaction, solved wake-free:
- stats for an asleep hibernating space answer read-only from its local DB
  (the warm cache — already how instanceStatus works; now pinned by tests
  with a socket tripwire);
- stats for a NEVER-woken hibernating space (no DB at all) are definitional,
  answered from the registry row: members 0, setup_complete false,
  has_content false — so it is reapable without ever being woken. Missing DB
  on a non-hibernating row stays 404 (reaper skips, as before).
- deleteInstance treats the unit stop as best-effort for hibernating rows
  (there is no unit; the route delete + router reload retires an awake child
  via the activator's reconcile), waits bounded for the socket to go quiet
  before archiving, and also archives the litestream meta dir. The S3 prefix
  is deliberately NOT deleted (controld holds no bucket creds; versioned
  bucket + noncurrent expiry; orphan-prefix sweep deferred to the S3
  lifecycle work — documented).

Capacity under the flag: LCHAT_MAX_INSTANCES counts MATERIALIZED
(non-hibernating) instances only; LCHAT_MAX_REGISTERED (default 10x) is a
sanity ceiling on total rows; the disk floor is unchanged. Flag off =
byte-identical behavior. Direct 'lchat add-instance' keeps its explicit
--hibernate semantics.

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

Paul Campbell 3 months ago 43c4d716f99382d9f9cf871aa36b20b618e008fc parent 3492e3f

6 files changed, +522 −22

control.go +37 −2
controld.go +82 −16
controld_test.go +288 −0
docs/cloud-provisioner.md +39 −0
reaper.go +15 −4
reaper_test.go +61 −0

diff --git a/control.go b/control.go

index e1ed366..c923357 100644

--- a/control.go

+++ b/control.go

@@ -17,6 +17,8 @@ import (

"strings"

"syscall"

"time"

+ "github.com/benbjohnson/litestream"

)

// The control subcommands automate what an operator would otherwise do by hand to

@@ -417,6 +419,15 @@ var errUnknownHost = fmt.Errorf("unknown host")

// /data/trash/<host>-<unix>/ rather than being destroyed, so an operator can

// still recover a space deleted in error until the trash is purged. The cert and

// live symlink are removed (both are re-creatable).

+//

+// A hibernating space's S3 replica (s3://<bucket>/<prefix>/<host>/) is

+// deliberately NOT deleted here: controld holds no bucket credentials (only the

+// router process does, and widening the root daemon's reach for this buys

+// nothing), the prefix is opaque ciphertext of a DB whose authoritative copy

+// just moved to trash, and the production bucket is versioned with a

+// noncurrent-version expiry, so an orphaned prefix ages rather than grows.

+// Reclaiming orphaned prefixes is deferred to the S3 lifecycle work — an

+// offline sweep comparing bucket prefixes against the registry.

func deleteInstance(reg *Registry, host string) error {

host = strings.ToLower(strings.TrimSpace(host))

if !validHost(host) {

@@ -433,7 +444,14 @@ func deleteInstance(reg *Registry, host string) error {

return fmt.Errorf("%q is a %s route, not an instance", host, rt.Kind)

}

_ = systemctl("disable", "--now", "lchat-bot@"+host) // best-effort: a sidecar may not exist

- if err := systemctl("disable", "--now", "lchat@"+host); err != nil {

+ if rt.Hibernate {

+ // Activator-managed: there is no lchat@ unit — stopping one is

+ // best-effort only, covering a half-flipped host. If the child is

+ // awake it's the ROUTER that puts it down: the route delete + reload

+ // below make the reload-time reconcile (retireExcept, activator.go)

+ // stop the child, take the final checkpoint, and release the S3 lease.

+ _ = systemctl("disable", "--now", "lchat@"+host)

+ } else if err := systemctl("disable", "--now", "lchat@"+host); err != nil {

return fmt.Errorf("stop unit: %v", err)

}

if err := reg.delete(host); err != nil {

@@ -442,12 +460,24 @@ func deleteInstance(reg *Registry, host string) error {

if err := systemctl("reload", "lchat-router"); err != nil {

return fmt.Errorf("reload router: %v", err)

}

+ if rt.Hibernate {

+ // The retire is asynchronous inside the router process; wait (bounded)

+ // for the child to release the socket so the archive renames below

+ // don't race its final checkpoint. Instant in the common case — the

+ // reaper's targets are asleep or never woken, so nothing listens.

+ // Best-effort on timeout: open fds follow the renamed inode, so a

+ // straggling checkpoint lands harmlessly inside the archive.

+ _ = waitSocketQuiet(rt.Socket, deleteReleaseTimeout)

+ }

// Archive the data (never destroy it here — the trash dir is the safety net).

+ // The litestream meta dir rides along: it describes the archived DB, and a

+ // leftover would only confuse a future re-create of the same host (the wake

+ // path clears a stale one, but don't rely on it).

trash := filepath.Join(instanceDataDir, "trash", fmt.Sprintf("%s-%d", host, time.Now().Unix()))

if err := os.MkdirAll(trash, 0o700); err != nil {

return fmt.Errorf("trash dir: %v", err)

}

- for _, name := range []string{host + ".db", host + ".db-wal", host + ".db-shm", host + ".env"} {

+ for _, name := range []string{host + ".db", host + ".db-wal", host + ".db-shm", host + ".db" + litestream.MetaDirSuffix, host + ".env"} {

src := filepath.Join(instanceDataDir, name)

if _, err := os.Stat(src); err == nil {

if err := os.Rename(src, filepath.Join(trash, name)); err != nil {

@@ -460,6 +490,11 @@ func deleteInstance(reg *Registry, host string) error {

return nil

}

+// deleteReleaseTimeout bounds how long deleteInstance waits for the activator

+// to release a hibernating instance's socket before archiving. A var so tests

+// can shrink the (rare) still-listening path.

+var deleteReleaseTimeout = 30 * time.Second

// runRemove: `lchat remove <host>` — stop routing a host. Data-safe by default:

// it deletes the route and reloads the router but leaves the instance running and

// its DB/env in place; --stop also stops+disables an instance's unit.

diff --git a/controld.go b/controld.go

index 3ccb6bb..1641203 100644

--- a/controld.go

+++ b/controld.go

@@ -55,7 +55,21 @@ type controld struct {

tokenHash string

maxInstances int

minFreeGB int

- mu sync.Mutex

+ // hibernateCreates (env LCHAT_CONTROL_HIBERNATE_CREATES=1) makes every

+ // provisioner-created space born hibernating (#550): registered + routed,

+ // env file + staged binary in place, but no systemd unit and no resident

+ // process — the router's activator wakes it on demand. Off by default: the

+ // flip to hibernate-by-default is this one env var on the box, not a code

+ // deploy. Direct `lchat add-instance` is unaffected either way (its

+ // --hibernate flag stays an explicit per-invocation choice). Requires the

+ // router to run with LCHAT_HIBERNATE_BUCKET, or created spaces never answer.

+ hibernateCreates bool

+ // maxRegistered caps TOTAL instance rows (materialized + hibernating) when

+ // hibernateCreates is on — a sanity ceiling on registry rows/env files, not

+ // a resource cap (hibernating rows hold no process). <=0 defaults to

+ // 10x maxInstances.

+ maxRegistered int

+ mu sync.Mutex

}

// controlCreateReq mirrors addInstanceOpts across the socket. AdminToken is the

@@ -102,6 +116,22 @@ func (c *controld) statsHandler(w http.ResponseWriter, r *http.Request) {

members, setupComplete, hasContent, err := instanceStatus(host)

if err != nil {

if os.IsNotExist(err) {

+ // A hibernating space that has never woken has no local DB at all —

+ // the first wake boots it fresh, and the setup-invite mint seeds the

+ // admin user. Its stats are therefore DEFINITIONAL, answered from

+ // the registry row alone: no users, never set up, no content. No

+ // socket is dialed and no wake is triggered — a reaper sweep over

+ // thousands of dormant spaces must stay a pure metadata read, never

+ // a wave of reaper-induced wakes. (A missing DB on a NON-hibernating

+ // row stays a 404: a systemd-managed instance creates its DB at

+ // first boot, so "no DB" there means a failed create — the reaper

+ // skips those, as before.)

+ if c.reg != nil {

+ if rt, rerr := c.reg.get(host); rerr == nil && rt != nil && rt.Kind == "instance" && rt.Hibernate {

+ writeJSON(w, http.StatusOK, map[string]any{"host": host, "members": 0, "setup_complete": false, "has_content": false})

+ return

+ }

writeControlErr(w, http.StatusNotFound, "unknown_host", "no such instance")

return

}

@@ -156,14 +186,34 @@ func (c *controld) atCapacity() (string, error) {

if err != nil {

return "", err

}

- n := 0

+ total, materialized := 0, 0

for _, r := range routes {

- if r.Kind == "instance" {

- n++

+ if r.Kind != "instance" {

+ continue

+ }

+ total++

+ if !r.Hibernate {

+ materialized++

}

- if n >= c.maxInstances {

- return fmt.Sprintf("instance limit reached (%d/%d)", n, c.maxInstances), nil

+ if c.hibernateCreates {

+ // Hibernating rows hold no unit, no process, no RAM — the per-instance

+ // resource maxInstances guards is what's MATERIALIZED (systemd-managed,

+ // always-on), so only those count against it. Registrations still get a

+ // much higher sanity ceiling (rows, env files, staged symlinks are not

+ // free forever), and disk stays the real floor below.

+ if materialized >= c.maxInstances {

+ return fmt.Sprintf("materialized-instance limit reached (%d/%d)", materialized, c.maxInstances), nil

+ }

+ maxReg := c.maxRegistered

+ if maxReg <= 0 {

+ maxReg = 10 * c.maxInstances

+ }

+ if total >= maxReg {

+ return fmt.Sprintf("registered-space limit reached (%d/%d)", total, maxReg), nil

+ }

+ } else if total >= c.maxInstances {

+ return fmt.Sprintf("instance limit reached (%d/%d)", total, c.maxInstances), nil

}

free, err := freeDiskBytes(instanceDataDir)

if err != nil {

@@ -205,9 +255,13 @@ func (c *controld) createHandler(w http.ResponseWriter, r *http.Request) {

HomeOrigin: req.HomeOrigin,

PushRelayURL: req.PushRelayURL,

PushRelayToken: req.PushRelayToken,

- // Provisioner-created spaces stay systemd-managed (Hibernate=false) for

- // now; hibernate-by-default cloud spaces are a later, deliberate flip.

- Hibernate: false,

+ // Hibernate-by-default (#550): with LCHAT_CONTROL_HIBERNATE_CREATES=1

+ // every provisioner-created space is born activator-managed — the

+ // create's public-readiness probe is its one (first) wake, minting the

+ // setup link and warming the cert, and the idle window then puts it to

+ // sleep. A never-claimed space costs zero RAM from that point on. Off

+ // by default so the flip stays a per-box config decision.

+ Hibernate: c.hibernateCreates,

})

switch {

case errors.Is(err, errInvalidHost):

@@ -254,9 +308,11 @@ func runControld(args []string) {

registryPath := fs.String("registry", envOr("LCHAT_REGISTRY", "/data/registry.db"), "router registry SQLite path")

socketPath := fs.String("socket", envOr("LCHAT_CONTROL_SOCKET", "/run/lchat/control.sock"), "unix socket to serve the control API on")

tokenFile := fs.String("token-file", envOr("LCHAT_CONTROL_TOKEN_FILE", controlTokenPath), "file holding the control bearer token")

- maxInstances := fs.Int("max-instances", envInt("LCHAT_MAX_INSTANCES", 40), "refuse creation beyond this many instances on the box")

+ maxInstances := fs.Int("max-instances", envInt("LCHAT_MAX_INSTANCES", 40), "refuse creation beyond this many instances on the box (materialized only under -hibernate-creates)")

minFreeGB := fs.Int("min-free-gb", envInt("LCHAT_MIN_FREE_GB", 5), "refuse creation when the data disk has less than this free (GB)")

trashKeepDays := fs.Int("trash-keep-days", envInt("LCHAT_TRASH_KEEP_DAYS", 0), "purge archived (deleted) spaces from /data/trash after this many days; 0 keeps them forever")

+ hibernateCreates := fs.Bool("hibernate-creates", os.Getenv("LCHAT_CONTROL_HIBERNATE_CREATES") == "1", "create provisioner spaces as hibernating (activator-managed, no systemd unit) — requires the router to run with LCHAT_HIBERNATE_BUCKET")

+ maxRegistered := fs.Int("max-registered", envInt("LCHAT_MAX_REGISTERED", 0), "with -hibernate-creates: refuse registration beyond this many total spaces, hibernating included (0 = 10x -max-instances)")

_ = fs.Parse(args)

tok, err := os.ReadFile(*tokenFile)

@@ -268,10 +324,12 @@ func runControld(args []string) {

die("controld: open registry: %v", err)

}

c := &controld{

- reg: reg,

- tokenHash: hashToken(string(trimNL(tok))),

- maxInstances: *maxInstances,

- minFreeGB: *minFreeGB,

+ reg: reg,

+ tokenHash: hashToken(string(trimNL(tok))),

+ maxInstances: *maxInstances,

+ minFreeGB: *minFreeGB,

+ hibernateCreates: *hibernateCreates,

+ maxRegistered: *maxRegistered,

}

if err := os.Remove(*socketPath); err != nil && !os.IsNotExist(err) {

@@ -294,8 +352,16 @@ func runControld(args []string) {

go runTrashPurge(time.Duration(*trashKeepDays) * 24 * time.Hour)

- log.Printf("controld: serving on %s (registry %s, max %d instances, %d GB disk floor)",

- *socketPath, *registryPath, *maxInstances, *minFreeGB)

+ mode := ""

+ if c.hibernateCreates {

+ maxReg := c.maxRegistered

+ if maxReg <= 0 {

+ maxReg = 10 * c.maxInstances

+ }

+ mode = fmt.Sprintf(", hibernate-creates on (max %d registered)", maxReg)

+ }

+ log.Printf("controld: serving on %s (registry %s, max %d instances, %d GB disk floor%s)",

+ *socketPath, *registryPath, *maxInstances, *minFreeGB, mode)

srv := &http.Server{Handler: c.handler(), ReadHeaderTimeout: 10 * time.Second}

if err := srv.Serve(ln); err != nil {

log.Fatal(err)

diff --git a/controld_test.go b/controld_test.go

index e83d41e..2ebfefb 100644

--- a/controld_test.go

+++ b/controld_test.go

@@ -9,6 +9,7 @@ import (

"os"

"path/filepath"

"strings"

+ "sync/atomic"

"testing"

)

@@ -107,6 +108,53 @@ func decodeBody(t *testing.T, resp *http.Response) map[string]any {

return m

}

+// wakeOnProbe replaces confirmPublicReady with a fake first wake: it binds the

+// host's socket and answers like a freshly spawned child (health check + the

+// setup-invite mint) — which is exactly what the router's activator does when

+// the create-path readiness probe hits a hibernating host. controlTestEnv's

+// cleanup restores the real confirmPublicReady.

+func wakeOnProbe(t *testing.T) {

+ t.Helper()

+ confirmPublicReady = func(h, _ string) {

+ ln, err := net.Listen("unix", filepath.Join(instanceSocketDir, h+".sock"))

+ if err != nil {

+ t.Errorf("bind woken socket: %v", err)

+ return

+ }

+ mux := http.NewServeMux()

+ mux.HandleFunc("GET /", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(200) })

+ mux.HandleFunc("POST /api/admin/key-invites", func(w http.ResponseWriter, r *http.Request) {

+ writeJSON(w, 200, map[string]any{"ok": true})

+ })

+ srv := &http.Server{Handler: mux}

+ go srv.Serve(ln)

+ t.Cleanup(func() { srv.Close() })

+ }

+// socketTripwire binds a host's instance socket and counts connections — proof

+// that a code path stayed a pure metadata read (dialed no socket, woke nothing).

+func socketTripwire(t *testing.T, host string) *atomic.Int32 {

+ t.Helper()

+ var dials atomic.Int32

+ ln, err := net.Listen("unix", filepath.Join(instanceSocketDir, host+".sock"))

+ if err != nil {

+ t.Fatal(err)

+ }

+ t.Cleanup(func() { ln.Close() })

+ go func() {

+ for {

+ c, err := ln.Accept()

+ if err != nil {

+ return

+ }

+ dials.Add(1)

+ c.Close()

+ }

+ }()

+ return &dials

func TestControldRequiresToken(t *testing.T) {

reg := controlTestEnv(t)

srv, _ := newTestControld(t, reg, 40)

@@ -288,3 +336,243 @@ func TestControldDeleteInstance(t *testing.T) {

t.Errorf("archived env not found in trash (matches=%v err=%v)", matches, err)

}

+// ---- hibernate-by-default creates (#550) ---------------------------------------

+// With LCHAT_CONTROL_HIBERNATE_CREATES on, a provisioner create is born

+// hibernating: routed + env + staged binary, setup link minted through the

+// probe-triggered first wake — but no systemd unit and no DB (the wake boots

+// fresh).

+func TestControldHibernateCreates(t *testing.T) {

+ reg := controlTestEnv(t)

+ // Pure recorder: the hibernating path must never enable a unit, so nothing

+ // here binds the socket for it — only the fake wake below does.

+ var calls []string

+ systemctl = func(args ...string) error {

+ calls = append(calls, strings.Join(args, " "))

+ return nil

+ }

+ wakeOnProbe(t)

+ token := "test-control-token"

+ c := &controld{reg: reg, tokenHash: hashToken(token), maxInstances: 40, minFreeGB: 5, hibernateCreates: true}

+ srv := httptest.NewServer(c.handler())

+ t.Cleanup(srv.Close)

+ resp := controlCreate(t, srv, token, "hib.11ch.at")

+ if resp.StatusCode != http.StatusCreated {

+ t.Fatalf("status = %d, want 201", resp.StatusCode)

+ }

+ m := decodeBody(t, resp)

+ if setup, _ := m["setup_link"].(string); !strings.HasPrefix(setup, "https://hib.11ch.at/join/setup#") {

+ t.Errorf("setup_link = %q — the probe-woken instance should have minted it", setup)

+ }

+ rt, err := reg.get("hib.11ch.at")

+ if err != nil || rt == nil || rt.Status != "ready" || !rt.Hibernate {

+ t.Fatalf("route = %+v, err %v; want a ready hibernating instance", rt, err)

+ }

+ for _, call := range calls {

+ if strings.HasPrefix(call, "enable ") {

+ t.Errorf("a systemd unit was touched: %q", call)

+ }

+ if _, err := os.Stat(filepath.Join(instanceDataDir, "hib.11ch.at.db")); !os.IsNotExist(err) {

+ t.Error("a DB was created — the first wake should boot fresh")

+ }

+ // The env file IS written — the future wake's spawn layers it (admin token).

+ if _, err := os.Stat(filepath.Join(instanceDataDir, "hib.11ch.at.env")); err != nil {

+ t.Errorf("env file: %v", err)

+ }

+// The flag defaults OFF: without it, provisioner creates stay systemd-managed —

+// the flip is a config decision, never a side effect of deploying this code.

+func TestControldHibernateCreatesDefaultOff(t *testing.T) {

+ reg := controlTestEnv(t)

+ srv, token := newTestControld(t, reg, 40)

+ if resp := controlCreate(t, srv, token, "plain.11ch.at"); resp.StatusCode != http.StatusCreated {

+ t.Fatalf("status = %d, want 201", resp.StatusCode)

+ }

+ rt, _ := reg.get("plain.11ch.at")

+ if rt == nil || rt.Hibernate {

+ t.Fatalf("route = %+v; the default create must stay systemd-managed", rt)

+ }

+// A hibernating space that has never woken has no local DB — its stats are

+// definitional (nothing to count, nothing set up, no content), answered from

+// the registry row alone. The tripwire proves no socket was dialed and the

+// recorder proves no unit was touched: a reaper sweep over dormant spaces must

+// never wake them.

+func TestControldStatsNeverWokenHibernating(t *testing.T) {

+ reg := controlTestEnv(t)

+ var calls []string

+ systemctl = func(args ...string) error {

+ calls = append(calls, strings.Join(args, " "))

+ return nil

+ }

+ const host = "dormant.11ch.at"

+ sock := filepath.Join(instanceSocketDir, host+".sock")

+ if err := reg.upsert(Route{Host: host, Kind: "instance", Socket: sock, Status: "ready", Hibernate: true}); err != nil {

+ t.Fatal(err)

+ }

+ dials := socketTripwire(t, host)

+ srv, token := newTestControld(t, reg, 40)

+ req, _ := http.NewRequest("GET", srv.URL+"/instances/"+host+"/stats", nil)

+ req.Header.Set("Authorization", "Bearer "+token)

+ resp, err := http.DefaultClient.Do(req)

+ if err != nil {

+ t.Fatal(err)

+ }

+ defer resp.Body.Close()

+ if resp.StatusCode != http.StatusOK {

+ t.Fatalf("stats = HTTP %d, want 200 (definitional, not unknown_host)", resp.StatusCode)

+ }

+ var body struct {

+ Members int `json:"members"`

+ SetupComplete bool `json:"setup_complete"`

+ HasContent bool `json:"has_content"`

+ }

+ if err := json.NewDecoder(resp.Body).Decode(&body); err != nil {

+ t.Fatal(err)

+ }

+ if body.Members != 0 || body.SetupComplete || body.HasContent {

+ t.Errorf("stats = %+v, want the definitional 0/false/false", body)

+ }

+ if n := dials.Load(); n != 0 {

+ t.Errorf("stats dialed the instance socket %d time(s) — it must stay a pure metadata read", n)

+ }

+ if len(calls) != 0 {

+ t.Errorf("stats touched systemd: %v", calls)

+ }

+// An asleep hibernating space (woken before, now stopped) answers from its

+// local DB — the warm cache the activator left at the last sleep — read-only,

+// again with zero socket touches.

+func TestControldStatsAsleepHibernatingAnswersFromFile(t *testing.T) {

+ reg := controlTestEnv(t)

+ const host = "napping.11ch.at"

+ // The warm-cache DB: the pre-seeded admin, never set up, no content.

+ db, err := openDB(filepath.Join(instanceDataDir, host+".db"))

+ if err != nil {

+ t.Fatal(err)

+ }

+ if _, err := db.sql.Exec(`INSERT INTO users(id, created_at) VALUES('admin',1)`); err != nil {

+ t.Fatal(err)

+ }

+ db.sql.Close()

+ sock := filepath.Join(instanceSocketDir, host+".sock")

+ if err := reg.upsert(Route{Host: host, Kind: "instance", Socket: sock, Status: "ready", Hibernate: true}); err != nil {

+ t.Fatal(err)

+ }

+ dials := socketTripwire(t, host)

+ srv, token := newTestControld(t, reg, 40)

+ req, _ := http.NewRequest("GET", srv.URL+"/instances/"+host+"/stats", nil)

+ req.Header.Set("Authorization", "Bearer "+token)

+ resp, err := http.DefaultClient.Do(req)

+ if err != nil {

+ t.Fatal(err)

+ }

+ defer resp.Body.Close()

+ if resp.StatusCode != http.StatusOK {

+ t.Fatalf("stats = HTTP %d, want 200", resp.StatusCode)

+ }

+ var body struct {

+ Members int `json:"members"`

+ SetupComplete bool `json:"setup_complete"`

+ HasContent bool `json:"has_content"`

+ }

+ if err := json.NewDecoder(resp.Body).Decode(&body); err != nil {

+ t.Fatal(err)

+ }

+ if body.Members != 1 || body.SetupComplete || body.HasContent {

+ t.Errorf("stats = %+v, want 1/false/false (from the warm-cache DB)", body)

+ }

+ if n := dials.Load(); n != 0 {

+ t.Errorf("stats dialed the instance socket %d time(s) — the DB file is the source", n)

+ }

+// The definitional answer is ONLY for hibernating rows: a host with no route at

+// all, and a registered systemd-managed instance whose DB is missing (a failed

+// create), both stay 404 — the reaper keeps skipping those, as before.

+func TestControldStatsMissingStays404(t *testing.T) {

+ reg := controlTestEnv(t)

+ srv, token := newTestControld(t, reg, 40)

+ stat := func(host string) int {

+ req, _ := http.NewRequest("GET", srv.URL+"/instances/"+host+"/stats", nil)

+ req.Header.Set("Authorization", "Bearer "+token)

+ resp, err := http.DefaultClient.Do(req)

+ if err != nil {

+ t.Fatal(err)

+ }

+ resp.Body.Close()

+ return resp.StatusCode

+ }

+ if code := stat("never-was.11ch.at"); code != http.StatusNotFound {

+ t.Errorf("unregistered host: %d, want 404", code)

+ }

+ if err := reg.upsert(Route{Host: "broken.11ch.at", Kind: "instance", Socket: "/x.sock", Status: "failed"}); err != nil {

+ t.Fatal(err)

+ }

+ if code := stat("broken.11ch.at"); code != http.StatusNotFound {

+ t.Errorf("non-hibernating row without a DB: %d, want 404", code)

+ }

+// Under hibernate-creates the capacity cap counts MATERIALIZED (systemd-managed)

+// instances only — hibernating rows cost no process, so they don't consume the

+// box's RAM budget; a separate, much higher ceiling bounds total registrations.

+func TestControldCapCountsMaterializedOnlyWhenHibernateCreates(t *testing.T) {

+ reg := controlTestEnv(t)

+ // Three hibernating rows against maxInstances=1: with the flag on they

+ // don't count, so a (hibernating) create still succeeds…

+ for _, h := range []string{"h1.11ch.at", "h2.11ch.at", "h3.11ch.at"} {

+ if err := reg.upsert(Route{Host: h, Kind: "instance", Socket: "/run/" + h + ".sock", Status: "ready", Hibernate: true}); err != nil {

+ t.Fatal(err)

+ }

+ token := "test-control-token"

+ c := &controld{reg: reg, tokenHash: hashToken(token), maxInstances: 1, maxRegistered: 10, minFreeGB: 5, hibernateCreates: true}

+ srv := httptest.NewServer(c.handler())

+ t.Cleanup(srv.Close)

+ if resp := controlCreate(t, srv, token, "new1.11ch.at"); resp.StatusCode != http.StatusCreated {

+ t.Fatalf("create with 3 hibernating rows: %d, want 201 (they must not count)", resp.StatusCode)

+ }

+ // …a materialized row DOES hold the maxInstances cap…

+ if err := reg.upsert(Route{Host: "always-on.11ch.at", Kind: "instance", Socket: "/run/a.sock", Status: "ready"}); err != nil {

+ t.Fatal(err)

+ }

+ resp := controlCreate(t, srv, token, "new2.11ch.at")

+ if resp.StatusCode != http.StatusServiceUnavailable {

+ t.Fatalf("create at the materialized cap: %d, want 503", resp.StatusCode)

+ }

+ if m := decodeBody(t, resp); m["error"] != "at_capacity" {

+ t.Errorf("error = %v, want at_capacity", m["error"])

+ }

+ // …and the registration ceiling bounds total rows, hibernating included

+ // (5 instance rows now: h1-h3 + new1 hibernating, always-on materialized).

+ c2 := &controld{reg: reg, tokenHash: hashToken(token), maxInstances: 40, maxRegistered: 5, minFreeGB: 5, hibernateCreates: true}

+ srv2 := httptest.NewServer(c2.handler())

+ t.Cleanup(srv2.Close)

+ resp2 := controlCreate(t, srv2, token, "new3.11ch.at")

+ if resp2.StatusCode != http.StatusServiceUnavailable {

+ t.Fatalf("create at the registration ceiling: %d, want 503", resp2.StatusCode)

+ }

+ if m := decodeBody(t, resp2); m["error"] != "at_capacity" {

+ t.Errorf("error = %v, want at_capacity", m["error"])

+ }

diff --git a/docs/cloud-provisioner.md b/docs/cloud-provisioner.md

index d506f7d..d3927e4 100644

--- a/docs/cloud-provisioner.md

+++ b/docs/cloud-provisioner.md

@@ -81,6 +81,30 @@ Past either limit the API answers `503 {error: "at_capacity"}` and the cloud

layer relays the member-facing message: *"We can't create a new Eleven space

right now — please try again later."* Clients show it verbatim and stay calm.

+### Hibernate-by-default creates (`LCHAT_CONTROL_HIBERNATE_CREATES=1`)

+With this env set on `lchat-control` (off by default — the flip is a per-box

+config decision, not a code deploy), every provisioner-created space is born

+**hibernating** (#550): the create registers the route, writes the env file,

+stages the binary — but starts **no systemd unit and no process**. The

+create-path readiness probe is the space's one *first wake* (it mints the setup

+link and warms the cert through the router's activator); after the idle window

+the activator puts it to sleep, and a never-claimed space costs **zero RAM**

+from then on. It requires the **router** to be running with

+`LCHAT_HIBERNATE_BUCKET`, or created spaces never answer. Direct

+`lchat add-instance` keeps its explicit `--hibernate` flag either way.

+Under the flag the capacity policy changes shape:

+- **`LCHAT_MAX_INSTANCES` counts *materialized* instances only** — the

+ systemd-managed, always-on ones that actually hold RAM. Hibernating rows

+ don't consume it, so the cap stops being the constraint on self-serve

+ creation.

+- **`LCHAT_MAX_REGISTERED`** (default **10× `LCHAT_MAX_INSTANCES`**) is a

+ sanity ceiling on *total* instance rows, hibernating included — registry

+ rows, env files, and staged symlinks aren't free forever.

+- The **disk floor** applies unchanged; disk stays the real long-run bound.

## Addresses

Spaces get generated friendly slugs — `amber-fox-42.<base-domain>` — from small

@@ -242,6 +266,21 @@ would otherwise sit there forever: empty, but holding a slot and a disk. The

Because the reaper only ever considers hosts **in the cloud ledger**, operator-run

instances (family.…, home, push, …) are never candidates.

+**Hibernating spaces never wake for the reaper.** Stats always come from local

+state, not from the instance: an *asleep* space is read from its on-disk DB (the

+warm cache the activator left at its last sleep — on one box, the newest truth),

+and a space that **never woke** has no DB at all, so its stats are definitional —

+`members 0, setup_complete false, has_content false`, answered from the registry

+row alone. A sweep over thousands of dormant spaces stays a pure metadata read.

+Reaping a hibernating space follows the same delete path (unit-stop becomes

+best-effort — there is no unit; the route delete + router reload retires any

+awake child via the activator's reconcile) and archives to `/data/trash` as

+usual. The space's **S3 replica prefix is deliberately not deleted**: controld

+holds no bucket credentials, the prefix is opaque ciphertext of a DB whose

+authoritative copy just moved to trash, and the versioned bucket's

+noncurrent-version expiry means it ages rather than grows — orphan-prefix

+cleanup is deferred to the S3 lifecycle work.

A companion **trash purge** in `controld` reclaims the archive after a longer

window: `LCHAT_TRASH_KEEP_DAYS` (opt-in — 0/unset keeps trash forever) removes

`/data/trash/<host>-<unix>` entries older than that, swept daily. Keep it well

diff --git a/reaper.go b/reaper.go

index f03205e..1636cb9 100644

--- a/reaper.go

+++ b/reaper.go

@@ -33,14 +33,25 @@ import (

// ---- "never claimed" signal (controld-side, root) -----------------------------

// instanceStatus reads a space's claimed-ness from its DB, READ-ONLY (safe

-// against the live instance — no migrations, no writes).

+// against the live instance — no migrations, no writes). For a hibernating

+// space this is also wake-free by construction: an asleep instance's local DB

+// is the warm cache the activator left at its last sleep (on one box it is the

+// newest truth — S3 only ever holds what was checkpointed FROM it), so the

+// answer comes straight off the file with no socket touched. A hibernating

+// space that has NEVER woken has no DB at all — os.Stat propagates not-exist

+// and the stats handler (controld.go) answers definitionally from the registry

+// instead: members 0, never set up, no content.

// "Never claimed" is NOT "zero members": add-instance pre-seeds the admin user

// so the setup invite has an identity to ship, and claiming BECOMES that same

// user — so a space has exactly one member whether or not anyone ever set it up.

-// The real signal is `setup_complete`: absent until the admin finishes the setup

-// conversation (markSetupComplete), then "1". members is returned too, as a

-// belt-and-suspenders guard (don't reap if anyone beyond the seeded admin joined).

+// (The pre-seed happens at setup-invite mint time, i.e. against a RUNNING

+// instance: for a hibernating create that's the create-path's readiness probe —

+// its first wake — so a woken-at-create space has the one admin; a never-woken

+// one has none.) The real signal is `setup_complete`: absent until the admin

+// finishes the setup conversation (markSetupComplete), then "1". members is

+// returned too, as a belt-and-suspenders guard (don't reap if anyone beyond the

+// seeded admin joined).

func instanceStatus(host string) (members int, setupComplete, hasContent bool, err error) {

path := filepath.Join(instanceDataDir, host+".db")

if _, err := os.Stat(path); err != nil {

diff --git a/reaper_test.go b/reaper_test.go

index 168c201..ca187aa 100644

--- a/reaper_test.go

+++ b/reaper_test.go

@@ -2,11 +2,13 @@ package main

import (

"encoding/json"

+ "fmt"

"net/http"

"net/http/httptest"

"os"

"path/filepath"

"strconv"

+ "strings"

"testing"

"time"

)

@@ -215,6 +217,65 @@ func TestReapOnceSkipsWhenStatsUnavailable(t *testing.T) {

}

+// ---- reaping a hibernating space -----------------------------------------------

+// End-to-end through the REAL control daemon: a hibernate-created space that

+// never woke (route + env, no DB, no unit, no process) must be reapable — its

+// stats answer definitionally from the registry, and the delete must not depend

+// on a systemd unit existing.

+func TestReapNeverWokenHibernatingSpace(t *testing.T) {

+ reg := controlTestEnv(t)

+ const host = "dead.11ch.at"

+ // Systemd stand-in: stopping the (nonexistent) instance unit FAILS, as a

+ // systemd that errors on unknown units would — the hibernating delete path

+ // must shrug (best-effort), not abort the reap.

+ var calls []string

+ systemctl = func(args ...string) error {

+ call := strings.Join(args, " ")

+ calls = append(calls, call)

+ if call == "disable --now lchat@"+host {

+ return fmt.Errorf("unit lchat@%s not loaded", host)

+ }

+ return nil

+ }

+ // The register-only footprint a hibernate-create leaves when its first wake

+ // never happened: a routed row + env file. No DB, no unit, no process.

+ if err := reg.upsert(Route{Host: host, Kind: "instance",

+ Socket: filepath.Join(instanceSocketDir, host+".sock"), Status: "ready", Hibernate: true}); err != nil {

+ t.Fatal(err)

+ }

+ if err := writeEnvFile(filepath.Join(instanceDataDir, host+".env"), map[string]string{"LCHAT_ADMIN_TOKEN": "tok"}); err != nil {

+ t.Fatal(err)

+ }

+ token := "ctl"

+ c := &controld{reg: reg, tokenHash: hashToken(token), maxInstances: 40, minFreeGB: 5}

+ sock := fakeControl(t, c.handler().ServeHTTP)

+ db := openTestCloudDB(t)

+ now := time.Unix(2_000_000_000, 0)

+ id := insertSpaceRow(t, db, host, now.Add(-20*24*time.Hour).Unix())

+ s := &cloudServer{db: db, control: controlClient(sock), controlToken: token, reapAfter: 14 * 24 * time.Hour}

+ reaped, kept := s.reapOnce(now)

+ if reaped != 1 || kept != 0 {

+ t.Fatalf("reaped=%d kept=%d, want 1/0 (calls: %v)", reaped, kept, calls)

+ }

+ // Route gone, env archived (recoverable), ledger row cleared.

+ if rt, _ := reg.get(host); rt != nil {

+ t.Errorf("route still present: %+v", rt)

+ }

+ matches, err := filepath.Glob(filepath.Join(instanceDataDir, "trash", host+"-*", host+".env"))

+ if err != nil || len(matches) != 1 {

+ t.Errorf("archived env not in trash (matches=%v err=%v)", matches, err)

+ }

+ if _, ok, _ := db.spaceOwnedByID(t, id); ok {

+ t.Error("ledger row should be cleared")

+ }

// ---- instanceStatus + the controld stats endpoint -----------------------------

func TestInstanceStatusAndStatsHandler(t *testing.T) {