amadan
server: materialize a route's first wake via controld, not directly (#844)
Found live during the mesh flip rehearsal: a mesh-replicated route
(meshreplicate.go's meshRouteHandler) only ever inserts a registry
row on the receiving box, never a live symlink — that's normally
materializeInstance's job, at add-instance time. So its first wake
there hit s3Backend.start's fallback path (stage the running binary +
point the live symlink), which tried to write to /opt/lchat directly
— a write this sandboxed process (ProtectSystem=strict,
ReadWritePaths=/data /run/lchat only) has never actually been able to
make. It only ever "worked" because every existing box A instance's
symlink already existed from the unsandboxed add-instance flow, so
this branch was dead code until mesh-seed exercised it for the first
time — two real customer spaces 503'd for several minutes as a result.
Considered widening ReadWritePaths to include /opt/lchat instead: real
downside is that it's not just a data-blast-radius question — it would
mean a bug in request handling could, for the first time, let an
attacker plant a binary the router later executes as a child process,
which ProtectSystem=strict specifically exists to prevent.
Routes the write through the existing control-daemon pattern instead:
controld already runs fully unsandboxed as root for exactly this class
of privileged action (it's what add-instance already goes through).
Added POST /instances/{host}/materialize (controld.go) — stages the
binary + points the live symlink for an already-registered route,
idempotent. The router calls it via a new optional s3Backend.materialize
hook (cloud.go's controldMaterializeFunc, activator.go's extracted
ensureLiveLink) instead of doing the write itself, authenticated with
the control token via systemd LoadCredential — the same secure handoff
lchat-cloud.service already uses, granting read access to one token
file, never a filesystem write grant.
Deliberately NOT added to the universal lchat-router.service template
(every self-hoster gets that on `lchat bootstrap`) — LoadCredential
fails a unit to start outright if its source path is missing, and most
boxes don't run controld at all. Instead: deploy/systemd/lchat-router-
control-token.conf, an optional drop-in for boxes that run controld AND
opt into the mesh. Without it, the router logs once at startup and
falls back to the direct calls — today's exact behavior, unchanged.
Co-authored-by: Claude <noreply@anthropic.com>
7 files changed,
+339
−12
activator.go+44 −11activator_test.go+95 −0cloud.go+40 −0controld.go+46 −0controld_test.go+83 −0deploy/systemd/lchat-router-control-token.conf+20 −0router.go+11 −1
diff --git a/activator.go b/activator.go| index 4552ede..53e4bc5 100644 |
| --- a/activator.go |
| +++ b/activator.go |
| @@ -1048,6 +1048,13 @@ func (a *activatorSvc) serveHTTP(w http.ResponseWriter, r *http.Request, rt Rout |
| // versioned binary the systemd unit would run. |
| type s3Backend struct { |
| cfg hibernateConfig |
| + // materialize, when non-nil, stages the running binary and points a |
| + // host's live symlink at it via the control daemon (cloud.go's |
| + // controldMaterializeFunc) instead of doing it directly — needed because |
| + // this process runs sandboxed and can't write to /opt/lchat itself. nil |
| + // falls back to the direct calls (start's else branch), today's |
| + // long-standing behavior for any box that hasn't opted in. |
| + materialize func(ctx context.Context, host string) error |
| } |
| func (b *s3Backend) acquire(ctx context.Context, host string) (actLease, error) { |
| @@ -1142,6 +1149,39 @@ func (b *s3Backend) nextDue(ctx context.Context, rt Route) (int64, error) { |
| // socket and answer. A var so tests can shrink the failure path. |
| var spawnHealthTimeout = 30 * time.Second |
| +// ensureLiveLink materializes host's live symlink (/opt/lchat/live/<host> → |
| +// a staged version) if this box hasn't already — the future create-is-a-row |
| +// flow, or (found live, 2026-07-13) a mesh-replicated route, which only ever |
| +// inserts a registry row (meshreplicate.go), never a symlink. Split out of |
| +// s3Backend.start for focused testing: exercising this doesn't need a real |
| +// spawned process, just the filesystem. |
| +// |
| +// Prefers routing the write through controld (materialize, non-nil) over |
| +// doing it directly: this process runs sandboxed (ProtectSystem=strict) and |
| +// can't write to /opt/lchat itself — a direct call here only ever worked by |
| +// accident, when some other, unsandboxed path (add-instance) had already |
| +// materialized every host this box would ever wake. |
| +func ensureLiveLink(ctx context.Context, host string, materialize func(ctx context.Context, host string) error) error { |
| + bin := liveLink(host) |
| + if _, err := os.Stat(bin); err == nil { |
| + return nil |
| + } |
| + if materialize != nil { |
| + if err := materialize(ctx, host); err != nil { |
| + return fmt.Errorf("materialize via controld: %w", err) |
| + } |
| + return nil |
| + } |
| + ver, err := ensureSelfStaged() |
| + if err != nil { |
| + return fmt.Errorf("stage binary: %w", err) |
| + } |
| + if err := pointLive(host, ver); err != nil { |
| + return fmt.Errorf("point live symlink: %w", err) |
| + } |
| + return nil |
| +} |
| + |
| func (b *s3Backend) start(ctx context.Context, rt Route) (actProc, string, error) { |
| host := rt.Host |
| dbPath := rt.DBPath |
| @@ -1151,18 +1191,11 @@ func (b *s3Backend) start(ctx context.Context, rt Route) (actProc, string, error |
| // Exec exactly what the systemd unit would: the per-instance live symlink |
| // (/opt/lchat/live/<host> → a staged version), same --socket/--db flags, |
| - // same EnvironmentFile. If the host was registered but never materialized |
| - // (the future create-is-a-row flow), stage the running binary and point |
| - // the link, exactly as materializeInstance would have. |
| + // same EnvironmentFile. ensureLiveLink materializes it first if this box |
| + // never has (a mesh-replicated route, or the future create-is-a-row flow). |
| bin := liveLink(host) |
| - if _, err := os.Stat(bin); err != nil { |
| - ver, err := ensureSelfStaged() |
| - if err != nil { |
| - return nil, "", fmt.Errorf("stage binary: %w", err) |
| - } |
| - if err := pointLive(host, ver); err != nil { |
| - return nil, "", fmt.Errorf("point live symlink: %w", err) |
| - } |
| + if err := ensureLiveLink(ctx, host, b.materialize); err != nil { |
| + return nil, "", err |
| } |
| // The env file is optional, like the unit's "-" prefix. |
diff --git a/activator_test.go b/activator_test.go| index fd70fb5..9f4ec0d 100644 |
| --- a/activator_test.go |
| +++ b/activator_test.go |
| @@ -9,10 +9,12 @@ package main |
| import ( |
| "context" |
| "errors" |
| + "fmt" |
| "io" |
| "net" |
| "net/http" |
| "net/http/httptest" |
| + "os" |
| "path/filepath" |
| "sync" |
| "sync/atomic" |
| @@ -1427,3 +1429,96 @@ func TestHibernateConfigFromEnv(t *testing.T) { |
| t.Fatal("bad idle duration should error, not default silently") |
| } |
| } |
| + |
| +// --- ensureLiveLink (the controld-mediated fix for mesh-replicated routes) --- |
| + |
| +// ensureLiveLinkTestEnv redirects optDir/selfPath into temp dirs, same seam |
| +// controld_test.go's controlTestEnv uses, so ensureSelfStaged/pointLive run |
| +// for real against throwaway paths. |
| +func ensureLiveLinkTestEnv(t *testing.T) { |
| + t.Helper() |
| + oldOpt, oldSelf := optDir, selfPath |
| + t.Cleanup(func() { optDir, selfPath = oldOpt, oldSelf }) |
| + optDir = t.TempDir() |
| + stub := filepath.Join(t.TempDir(), "stub") |
| + if err := os.WriteFile(stub, []byte("#!/bin/sh\n"), 0o755); err != nil { |
| + t.Fatal(err) |
| + } |
| + selfPath = func() (string, error) { return stub, nil } |
| +} |
| + |
| +// Already-materialized: the common case for every existing box A instance — |
| +// neither the direct path nor materialize is ever invoked. |
| +func TestEnsureLiveLinkAlreadyMaterialized(t *testing.T) { |
| + ensureLiveLinkTestEnv(t) |
| + ver, err := ensureSelfStaged() |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := pointLive("existing.example.com", ver); err != nil { |
| + t.Fatal(err) |
| + } |
| + called := false |
| + materialize := func(context.Context, string) error { called = true; return nil } |
| + if err := ensureLiveLink(context.Background(), "existing.example.com", materialize); err != nil { |
| + t.Fatalf("ensureLiveLink: %v", err) |
| + } |
| + if called { |
| + t.Error("materialize was called for an already-materialized host") |
| + } |
| +} |
| + |
| +// No materialize hook configured (no control token — the common case for any |
| +// box that hasn't opted in): falls back to the direct calls, today's |
| +// long-standing behavior. |
| +func TestEnsureLiveLinkFallsBackWithoutMaterialize(t *testing.T) { |
| + ensureLiveLinkTestEnv(t) |
| + if err := ensureLiveLink(context.Background(), "fresh.example.com", nil); err != nil { |
| + t.Fatalf("ensureLiveLink: %v", err) |
| + } |
| + if _, err := os.Stat(liveLink("fresh.example.com")); err != nil { |
| + t.Fatalf("live symlink was not created: %v", err) |
| + } |
| +} |
| + |
| +// Missing symlink + materialize configured: routed through materialize, not |
| +// the direct (sandboxed-and-failing, in production) calls. |
| +func TestEnsureLiveLinkUsesMaterializeWhenConfigured(t *testing.T) { |
| + ensureLiveLinkTestEnv(t) |
| + var gotHost string |
| + materialize := func(_ context.Context, host string) error { |
| + gotHost = host |
| + // The real controld handler stages + points live; a fake only needs |
| + // to prove IT was the one asked, and that its result is honored. |
| + ver, err := ensureSelfStaged() |
| + if err != nil { |
| + return err |
| + } |
| + return pointLive(host, ver) |
| + } |
| + if err := ensureLiveLink(context.Background(), "seeded.11ch.at", materialize); err != nil { |
| + t.Fatalf("ensureLiveLink: %v", err) |
| + } |
| + if gotHost != "seeded.11ch.at" { |
| + t.Fatalf("materialize called with %q, want seeded.11ch.at", gotHost) |
| + } |
| + if _, err := os.Stat(liveLink("seeded.11ch.at")); err != nil { |
| + t.Fatalf("live symlink was not created via materialize: %v", err) |
| + } |
| +} |
| + |
| +// A materialize failure (e.g. controld unreachable) must propagate — the |
| +// exact live symptom found 2026-07-13 (a 503 "instance failed to wake"), |
| +// never silently swallowed into "looks fine, wakes anyway." |
| +func TestEnsureLiveLinkPropagatesMaterializeError(t *testing.T) { |
| + ensureLiveLinkTestEnv(t) |
| + wantErr := fmt.Errorf("controld materialize seeded.11ch.at: HTTP 502") |
| + materialize := func(context.Context, string) error { return wantErr } |
| + err := ensureLiveLink(context.Background(), "seeded.11ch.at", materialize) |
| + if err == nil { |
| + t.Fatal("expected an error, got nil") |
| + } |
| + if _, statErr := os.Stat(liveLink("seeded.11ch.at")); statErr == nil { |
| + t.Fatal("live symlink should not exist after a failed materialize") |
| + } |
| +} |
diff --git a/cloud.go b/cloud.go| index a011b64..d8c6f20 100644 |
| --- a/cloud.go |
| +++ b/cloud.go |
| @@ -499,3 +499,43 @@ func controlClient(socket string) *http.Client { |
| }, |
| } |
| } |
| + |
| +// controldMaterializeFunc builds the router's s3Backend.materialize hook |
| +// (activator.go): a POST to controld's privileged /instances/{host}/ |
| +// materialize endpoint over its local socket, instead of staging the binary |
| +// and pointing the live symlink directly — the router runs sandboxed |
| +// (ProtectSystem=strict, no write access outside /data and /run/lchat) and |
| +// can't do that write itself; controld always runs unsandboxed as root. |
| +// |
| +// Returns nil when the control token isn't available — no systemd |
| +// LoadCredential=control-token:… on this unit, the case for any box that |
| +// doesn't run controld (self-hosters, and any box before it's specifically |
| +// opted into this via the drop-in — see |
| +// deploy/systemd/lchat-router-control-token.conf). A nil hook means |
| +// s3Backend.start falls back to the direct calls, today's behavior, |
| +// unchanged — so this is purely additive for boxes that don't opt in. |
| +func controldMaterializeFunc() func(ctx context.Context, host string) error { |
| + token, err := readControlToken() |
| + if err != nil { |
| + return nil |
| + } |
| + socket := envOr("LCHAT_CONTROL_SOCKET", "/run/lchat/control.sock") |
| + client := controlClient(socket) |
| + return func(ctx context.Context, host string) error { |
| + req, err := http.NewRequestWithContext(ctx, http.MethodPost, "http://control/instances/"+host+"/materialize", nil) |
| + if err != nil { |
| + return err |
| + } |
| + req.Header.Set("Authorization", "Bearer "+token) |
| + resp, err := client.Do(req) |
| + if err != nil { |
| + return err |
| + } |
| + defer resp.Body.Close() |
| + if resp.StatusCode != http.StatusOK { |
| + body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<12)) |
| + return fmt.Errorf("controld materialize %s: HTTP %d %s", host, resp.StatusCode, body) |
| + } |
| + return nil |
| + } |
| +} |
diff --git a/controld.go b/controld.go| index f2172ef..9062879 100644 |
| --- a/controld.go |
| +++ b/controld.go |
| @@ -103,9 +103,55 @@ func (c *controld) handler() http.Handler { |
| mux.HandleFunc("DELETE /instances/{host}", c.auth(c.deleteHandler)) |
| mux.HandleFunc("GET /instances/{host}/stats", c.auth(c.statsHandler)) |
| mux.HandleFunc("POST /instances/{host}/quota", c.auth(c.quotaHandler)) |
| + mux.HandleFunc("POST /instances/{host}/materialize", c.auth(c.materializeHandler)) |
| return mux |
| } |
| +// materializeHandler stages the running control binary and points host's live |
| +// symlink at it, if not already done. This is the privileged half of a route's |
| +// first wake on a box that never locally created it (control.go's |
| +// materializeInstance normally does this at add-instance time, but a |
| +// mesh-replicated route — meshreplicate.go — only ever inserts a registry row, |
| +// never a symlink, since the receiving box didn't create the space). |
| +// activator.go's s3Backend.start can't do this itself: the router runs |
| +// sandboxed (ProtectSystem=strict, no write access outside /data and |
| +// /run/lchat — see docs/instance-router.md § Hardening), so only this |
| +// always-unsandboxed root daemon can. Idempotent: safe to call even if the |
| +// host is already materialized (pointLive happily repoints at the same or a |
| +// newer version), and safe to call for a host this box didn't create — the |
| +// row already exists (checked below) is the only prerequisite. |
| +func (c *controld) materializeHandler(w http.ResponseWriter, r *http.Request) { |
| + host := strings.ToLower(strings.TrimSpace(r.PathValue("host"))) |
| + if !validHost(host) { |
| + writeControlErr(w, http.StatusBadRequest, "invalid_host", "not a valid hostname") |
| + return |
| + } |
| + if c.reg != nil { |
| + rt, err := c.reg.get(host) |
| + if err != nil { |
| + log.Printf("controld: materialize %s: lookup: %v", host, err) |
| + writeControlErr(w, http.StatusInternalServerError, "lookup_failed", "db error") |
| + return |
| + } |
| + if rt == nil { |
| + writeControlErr(w, http.StatusNotFound, "unknown_host", "no such route") |
| + return |
| + } |
| + } |
| + ver, err := ensureSelfStaged() |
| + if err != nil { |
| + log.Printf("controld: materialize %s: stage: %v", host, err) |
| + writeControlErr(w, http.StatusInternalServerError, "stage_failed", err.Error()) |
| + return |
| + } |
| + if err := pointLive(host, ver); err != nil { |
| + log.Printf("controld: materialize %s: point live: %v", host, err) |
| + writeControlErr(w, http.StatusInternalServerError, "point_live_failed", err.Error()) |
| + return |
| + } |
| + writeJSON(w, http.StatusOK, map[string]any{"ok": true, "version": ver}) |
| +} |
| + |
| // quotaHandler applies a storage cap to one instance: upsert |
| // LCHAT_BLOB_QUOTA_BYTES in its env file (0 = back to the compiled default), |
| // mirror the billing entitlement alongside it when the caller sends one, and |
diff --git a/controld_test.go b/controld_test.go| index 2ebfefb..116ff57 100644 |
| --- a/controld_test.go |
| +++ b/controld_test.go |
| @@ -576,3 +576,86 @@ func TestControldCapCountsMaterializedOnlyWhenHibernateCreates(t *testing.T) { |
| t.Errorf("error = %v, want at_capacity", m["error"]) |
| } |
| } |
| + |
| +// --- materialize (the controld-mediated fix for mesh-replicated routes' first wake) --- |
| + |
| +func controlMaterialize(t *testing.T, srv *httptest.Server, token, host string) *http.Response { |
| + t.Helper() |
| + req, _ := http.NewRequest("POST", srv.URL+"/instances/"+host+"/materialize", nil) |
| + if token != "" { |
| + req.Header.Set("Authorization", "Bearer "+token) |
| + } |
| + resp, err := http.DefaultClient.Do(req) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + t.Cleanup(func() { resp.Body.Close() }) |
| + return resp |
| +} |
| + |
| +func TestControldMaterializeRequiresToken(t *testing.T) { |
| + reg := controlTestEnv(t) |
| + srv, _ := newTestControld(t, reg, 40) |
| + if resp := controlMaterialize(t, srv, "", "a.example.com"); resp.StatusCode != http.StatusUnauthorized { |
| + t.Errorf("no token: status = %d, want 401", resp.StatusCode) |
| + } |
| +} |
| + |
| +func TestControldMaterializeUnknownHost(t *testing.T) { |
| + reg := controlTestEnv(t) |
| + srv, token := newTestControld(t, reg, 40) |
| + resp := controlMaterialize(t, srv, token, "never-registered.example.com") |
| + if resp.StatusCode != http.StatusNotFound { |
| + t.Fatalf("status = %d, want 404", resp.StatusCode) |
| + } |
| + if m := decodeBody(t, resp); m["error"] != "unknown_host" { |
| + t.Errorf("error = %v, want unknown_host", m["error"]) |
| + } |
| +} |
| + |
| +// Mirrors the exact scenario found live 2026-07-13: a route with a registry |
| +// row but no live symlink (meshreplicate.go's meshRouteHandler only ever |
| +// upserts the row) must come out materialized — staged binary + live symlink |
| +// — after one call, ready for s3Backend.start to exec. |
| +func TestControldMaterializeStagesAndPointsLive(t *testing.T) { |
| + reg := controlTestEnv(t) |
| + if err := reg.upsert(Route{Host: "seeded.11ch.at", Kind: "instance", Socket: "/run/seeded.sock", Hibernate: true}); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := os.Stat(liveLink("seeded.11ch.at")); err == nil { |
| + t.Fatal("test setup: live symlink should not exist yet") |
| + } |
| + |
| + srv, token := newTestControld(t, reg, 40) |
| + resp := controlMaterialize(t, srv, token, "seeded.11ch.at") |
| + if resp.StatusCode != http.StatusOK { |
| + t.Fatalf("status = %d, want 200", resp.StatusCode) |
| + } |
| + m := decodeBody(t, resp) |
| + if m["ok"] != true || m["version"] == "" { |
| + t.Fatalf("body = %v, want ok:true + a version", m) |
| + } |
| + |
| + ver, err := liveVersion("seeded.11ch.at") |
| + if err != nil || ver == "" { |
| + t.Fatalf("liveVersion after materialize: %v, %v — symlink was not created", ver, err) |
| + } |
| + if _, err := os.Stat(versionBinPath(ver)); err != nil { |
| + t.Fatalf("staged binary missing at the version path: %v", err) |
| + } |
| +} |
| + |
| +// A second call (the retry a wake would make) must not fail just because the |
| +// symlink already exists — pointLive happily repoints at the same version. |
| +func TestControldMaterializeIdempotent(t *testing.T) { |
| + reg := controlTestEnv(t) |
| + if err := reg.upsert(Route{Host: "seeded2.11ch.at", Kind: "instance", Socket: "/run/seeded2.sock", Hibernate: true}); err != nil { |
| + t.Fatal(err) |
| + } |
| + srv, token := newTestControld(t, reg, 40) |
| + for i := 0; i < 2; i++ { |
| + if resp := controlMaterialize(t, srv, token, "seeded2.11ch.at"); resp.StatusCode != http.StatusOK { |
| + t.Fatalf("call %d: status = %d, want 200", i, resp.StatusCode) |
| + } |
| + } |
| +} |
diff --git a/deploy/systemd/lchat-router-control-token.conf b/deploy/systemd/lchat-router-control-token.conf| new file mode 100644 |
| index 0000000..4720b73 |
| --- /dev/null |
| +++ b/deploy/systemd/lchat-router-control-token.conf |
| @@ -0,0 +1,20 @@ |
| +[Service] |
| +# Optional drop-in: apply ONLY on a box that already runs controld AND opts |
| +# into the activator mesh (docs/planet-scale.md, mesh.go). Grants the router |
| +# read-only access to the control token via systemd's LoadCredential — not a |
| +# filesystem write grant. Without it, a mesh-replicated route (one that only |
| +# ever gets a registry row on this box, never a live symlink — see |
| +# meshreplicate.go) fails its first wake here: staging its binary needs a |
| +# write to /opt/lchat, which this unit's own ProtectSystem=strict sandbox |
| +# refuses, and only controld's always-unsandboxed root process can do (see |
| +# activator.go's s3Backend.materialize, controld.go's materializeHandler). |
| +# |
| +# Apply: |
| +# cp deploy/systemd/lchat-router-control-token.conf \ |
| +# /etc/systemd/system/lchat-router.service.d/control-token.conf |
| +# systemctl daemon-reload && systemctl restart lchat-router |
| +# |
| +# Requires /etc/lchat/control.token to already exist (controld already set |
| +# up first) — LoadCredential fails the WHOLE unit to start if its source path |
| +# is missing, so do not apply this on a box that doesn't run controld. |
| +LoadCredential=control-token:/etc/lchat/control.token |
diff --git a/router.go b/router.go| index ba2404a..9eaebd9 100644 |
| --- a/router.go |
| +++ b/router.go |
| @@ -248,10 +248,20 @@ func runRouter(args []string) { |
| log.Fatalf("router: %v", err) |
| } |
| if hc != nil { |
| + // materialize (cloud.go's controldMaterializeFunc) routes a route's |
| + // first-wake staging through the control daemon instead of writing |
| + // /opt/lchat directly, which this sandboxed process can't do (see |
| + // s3Backend.materialize's doc comment). nil — the common case, no |
| + // LoadCredential=control-token:… on this unit — falls back to the |
| + // direct calls, today's long-standing behavior, unchanged. |
| + materialize := controldMaterializeFunc() |
| + if materialize == nil { |
| + log.Printf("router: no control token available — a route materialized elsewhere (e.g. a mesh-replicated route) will fail its first wake here; see deploy/systemd/lchat-router-control-token.conf") |
| + } |
| // The registry doubles as the due-time index's store (routes.next_due), |
| // so scheduled deliveries survive router restarts and fire on time even |
| // for sleeping instances. |
| - act = newActivator(*hc, &s3Backend{cfg: *hc}, reg) |
| + act = newActivator(*hc, &s3Backend{cfg: *hc, materialize: materialize}, reg) |
| if mesh != nil { |
| // Wired and beating before act.start(): the sweeper consults the |
| // mesh from its first tick, and the first request deserves a |