amadan
server: mesh-placed creates were born dark — replicate env, fix probe + mint (#1337)
Root cause of tonight's failed setup links (56c/vmb.11ch.at): controld's unit never saw the mesh env, so fanOutMeshRoute silently no-opped on every create; a host HRW-placed on the peer box then 404'd everywhere (the peer had no route row), the readiness probe treated that 404 as ready, and the socket mint had nothing to dial. Fixes, layered: - lchat-control.service reads /data/router.env (controld must see the mesh exactly as the router does) + controld logs its replication posture at startup and any setup-link mint failure — the silent no-op is what let this hide for weeks. - The instance env (admin token, entitlement, seeded name) now RIDES route replication and mesh-seed, and the receiver writes /data/<host>.env — a peer-woken instance boots with the token its creator minted, instead of token-less. relocate fans the env out too. Token-less payloads never clobber an existing env. - confirmPublicReady no longer counts 404 (the router's unknown-host) as ready (publicReadyStatus). - The setup-link mint falls back to the public path when the local socket fails — the same router hop members use, which follows the mesh to wherever the instance actually woke. - mesh-seed gains --env-file (borrow the router's env under sudo) and ops gains 'mesh seed <peer> [box]'. Fleet guide updated.
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
10 files changed,
+330
−32
control.go+66 −16control_test.go+52 −0controld.go+18 −0deploy/ops+3 −0deploy/ops-mesh.sh+48 −0deploy/systemd/lchat-control.service+7 −0docs/planet-scale-guide/mesh.html+3 −2meshreplicate.go+67 −10meshreplicate_test.go+59 −2relocate.go+7 −2
diff --git a/control.go b/control.go| index ce6425e..5030736 100644 |
| --- a/control.go |
| +++ b/control.go |
| @@ -265,19 +265,8 @@ func materializeInstance(reg *Registry, route Route, token string, opts addInsta |
| // The instance stores only the token's hash, so this is the one chance to |
| // capture the plaintext. Persist it (and per-instance config) to an |
| // EnvironmentFile the unit reads — that keeps it recoverable via `admin-link`. |
| - env := map[string]string{"LCHAT_ADMIN_TOKEN": token} |
| - for k, v := range map[string]string{ |
| - "LCHAT_APP_STORE_URL": opts.AppStoreURL, |
| - "LCHAT_HOME_ORIGIN": opts.HomeOrigin, |
| - "LCHAT_PUSH_RELAY_URL": opts.PushRelayURL, |
| - "LCHAT_PUSH_RELAY_TOKEN": opts.PushRelayToken, |
| - "LCHAT_ENTITLEMENT": opts.Entitlement, |
| - "LCHAT_GROUP_NAME": opts.GroupName, |
| - } { |
| - if v != "" { |
| - env[k] = v |
| - } |
| - } |
| + env := instanceEnvMap(token, opts.AppStoreURL, opts.HomeOrigin, |
| + opts.PushRelayURL, opts.PushRelayToken, opts.Entitlement, opts.GroupName) |
| envPath := filepath.Join(instanceDataDir, host+".env") |
| if err := writeEnvFile(envPath, env); err != nil { |
| return nil, fmt.Errorf("write env file: %v", err) |
| @@ -346,7 +335,7 @@ func materializeInstance(reg *Registry, route Route, token string, opts addInsta |
| if hc, hcErr := hibernateConfigFromEnv(); hcErr != nil { |
| log.Printf("mesh: skipping route replication for %s: %v", host, hcErr) |
| } else { |
| - fanOutMeshRoute(hc, route) |
| + fanOutMeshRoute(hc, route, env) |
| } |
| } |
| if err := systemctl("reload", routerUnit); err != nil { |
| @@ -376,11 +365,41 @@ func materializeInstance(reg *Registry, route Route, token string, opts addInsta |
| res := &addInstanceResult{Host: host, AdminToken: token} |
| res.SetupLink, res.SetupErr = mintSetupInvite(socketClient(socket), "http://lchat", "https://"+host, token) |
| if res.SetupErr != nil { |
| - res.SetupLink = "" |
| + // A hibernating instance may live behind the local socket in name only: |
| + // the mesh can place it on a peer box, where the readiness probe above |
| + // woke it — locally nothing listens, ever. Mint through the public path |
| + // instead (the same router hop the member will use), which follows the |
| + // mesh to wherever the instance actually runs; the replicated env means |
| + // it holds the same admin token. Non-mesh boxes just get a second local |
| + // try through the router, which is harmless. |
| + link, perr := mintPublicSetupInvite(host, token) |
| + if perr != nil { |
| + res.SetupErr = fmt.Errorf("socket: %v; public: %v", res.SetupErr, perr) |
| + res.SetupLink = "" |
| + } else { |
| + res.SetupLink, res.SetupErr = link, nil |
| + } |
| } |
| return res, nil |
| } |
| +// mintPublicSetupInvite mints the setup link through the local router's public |
| +// HTTPS front (SNI-pinned like confirmPublicReady, so the real route + cert |
| +// are exercised) — the path that works no matter which mesh box the instance |
| +// woke on. A var so the add-instance tests, which run no TLS router, can stub |
| +// it alongside confirmPublicReady. |
| +var mintPublicSetupInvite = func(host, adminToken string) (string, error) { |
| + client := &http.Client{ |
| + Timeout: 15 * time.Second, |
| + Transport: &http.Transport{ |
| + DialTLSContext: func(ctx context.Context, _, _ string) (net.Conn, error) { |
| + return (&tls.Dialer{Config: &tls.Config{ServerName: host}}).DialContext(ctx, "tcp", routerHTTPSAddr) |
| + }, |
| + }, |
| + } |
| + return mintSetupInvite(client, "https://"+host, "https://"+host, adminToken) |
| +} |
| + |
| // runAddInstance: `lchat add-instance <host>` — mint token + env file, start the |
| // instance unit, register the route, reload the router. |
| func runAddInstance(args []string) { |
| @@ -1444,7 +1463,7 @@ var confirmPublicReady = func(host, httpsAddr string) { |
| resp, err := client.Get("https://" + host + "/") |
| if err == nil { |
| resp.Body.Close() |
| - if resp.StatusCode < 500 { |
| + if publicReadyStatus(resp.StatusCode) { |
| return |
| } |
| } |
| @@ -1452,6 +1471,16 @@ var confirmPublicReady = func(host, httpsAddr string) { |
| } |
| } |
| +// publicReadyStatus classifies one probe response: anything the instance |
| +// itself could have answered counts as ready, but 404 is the ROUTER's |
| +// "unknown host" (an instance's "/" always serves a page) — the route hasn't |
| +// landed on whichever box owns this host yet (mesh replication in flight). |
| +// Treating 404 as ready is exactly how a create once handed out a dead space, |
| +// so it waits like any 5xx. |
| +func publicReadyStatus(code int) bool { |
| + return code < 500 && code != http.StatusNotFound |
| +} |
| + |
| // socketClient returns an HTTP client that dials an instance's unix socket |
| // (request URLs use the placeholder host http://lchat). |
| func socketClient(socket string) *http.Client { |
| @@ -1579,6 +1608,27 @@ func matchOwner(path, dir string) error { |
| // writeEnvFile writes KEY=value lines (sorted for a stable file) at 0600 — it |
| // holds the admin token, so it's operator-secret, like the platform env it |
| // replaces. |
| +// instanceEnvMap builds a per-instance EnvironmentFile's contents — the ONE |
| +// definition, shared by materializeInstance and the mesh route receiver |
| +// (meshreplicate.go), so a replicated instance wakes with exactly the env its |
| +// creator wrote. Empty values are omitted, like the unit's "-" prefix. |
| +func instanceEnvMap(adminToken, appStoreURL, homeOrigin, pushRelayURL, pushRelayToken, entitlement, groupName string) map[string]string { |
| + env := map[string]string{"LCHAT_ADMIN_TOKEN": adminToken} |
| + for k, v := range map[string]string{ |
| + "LCHAT_APP_STORE_URL": appStoreURL, |
| + "LCHAT_HOME_ORIGIN": homeOrigin, |
| + "LCHAT_PUSH_RELAY_URL": pushRelayURL, |
| + "LCHAT_PUSH_RELAY_TOKEN": pushRelayToken, |
| + "LCHAT_ENTITLEMENT": entitlement, |
| + "LCHAT_GROUP_NAME": groupName, |
| + } { |
| + if v != "" { |
| + env[k] = v |
| + } |
| + } |
| + return env |
| +} |
| + |
| func writeEnvFile(path string, kv map[string]string) error { |
| keys := make([]string, 0, len(kv)) |
| for k := range kv { |
diff --git a/control_test.go b/control_test.go| index b4e172d..428d18a 100644 |
| --- a/control_test.go |
| +++ b/control_test.go |
| @@ -3,6 +3,7 @@ package main |
| import ( |
| "database/sql" |
| "flag" |
| + "fmt" |
| "net" |
| "net/http" |
| "os" |
| @@ -1166,3 +1167,54 @@ func TestApplySeededName(t *testing.T) { |
| t.Errorf("after re-seed, name = %q, want Book Club (seed-only)", got) |
| } |
| } |
| + |
| +// The readiness probe's status vocabulary: an instance's own answers (2xx-4xx |
| +// bar 404) mean ready; 404 is the router's "unknown host" (a route still |
| +// replicating to its owner box) and must keep the probe waiting, like any 5xx. |
| +func TestPublicReadyStatus(t *testing.T) { |
| + for code, want := range map[int]bool{200: true, 302: true, 401: true, 404: false, 500: false, 502: false} { |
| + if got := publicReadyStatus(code); got != want { |
| + t.Errorf("publicReadyStatus(%d) = %v, want %v", code, got, want) |
| + } |
| + } |
| +} |
| + |
| +// A hibernating instance the mesh placed on a peer box never listens on the |
| +// local socket — the setup link then mints through the public path (the same |
| +// router hop members use, which follows the mesh to wherever the instance |
| +// woke). Both paths failing surfaces both errors and no link. |
| +func TestAddInstanceSetupLinkPublicFallback(t *testing.T) { |
| + reg := controlTestEnv(t) |
| + systemctl = func(args ...string) error { return nil } |
| + oldMint := mintPublicSetupInvite |
| + defer func() { mintPublicSetupInvite = oldMint }() |
| + confirmPublicReady = func(string, string) { /* woke on the peer — nothing local */ } |
| + var mintedHost, mintedToken string |
| + mintPublicSetupInvite = func(host, token string) (string, error) { |
| + mintedHost, mintedToken = host, token |
| + return "https://" + host + "/join/setup#PUB", nil |
| + } |
| + |
| + res, err := addInstance(reg, addInstanceOpts{Host: "peerowned.example.com", Hibernate: true}) |
| + if err != nil { |
| + t.Fatalf("addInstance: %v", err) |
| + } |
| + if res.SetupErr != nil || res.SetupLink != "https://peerowned.example.com/join/setup#PUB" { |
| + t.Fatalf("setup link = %q (err %v), want the public-path mint", res.SetupLink, res.SetupErr) |
| + } |
| + if mintedHost != "peerowned.example.com" || mintedToken != res.AdminToken { |
| + t.Errorf("public mint saw host %q, token match %v", mintedHost, mintedToken == res.AdminToken) |
| + } |
| + |
| + mintPublicSetupInvite = func(string, string) (string, error) { return "", fmt.Errorf("no route") } |
| + res2, err := addInstance(reg, addInstanceOpts{Host: "dark.example.com", Hibernate: true}) |
| + if err != nil { |
| + t.Fatalf("addInstance: %v", err) |
| + } |
| + if res2.SetupLink != "" || res2.SetupErr == nil { |
| + t.Fatalf("double failure: link %q, err %v — want no link + a surfaced error", res2.SetupLink, res2.SetupErr) |
| + } |
| + if !strings.Contains(res2.SetupErr.Error(), "public") { |
| + t.Errorf("error should name both paths: %v", res2.SetupErr) |
| + } |
| +} |
diff --git a/controld.go b/controld.go| index 1d7321c..6181434 100644 |
| --- a/controld.go |
| +++ b/controld.go |
| @@ -615,6 +615,11 @@ func (c *controld) createHandler(w http.ResponseWriter, r *http.Request) { |
| return |
| } |
| log.Printf("controld: created %s", res.Host) |
| + if res.SetupErr != nil { |
| + // The admin link still bootstraps, but a create whose setup link died |
| + // is a degraded first-run — it must be visible in the journal. |
| + log.Printf("controld: %s: setup link mint FAILED: %v", res.Host, res.SetupErr) |
| + } |
| writeJSON(w, http.StatusCreated, map[string]any{ |
| "host": res.Host, |
| "url": "https://" + res.Host, |
| @@ -703,6 +708,19 @@ func runControld(args []string) { |
| } |
| log.Printf("controld: serving on %s (registry %s, max %d instances, %d GB disk floor%s)", |
| *socketPath, *registryPath, *maxInstances, *minFreeGB, mode) |
| + // Say the mesh posture out loud: on a mesh box this daemon MUST see the |
| + // router's mesh env (the unit reads /data/router.env), or every create's |
| + // route replication silently no-ops and peer-owned hosts are born dark — |
| + // the failure that hid for weeks precisely because nothing logged it. |
| + if hc, hcErr := hibernateConfigFromEnv(); hcErr != nil { |
| + log.Printf("controld: mesh route replication OFF (%v)", hcErr) |
| + } else if mesh, meshErr := meshFromEnv(hc); meshErr != nil { |
| + log.Printf("controld: mesh route replication OFF (%v)", meshErr) |
| + } else if mesh == nil { |
| + log.Printf("controld: mesh route replication off (no LCHAT_MESH_SELF — fine off-mesh; on a mesh box check the unit reads /data/router.env)") |
| + } else { |
| + log.Printf("controld: mesh route replication on (self %s, domains %v)", mesh.self, mesh.domains) |
| + } |
| srv := &http.Server{Handler: c.handler(), ReadHeaderTimeout: 10 * time.Second} |
| if err := srv.Serve(ln); err != nil { |
| log.Fatal(err) |
diff --git a/deploy/ops b/deploy/ops| index ff9e3d7..c44a981 100755 |
| --- a/deploy/ops |
| +++ b/deploy/ops |
| @@ -29,6 +29,8 @@ ops — the operator command surface |
| ./deploy/ops dev <plan|apply|deploy|rebuild> |
| the two-node dev mesh (tofu / scp+stage+swap) |
| ./deploy/ops ledger "why..." draft the public SSH-access why-entry |
| + ./deploy/ops mesh seed <peer> [box] |
| + replicate routes (+env) to a mesh peer |
| Start with `ops doctor` — it checks every credential and access path the other |
| verbs need, and prints the fix under each failure. `ops validate` is its |
| @@ -59,6 +61,7 @@ case "$verb" in |
| canary) exec "$here/ops-canary.sh" "$@" ;; |
| dev) exec "$here/ops-dev.sh" "$@" ;; |
| ledger) exec "$here/ops-ledger.sh" "$@" ;; |
| + mesh) exec "$here/ops-mesh.sh" "$@" ;; |
| ""|-h|--help|help) usage ;; |
| *) |
| printf 'ops: unknown verb %s\n\n' "$verb" >&2 |
diff --git a/deploy/ops-mesh.sh b/deploy/ops-mesh.sh| new file mode 100755 |
| index 0000000..ecc1f4a |
| --- /dev/null |
| +++ b/deploy/ops-mesh.sh |
| @@ -0,0 +1,48 @@ |
| +#!/usr/bin/env sh |
| +# 🤖 ops mesh — mesh-membership maintenance for the activator mesh |
| +# (docs/planet-scale.md; meshreplicate.go). |
| +# |
| +# ./deploy/ops mesh seed <peer-base-url> [box] # replicate every in-scope |
| +# # hibernating route (+ its |
| +# # instance env) to a peer |
| +# |
| +# Why this verb exists: a box joins the mesh with an EMPTY registry, but HRW |
| +# placement starts assigning it hosts the moment it heartbeats — any host it |
| +# doesn't know answers "unknown host", so an unseeded member silently blackholes |
| +# ~1/N of the placement-scoped fleet, new creates included. `lchat mesh-seed` is |
| +# the catch-up; this verb runs it on the source box with the router's own env |
| +# borrowed via --env-file (the mesh token/domains live in /data/router.env, |
| +# which operators deliberately cannot read — the sudo'd binary can). |
| +# |
| +# Run it FROM the box that has the routes (the established member), pointing at |
| +# the box that needs them. Idempotent: re-seeding overwrites with the same |
| +# content, so run it again whenever in doubt — e.g. after a peer rebuild, or |
| +# after this repo gains new replicated fields (the env began riding along |
| +# 2026-07-24; a pre-env seed left peers waking instances token-less). |
| +set -eu |
| + |
| +sub="${1:-}" |
| +[ $# -gt 0 ] && shift |
| + |
| +case "$sub" in |
| + seed) |
| + peer="${1:-}" |
| + box="${2:-${LCHAT_MESH_SOURCE_BOX:-elevenbox}}" |
| + if [ -z "$peer" ]; then |
| + echo "usage: ops mesh seed <peer-base-url> [box] (e.g. ops mesh seed https://77.68.125.53)" >&2 |
| + exit 2 |
| + fi |
| + echo "== seeding mesh routes (+env) from $box to $peer" |
| + ssh "$box" "sudo -n /opt/lchat/lchat mesh-seed --env-file /data/router.env '$peer'" |
| + ;; |
| + ""|-h|--help|help|*) |
| + cat <<'EOF' >&2 |
| +ops mesh — mesh-membership maintenance |
| + |
| + ./deploy/ops mesh seed <peer-base-url> [box] replicate all in-scope hibernating |
| + routes (+ instance env) to a peer |
| + (box defaults to elevenbox) |
| +EOF |
| + [ "$sub" = "" ] || [ "$sub" = "-h" ] || [ "$sub" = "--help" ] || [ "$sub" = "help" ] || exit 2 |
| + ;; |
| +esac |
diff --git a/deploy/systemd/lchat-control.service b/deploy/systemd/lchat-control.service| index d8d40b7..78e4163 100644 |
| --- a/deploy/systemd/lchat-control.service |
| +++ b/deploy/systemd/lchat-control.service |
| @@ -4,6 +4,13 @@ After=network-online.target |
| Wants=network-online.target |
| [Service] |
| +# The router's env FIRST: controld executes add-instance, whose mesh route |
| +# replication (fanOutMeshRoute) must see the box's mesh identity exactly as |
| +# the router does (LCHAT_MESH_SELF/_TOKEN/_DOMAINS, the hibernate bucket + |
| +# store credentials). Without this line the fan-out silently no-ops and any |
| +# create HRW-placed on a peer box is born dark — the peer answers "unknown |
| +# host" for it. control.env second, so control-specific values still win. |
| +EnvironmentFile=-/data/router.env |
| # Optional overrides: LCHAT_MAX_INSTANCES / LCHAT_MIN_FREE_GB (capacity policy). |
| EnvironmentFile=-/data/control.env |
| ExecStart=/opt/lchat/lchat controld |
diff --git a/docs/planet-scale-guide/mesh.html b/docs/planet-scale-guide/mesh.html| index 8eb76ab..5679360 100644 |
| --- a/docs/planet-scale-guide/mesh.html |
| +++ b/docs/planet-scale-guide/mesh.html |
| @@ -85,7 +85,7 @@ echo aHR0cHM6Ly9ib3gtMS5leGFtcGxlLm5ldA | base64 -d |
| <h2 id="mesh-replication">Route replication: every box learns about every host</h2> |
| <p>The router looks a request’s hostname up in its <strong>own local registry</strong> before any mesh logic runs at all, and answers <code>404</code> immediately on a miss. So placement (hashing, forwarding) is only half the story: a box can only ever serve — or even correctly forward — a host it has a registry row for. Two mechanisms keep every mesh member’s registry in sync, both authenticated with <code>LCHAT_MESH_TOKEN</code>:</p> |
| <ul> |
| - <li><strong>Automatic fan-out on creation.</strong> The moment a new in-scope hibernating host is registered anywhere, that box replicates the row to every <em>currently live</em> mesh peer (found the same way placement does — the membership heartbeats). Only a narrow, purpose-built payload crosses the wire — host, kind, version, hibernate flag, and a handful of routing-relevant fields — never the receiving box’s own socket path or database path, which each box always derives locally. A peer that is down at that moment simply misses the row; fan-out is best-effort, logged, and never blocks the create.</li> |
| + <li><strong>Automatic fan-out on creation.</strong> The moment a new in-scope hibernating host is registered anywhere, that box replicates the row to every <em>currently live</em> mesh peer (found the same way placement does — the membership heartbeats). Only a narrow, purpose-built payload crosses the wire — host, kind, version, hibernate flag, a handful of routing-relevant fields, <em>and the instance’s own env values</em> (admin token, entitlement, seeded name) so a peer-woken instance boots with the token its creator minted — never the receiving box’s own socket path or database path, which each box always derives locally. A peer that is down at that moment simply misses the row; fan-out is best-effort, logged, and never blocks the create. <strong>The daemon doing the creating must see the mesh env:</strong> <code>lchat-control.service</code> reads <code>/data/router.env</code> for exactly this (a controld without <code>LCHAT_MESH_SELF</code> silently skips fan-out, and any create HRW-placed on a peer is born dark — the peer answers <code>404 unknown host</code> for it; a live production incident, 2026-07-24). controld logs its replication posture at startup.</li> |
| <li><strong><code>lchat mesh-seed <peer-url></code> — the one-time catch-up.</strong> Run on an existing box and pointed at a newly joined one, this bulk-replicates every in-scope hibernating route the existing box already knows about, in one pass. This is the command that makes joining a box a single step instead of hand-registering every host (see <a href="#add-box">adding a box</a>, below).</li> |
| </ul> |
| <div class="why"> |
| @@ -115,7 +115,8 @@ LCHAT_MESH_TOKEN=<the same shared secret as every other box> |
| <p>Within one heartbeat interval the peers see the new member.</p></li> |
| <li><p class="step-title">Seed its registry from an existing box.</p> |
| <figure class="cmd"><figcaption>Run on an EXISTING mesh box, pointed at the new one</figcaption> |
| - <pre><code>lchat mesh-seed https://box-2.example.net</code></pre></figure> |
| + <pre><code>lchat mesh-seed --env-file /data/router.env https://box-2.example.net</code></pre> |
| + <p>(<code>--env-file</code> borrows the router’s <code>LCHAT_MESH_*</code> values — the seed can run under a sudo’d operator session that has no mesh env of its own. The encoded operator path is <code>./deploy/ops mesh seed <peer-url> [box]</code>.)</p></figure> |
| <p>Bulk-replicates every in-scope hibernating route in one pass, so the new box can immediately answer for (or correctly proxy) any of them. After this, roughly <em>1/N</em> of the in-scope spaces re-home to the new box at their next sleep/wake boundary — nothing needs announcing beyond this one command; the guest book and the registry rows do the rest.</p></li> |
| </ol> |
| <h3 id="drain-box">Draining a box (deploys, maintenance, scale-down)</h3> |
diff --git a/meshreplicate.go b/meshreplicate.go| index 55ec791..c30d04e 100644 |
| --- a/meshreplicate.go |
| +++ b/meshreplicate.go |
| @@ -48,7 +48,14 @@ import ( |
| ) |
| // meshRoutePayload is the wire shape for one replicated route — deliberately |
| -// narrower than Route (see file doc comment). |
| +// narrower than Route (see file doc comment). The Env* fields carry the |
| +// per-instance EnvironmentFile values that are NOT route fields (admin token, |
| +// entitlement, seeded name): a hibernating route's first wake on the receiving |
| +// box reads /data/<host>.env exactly as a local wake would (activator.go |
| +// s3Backend.start), so without them a peer-woken instance would boot with no |
| +// admin token and the creating box's setup link could never authenticate. |
| +// They ride the same verified-TLS, mesh-token-authenticated hop the |
| +// PushRelayToken already does. |
| type meshRoutePayload struct { |
| Host string |
| Kind string |
| @@ -61,6 +68,9 @@ type meshRoutePayload struct { |
| HomeRegion string |
| Provider string |
| Version string |
| + EnvAdminToken string |
| + EnvEntitlement string |
| + EnvGroupName string |
| } |
| func routeToPayload(r Route) meshRoutePayload { |
| @@ -72,6 +82,18 @@ func routeToPayload(r Route) meshRoutePayload { |
| } |
| } |
| +// payloadFromRouteEnv is routeToPayload plus the instance env values the |
| +// receiving box must reproduce (see the meshRoutePayload doc). env is the |
| +// same map materializeInstance writes to /data/<host>.env (or a readEnvFile |
| +// of it, for the seed). |
| +func payloadFromRouteEnv(r Route, env map[string]string) meshRoutePayload { |
| + p := routeToPayload(r) |
| + p.EnvAdminToken = env["LCHAT_ADMIN_TOKEN"] |
| + p.EnvEntitlement = env["LCHAT_ENTITLEMENT"] |
| + p.EnvGroupName = env["LCHAT_GROUP_NAME"] |
| + return p |
| +} |
| + |
| // toLocalRoute rebuilds a full Route from a peer's payload, deriving the |
| // box-local Socket/DBPath itself — never trusting a peer's filesystem layout |
| // — exactly as registerInstance does for a route created locally. |
| @@ -113,16 +135,16 @@ func (p meshRoutePayload) toLocalRoute() (Route, error) { |
| // subdomain still completes TLS" property wildcard.go relies on). Callers |
| // only ever pass an in-scope route (fanOutMeshRoute/runMeshSeed both check |
| // first), so this is never asked to authenticate as an out-of-scope host. |
| -func meshReplicateRoute(ctx context.Context, peer, token string, route Route) error { |
| +func meshReplicateRoute(ctx context.Context, peer, token string, payload meshRoutePayload) error { |
| peerURL, err := url.Parse(peer) |
| if err != nil || peerURL.Host == "" { |
| return fmt.Errorf("bad peer URL %q: %w", peer, err) |
| } |
| - body, err := json.Marshal(routeToPayload(route)) |
| + body, err := json.Marshal(payload) |
| if err != nil { |
| return err |
| } |
| - reqURL := url.URL{Scheme: peerURL.Scheme, Host: route.Host, Path: "/api/mesh/route"} |
| + reqURL := url.URL{Scheme: peerURL.Scheme, Host: payload.Host, Path: "/api/mesh/route"} |
| req, err := http.NewRequestWithContext(ctx, http.MethodPost, reqURL.String(), bytes.NewReader(body)) |
| if err != nil { |
| return err |
| @@ -142,7 +164,7 @@ func meshReplicateRoute(ctx context.Context, peer, token string, route Route) er |
| if err != nil { |
| return nil, err |
| } |
| - tlsConn := tls.Client(conn, &tls.Config{ServerName: route.Host}) |
| + tlsConn := tls.Client(conn, &tls.Config{ServerName: payload.Host}) |
| if err := tlsConn.HandshakeContext(ctx); err != nil { |
| conn.Close() |
| return nil, err |
| @@ -170,21 +192,24 @@ func meshReplicateRoute(ctx context.Context, peer, token string, route Route) er |
| // identical to today — whenever the mesh is off or the host is out of |
| // LCHAT_MESH_DOMAINS scope, so self-hosters and non-mesh boxes are untouched. |
| // Called right after registerInstance's local reg.upsert. |
| -func fanOutMeshRoute(hc *hibernateConfig, route Route) { |
| +func fanOutMeshRoute(hc *hibernateConfig, route Route, env map[string]string) { |
| mesh, err := meshFromEnv(hc) |
| if err != nil || mesh == nil || !mesh.inScope(route.Host) { |
| return |
| } |
| + payload := payloadFromRouteEnv(route, env) |
| mesh.beat() // one synchronous PUT+LIST for a fresh membership view |
| for _, peer := range mesh.liveBoxes() { |
| if peer == mesh.self { |
| continue |
| } |
| ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
| - err := meshReplicateRoute(ctx, peer, mesh.token, route) |
| + err := meshReplicateRoute(ctx, peer, mesh.token, payload) |
| cancel() |
| if err != nil { |
| log.Printf("mesh: replicate %s to %s failed: %v — it can't serve this host until a later replication succeeds", route.Host, peer, err) |
| + } else { |
| + log.Printf("mesh: replicated %s to %s", route.Host, peer) |
| } |
| } |
| } |
| @@ -217,6 +242,19 @@ func meshRouteHandler(reg *Registry, rt *router, token string) http.HandlerFunc |
| http.Error(w, "internal error", http.StatusInternalServerError) |
| return |
| } |
| + // The instance env travels with the route (meshRoutePayload doc): a |
| + // peer-woken instance must boot with the SAME admin token its creator |
| + // minted, or the setup link dies and the admin console is orphaned. |
| + // The sender is canonical, so a re-replication may overwrite — but a |
| + // payload WITHOUT a token (an older peer) never clobbers a file that |
| + // exists. |
| + if payload.EnvAdminToken != "" { |
| + env := instanceEnvMap(payload.EnvAdminToken, payload.AppStoreURL, payload.HomeOrigin, |
| + payload.PushRelayURL, payload.PushRelayToken, payload.EnvEntitlement, payload.EnvGroupName) |
| + if err := writeEnvFile(filepath.Join(instanceDataDir, route.Host+".env"), env); err != nil { |
| + log.Printf("mesh: route replication env write %s failed: %v — its first wake here will boot token-less", route.Host, err) |
| + } |
| + } |
| if err := rt.reload(); err != nil { |
| log.Printf("mesh: route replication reload after %s failed: %v", route.Host, err) |
| } |
| @@ -235,10 +273,22 @@ func meshRouteHandler(reg *Registry, rt *router, token string) http.HandlerFunc |
| func runMeshSeed(args []string) { |
| fs := flag.NewFlagSet("mesh-seed", flag.ExitOnError) |
| registryPath := fs.String("registry", envOr("LCHAT_REGISTRY", "/data/registry.db"), "router registry SQLite path") |
| - _ = fs.Parse(args) |
| + envFile := fs.String("env-file", "", "load LCHAT_MESH_* (and anything else) from this KEY=value file first — lets a sudo'd operator run borrow the router's env (e.g. /data/router.env) without a root shell") |
| + _ = fs.Parse(reorderFlagsFirst(fs, args)) |
| + if *envFile != "" { |
| + kv, err := readEnvFile(*envFile) |
| + if err != nil { |
| + die("mesh-seed: read --env-file: %v", err) |
| + } |
| + for k, v := range kv { |
| + if os.Getenv(k) == "" { // real env always wins over the file |
| + os.Setenv(k, v) |
| + } |
| + } |
| + } |
| peer := fs.Arg(0) |
| if peer == "" { |
| - die("mesh-seed: usage: lchat mesh-seed <peer-base-url>") |
| + die("mesh-seed: usage: lchat mesh-seed [--env-file /data/router.env] <peer-base-url>") |
| } |
| domains := splitDomains(os.Getenv("LCHAT_MESH_DOMAINS")) |
| if len(domains) == 0 { |
| @@ -262,8 +312,15 @@ func runMeshSeed(args []string) { |
| if !route.Hibernate || !hostInDomains(route.Host, domains) { |
| continue |
| } |
| + // The instance env rides along (meshRoutePayload doc) — without it the |
| + // peer wakes this host token-less. Missing file (never materialized |
| + // here either) just seeds the bare route, exactly as before. |
| + env, envErr := readEnvFile(filepath.Join(instanceDataDir, route.Host+".env")) |
| + if envErr != nil && !os.IsNotExist(envErr) { |
| + fmt.Printf(" %s: env unreadable (%v) — seeding the bare route\n", route.Host, envErr) |
| + } |
| ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
| - err := meshReplicateRoute(ctx, peer, token, route) |
| + err := meshReplicateRoute(ctx, peer, token, payloadFromRouteEnv(route, env)) |
| cancel() |
| if err != nil { |
| fmt.Printf(" %s: FAILED (%v)\n", route.Host, err) |
diff --git a/meshreplicate_test.go b/meshreplicate_test.go| index 2a3d7cf..f6075e1 100644 |
| --- a/meshreplicate_test.go |
| +++ b/meshreplicate_test.go |
| @@ -142,14 +142,71 @@ func TestMeshReplicateRouteOverHTTP(t *testing.T) { |
| defer srv.Close() |
| route := testRoute("space3.11ch.at") |
| - if err := meshReplicateRoute(context.Background(), srv.URL, "shared-secret", route); err != nil { |
| + if err := meshReplicateRoute(context.Background(), srv.URL, "shared-secret", routeToPayload(route)); err != nil { |
| t.Fatalf("meshReplicateRoute: %v", err) |
| } |
| if !rt.allowed(route.Host) { |
| t.Fatal("route was not replicated") |
| } |
| - if err := meshReplicateRoute(context.Background(), srv.URL, "wrong-secret", route); err == nil { |
| + if err := meshReplicateRoute(context.Background(), srv.URL, "wrong-secret", routeToPayload(route)); err == nil { |
| t.Fatal("wrong token should be rejected") |
| } |
| } |
| + |
| +// The instance env rides replication: the receiving box writes /data/<host>.env |
| +// from the payload, so its first wake of the host boots with the SAME admin |
| +// token the creating box minted — without it, the creator's setup link could |
| +// never authenticate against a peer-woken instance. A payload without a token |
| +// (an older peer) must never clobber an env that already exists. |
| +func TestMeshReplicationCarriesInstanceEnv(t *testing.T) { |
| + dir := t.TempDir() |
| + oldData := instanceDataDir |
| + instanceDataDir = dir |
| + defer func() { instanceDataDir = oldData }() |
| + |
| + reg, err := openRegistry(filepath.Join(t.TempDir(), "registry.db")) |
| + if err != nil { |
| + t.Fatalf("open registry: %v", err) |
| + } |
| + rt, err := newRouter(reg, nil) |
| + if err != nil { |
| + t.Fatalf("new router: %v", err) |
| + } |
| + mux := http.NewServeMux() |
| + mux.Handle("POST /api/mesh/route", meshRouteHandler(reg, rt, "shared-secret")) |
| + srv := httptest.NewServer(mux) |
| + defer srv.Close() |
| + |
| + route := testRoute("space4.11ch.at") |
| + route.PushRelayURL = "https://push.example.com" |
| + env := map[string]string{ |
| + "LCHAT_ADMIN_TOKEN": "tok-abc", |
| + "LCHAT_ENTITLEMENT": "paid", |
| + "LCHAT_GROUP_NAME": "The Campbells", |
| + } |
| + if err := meshReplicateRoute(context.Background(), srv.URL, "shared-secret", payloadFromRouteEnv(route, env)); err != nil { |
| + t.Fatalf("meshReplicateRoute: %v", err) |
| + } |
| + got, err := readEnvFile(filepath.Join(dir, route.Host+".env")) |
| + if err != nil { |
| + t.Fatalf("receiver wrote no env file: %v", err) |
| + } |
| + if got["LCHAT_ADMIN_TOKEN"] != "tok-abc" || got["LCHAT_ENTITLEMENT"] != "paid" || got["LCHAT_GROUP_NAME"] != "The Campbells" { |
| + t.Errorf("replicated env = %v", got) |
| + } |
| + // Route fields that are ALSO env values must land through the shared |
| + // builder, exactly as materializeInstance writes them locally. |
| + if got["LCHAT_PUSH_RELAY_URL"] != "https://push.example.com" { |
| + t.Errorf("push relay url = %q, want the route's", got["LCHAT_PUSH_RELAY_URL"]) |
| + } |
| + |
| + // A token-less payload (older peer, or a host with no env at the source) |
| + // upserts the route but leaves the existing env alone. |
| + if err := meshReplicateRoute(context.Background(), srv.URL, "shared-secret", routeToPayload(route)); err != nil { |
| + t.Fatalf("re-replicate: %v", err) |
| + } |
| + if got, _ := readEnvFile(filepath.Join(dir, route.Host+".env")); got["LCHAT_ADMIN_TOKEN"] != "tok-abc" { |
| + t.Errorf("token-less payload clobbered the env: %v", got) |
| + } |
| +} |
diff --git a/relocate.go b/relocate.go| index 1d57ae1..60f864b 100644 |
| --- a/relocate.go |
| +++ b/relocate.go |
| @@ -68,6 +68,7 @@ import ( |
| awsconfig "github.com/aws/aws-sdk-go-v2/config" |
| "github.com/aws/aws-sdk-go-v2/credentials" |
| awss3 "github.com/aws/aws-sdk-go-v2/service/s3" |
| + "path/filepath" |
| ) |
| // storeLocation names one region's object store for a space's replica prefix. |
| @@ -256,11 +257,15 @@ func relocate(reg *Registry, mesh *meshSvc, hc *hibernateConfig, opts relocateOp |
| // target, the DESTINATION holds the authoritative copy. Re-copying source→dest |
| // would clobber newer destination writes, so DON'T. Just heal any peer that |
| // missed the earlier flip by re-fanning-out the route, and return. |
| + // The instance env rides every fan-out (meshRoutePayload doc) so the new |
| + // owner wakes this space with its real admin token, not a token-less boot. |
| + instEnv, _ := readEnvFile(filepath.Join(instanceDataDir, host+".env")) |
| + |
| if rt.HomeRegion == region { |
| res.AlreadyHome = true |
| here := storeLocation{Bucket: hc.Bucket, Prefix: hc.hibernatePrefix(host)} |
| res.Plan = relocationStorePlan(here, here) // same store → Skip |
| - fanOutMeshRoute(hc, *rt) // best-effort heal; a no-op when the mesh is off |
| + fanOutMeshRoute(hc, *rt, instEnv) // best-effort heal; a no-op when the mesh is off |
| return res, nil |
| } |
| @@ -297,7 +302,7 @@ func relocate(reg *Registry, mesh *meshSvc, hc *hibernateConfig, opts relocateOp |
| } |
| updated := *rt |
| updated.HomeRegion = region |
| - fanOutMeshRoute(hc, updated) |
| + fanOutMeshRoute(hc, updated, instEnv) |
| // Step 3 — optional A-record move. |
| if opts.MoveDNS { |