| 1 | package main |
| 2 | |
| 3 | // Route replication across the activator mesh (mesh.go). registerInstance's |
| 4 | // reg.upsert only ever writes ONE box's registry — but a host under |
| 5 | // LCHAT_MESH_DOMAINS needs a routes row on every live mesh member before HRW |
| 6 | // can honestly place it there (mesh.go's inScope doc comment: a box with no |
| 7 | // row for a host 404s it, never wakes it). This file is the fan-out: |
| 8 | // |
| 9 | // - An admin RPC, POST /api/mesh/route (meshRouteHandler, wired by runRouter |
| 10 | // only when the mesh is on), inserts or updates one route. |
| 11 | // - registerInstance calls fanOutMeshRoute right after its local upsert, |
| 12 | // best-effort against whichever peers are live right now. |
| 13 | // - `lchat mesh-seed [--region <cc>] <peer>` (runMeshSeed) bulk-replicates |
| 14 | // every existing route the joining box owns once — the catch-up a freshly |
| 15 | // joined box needs before it can serve anything, since it joins with an |
| 16 | // empty registry. |
| 17 | // |
| 18 | // Which peers get a route is the UNION of two sets (docs/multi-edge.md, Phase 1): |
| 19 | // the boxes covering the host by DOMAIN (its LCHAT_MESH_DOMAINS — today's |
| 20 | // fleet-wide fan-out for an in-domain host) and the boxes owning it by REGION |
| 21 | // (those advertising the route's home_region). The region arm is the multi-edge |
| 22 | // fix: a space placed by home_region onto a box whose per-country domain does NOT |
| 23 | // cover the host (mesh.go candidatesFor) still needs a registry row THERE to |
| 24 | // serve when the DNS-target box proxies to it — domain scope alone would strand |
| 25 | // it. With no LCHAT_MESH_REGION and no home_region anywhere the region arm is |
| 26 | // always empty, so this stays byte-identical to the pre-region, domain-only |
| 27 | // fan-out. (Completing TLS to a region-owner box for a host outside ITS domain |
| 28 | // is a separate, still-unbuilt slice — see meshReplicateRoute.) |
| 29 | // |
| 30 | // Both paths send only the fields a route's CONTENT actually needs (host, |
| 31 | // kind, hibernate flag, per-instance config) — never Socket/DBPath, which are |
| 32 | // box-local filesystem layout the receiving box must derive itself exactly as |
| 33 | // registerInstance does for a route created locally, and never |
| 34 | // Status/NextDue/CreatedAt, which are receiver-local lifecycle state. |
| 35 | // |
| 36 | // Deletions replicate too (#1346): both delete paths (controld's |
| 37 | // deleteInstance and `lchat remove`) fan a tombstone |
| 38 | // (fanOutMeshRouteDelete → DELETE /api/mesh/route/{host}), and |
| 39 | // `mesh-seed --prune` reconciles whatever a best-effort fan-out missed — |
| 40 | // without this, a destroyed space lived on (and WOKE, from a stale replica) |
| 41 | // on any peer that held its row, violating the destroy promise. Live case: |
| 42 | // six zombies swept off kepler 2026-07-25. Deletion fan-out and --prune are |
| 43 | // still DOMAIN-scoped only: a region-owned, out-of-domain row's tombstone is |
| 44 | // the next multi-edge slice (alongside out-of-domain TLS, below) — inert today |
| 45 | // because nothing region-places out-of-domain hosts until that TLS lands. |
| 46 | // |
| 47 | // Honest gap, narrowed: a HIBERNATE FLIP still doesn't replicate — and must |
| 48 | // not naively: a peer holding Hibernate=false would proxy to a local socket |
| 49 | // that doesn't exist instead of forwarding. A flipped-to-systemd host needs |
| 50 | // a "lives elsewhere" route shape peers don't have yet. Renames don't exist |
| 51 | // (host is the key). |
| 52 | |
| 53 | import ( |
| 54 | "bytes" |
| 55 | "context" |
| 56 | "crypto/subtle" |
| 57 | "crypto/tls" |
| 58 | "encoding/json" |
| 59 | "flag" |
| 60 | "fmt" |
| 61 | "io" |
| 62 | "log" |
| 63 | "net" |
| 64 | "net/http" |
| 65 | "net/url" |
| 66 | "os" |
| 67 | "path/filepath" |
| 68 | "strings" |
| 69 | "time" |
| 70 | ) |
| 71 | |
| 72 | // meshRoutePayload is the wire shape for one replicated route — deliberately |
| 73 | // narrower than Route (see file doc comment). The Env* fields carry the |
| 74 | // per-instance EnvironmentFile values that are NOT route fields (admin token, |
| 75 | // entitlement, seeded name): a hibernating route's first wake on the receiving |
| 76 | // box reads /data/<host>.env exactly as a local wake would (activator.go |
| 77 | // s3Backend.start), so without them a peer-woken instance would boot with no |
| 78 | // admin token and the creating box's setup link could never authenticate. |
| 79 | // They ride the same verified-TLS, mesh-token-authenticated hop the |
| 80 | // PushRelayToken already does. |
| 81 | type meshRoutePayload struct { |
| 82 | Host string |
| 83 | Kind string |
| 84 | Hibernate bool |
| 85 | Replication string |
| 86 | AppStoreURL string |
| 87 | HomeOrigin string |
| 88 | PushRelayURL string |
| 89 | PushRelayToken string |
| 90 | HomeRegion string |
| 91 | Provider string |
| 92 | Version string |
| 93 | EnvAdminToken string |
| 94 | EnvEntitlement string |
| 95 | EnvGroupName string |
| 96 | } |
| 97 | |
| 98 | func routeToPayload(r Route) meshRoutePayload { |
| 99 | return meshRoutePayload{ |
| 100 | Host: r.Host, Kind: r.Kind, Hibernate: r.Hibernate, Replication: r.Replication, |
| 101 | AppStoreURL: r.AppStoreURL, HomeOrigin: r.HomeOrigin, |
| 102 | PushRelayURL: r.PushRelayURL, PushRelayToken: r.PushRelayToken, |
| 103 | HomeRegion: r.HomeRegion, Provider: r.Provider, Version: r.Version, |
| 104 | } |
| 105 | } |
| 106 | |
| 107 | // payloadFromRouteEnv is routeToPayload plus the instance env values the |
| 108 | // receiving box must reproduce (see the meshRoutePayload doc). env is the |
| 109 | // same map materializeInstance writes to /data/<host>.env (or a readEnvFile |
| 110 | // of it, for the seed). |
| 111 | func payloadFromRouteEnv(r Route, env map[string]string) meshRoutePayload { |
| 112 | p := routeToPayload(r) |
| 113 | p.EnvAdminToken = env["LCHAT_ADMIN_TOKEN"] |
| 114 | p.EnvEntitlement = env["LCHAT_ENTITLEMENT"] |
| 115 | p.EnvGroupName = env["LCHAT_GROUP_NAME"] |
| 116 | return p |
| 117 | } |
| 118 | |
| 119 | // toLocalRoute rebuilds a full Route from a peer's payload, deriving the |
| 120 | // box-local Socket/DBPath itself — never trusting a peer's filesystem layout |
| 121 | // — exactly as registerInstance does for a route created locally. |
| 122 | func (p meshRoutePayload) toLocalRoute() (Route, error) { |
| 123 | host := strings.ToLower(strings.TrimSpace(p.Host)) |
| 124 | if !validHost(host) { |
| 125 | return Route{}, fmt.Errorf("%w: %q", errInvalidHost, host) |
| 126 | } |
| 127 | socket := filepath.Join(instanceSocketDir, host+".sock") |
| 128 | if err := checkSocketLen(socket); err != nil { |
| 129 | return Route{}, err |
| 130 | } |
| 131 | return Route{ |
| 132 | Host: host, Kind: p.Kind, Socket: socket, |
| 133 | DBPath: filepath.Join(instanceDataDir, host+".db"), |
| 134 | AppStoreURL: p.AppStoreURL, |
| 135 | HomeOrigin: p.HomeOrigin, |
| 136 | PushRelayURL: p.PushRelayURL, |
| 137 | PushRelayToken: p.PushRelayToken, |
| 138 | HomeRegion: p.HomeRegion, |
| 139 | Provider: p.Provider, |
| 140 | Hibernate: p.Hibernate, |
| 141 | Replication: p.Replication, |
| 142 | Version: p.Version, |
| 143 | }, nil |
| 144 | } |
| 145 | |
| 146 | // meshReplicateRoute POSTs one route to peer's /api/mesh/route, authenticated |
| 147 | // with the mesh's shared token. Used both by fanOutMeshRoute and by |
| 148 | // runMeshSeed's bulk seed. |
| 149 | // The peer's own base URL (LCHAT_MESH_SELF, typically a bare address like |
| 150 | // https://10.0.0.5) was never issued a certificate of its own — only |
| 151 | // registered, wildcard-covered instance hostnames get one. So — exactly like |
| 152 | // mesh.go's own newPeerProxy authenticates a proxied hop by the INSTANCE |
| 153 | // hostname, never the peer's bare identity — this dials the peer's raw |
| 154 | // address but performs TLS (and sends Host) as route.Host: a wildcard-covered |
| 155 | // hostname the receiving box's wildcard manager hands back a valid cert for |
| 156 | // regardless of whether that host is registered THERE yet (the "unknown |
| 157 | // subdomain still completes TLS" property wildcard.go relies on). For a |
| 158 | // DOMAIN-covered target that cert is the box's own wildcard, so TLS completes |
| 159 | // exactly as before. Region replication (fanOutMeshRoute's union with the |
| 160 | // route's home_region owners) can now also target a box whose per-country |
| 161 | // LCHAT_MESH_DOMAINS does NOT cover payload.Host — an out-of-domain host that |
| 162 | // box may hold no wildcard for, so this handshake can fail there. Making a box |
| 163 | // present a valid cert for an out-of-domain host it now owns by region is the |
| 164 | // NEXT multi-edge slice (docs/multi-edge.md), deliberately not built here; a |
| 165 | // same-domain fleet (production today) is unaffected. |
| 166 | func meshReplicateRoute(ctx context.Context, peer, token string, payload meshRoutePayload) error { |
| 167 | body, err := json.Marshal(payload) |
| 168 | if err != nil { |
| 169 | return err |
| 170 | } |
| 171 | resp, err := meshPeerRequest(ctx, peer, token, http.MethodPost, "/api/mesh/route", payload.Host, body) |
| 172 | if err != nil { |
| 173 | return err |
| 174 | } |
| 175 | defer resp.Body.Close() |
| 176 | if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { |
| 177 | return fmt.Errorf("peer %s: %s", peer, resp.Status) |
| 178 | } |
| 179 | return nil |
| 180 | } |
| 181 | |
| 182 | // replicationTargets is the set of OTHER live boxes that must hold route's |
| 183 | // registry row: the boxes covering host by DOMAIN (this box's |
| 184 | // LCHAT_MESH_DOMAINS — today's fleet-wide fan-out for an in-domain host) UNION |
| 185 | // the boxes owning host by REGION (those advertising route.HomeRegion). The |
| 186 | // union is the multi-edge fix (docs/multi-edge.md): a space placed by home_region |
| 187 | // onto a box whose per-country domain does NOT cover the host still needs a row |
| 188 | // there to serve when the DNS-target box proxies to it — domain scope alone would |
| 189 | // strand it. Self is excluded (registerInstance already upserted the local row), |
| 190 | // and a box qualifying by BOTH domain and region is sent to once. |
| 191 | // |
| 192 | // Region targets are non-empty only on a region-aware box (one that fetched peer |
| 193 | // regions; m.regions is empty otherwise) and only for a route carrying a |
| 194 | // home_region, so with no LCHAT_MESH_REGION and no home_region anywhere this is |
| 195 | // exactly the domain-scoped set — byte-identical to pre-region replication. |
| 196 | func (m *meshSvc) replicationTargets(route Route) []string { |
| 197 | seen := map[string]bool{m.self: true} // never replicate to ourselves |
| 198 | var targets []string |
| 199 | add := func(boxes []string) { |
| 200 | for _, b := range boxes { |
| 201 | if !seen[b] { |
| 202 | seen[b] = true |
| 203 | targets = append(targets, b) |
| 204 | } |
| 205 | } |
| 206 | } |
| 207 | if m.inScope(route.Host) { |
| 208 | add(m.liveBoxes()) // domain-scoped: the whole live fleet, as before regions |
| 209 | } |
| 210 | add(m.boxesInRegion(route.HomeRegion)) // region-owned: the owner box(es) |
| 211 | return targets |
| 212 | } |
| 213 | |
| 214 | // boxesInRegion returns the live boxes whose advertised region matches region — |
| 215 | // the mesh members that own a route with this home_region. It is candidatesFor's |
| 216 | // region filter keyed on a region string (rather than a host's registry row), |
| 217 | // reusing slice 1's peer-region view (m.regions, self via m.region — never a |
| 218 | // second advertisement mechanism). An empty region, or a region-agnostic box |
| 219 | // (m.regions empty because it never fetched peer regions), yields no matches, |
| 220 | // which is what keeps region replication inert until the fleet opts into regions. |
| 221 | func (m *meshSvc) boxesInRegion(region string) []string { |
| 222 | if region == "" { |
| 223 | return nil |
| 224 | } |
| 225 | live := m.liveBoxes() |
| 226 | m.mu.Lock() |
| 227 | regions := m.regions |
| 228 | m.mu.Unlock() |
| 229 | var matched []string |
| 230 | for _, b := range live { |
| 231 | r := regions[b] |
| 232 | if b == m.self { |
| 233 | r = m.region // our own region is known locally, never via S3 |
| 234 | } |
| 235 | if r == region { |
| 236 | matched = append(matched, b) |
| 237 | } |
| 238 | } |
| 239 | return matched |
| 240 | } |
| 241 | |
| 242 | // fanOutMeshRoute replicates route to every OTHER box that must hold its |
| 243 | // registry row — the DOMAIN ∪ REGION union replicationTargets computes |
| 244 | // (docs/multi-edge.md) — best-effort: a peer that's down or errors just can't |
| 245 | // yet serve this host (see the file doc comment's honest gap on mutations — |
| 246 | // creation retries only via a later add-instance or an operator's mesh-seed |
| 247 | // rerun). A no-op — byte-identical to today — whenever the mesh is off, or the |
| 248 | // host is out of LCHAT_MESH_DOMAINS scope AND this box can't region-place it |
| 249 | // (no LCHAT_MESH_REGION, or the route carries no home_region), so self-hosters, |
| 250 | // non-mesh boxes, and every region-agnostic box are untouched. Called right |
| 251 | // after registerInstance's local reg.upsert. |
| 252 | func fanOutMeshRoute(hc *hibernateConfig, route Route, env map[string]string) { |
| 253 | mesh, err := meshFromEnv(hc) |
| 254 | if err != nil || mesh == nil { |
| 255 | return |
| 256 | } |
| 257 | // Pre-region early-out, kept byte-identical: an out-of-domain host on a box |
| 258 | // that can't region-place it (region unset, or the route has no home_region) |
| 259 | // replicates nowhere, and — crucially — doesn't even beat. An agnostic box |
| 260 | // always lands here for an out-of-scope host, exactly as before regions. |
| 261 | if !mesh.inScope(route.Host) && (mesh.region == "" || route.HomeRegion == "") { |
| 262 | return |
| 263 | } |
| 264 | payload := payloadFromRouteEnv(route, env) |
| 265 | mesh.beat() // one synchronous PUT+LIST(+region refresh) for a fresh view |
| 266 | for _, peer := range mesh.replicationTargets(route) { |
| 267 | ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
| 268 | err := meshReplicateRoute(ctx, peer, mesh.token, payload) |
| 269 | cancel() |
| 270 | if err != nil { |
| 271 | log.Printf("mesh: replicate %s to %s failed: %v — it can't serve this host until a later replication succeeds", route.Host, peer, err) |
| 272 | } else { |
| 273 | log.Printf("mesh: replicated %s to %s", route.Host, peer) |
| 274 | } |
| 275 | } |
| 276 | } |
| 277 | |
| 278 | // meshPeerRequest performs one authenticated request against a mesh peer's |
| 279 | // raw address, TLS'd (and Host'd) as sniHost — the wildcard-covered instance |
| 280 | // hostname trick meshReplicateRoute documents above. Shared by the route |
| 281 | // upsert, the tombstone, and the reconciliation listing. |
| 282 | func meshPeerRequest(ctx context.Context, peer, token, method, path, sniHost string, body []byte) (*http.Response, error) { |
| 283 | peerURL, err := url.Parse(peer) |
| 284 | if err != nil || peerURL.Host == "" { |
| 285 | return nil, fmt.Errorf("bad peer URL %q: %w", peer, err) |
| 286 | } |
| 287 | reqURL := url.URL{Scheme: peerURL.Scheme, Host: sniHost, Path: path} |
| 288 | var rd io.Reader |
| 289 | if body != nil { |
| 290 | rd = bytes.NewReader(body) |
| 291 | } |
| 292 | req, err := http.NewRequestWithContext(ctx, method, reqURL.String(), rd) |
| 293 | if err != nil { |
| 294 | return nil, err |
| 295 | } |
| 296 | req.Header.Set("Content-Type", "application/json") |
| 297 | req.Header.Set("Authorization", "Bearer "+token) |
| 298 | dialAddr := peerDialAddr(peerURL) |
| 299 | transport := &http.Transport{ |
| 300 | DialContext: func(ctx context.Context, network, _ string) (net.Conn, error) { |
| 301 | return (&net.Dialer{Timeout: 10 * time.Second}).DialContext(ctx, network, dialAddr) |
| 302 | }, |
| 303 | } |
| 304 | if peerURL.Scheme == "https" { |
| 305 | transport.DialTLSContext = func(ctx context.Context, network, _ string) (net.Conn, error) { |
| 306 | conn, err := (&net.Dialer{Timeout: 10 * time.Second}).DialContext(ctx, network, dialAddr) |
| 307 | if err != nil { |
| 308 | return nil, err |
| 309 | } |
| 310 | tlsConn := tls.Client(conn, &tls.Config{ServerName: sniHost}) |
| 311 | if err := tlsConn.HandshakeContext(ctx); err != nil { |
| 312 | conn.Close() |
| 313 | return nil, err |
| 314 | } |
| 315 | return tlsConn, nil |
| 316 | } |
| 317 | } |
| 318 | return (&http.Client{Transport: transport}).Do(req) |
| 319 | } |
| 320 | |
| 321 | // meshDeleteRoute sends host's tombstone to one peer — the deletion half of |
| 322 | // replication (#1346). Idempotent on the receiving side, so re-sends and |
| 323 | // reconciliation sweeps are always safe. |
| 324 | func meshDeleteRoute(ctx context.Context, peer, token, host string) error { |
| 325 | resp, err := meshPeerRequest(ctx, peer, token, http.MethodDelete, "/api/mesh/route/"+host, host, nil) |
| 326 | if err != nil { |
| 327 | return err |
| 328 | } |
| 329 | defer resp.Body.Close() |
| 330 | if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { |
| 331 | return fmt.Errorf("peer %s: %s", peer, resp.Status) |
| 332 | } |
| 333 | return nil |
| 334 | } |
| 335 | |
| 336 | // meshListRoutes reads a peer's in-scope hibernating route hosts — what |
| 337 | // mesh-seed --prune diffs against the source registry. The SNI rides any |
| 338 | // wildcard-covered host the caller knows (its own first route works); the |
| 339 | // peer answers from its registry either way. |
| 340 | func meshListRoutes(ctx context.Context, peer, token, sniHost string) ([]string, error) { |
| 341 | resp, err := meshPeerRequest(ctx, peer, token, http.MethodGet, "/api/mesh/routes", sniHost, nil) |
| 342 | if err != nil { |
| 343 | return nil, err |
| 344 | } |
| 345 | defer resp.Body.Close() |
| 346 | if resp.StatusCode != http.StatusOK { |
| 347 | return nil, fmt.Errorf("peer %s: %s", peer, resp.Status) |
| 348 | } |
| 349 | var hosts []string |
| 350 | if err := json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&hosts); err != nil { |
| 351 | return nil, err |
| 352 | } |
| 353 | return hosts, nil |
| 354 | } |
| 355 | |
| 356 | // fanOutMeshRouteDelete replicates a deletion to every live mesh peer — the |
| 357 | // tombstone counterpart of fanOutMeshRoute, called from both delete paths |
| 358 | // (controld's deleteInstance and `lchat remove`) after the local row is gone. |
| 359 | // Best-effort exactly like creation fan-out: a peer that's down keeps its |
| 360 | // stale row until a `mesh-seed --prune` reconciles it. |
| 361 | func fanOutMeshRouteDelete(hc *hibernateConfig, host string) { |
| 362 | mesh, err := meshFromEnv(hc) |
| 363 | if err != nil || mesh == nil || !mesh.inScope(host) { |
| 364 | return |
| 365 | } |
| 366 | mesh.beat() |
| 367 | for _, peer := range mesh.liveBoxes() { |
| 368 | if peer == mesh.self { |
| 369 | continue |
| 370 | } |
| 371 | ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
| 372 | err := meshDeleteRoute(ctx, peer, mesh.token, host) |
| 373 | cancel() |
| 374 | if err != nil { |
| 375 | log.Printf("mesh: delete %s on %s failed: %v — its stale row stays until a mesh-seed --prune", host, peer, err) |
| 376 | } else { |
| 377 | log.Printf("mesh: deleted %s on %s", host, peer) |
| 378 | } |
| 379 | } |
| 380 | } |
| 381 | |
| 382 | // meshRouteHandler is the receiving side: POST /api/mesh/route, wired by |
| 383 | // runRouter only when the mesh is on. Bearer-authenticated with the same |
| 384 | // shared token every mesh member is configured with — without this, any |
| 385 | // caller could hand this box arbitrary routes. |
| 386 | func meshRouteHandler(reg *Registry, rt *router, token string) http.HandlerFunc { |
| 387 | return func(w http.ResponseWriter, r *http.Request) { |
| 388 | const prefix = "Bearer " |
| 389 | auth := r.Header.Get("Authorization") |
| 390 | if !strings.HasPrefix(auth, prefix) || |
| 391 | subtle.ConstantTimeCompare([]byte(strings.TrimPrefix(auth, prefix)), []byte(token)) != 1 { |
| 392 | http.Error(w, "unauthorized", http.StatusUnauthorized) |
| 393 | return |
| 394 | } |
| 395 | var payload meshRoutePayload |
| 396 | if err := json.NewDecoder(io.LimitReader(r.Body, 1<<16)).Decode(&payload); err != nil { |
| 397 | http.Error(w, "bad request", http.StatusBadRequest) |
| 398 | return |
| 399 | } |
| 400 | route, err := payload.toLocalRoute() |
| 401 | if err != nil { |
| 402 | http.Error(w, err.Error(), http.StatusBadRequest) |
| 403 | return |
| 404 | } |
| 405 | if err := reg.upsert(route); err != nil { |
| 406 | log.Printf("mesh: route replication upsert %s failed: %v", route.Host, err) |
| 407 | http.Error(w, "internal error", http.StatusInternalServerError) |
| 408 | return |
| 409 | } |
| 410 | // The instance env travels with the route (meshRoutePayload doc): a |
| 411 | // peer-woken instance must boot with the SAME admin token its creator |
| 412 | // minted, or the setup link dies and the admin console is orphaned. |
| 413 | // The sender is canonical, so a re-replication may overwrite — but a |
| 414 | // payload WITHOUT a token (an older peer) never clobbers a file that |
| 415 | // exists. |
| 416 | if payload.EnvAdminToken != "" { |
| 417 | env := instanceEnvMap(payload.EnvAdminToken, payload.AppStoreURL, payload.HomeOrigin, |
| 418 | payload.PushRelayURL, payload.PushRelayToken, payload.EnvEntitlement, payload.EnvGroupName) |
| 419 | if err := writeEnvFile(filepath.Join(instanceDataDir, route.Host+".env"), env); err != nil { |
| 420 | log.Printf("mesh: route replication env write %s failed: %v — its first wake here will boot token-less", route.Host, err) |
| 421 | } |
| 422 | } |
| 423 | if err := rt.reload(); err != nil { |
| 424 | log.Printf("mesh: route replication reload after %s failed: %v", route.Host, err) |
| 425 | } |
| 426 | w.WriteHeader(http.StatusNoContent) |
| 427 | } |
| 428 | } |
| 429 | |
| 430 | // meshAuthed wraps a mesh endpoint behind the shared bearer token — the same |
| 431 | // constant-time gate meshRouteHandler carries inline. |
| 432 | func meshAuthed(token string, next http.HandlerFunc) http.HandlerFunc { |
| 433 | return func(w http.ResponseWriter, r *http.Request) { |
| 434 | const prefix = "Bearer " |
| 435 | auth := r.Header.Get("Authorization") |
| 436 | if !strings.HasPrefix(auth, prefix) || |
| 437 | subtle.ConstantTimeCompare([]byte(strings.TrimPrefix(auth, prefix)), []byte(token)) != 1 { |
| 438 | http.Error(w, "unauthorized", http.StatusUnauthorized) |
| 439 | return |
| 440 | } |
| 441 | next(w, r) |
| 442 | } |
| 443 | } |
| 444 | |
| 445 | // meshRouteDeleteHandler: DELETE /api/mesh/route/{host} — a peer says this |
| 446 | // space was destroyed. Idempotent (an absent row still answers 204, so |
| 447 | // re-sends and prune sweeps never error). Deletes the row, the instance's |
| 448 | // webhook-ingest row (in.<host> dies with the space, mirroring lchat remove), |
| 449 | // and the REPLICATED env file — it holds a destroyed space's admin token and |
| 450 | // arrived via replication, so it's ours to take back; the db/blob files are |
| 451 | // member-data-shaped and stay, exactly like every other delete path (the |
| 452 | // data-safety rule). The reload's retire pass stops any live activator child. |
| 453 | func meshRouteDeleteHandler(reg *Registry, rt *router, token string) http.HandlerFunc { |
| 454 | return meshAuthed(token, func(w http.ResponseWriter, r *http.Request) { |
| 455 | host := strings.ToLower(strings.TrimSpace(r.PathValue("host"))) |
| 456 | if !validHost(host) { |
| 457 | http.Error(w, "bad host", http.StatusBadRequest) |
| 458 | return |
| 459 | } |
| 460 | if err := reg.delete(host); err != nil { |
| 461 | log.Printf("mesh: tombstone delete %s failed: %v", host, err) |
| 462 | http.Error(w, "internal error", http.StatusInternalServerError) |
| 463 | return |
| 464 | } |
| 465 | if err := reg.delete("in." + host); err != nil { |
| 466 | log.Printf("mesh: tombstone delete in.%s failed: %v", host, err) |
| 467 | } |
| 468 | if err := os.Remove(filepath.Join(instanceDataDir, host+".env")); err != nil && !os.IsNotExist(err) { |
| 469 | log.Printf("mesh: tombstone env removal for %s failed: %v", host, err) |
| 470 | } |
| 471 | if err := rt.reload(); err != nil { |
| 472 | log.Printf("mesh: reload after tombstone %s failed: %v", host, err) |
| 473 | } |
| 474 | log.Printf("mesh: tombstoned %s (peer says it was destroyed)", host) |
| 475 | w.WriteHeader(http.StatusNoContent) |
| 476 | }) |
| 477 | } |
| 478 | |
| 479 | // meshRouteListHandler: GET /api/mesh/routes — this box's in-scope hibernating |
| 480 | // instance hosts, the set a peer's mesh-seed --prune may reconcile. Routing |
| 481 | // metadata only (hostnames), gated by the same shared token as everything |
| 482 | // else on this surface. |
| 483 | func meshRouteListHandler(reg *Registry, rt *router, token string, domains []string) http.HandlerFunc { |
| 484 | return meshAuthed(token, func(w http.ResponseWriter, r *http.Request) { |
| 485 | routes, err := reg.list() |
| 486 | if err != nil { |
| 487 | http.Error(w, "internal error", http.StatusInternalServerError) |
| 488 | return |
| 489 | } |
| 490 | hosts := []string{} |
| 491 | for _, t := range routes { |
| 492 | if t.Kind == "instance" && t.Hibernate && hostInDomains(t.Host, domains) { |
| 493 | hosts = append(hosts, t.Host) |
| 494 | } |
| 495 | } |
| 496 | w.Header().Set("Content-Type", "application/json") |
| 497 | json.NewEncoder(w).Encode(hosts) |
| 498 | }) |
| 499 | } |
| 500 | |
| 501 | // pruneCandidates picks the peer hosts a reconciling seed may tombstone: |
| 502 | // in-scope peer rows whose host the SOURCE registry doesn't know at all (any |
| 503 | // kind, any flag — existing anywhere on the source means alive). The zero |
| 504 | // guard is load-bearing: a source that sees none of its own in-scope routes |
| 505 | // (wrong registry path, empty box) must never be allowed to conclude the |
| 506 | // peer's whole set is dead. |
| 507 | func pruneCandidates(sourceHosts, peerHosts, domains []string, sourceInScope int) []string { |
| 508 | if sourceInScope == 0 { |
| 509 | return nil |
| 510 | } |
| 511 | known := make(map[string]bool, len(sourceHosts)) |
| 512 | for _, h := range sourceHosts { |
| 513 | known[h] = true |
| 514 | } |
| 515 | var out []string |
| 516 | for _, h := range peerHosts { |
| 517 | if !known[h] && hostInDomains(h, domains) { |
| 518 | out = append(out, h) |
| 519 | } |
| 520 | } |
| 521 | return out |
| 522 | } |
| 523 | |
| 524 | // runMeshSeed is `lchat mesh-seed [--region <cc>] <peer-base-url>`: |
| 525 | // bulk-replicates the hibernating routes one peer must hold — the one-time |
| 526 | // catch-up a freshly joined box needs (it starts with an empty registry) before |
| 527 | // HRW can honestly place any host there. It seeds by the same DOMAIN ∪ REGION |
| 528 | // union fanOutMeshRoute uses (docs/multi-edge.md): a route whose host is under |
| 529 | // this seed's LCHAT_MESH_DOMAINS, PLUS — when --region names the joining box's |
| 530 | // placement region (its LCHAT_MESH_REGION) — a route whose home_region matches |
| 531 | // it, even if out of domain. Without --region the filter is exactly domain scope, |
| 532 | // so an existing same-domain fleet seeds byte-identically to before. |
| 533 | // |
| 534 | // Reads LCHAT_MESH_DOMAINS/LCHAT_MESH_TOKEN directly rather than through |
| 535 | // meshFromEnv: the CALLER (the already-running box doing the seeding) need not |
| 536 | // have LCHAT_MESH_SELF set yet — seeding is an operator step that runs before |
| 537 | // the mesh itself is flipped on (see docs/instance-router.md). The joining box's |
| 538 | // region is passed explicitly rather than looked up, so the seed needs no |
| 539 | // bucket/heartbeat wiring — the operator who just set LCHAT_MESH_REGION on the |
| 540 | // new box supplies the same value here. |
| 541 | func runMeshSeed(args []string) { |
| 542 | fs := flag.NewFlagSet("mesh-seed", flag.ExitOnError) |
| 543 | registryPath := fs.String("registry", envOr("LCHAT_REGISTRY", "/data/registry.db"), "router registry SQLite path") |
| 544 | envFile := fs.String("env-file", "", "load LCHAT_MESH_* (and anything else) from this KEY=value file first — lets a sudo'd operator run borrow the router's env (e.g. /data/router.env) without a root shell") |
| 545 | prune := fs.Bool("prune", false, "after seeding, tombstone the peer's in-scope hibernating rows this registry doesn't know — reconciles deletions a best-effort fan-out missed (#1346); refuses on an empty source view") |
| 546 | peerRegion := fs.String("region", "", "the JOINING box's placement region (its LCHAT_MESH_REGION); when set, routes whose home_region matches are seeded too, even if out of LCHAT_MESH_DOMAINS") |
| 547 | _ = fs.Parse(reorderFlagsFirst(fs, args)) |
| 548 | if *envFile != "" { |
| 549 | kv, err := readEnvFile(*envFile) |
| 550 | if err != nil { |
| 551 | die("mesh-seed: read --env-file: %v", err) |
| 552 | } |
| 553 | for k, v := range kv { |
| 554 | if os.Getenv(k) == "" { // real env always wins over the file |
| 555 | os.Setenv(k, v) |
| 556 | } |
| 557 | } |
| 558 | } |
| 559 | peer := fs.Arg(0) |
| 560 | if peer == "" { |
| 561 | die("mesh-seed: usage: lchat mesh-seed [--env-file /data/router.env] [--region <cc>] [--prune] <peer-base-url>") |
| 562 | } |
| 563 | domains := splitDomains(os.Getenv("LCHAT_MESH_DOMAINS")) |
| 564 | if len(domains) == 0 { |
| 565 | die("mesh-seed: LCHAT_MESH_DOMAINS must be set to the domains this seed covers") |
| 566 | } |
| 567 | token := strings.TrimSpace(os.Getenv("LCHAT_MESH_TOKEN")) |
| 568 | if token == "" { |
| 569 | die("mesh-seed: LCHAT_MESH_TOKEN must be set") |
| 570 | } |
| 571 | region := strings.TrimSpace(*peerRegion) |
| 572 | |
| 573 | reg, err := openRegistry(*registryPath) |
| 574 | if err != nil { |
| 575 | die("mesh-seed: open registry: %v", err) |
| 576 | } |
| 577 | routes, err := reg.list() |
| 578 | if err != nil { |
| 579 | die("mesh-seed: list routes: %v", err) |
| 580 | } |
| 581 | seeded := 0 |
| 582 | for _, route := range routes { |
| 583 | if !route.Hibernate { |
| 584 | continue |
| 585 | } |
| 586 | // DOMAIN ∪ REGION, mirroring fanOutMeshRoute/replicationTargets: seed a |
| 587 | // route the joining box covers by domain, or (when --region is given) one |
| 588 | // it owns by home_region even though it's out of domain. |
| 589 | domainCovered := hostInDomains(route.Host, domains) |
| 590 | regionOwned := region != "" && route.HomeRegion == region |
| 591 | if !domainCovered && !regionOwned { |
| 592 | continue |
| 593 | } |
| 594 | // The instance env rides along (meshRoutePayload doc) — without it the |
| 595 | // peer wakes this host token-less. Missing file (never materialized |
| 596 | // here either) just seeds the bare route, exactly as before. |
| 597 | env, envErr := readEnvFile(filepath.Join(instanceDataDir, route.Host+".env")) |
| 598 | if envErr != nil && !os.IsNotExist(envErr) { |
| 599 | fmt.Printf(" %s: env unreadable (%v) — seeding the bare route\n", route.Host, envErr) |
| 600 | } |
| 601 | ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
| 602 | err := meshReplicateRoute(ctx, peer, token, payloadFromRouteEnv(route, env)) |
| 603 | cancel() |
| 604 | if err != nil { |
| 605 | fmt.Printf(" %s: FAILED (%v)\n", route.Host, err) |
| 606 | continue |
| 607 | } |
| 608 | fmt.Printf(" %s: seeded\n", route.Host) |
| 609 | seeded++ |
| 610 | } |
| 611 | fmt.Printf("seeded %d/%d route(s) the joining box owns to %s\n", seeded, len(routes), peer) |
| 612 | |
| 613 | if *prune { |
| 614 | // Reconcile: anything the peer serves in-scope that this registry has |
| 615 | // no row for AT ALL (any kind, any flag) was deleted here — tombstone |
| 616 | // it there. The zero guard in pruneCandidates keeps an empty or |
| 617 | // misread source from ever mass-deleting a peer. |
| 618 | var all []string |
| 619 | inScope := 0 |
| 620 | var sni string |
| 621 | for _, route := range routes { |
| 622 | all = append(all, route.Host) |
| 623 | if route.Hibernate && hostInDomains(route.Host, domains) { |
| 624 | inScope++ |
| 625 | if sni == "" { |
| 626 | sni = route.Host |
| 627 | } |
| 628 | } |
| 629 | } |
| 630 | if sni == "" { |
| 631 | die("mesh-seed: --prune refused — this registry has no in-scope hibernating routes to speak for") |
| 632 | } |
| 633 | ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) |
| 634 | peerHosts, err := meshListRoutes(ctx, peer, token, sni) |
| 635 | cancel() |
| 636 | if err != nil { |
| 637 | die("mesh-seed: --prune: list peer routes: %v", err) |
| 638 | } |
| 639 | pruned := 0 |
| 640 | for _, h := range pruneCandidates(all, peerHosts, domains, inScope) { |
| 641 | ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
| 642 | err := meshDeleteRoute(ctx, peer, token, h) |
| 643 | cancel() |
| 644 | if err != nil { |
| 645 | fmt.Printf(" %s: prune FAILED (%v)\n", h, err) |
| 646 | continue |
| 647 | } |
| 648 | fmt.Printf(" %s: pruned (deleted here, tombstoned there)\n", h) |
| 649 | pruned++ |
| 650 | } |
| 651 | fmt.Printf("pruned %d stale route(s) on %s\n", pruned, peer) |
| 652 | } |
| 653 | } |
| 654 | |
amadan