eleven / messenger Public

server: mesh route deletions replicate — tombstone fan-out + reconciling seed (#1346) (#1348)

A destroyed space could live on (and WAKE, from a stale replica) on any mesh
peer holding its replicated row — six such zombies were swept off kepler by
hand this morning. Now both delete paths (controld's deleteInstance and
lchat remove) fan a tombstone to live peers after the local row is gone:
DELETE /api/mesh/route/{host}, idempotent, removing the row, the in.<host>
ingest row, and the replicated env file (a dead space's admin token) —
never member-data-shaped files. For fan-outs a down peer missed,
mesh-seed --prune reconciles: diff the peer's in-scope listing
(GET /api/mesh/routes) against the source registry and tombstone the
difference, with a zero-source guard so a misread registry can never
mass-delete a peer. ops mesh seed gains --prune. The shared peer transport
is extracted (meshPeerRequest); guide's mesh page updated — the remaining
honest gap narrows to hibernate flips (which must NOT replicate naively:
a peer holding Hibernate=false would dead-proxy instead of forwarding).

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

Paul Campbell 3 months ago fb9ed1fb37897e48da2925cd614d9b37cda6655d parent 4a8ecdc

7 files changed, +388 −39

control.go +13 −0
deploy/ops +1 −1
deploy/ops-mesh.sh +12 −5
docs/planet-scale-guide/mesh.html +2 −2
meshreplicate.go +243 −31
meshreplicate_test.go +113 −0
router.go +4 −0

diff --git a/control.go b/control.go

index 5030736..6f7dfab 100644

--- a/control.go

+++ b/control.go

@@ -552,6 +552,13 @@ func deleteInstance(reg *Registry, host string, discard bool) error {

if err := systemctl("reload", "lchat-router"); err != nil {

return fmt.Errorf("reload router: %v", err)

}

+ // The row is authoritatively gone here — tell the mesh peers so a

+ // destroyed space can't live on (or wake!) from a replicated copy

+ // (#1346). Best-effort like creation fan-out; a missed peer reconciles at

+ // the next mesh-seed --prune. No-op off-mesh.

+ if hc, hcErr := hibernateConfigFromEnv(); hcErr == nil {

+ fanOutMeshRouteDelete(hc, host)

+ }

if rt.Hibernate {

// The retire is asynchronous inside the router process; wait (bounded)

// for the child to release the socket so the archive renames below

@@ -639,6 +646,12 @@ func runRemove(args []string) {

if err := systemctl("reload", *routerUnit); err != nil {

die("remove: reload %s: %v", *routerUnit, err)

}

+ // Tombstone the mesh peers too (#1346) — same reasoning as deleteInstance.

+ if rt.Kind == "instance" {

+ if hc, hcErr := hibernateConfigFromEnv(); hcErr == nil {

+ fanOutMeshRouteDelete(hc, host)

+ }

if *stop && rt.Kind == "instance" {

_ = systemctl("disable", "--now", *botUnit+host) // best-effort: a sidecar may not exist

_ = systemctl("disable", "--now", "lchat-hooks@"+host) // best-effort: same for webhook ingest

diff --git a/deploy/ops b/deploy/ops

index c44a981..d7c5013 100755

--- a/deploy/ops

+++ b/deploy/ops

@@ -29,7 +29,7 @@ ops — the operator command surface

./deploy/ops dev <plan|apply|deploy|rebuild>

the two-node dev mesh (tofu / scp+stage+swap)

./deploy/ops ledger "why..." draft the public SSH-access why-entry

- ./deploy/ops mesh seed <peer> [box]

+ ./deploy/ops mesh seed [--prune] <peer> [box]

replicate routes (+env) to a mesh peer

Start with `ops doctor` — it checks every credential and access path the other

diff --git a/deploy/ops-mesh.sh b/deploy/ops-mesh.sh

index ecc1f4a..0522001 100755

--- a/deploy/ops-mesh.sh

+++ b/deploy/ops-mesh.sh

@@ -26,21 +26,28 @@ sub="${1:-}"

case "$sub" in

seed)

+ # `seed --prune` also tombstones peer rows the source no longer has —

+ # the reconciliation for deletions a best-effort fan-out missed (#1346).

+ PRUNE=""

+ if [ "${1:-}" = "--prune" ]; then PRUNE="--prune"; shift; fi

peer="${1:-}"

box="${2:-${LCHAT_MESH_SOURCE_BOX:-elevenbox}}"

if [ -z "$peer" ]; then

- echo "usage: ops mesh seed <peer-base-url> [box] (e.g. ops mesh seed https://77.68.125.53)" >&2

+ echo "usage: ops mesh seed [--prune] <peer-base-url> [box] (e.g. ops mesh seed https://77.68.125.53)" >&2

exit 2

- echo "== seeding mesh routes (+env) from $box to $peer"

- ssh "$box" "sudo -n /opt/lchat/lchat mesh-seed --env-file /data/router.env '$peer'"

+ echo "== seeding mesh routes (+env) from $box to $peer${PRUNE:+, pruning stale peer rows}"

+ ssh "$box" "sudo -n /opt/lchat/lchat mesh-seed --env-file /data/router.env $PRUNE '$peer'"

;;

""|-h|--help|help|*)

cat <<'EOF' >&2

ops mesh — mesh-membership maintenance

- ./deploy/ops mesh seed <peer-base-url> [box] replicate all in-scope hibernating

- routes (+ instance env) to a peer

+ ./deploy/ops mesh seed [--prune] <peer-base-url> [box]

+ replicate all in-scope hibernating

+ routes (+ instance env) to a peer;

+ --prune also tombstones peer rows

+ the source no longer has

(box defaults to elevenbox)

EOF

[ "$sub" = "" ] || [ "$sub" = "-h" ] || [ "$sub" = "--help" ] || [ "$sub" = "help" ] || exit 2

diff --git a/docs/planet-scale-guide/mesh.html b/docs/planet-scale-guide/mesh.html

index 5679360..6f2713a 100644

--- a/docs/planet-scale-guide/mesh.html

+++ b/docs/planet-scale-guide/mesh.html

@@ -116,7 +116,7 @@ LCHAT_MESH_TOKEN=<the same shared secret as every other box>

<li>Seed its registry from an existing box.

<figure class="cmd"><figcaption>Run on an EXISTING mesh box, pointed at the new one</figcaption>

<pre><code>lchat mesh-seed --env-file /data/router.env https://box-2.example.net</code></pre>

- (<code>--env-file</code> borrows the router’s <code>LCHAT_MESH_*</code> values — the seed can run under a sudo’d operator session that has no mesh env of its own. The encoded operator path is <code>./deploy/ops mesh seed <peer-url> [box]</code>.)</figure>

+ (<code>--env-file</code> borrows the router’s <code>LCHAT_MESH_*</code> values — the seed can run under a sudo’d operator session that has no mesh env of its own. Add <code>--prune</code> to also tombstone peer rows this registry no longer has — the reconciliation half of deletion replication. The encoded operator path is <code>./deploy/ops mesh seed [--prune] <peer-url> [box]</code>.)</figure>

Bulk-replicates every in-scope hibernating route in one pass, so the new box can immediately answer for (or correctly proxy) any of them. After this, roughly 1/N of the in-scope spaces re-home to the new box at their next sleep/wake boundary — nothing needs announcing beyond this one command; the guest book and the registry rows do the rest.</li>

</ol>

<h3 id="drain-box">Draining a box (deploys, maintenance, scale-down)</h3>

@@ -178,7 +178,7 @@ LCHAT_MESH_TOKEN=<the same shared secret as every other box>

<h2 id="mesh-limits">The honest limits: what stands between here and a true multi-box fleet</h2>

Two gaps remain. Neither is speculative — each corresponds to a concrete piece of code that is reserved or not yet written. (Two things this section used to list — registry replication and fleet-wide status aggregation — have since shipped and are covered above and in <a href="operations.html#observability">part 08</a>.)

<ul>

- <li>partially built Route mutations after creation do not fully replicate. The initial fan-out (<a href="#mesh-replication">above</a>) and <code>mesh-seed</code> cover creation and joining, and <a href="#relocate"><code>lchat relocate</code></a> now explicitly re-fans-out the one route field it changes (<code>home_region</code>). But a later rename, removal, or hibernation-flag flip on one box is still not propagated to its peers — those must be repeated by hand on every mesh member. Fine for a pool of auto-provisioned, rarely-renamed spaces; worth closing before mesh placement carries more than that.</li>

+ <li>partially built Route mutations: deletions now replicate; hibernate flips still don’t. Since #1346 both delete paths (controld’s <code>deleteInstance</code> and <code>lchat remove</code>) fan a tombstone to every live peer (<code>DELETE /api/mesh/route/{host}</code> — idempotent; removes the row, the <code>in.<host></code> ingest row, and the replicated env file, never member-data-shaped files), and <code>mesh-seed --prune</code> reconciles whatever a best-effort fan-out missed by diffing the peer’s in-scope listing (<code>GET /api/mesh/routes</code>) against the source registry — with a zero-source guard so a misread registry can never mass-delete a peer. Without this, a destroyed space lived on — and could wake, from a stale replica — on any peer holding its row (six such zombies were swept off a production box by hand on 2026-07-25). Still open: a hibernate flip doesn’t replicate, and must not naively — a peer holding <code>Hibernate=false</code> would proxy to a local socket that doesn’t exist instead of forwarding; a flipped-to-systemd host needs a “lives elsewhere” route shape peers don’t have yet. Renames don’t exist (the host is the key).</li>

<li>built, not yet proven in production The cross-region store copy is exercised only as a no-op. <a href="#relocate">Relocation</a>’s per-region-bucket copy path is written and unit-tested, but the dev mesh shares one bucket, so live proof only covers the no-op skip path. Before a real cross-region move in production it needs: the source quiesced (asleep) before the copy so a live-writing source can’t be copied mid-write; the copy client signed for the destination bucket’s region; and a download+reupload variant for cross-account or R2 destinations (<code>CopyObject</code> covers only same-account, same-provider). All three are noted in <code>docs/multi-edge.md</code>.</li>

<li>partially built The certificate cache is per box. The wildcard certificate makes mesh domains workable (each box independently obtains its own copy), but per-host certificates for domains outside the wildcard cannot follow a space between boxes; the S3-backed shared cache that would fix this is reserved (<code>LCHAT_CERT_CACHE=s3</code>) and deliberately refuses to start. See <a href="router.html#tls">part 02</a>.</li>

</ul>

diff --git a/meshreplicate.go b/meshreplicate.go

index c30d04e..aa61f8a 100644

--- a/meshreplicate.go

+++ b/meshreplicate.go

@@ -20,13 +20,19 @@ package main

// registerInstance does for a route created locally, and never

// Status/NextDue/CreatedAt, which are receiver-local lifecycle state.

-// Honest gap: route MUTATIONS after creation (rename, remove, a hibernate

-// flip) are not replicated — only the initial registerInstance fan-out and

-// the one-time seed. A box whose peer never saw a later change either serves

-// stale config (rare per-instance fields) or, for a removed host, still has a

-// row an out-of-scope hash never assigns it anyway (mesh.go's inScope). Worth

-// closing before mesh placement is load-bearing for more than the initial

-// auto-provisioned space pool.

+// Deletions replicate too (#1346): both delete paths (controld's

+// deleteInstance and `lchat remove`) fan a tombstone

+// (fanOutMeshRouteDelete → DELETE /api/mesh/route/{host}), and

+// `mesh-seed --prune` reconciles whatever a best-effort fan-out missed —

+// without this, a destroyed space lived on (and WOKE, from a stale replica)

+// on any peer that held its row, violating the destroy promise. Live case:

+// six zombies swept off kepler 2026-07-25.

+//

+// Honest gap, narrowed: a HIBERNATE FLIP still doesn't replicate — and must

+// not naively: a peer holding Hibernate=false would proxy to a local socket

+// that doesn't exist instead of forwarding. A flipped-to-systemd host needs

+// a "lives elsewhere" route shape peers don't have yet. Renames don't exist

+// (host is the key).

import (

"bytes"

@@ -136,22 +142,70 @@ func (p meshRoutePayload) toLocalRoute() (Route, error) {

// only ever pass an in-scope route (fanOutMeshRoute/runMeshSeed both check

// first), so this is never asked to authenticate as an out-of-scope host.

func meshReplicateRoute(ctx context.Context, peer, token string, payload meshRoutePayload) error {

- peerURL, err := url.Parse(peer)

- if err != nil || peerURL.Host == "" {

- return fmt.Errorf("bad peer URL %q: %w", peer, err)

- }

body, err := json.Marshal(payload)

if err != nil {

return err

}

- reqURL := url.URL{Scheme: peerURL.Scheme, Host: payload.Host, Path: "/api/mesh/route"}

- req, err := http.NewRequestWithContext(ctx, http.MethodPost, reqURL.String(), bytes.NewReader(body))

+ resp, err := meshPeerRequest(ctx, peer, token, http.MethodPost, "/api/mesh/route", payload.Host, body)

if err != nil {

return err

}

+ defer resp.Body.Close()

+ if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent {

+ return fmt.Errorf("peer %s: %s", peer, resp.Status)

+ }

+ return nil

+// fanOutMeshRoute replicates route to every OTHER live mesh member,

+// best-effort: a peer that's down or errors just can't yet serve this host

+// (see the file doc comment's honest gap on mutations — creation retries only

+// via a later add-instance or an operator's mesh-seed rerun). A no-op — byte-

+// identical to today — whenever the mesh is off or the host is out of

+// LCHAT_MESH_DOMAINS scope, so self-hosters and non-mesh boxes are untouched.

+// Called right after registerInstance's local reg.upsert.

+func fanOutMeshRoute(hc *hibernateConfig, route Route, env map[string]string) {

+ mesh, err := meshFromEnv(hc)

+ if err != nil || mesh == nil || !mesh.inScope(route.Host) {

+ return

+ }

+ payload := payloadFromRouteEnv(route, env)

+ mesh.beat() // one synchronous PUT+LIST for a fresh membership view

+ for _, peer := range mesh.liveBoxes() {

+ if peer == mesh.self {

+ continue

+ }

+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)

+ err := meshReplicateRoute(ctx, peer, mesh.token, payload)

+ cancel()

+ if err != nil {

+ log.Printf("mesh: replicate %s to %s failed: %v — it can't serve this host until a later replication succeeds", route.Host, peer, err)

+ } else {

+ log.Printf("mesh: replicated %s to %s", route.Host, peer)

+ }

+// meshPeerRequest performs one authenticated request against a mesh peer's

+// raw address, TLS'd (and Host'd) as sniHost — the wildcard-covered instance

+// hostname trick meshReplicateRoute documents above. Shared by the route

+// upsert, the tombstone, and the reconciliation listing.

+func meshPeerRequest(ctx context.Context, peer, token, method, path, sniHost string, body []byte) (*http.Response, error) {

+ peerURL, err := url.Parse(peer)

+ if err != nil || peerURL.Host == "" {

+ return nil, fmt.Errorf("bad peer URL %q: %w", peer, err)

+ }

+ reqURL := url.URL{Scheme: peerURL.Scheme, Host: sniHost, Path: path}

+ var rd io.Reader

+ if body != nil {

+ rd = bytes.NewReader(body)

+ }

+ req, err := http.NewRequestWithContext(ctx, method, reqURL.String(), rd)

+ if err != nil {

+ return nil, err

+ }

req.Header.Set("Content-Type", "application/json")

req.Header.Set("Authorization", "Bearer "+token)

dialAddr := peerDialAddr(peerURL)

transport := &http.Transport{

DialContext: func(ctx context.Context, network, _ string) (net.Conn, error) {

@@ -164,7 +218,7 @@ func meshReplicateRoute(ctx context.Context, peer, token string, payload meshRou

if err != nil {

return nil, err

}

- tlsConn := tls.Client(conn, &tls.Config{ServerName: payload.Host})

+ tlsConn := tls.Client(conn, &tls.Config{ServerName: sniHost})

if err := tlsConn.HandshakeContext(ctx); err != nil {

conn.Close()

return nil, err

@@ -172,9 +226,14 @@ func meshReplicateRoute(ctx context.Context, peer, token string, payload meshRou

return tlsConn, nil

}

- client := &http.Client{Transport: transport}

+ return (&http.Client{Transport: transport}).Do(req)

- resp, err := client.Do(req)

+// meshDeleteRoute sends host's tombstone to one peer — the deletion half of

+// replication (#1346). Idempotent on the receiving side, so re-sends and

+// reconciliation sweeps are always safe.

+func meshDeleteRoute(ctx context.Context, peer, token, host string) error {

+ resp, err := meshPeerRequest(ctx, peer, token, http.MethodDelete, "/api/mesh/route/"+host, host, nil)

if err != nil {

return err

}

@@ -185,31 +244,48 @@ func meshReplicateRoute(ctx context.Context, peer, token string, payload meshRou

return nil

}

-// fanOutMeshRoute replicates route to every OTHER live mesh member,

-// best-effort: a peer that's down or errors just can't yet serve this host

-// (see the file doc comment's honest gap on mutations — creation retries only

-// via a later add-instance or an operator's mesh-seed rerun). A no-op — byte-

-// identical to today — whenever the mesh is off or the host is out of

-// LCHAT_MESH_DOMAINS scope, so self-hosters and non-mesh boxes are untouched.

-// Called right after registerInstance's local reg.upsert.

-func fanOutMeshRoute(hc *hibernateConfig, route Route, env map[string]string) {

+// meshListRoutes reads a peer's in-scope hibernating route hosts — what

+// mesh-seed --prune diffs against the source registry. The SNI rides any

+// wildcard-covered host the caller knows (its own first route works); the

+// peer answers from its registry either way.

+func meshListRoutes(ctx context.Context, peer, token, sniHost string) ([]string, error) {

+ resp, err := meshPeerRequest(ctx, peer, token, http.MethodGet, "/api/mesh/routes", sniHost, nil)

+ if err != nil {

+ return nil, err

+ }

+ defer resp.Body.Close()

+ if resp.StatusCode != http.StatusOK {

+ return nil, fmt.Errorf("peer %s: %s", peer, resp.Status)

+ }

+ var hosts []string

+ if err := json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&hosts); err != nil {

+ return nil, err

+ }

+ return hosts, nil

+// fanOutMeshRouteDelete replicates a deletion to every live mesh peer — the

+// tombstone counterpart of fanOutMeshRoute, called from both delete paths

+// (controld's deleteInstance and `lchat remove`) after the local row is gone.

+// Best-effort exactly like creation fan-out: a peer that's down keeps its

+// stale row until a `mesh-seed --prune` reconciles it.

+func fanOutMeshRouteDelete(hc *hibernateConfig, host string) {

mesh, err := meshFromEnv(hc)

- if err != nil || mesh == nil || !mesh.inScope(route.Host) {

+ if err != nil || mesh == nil || !mesh.inScope(host) {

return

}

- payload := payloadFromRouteEnv(route, env)

- mesh.beat() // one synchronous PUT+LIST for a fresh membership view

+ mesh.beat()

for _, peer := range mesh.liveBoxes() {

if peer == mesh.self {

continue

}

ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)

- err := meshReplicateRoute(ctx, peer, mesh.token, payload)

+ err := meshDeleteRoute(ctx, peer, mesh.token, host)

cancel()

if err != nil {

- log.Printf("mesh: replicate %s to %s failed: %v — it can't serve this host until a later replication succeeds", route.Host, peer, err)

+ log.Printf("mesh: delete %s on %s failed: %v — its stale row stays until a mesh-seed --prune", host, peer, err)

} else {

- log.Printf("mesh: replicated %s to %s", route.Host, peer)

+ log.Printf("mesh: deleted %s on %s", host, peer)

}

@@ -262,6 +338,100 @@ func meshRouteHandler(reg *Registry, rt *router, token string) http.HandlerFunc

}

+// meshAuthed wraps a mesh endpoint behind the shared bearer token — the same

+// constant-time gate meshRouteHandler carries inline.

+func meshAuthed(token string, next http.HandlerFunc) http.HandlerFunc {

+ return func(w http.ResponseWriter, r *http.Request) {

+ const prefix = "Bearer "

+ auth := r.Header.Get("Authorization")

+ if !strings.HasPrefix(auth, prefix) ||

+ subtle.ConstantTimeCompare([]byte(strings.TrimPrefix(auth, prefix)), []byte(token)) != 1 {

+ http.Error(w, "unauthorized", http.StatusUnauthorized)

+ return

+ }

+ next(w, r)

+ }

+// meshRouteDeleteHandler: DELETE /api/mesh/route/{host} — a peer says this

+// space was destroyed. Idempotent (an absent row still answers 204, so

+// re-sends and prune sweeps never error). Deletes the row, the instance's

+// webhook-ingest row (in.<host> dies with the space, mirroring lchat remove),

+// and the REPLICATED env file — it holds a destroyed space's admin token and

+// arrived via replication, so it's ours to take back; the db/blob files are

+// member-data-shaped and stay, exactly like every other delete path (the

+// data-safety rule). The reload's retire pass stops any live activator child.

+func meshRouteDeleteHandler(reg *Registry, rt *router, token string) http.HandlerFunc {

+ return meshAuthed(token, func(w http.ResponseWriter, r *http.Request) {

+ host := strings.ToLower(strings.TrimSpace(r.PathValue("host")))

+ if !validHost(host) {

+ http.Error(w, "bad host", http.StatusBadRequest)

+ return

+ }

+ if err := reg.delete(host); err != nil {

+ log.Printf("mesh: tombstone delete %s failed: %v", host, err)

+ http.Error(w, "internal error", http.StatusInternalServerError)

+ return

+ }

+ if err := reg.delete("in." + host); err != nil {

+ log.Printf("mesh: tombstone delete in.%s failed: %v", host, err)

+ }

+ if err := os.Remove(filepath.Join(instanceDataDir, host+".env")); err != nil && !os.IsNotExist(err) {

+ log.Printf("mesh: tombstone env removal for %s failed: %v", host, err)

+ }

+ if err := rt.reload(); err != nil {

+ log.Printf("mesh: reload after tombstone %s failed: %v", host, err)

+ }

+ log.Printf("mesh: tombstoned %s (peer says it was destroyed)", host)

+ w.WriteHeader(http.StatusNoContent)

+ })

+// meshRouteListHandler: GET /api/mesh/routes — this box's in-scope hibernating

+// instance hosts, the set a peer's mesh-seed --prune may reconcile. Routing

+// metadata only (hostnames), gated by the same shared token as everything

+// else on this surface.

+func meshRouteListHandler(reg *Registry, rt *router, token string, domains []string) http.HandlerFunc {

+ return meshAuthed(token, func(w http.ResponseWriter, r *http.Request) {

+ routes, err := reg.list()

+ if err != nil {

+ http.Error(w, "internal error", http.StatusInternalServerError)

+ return

+ }

+ hosts := []string{}

+ for _, t := range routes {

+ if t.Kind == "instance" && t.Hibernate && hostInDomains(t.Host, domains) {

+ hosts = append(hosts, t.Host)

+ }

+ w.Header().Set("Content-Type", "application/json")

+ json.NewEncoder(w).Encode(hosts)

+ })

+// pruneCandidates picks the peer hosts a reconciling seed may tombstone:

+// in-scope peer rows whose host the SOURCE registry doesn't know at all (any

+// kind, any flag — existing anywhere on the source means alive). The zero

+// guard is load-bearing: a source that sees none of its own in-scope routes

+// (wrong registry path, empty box) must never be allowed to conclude the

+// peer's whole set is dead.

+func pruneCandidates(sourceHosts, peerHosts, domains []string, sourceInScope int) []string {

+ if sourceInScope == 0 {

+ return nil

+ }

+ known := make(map[string]bool, len(sourceHosts))

+ for _, h := range sourceHosts {

+ known[h] = true

+ }

+ var out []string

+ for _, h := range peerHosts {

+ if !known[h] && hostInDomains(h, domains) {

+ out = append(out, h)

+ }

+ return out

// runMeshSeed is `lchat mesh-seed <peer-base-url>`: bulk-replicates every

// existing in-scope, hibernating route to one peer — the one-time catch-up a

// freshly joined box needs (it starts with an empty registry) before HRW can

@@ -274,6 +444,7 @@ func runMeshSeed(args []string) {

fs := flag.NewFlagSet("mesh-seed", flag.ExitOnError)

registryPath := fs.String("registry", envOr("LCHAT_REGISTRY", "/data/registry.db"), "router registry SQLite path")

envFile := fs.String("env-file", "", "load LCHAT_MESH_* (and anything else) from this KEY=value file first — lets a sudo'd operator run borrow the router's env (e.g. /data/router.env) without a root shell")

+ prune := fs.Bool("prune", false, "after seeding, tombstone the peer's in-scope hibernating rows this registry doesn't know — reconciles deletions a best-effort fan-out missed (#1346); refuses on an empty source view")

_ = fs.Parse(reorderFlagsFirst(fs, args))

if *envFile != "" {

kv, err := readEnvFile(*envFile)

@@ -330,4 +501,45 @@ func runMeshSeed(args []string) {

seeded++

}

fmt.Printf("seeded %d/%d in-scope hibernating route(s) to %s\n", seeded, len(routes), peer)

+ if *prune {

+ // Reconcile: anything the peer serves in-scope that this registry has

+ // no row for AT ALL (any kind, any flag) was deleted here — tombstone

+ // it there. The zero guard in pruneCandidates keeps an empty or

+ // misread source from ever mass-deleting a peer.

+ var all []string

+ inScope := 0

+ var sni string

+ for _, route := range routes {

+ all = append(all, route.Host)

+ if route.Hibernate && hostInDomains(route.Host, domains) {

+ inScope++

+ if sni == "" {

+ sni = route.Host

+ }

+ if sni == "" {

+ die("mesh-seed: --prune refused — this registry has no in-scope hibernating routes to speak for")

+ }

+ ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)

+ peerHosts, err := meshListRoutes(ctx, peer, token, sni)

+ cancel()

+ if err != nil {

+ die("mesh-seed: --prune: list peer routes: %v", err)

+ }

+ pruned := 0

+ for _, h := range pruneCandidates(all, peerHosts, domains, inScope) {

+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)

+ err := meshDeleteRoute(ctx, peer, token, h)

+ cancel()

+ if err != nil {

+ fmt.Printf(" %s: prune FAILED (%v)\n", h, err)

+ continue

+ }

+ fmt.Printf(" %s: pruned (deleted here, tombstoned there)\n", h)

+ pruned++

+ }

+ fmt.Printf("pruned %d stale route(s) on %s\n", pruned, peer)

+ }

}

diff --git a/meshreplicate_test.go b/meshreplicate_test.go

index f6075e1..c6cb9f0 100644

--- a/meshreplicate_test.go

+++ b/meshreplicate_test.go

@@ -14,6 +14,7 @@ import (

"encoding/json"

"net/http"

"net/http/httptest"

+ "os"

"path/filepath"

"testing"

)

@@ -210,3 +211,115 @@ func TestMeshReplicationCarriesInstanceEnv(t *testing.T) {

t.Errorf("token-less payload clobbered the env: %v", got)

}

+// The tombstone: a valid DELETE removes the row, the instance's in.<host>

+// row, and the REPLICATED env file (a destroyed space's admin token is not

+// ours to keep) — idempotently, so re-sends and prune sweeps never error.

+// Auth gates it like every mesh endpoint.

+func TestMeshRouteDeleteHandler(t *testing.T) {

+ dir := t.TempDir()

+ oldData := instanceDataDir

+ instanceDataDir = dir

+ defer func() { instanceDataDir = oldData }()

+ reg, err := openRegistry(filepath.Join(t.TempDir(), "registry.db"))

+ if err != nil {

+ t.Fatalf("open registry: %v", err)

+ }

+ rt, err := newRouter(reg, nil)

+ if err != nil {

+ t.Fatalf("new router: %v", err)

+ }

+ mux := http.NewServeMux()

+ mux.Handle("POST /api/mesh/route", meshRouteHandler(reg, rt, "shared-secret"))

+ mux.Handle("DELETE /api/mesh/route/{host}", meshRouteDeleteHandler(reg, rt, "shared-secret"))

+ srv := httptest.NewServer(mux)

+ defer srv.Close()

+ // Replicate a route (with env) + a webhook-ingest row for it.

+ route := testRoute("gone.11ch.at")

+ env := map[string]string{"LCHAT_ADMIN_TOKEN": "tok-dead"}

+ if err := meshReplicateRoute(context.Background(), srv.URL, "shared-secret", payloadFromRouteEnv(route, env)); err != nil {

+ t.Fatalf("replicate: %v", err)

+ }

+ if err := reg.upsert(Route{Host: "in.gone.11ch.at", Kind: "hooks", Socket: "/run/lchat/hooks.sock"}); err != nil {

+ t.Fatal(err)

+ }

+ // Wrong bearer never deletes.

+ if err := meshDeleteRoute(context.Background(), srv.URL, "wrong-secret", route.Host); err == nil {

+ t.Fatal("wrong token should be rejected")

+ }

+ if !rt.allowed(route.Host) {

+ t.Fatal("unauthorized tombstone deleted the route")

+ }

+ if err := meshDeleteRoute(context.Background(), srv.URL, "shared-secret", route.Host); err != nil {

+ t.Fatalf("tombstone: %v", err)

+ }

+ if rt.allowed(route.Host) {

+ t.Error("route survived its tombstone")

+ }

+ if got, _ := reg.get("in." + route.Host); got != nil {

+ t.Error("webhook-ingest row survived the tombstone")

+ }

+ if _, err := os.Stat(filepath.Join(dir, route.Host+".env")); !os.IsNotExist(err) {

+ t.Error("replicated env file survived the tombstone")

+ }

+ // Idempotent: tombstoning an absent host answers success.

+ if err := meshDeleteRoute(context.Background(), srv.URL, "shared-secret", route.Host); err != nil {

+ t.Errorf("re-tombstone of an absent host: %v", err)

+ }

+// The reconciliation listing exposes only in-scope hibernating instance hosts.

+func TestMeshRouteListHandler(t *testing.T) {

+ reg, err := openRegistry(filepath.Join(t.TempDir(), "registry.db"))

+ if err != nil {

+ t.Fatalf("open registry: %v", err)

+ }

+ rt, err := newRouter(reg, nil)

+ if err != nil {

+ t.Fatalf("new router: %v", err)

+ }

+ seed := func(host, kind string, hib bool) {

+ t.Helper()

+ if err := reg.upsert(Route{Host: host, Kind: kind, Socket: "/run/lchat/" + host + ".sock", Hibernate: hib}); err != nil {

+ t.Fatal(err)

+ }

+ seed("a.11ch.at", "instance", true) // in

+ seed("b.11ch.at", "instance", false) // out: not hibernating

+ seed("c.example.com", "instance", true) // out: out of scope

+ seed("d.11ch.at", "service", true) // out: not an instance

+ srv := httptest.NewServer(meshRouteListHandler(reg, rt, "shared-secret", []string{"11ch.at"}))

+ defer srv.Close()

+ hosts, err := meshListRoutes(context.Background(), srv.URL, "shared-secret", "a.11ch.at")

+ if err != nil {

+ t.Fatalf("list: %v", err)

+ }

+ if len(hosts) != 1 || hosts[0] != "a.11ch.at" {

+ t.Errorf("hosts = %v, want [a.11ch.at]", hosts)

+ }

+ if _, err := meshListRoutes(context.Background(), srv.URL, "wrong-secret", "a.11ch.at"); err == nil {

+ t.Error("wrong token should be rejected")

+ }

+// pruneCandidates: only in-scope peer hosts the source knows nothing about —

+// and NOTHING when the source's own in-scope view is empty (the guard that

+// keeps a misread registry from mass-deleting a peer).

+func TestPruneCandidates(t *testing.T) {

+ source := []string{"alive.11ch.at", "flipped.11ch.at"}

+ peer := []string{"alive.11ch.at", "flipped.11ch.at", "dead.11ch.at", "other.example.com"}

+ domains := []string{"11ch.at"}

+ got := pruneCandidates(source, peer, domains, 2)

+ if len(got) != 1 || got[0] != "dead.11ch.at" {

+ t.Errorf("candidates = %v, want [dead.11ch.at]", got)

+ }

+ if got := pruneCandidates(nil, peer, domains, 0); got != nil {

+ t.Errorf("zero-guard failed: %v", got)

+ }

diff --git a/router.go b/router.go

index 5d9c5a6..89e7425 100644

--- a/router.go

+++ b/router.go

@@ -438,6 +438,10 @@ func runRouter(args []string) {

if mesh != nil {

mux := http.NewServeMux()

mux.Handle("POST /api/mesh/route", meshRouteHandler(reg, rt, mesh.token))

+ // Deletion + reconciliation (#1346): a destroyed space's tombstone,

+ // and the route listing mesh-seed --prune diffs against.

+ mux.Handle("DELETE /api/mesh/route/{host}", meshRouteDeleteHandler(reg, rt, mesh.token))

+ mux.Handle("GET /api/mesh/routes", meshRouteListHandler(reg, rt, mesh.token, mesh.domains))

mux.Handle("/", rt)

httpsHandler = mux

}