amadan
server: scope activator mesh placement to LCHAT_MESH_DOMAINS (#820)
HRW placement (mesh.go) applied to every hibernating host with no domain restriction. A registry row only ever exists on the boxes control.go's registerInstance replicates it to, so hashing an out-of-scope host onto a box with no row for it would 404 there instead of waking it. Domain scoping also bounds placement to hosts every mesh member can independently get a matching cert for (the same domains as LCHAT_WILDCARD_DOMAINS's DNS-01 wildcard, unlike per-host HTTP-01 which is tied to whichever box DNS points at). meshFromEnv now requires LCHAT_MESH_DOMAINS whenever LCHAT_MESH_SELF is set, refusing to start otherwise — failing loudly beats silently letting HRW assign a host nobody but its original box can serve. Ships with LCHAT_MESH_SELF still unset everywhere, so today's running system is unaffected; this is prep for turning the mesh on.
Co-authored-by: Claude <noreply@anthropic.com>
4 files changed,
+173
−20
mesh.go+60 −7mesh_live_test.go+1 −1mesh_test.go+110 −10router.go+2 −2
diff --git a/mesh.go b/mesh.go| index 1e686a6..b27ff0f 100644 |
| --- a/mesh.go |
| +++ b/mesh.go |
| @@ -28,6 +28,18 @@ package main |
| // dead/stale owner drops out of the member set, so the hash reassigns and |
| // the request falls through to a local wake — failover IS the routine |
| // wake path. |
| +// 5. Domain scoping (LCHAT_MESH_DOMAINS) — HRW placement only ever applies to |
| +// hosts under a configured mesh domain; everything else always resolves |
| +// locally, on whichever box is asked. This isn't an optimization: a |
| +// registry row only ever exists on the boxes it was explicitly replicated |
| +// to (see registerInstance in control.go), so a host outside the |
| +// configured domains was never given a row anywhere but its original box, |
| +// and letting the hash "place" it elsewhere would just 404 there. It also |
| +// keeps placement bounded to hosts every mesh member can actually get a |
| +// matching TLS cert for (typically the same domains as |
| +// LCHAT_WILDCARD_DOMAINS's ACME DNS-01 cert, which — unlike per-host |
| +// HTTP-01 — any box can obtain independently of which box DNS currently |
| +// points at). |
| // |
| // Drain (activatorSvc.shutdown) deletes the heartbeat FIRST, then checkpoints |
| // and releases as today — peers take over immediately, and deploys, box |
| @@ -86,6 +98,7 @@ type meshStore interface { |
| type meshSvc struct { |
| self string // this box's peer base URL — its mesh identity |
| store meshStore |
| + domains []string // LCHAT_MESH_DOMAINS, lowercased — see inScope. Empty = unrestricted (test convenience only; meshFromEnv always supplies a non-empty list) |
| interval time.Duration |
| staleAfter time.Duration |
| now func() time.Time // a seam for the unit tests' clock |
| @@ -99,10 +112,11 @@ type meshSvc struct { |
| done chan struct{} |
| } |
| -func newMesh(self string, store meshStore) *meshSvc { |
| +func newMesh(self string, store meshStore, domains []string) *meshSvc { |
| return &meshSvc{ |
| self: self, |
| store: store, |
| + domains: domains, |
| interval: meshHeartbeatInterval, |
| staleAfter: meshStaleAfter, |
| now: time.Now, |
| @@ -117,7 +131,11 @@ func newMesh(self string, store meshStore) *meshSvc { |
| // network, or https where the box presents a verifiable cert (see |
| // newPeerProxy). Presence turns the mesh on; absent = single box, zero mesh |
| // codepaths. The mesh rides on the hibernation bucket, so it requires |
| -// LCHAT_HIBERNATE_BUCKET. |
| +// LCHAT_HIBERNATE_BUCKET. It also requires LCHAT_MESH_DOMAINS (same |
| +// comma-list shape as LCHAT_WILDCARD_DOMAINS, parsed by the same |
| +// splitDomains): failing loudly here beats silently letting HRW "place" a |
| +// host onto a box that has no registry row for it (a 404, not a graceful |
| +// wake) or that could never get it a matching cert — see inScope. |
| func meshFromEnv(hc *hibernateConfig) (*meshSvc, error) { |
| self := strings.TrimSpace(os.Getenv("LCHAT_MESH_SELF")) |
| if self == "" { |
| @@ -126,6 +144,10 @@ func meshFromEnv(hc *hibernateConfig) (*meshSvc, error) { |
| if hc == nil { |
| return nil, errors.New("LCHAT_MESH_SELF set but LCHAT_HIBERNATE_BUCKET missing — the mesh rides on the hibernation bucket and only places hibernating instances") |
| } |
| + domains := splitDomains(os.Getenv("LCHAT_MESH_DOMAINS")) |
| + if len(domains) == 0 { |
| + return nil, errors.New("LCHAT_MESH_SELF set but LCHAT_MESH_DOMAINS missing — mesh placement needs an explicit domain allowlist so HRW never assigns a host to a box that has no registry row for it or can't obtain it a matching cert; set it to the domains every mesh member can serve (typically the same value as LCHAT_WILDCARD_DOMAINS)") |
| + } |
| self = strings.TrimRight(self, "/") |
| u, err := url.Parse(self) |
| if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") || u.Path != "" { |
| @@ -135,7 +157,7 @@ func meshFromEnv(hc *hibernateConfig) (*meshSvc, error) { |
| if err != nil { |
| return nil, fmt.Errorf("mesh heartbeat store: %w", err) |
| } |
| - return newMesh(self, store), nil |
| + return newMesh(self, store, domains), nil |
| } |
| // start registers this box (one synchronous beat, so the first sweep and the |
| @@ -236,17 +258,48 @@ func (m *meshSvc) liveBoxes() []string { |
| return boxes |
| } |
| +// inScope reports whether host participates in HRW placement at all. Only |
| +// hosts under a configured LCHAT_MESH_DOMAINS domain are ever hashed across |
| +// boxes; everything else always resolves to whichever box is asked — correct |
| +// because registerInstance only ever replicates a route to the boxes covered |
| +// by those same domains (control.go), so no other box has a row for it |
| +// anyway, and this box (which was asked) is the only one that could possibly |
| +// serve it. An empty domain list is unrestricted — a test-construction |
| +// convenience (direct newMesh callers exercising bare HRW); meshFromEnv |
| +// always supplies a non-empty list in production. |
| +func (m *meshSvc) inScope(host string) bool { |
| + if len(m.domains) == 0 { |
| + return true |
| + } |
| + host = strings.ToLower(host) |
| + for _, d := range m.domains { |
| + if host == d || strings.HasSuffix(host, "."+d) { |
| + return true |
| + } |
| + } |
| + return false |
| +} |
| + |
| // owns reports whether this box is host's HRW owner in the live member set — |
| -// the due-time sweeper's gate: only the owner due-wakes a host. |
| +// the due-time sweeper's gate: only the owner due-wakes a host. Out-of-scope |
| +// hosts always report true: this box is the only one ever asked about them |
| +// (see inScope), so there is nothing to arbitrate. |
| func (m *meshSvc) owns(host string) bool { |
| + if !m.inScope(host) { |
| + return true |
| + } |
| return hrwOwner(host, m.liveBoxes()) == m.self |
| } |
| // remoteOwner returns the peer to proxy host to when its HRW owner is another |
| -// live box. ok=false means serve locally: we are the owner, or the previous |
| -// owner is dead/stale (it's out of the member set, so the hash has already |
| -// reassigned — possibly to us — and a local wake is the failover). |
| +// live box. ok=false means serve locally: host is out of mesh scope, we are |
| +// the owner, or the previous owner is dead/stale (it's out of the member set, |
| +// so the hash has already reassigned — possibly to us — and a local wake is |
| +// the failover). |
| func (m *meshSvc) remoteOwner(host string) (peer string, ok bool) { |
| + if !m.inScope(host) { |
| + return "", false |
| + } |
| if o := hrwOwner(host, m.liveBoxes()); o != m.self { |
| return o, true |
| } |
diff --git a/mesh_live_test.go b/mesh_live_test.go| index c3ade11..646fb6c 100644 |
| --- a/mesh_live_test.go |
| +++ b/mesh_live_test.go |
| @@ -146,7 +146,7 @@ func setupMeshLive(t *testing.T, host string) (boxA, boxB *liveMeshBox, adminTok |
| if err != nil { |
| t.Fatalf("heartbeat store: %v", err) |
| } |
| - box.mesh = newMesh(box.self, hb) |
| + box.mesh = newMesh(box.self, hb, nil) |
| box.mesh.interval = time.Second |
| box.mesh.staleAfter = 3 * time.Second |
| box.act.mesh = box.mesh |
diff --git a/mesh_test.go b/mesh_test.go| index 2929265..eaf6d8f 100644 |
| --- a/mesh_test.go |
| +++ b/mesh_test.go |
| @@ -120,7 +120,7 @@ type meshTestBox struct { |
| func (box *meshTestBox) self() string { return box.front.URL } |
| -func newMeshTestBox(t *testing.T, name string, store meshStore, clk *fakeClock) *meshTestBox { |
| +func newMeshTestBox(t *testing.T, name string, store meshStore, clk *fakeClock, domains []string) *meshTestBox { |
| t.Helper() |
| box := &meshTestBox{name: name, b: &fakeBackend{}} |
| @@ -154,7 +154,7 @@ func newMeshTestBox(t *testing.T, name string, store meshStore, clk *fakeClock) |
| box.front = httptest.NewServer(box.rt) |
| t.Cleanup(box.front.Close) |
| - box.mesh = newMesh(box.front.URL, store) |
| + box.mesh = newMesh(box.front.URL, store, domains) |
| box.mesh.now = clk.now |
| box.act.mesh = box.mesh |
| return box |
| @@ -281,7 +281,7 @@ func TestHRWRedistributionMinimal(t *testing.T) { |
| func TestMeshMembershipStaleness(t *testing.T) { |
| clk := &fakeClock{t: time.Unix(1_700_000_000, 0)} |
| store := newFakeMeshStore(clk.now) |
| - m := newMesh("http://a", store) |
| + m := newMesh("http://a", store, nil) |
| m.now = clk.now |
| // Before any beat: membership of one — the degenerate single box. |
| @@ -314,7 +314,7 @@ func TestMeshMembershipStaleness(t *testing.T) { |
| func TestMeshRoutingDecision(t *testing.T) { |
| clk := &fakeClock{t: time.Unix(1_700_000_000, 0)} |
| store := newFakeMeshStore(clk.now) |
| - m := newMesh("http://a", store) |
| + m := newMesh("http://a", store, nil) |
| m.now = clk.now |
| store.setBeat("http://b", clk.now()) |
| m.beat() |
| @@ -343,6 +343,102 @@ func TestMeshRoutingDecision(t *testing.T) { |
| } |
| } |
| +// --- domain scoping ------------------------------------------------------------ |
| + |
| +// inScope's matching rule: exact domain or a proper subdomain of one, case- |
| +// insensitive (hosts and configured domains alike); nothing else. |
| +func TestMeshInScope(t *testing.T) { |
| + m := newMesh("http://a", newFakeMeshStore(time.Now), []string{"11ch.at", "11relay.com"}) |
| + cases := []struct { |
| + host string |
| + want bool |
| + }{ |
| + {"11ch.at", true}, // the bare domain itself |
| + {"abc.11ch.at", true}, // a subdomain |
| + {"ABC.11CH.AT", true}, // case-insensitive |
| + {"11relay.com", true}, |
| + {"abc.11relay.com", true}, |
| + {"family.luan.chat", false}, // a different domain entirely |
| + {"evil11ch.at", false}, // must not match on a bare suffix |
| + {"11ch.at.evil.com", false}, // must not match as a prefix |
| + } |
| + for _, c := range cases { |
| + if got := m.inScope(c.host); got != c.want { |
| + t.Errorf("inScope(%q) = %v, want %v", c.host, got, c.want) |
| + } |
| + } |
| +} |
| + |
| +// An out-of-scope host always resolves locally, regardless of what HRW would |
| +// otherwise say — it's never proxied away, and this box always reports owning |
| +// it (the sweep's due-wake gate must never skip it). Domain scoping exists |
| +// because a registry row for an out-of-scope host is only ever replicated to |
| +// the box(es) it already lives on (control.go's registerInstance): a host in |
| +// scope legitimately has rows on every mesh member and needs real HRW |
| +// placement; a host out of scope does not, and hashing it elsewhere would |
| +// just 404 there. |
| +func TestMeshDomainScopingBypassesHRW(t *testing.T) { |
| + clk := &fakeClock{t: time.Unix(1_700_000_000, 0)} |
| + store := newFakeMeshStore(clk.now) |
| + m := newMesh("http://a", store, []string{"11ch.at"}) |
| + m.now = clk.now |
| + store.setBeat("http://b", clk.now()) |
| + m.beat() |
| + |
| + boxes := []string{"http://a", "http://b"} |
| + // A host under the in-scope domain that genuinely hashes to the peer. |
| + // hostOwnedBy's "space-N.test" hosts fall under no configured domain, so |
| + // build one under 11ch.at directly. |
| + var inScopeTheirs string |
| + for i := 0; ; i++ { |
| + h := fmt.Sprintf("space-%d.11ch.at", i) |
| + if hrwOwner(h, boxes) == "http://b" { |
| + inScopeTheirs = h |
| + break |
| + } |
| + } |
| + if peer, remote := m.remoteOwner(inScopeTheirs); !remote || peer != "http://b" { |
| + t.Fatalf("in-scope host: remoteOwner = %q, %v; want b, true", peer, remote) |
| + } |
| + |
| + // The same host name, but under a domain the mesh isn't scoped to. Find |
| + // one that HRW would (incorrectly, if honored) assign to the peer. |
| + var outOfScope string |
| + for i := 0; ; i++ { |
| + h := fmt.Sprintf("family-%d.luan.chat", i) |
| + if hrwOwner(h, boxes) == "http://b" { |
| + outOfScope = h |
| + break |
| + } |
| + } |
| + if _, remote := m.remoteOwner(outOfScope); remote { |
| + t.Fatalf("out-of-scope host %q was proxied away — domain scoping did not apply", outOfScope) |
| + } |
| + if !m.owns(outOfScope) { |
| + t.Fatalf("out-of-scope host %q: owns() = false — the due-wake gate would wrongly skip it", outOfScope) |
| + } |
| +} |
| + |
| +// meshFromEnv refuses to start the mesh without an explicit domain allowlist — |
| +// failing loudly beats silently letting HRW assign a host to a box that has |
| +// no registry row for it (see the inScope doc comment). |
| +func TestMeshFromEnvRequiresDomains(t *testing.T) { |
| + t.Setenv("LCHAT_MESH_SELF", "https://box-a.internal") |
| + t.Setenv("LCHAT_MESH_DOMAINS", "") |
| + if _, err := meshFromEnv(&hibernateConfig{Bucket: "b", Region: "eu-west-1"}); err == nil { |
| + t.Fatal("mesh without LCHAT_MESH_DOMAINS should be a config error") |
| + } |
| + |
| + t.Setenv("LCHAT_MESH_DOMAINS", "11ch.at, 11relay.com") |
| + m, err := meshFromEnv(&hibernateConfig{Bucket: "b", Region: "eu-west-1"}) |
| + if err != nil || m == nil { |
| + t.Fatalf("valid domains: %v, %v", m, err) |
| + } |
| + if len(m.domains) != 2 || m.domains[0] != "11ch.at" || m.domains[1] != "11relay.com" { |
| + t.Fatalf("domains = %v, want the parsed, trimmed, lowercased list", m.domains) |
| + } |
| +} |
| + |
| // --- the proxy hop ------------------------------------------------------------- |
| // Scheme/Host/SNI semantics of the peer hop: the instance hostname survives |
| @@ -396,8 +492,8 @@ func TestPeerProxyRewrite(t *testing.T) { |
| func TestMeshProxyToOwner(t *testing.T) { |
| clk := &fakeClock{t: time.Unix(1_700_000_000, 0)} |
| store := newFakeMeshStore(clk.now) |
| - box1 := newMeshTestBox(t, "box1", store, clk) |
| - box2 := newMeshTestBox(t, "box2", store, clk) |
| + box1 := newMeshTestBox(t, "box1", store, clk, nil) |
| + box2 := newMeshTestBox(t, "box2", store, clk, nil) |
| // Beat both (twice for box1, whose first beat predates box2's PUT) so |
| // each box sees the full membership. |
| box1.mesh.beat() |
| @@ -474,8 +570,8 @@ func TestMeshProxyToOwner(t *testing.T) { |
| func TestMeshOffTripwire(t *testing.T) { |
| clk := &fakeClock{t: time.Unix(1_700_000_000, 0)} |
| store := newFakeMeshStore(clk.now) |
| - box1 := newMeshTestBox(t, "box1", store, clk) |
| - box2 := newMeshTestBox(t, "box2", store, clk) |
| + box1 := newMeshTestBox(t, "box1", store, clk, nil) |
| + box2 := newMeshTestBox(t, "box2", store, clk, nil) |
| box1.act.mesh = nil // the tripwire: box1 is mesh-off |
| box2.mesh.beat() // box2 participates; its heartbeat is in the store |
| @@ -520,6 +616,10 @@ func TestMeshOffTripwire(t *testing.T) { |
| // meshFromEnv: unset = off (nil, nil — the zero-change default); set without |
| // the hibernation bucket = a loud config error; malformed URLs rejected. |
| func TestMeshFromEnv(t *testing.T) { |
| + // A valid domain allowlist throughout — TestMeshFromEnvRequiresDomains |
| + // covers the missing-domains case on its own. |
| + t.Setenv("LCHAT_MESH_DOMAINS", "11ch.at") |
| + |
| t.Setenv("LCHAT_MESH_SELF", "") |
| if m, err := meshFromEnv(&hibernateConfig{Bucket: "b"}); err != nil || m != nil { |
| t.Fatalf("unset: got %v, %v; want nil, nil (mesh off)", m, err) |
| @@ -557,7 +657,7 @@ func TestMeshDrainDeletesHeartbeatFirst(t *testing.T) { |
| a, b, clk, _ := testActivator(t, time.Hour) |
| store := newFakeMeshStore(clk.now) |
| store.rec = b.record // interleave heartbeat ops with the backend's order log |
| - m := newMesh("http://box-a", store) |
| + m := newMesh("http://box-a", store, nil) |
| m.now = clk.now |
| m.interval = time.Hour // the loop must never beat on its own mid-test |
| a.mesh = m |
| @@ -599,7 +699,7 @@ func TestMeshDrainDeletesHeartbeatFirst(t *testing.T) { |
| func TestMeshDueSweepOnlyOwner(t *testing.T) { |
| a, b, clk, reg := testActivator(t, 10*time.Second) |
| store := newFakeMeshStore(clk.now) |
| - m := newMesh("http://box-a", store) |
| + m := newMesh("http://box-a", store, nil) |
| m.now = clk.now |
| a.mesh = m |
| store.setBeat("http://box-b", clk.now()) |
diff --git a/router.go b/router.go| index 98c4abd..76a8173 100644 |
| --- a/router.go |
| +++ b/router.go |
| @@ -257,8 +257,8 @@ func runRouter(args []string) { |
| // membership view. |
| act.mesh = mesh |
| mesh.start() |
| - log.Printf("router: activator mesh on — this box is %s (heartbeats s3://%s/%s every %s, peers stale after %s)", |
| - mesh.self, hc.Bucket, hc.meshPrefix(), mesh.interval, mesh.staleAfter) |
| + log.Printf("router: activator mesh on — this box is %s (heartbeats s3://%s/%s every %s, peers stale after %s, placement scoped to %v)", |
| + mesh.self, hc.Bucket, hc.meshPrefix(), mesh.interval, mesh.staleAfter, mesh.domains) |
| } |
| act.start() |
| log.Printf("router: hibernation active (bucket %s, prefix %q, idle %s) — flagged instances wake on demand", |