eleven / ocho Public

server: scope activator mesh placement to LCHAT_MESH_DOMAINS (#820)

HRW placement (mesh.go) applied to every hibernating host with no
domain restriction. A registry row only ever exists on the boxes
control.go's registerInstance replicates it to, so hashing an
out-of-scope host onto a box with no row for it would 404 there
instead of waking it. Domain scoping also bounds placement to hosts
every mesh member can independently get a matching cert for (the
same domains as LCHAT_WILDCARD_DOMAINS's DNS-01 wildcard, unlike
per-host HTTP-01 which is tied to whichever box DNS points at).

meshFromEnv now requires LCHAT_MESH_DOMAINS whenever LCHAT_MESH_SELF
is set, refusing to start otherwise — failing loudly beats silently
letting HRW assign a host nobody but its original box can serve.
Ships with LCHAT_MESH_SELF still unset everywhere, so today's running
system is unaffected; this is prep for turning the mesh on.

Co-authored-by: Claude <noreply@anthropic.com>

James Adam 3 months ago 32ce9d67a998aa2135ad2a0617d9e9da73845bc9 parent fe82e20

4 files changed, +173 −20

mesh.go +60 −7
mesh_live_test.go +1 −1
mesh_test.go +110 −10
router.go +2 −2

diff --git a/mesh.go b/mesh.go

index 1e686a6..b27ff0f 100644

--- a/mesh.go

+++ b/mesh.go

@@ -28,6 +28,18 @@ package main

// dead/stale owner drops out of the member set, so the hash reassigns and

// the request falls through to a local wake — failover IS the routine

// wake path.

+// 5. Domain scoping (LCHAT_MESH_DOMAINS) — HRW placement only ever applies to

+// hosts under a configured mesh domain; everything else always resolves

+// locally, on whichever box is asked. This isn't an optimization: a

+// registry row only ever exists on the boxes it was explicitly replicated

+// to (see registerInstance in control.go), so a host outside the

+// configured domains was never given a row anywhere but its original box,

+// and letting the hash "place" it elsewhere would just 404 there. It also

+// keeps placement bounded to hosts every mesh member can actually get a

+// matching TLS cert for (typically the same domains as

+// LCHAT_WILDCARD_DOMAINS's ACME DNS-01 cert, which — unlike per-host

+// HTTP-01 — any box can obtain independently of which box DNS currently

+// points at).

// Drain (activatorSvc.shutdown) deletes the heartbeat FIRST, then checkpoints

// and releases as today — peers take over immediately, and deploys, box

@@ -86,6 +98,7 @@ type meshStore interface {

type meshSvc struct {

self string // this box's peer base URL — its mesh identity

store meshStore

+ domains []string // LCHAT_MESH_DOMAINS, lowercased — see inScope. Empty = unrestricted (test convenience only; meshFromEnv always supplies a non-empty list)

interval time.Duration

staleAfter time.Duration

now func() time.Time // a seam for the unit tests' clock

@@ -99,10 +112,11 @@ type meshSvc struct {

done chan struct{}

}

-func newMesh(self string, store meshStore) *meshSvc {

+func newMesh(self string, store meshStore, domains []string) *meshSvc {

return &meshSvc{

self: self,

store: store,

+ domains: domains,

interval: meshHeartbeatInterval,

staleAfter: meshStaleAfter,

now: time.Now,

@@ -117,7 +131,11 @@ func newMesh(self string, store meshStore) *meshSvc {

// network, or https where the box presents a verifiable cert (see

// newPeerProxy). Presence turns the mesh on; absent = single box, zero mesh

// codepaths. The mesh rides on the hibernation bucket, so it requires

-// LCHAT_HIBERNATE_BUCKET.

+// LCHAT_HIBERNATE_BUCKET. It also requires LCHAT_MESH_DOMAINS (same

+// comma-list shape as LCHAT_WILDCARD_DOMAINS, parsed by the same

+// splitDomains): failing loudly here beats silently letting HRW "place" a

+// host onto a box that has no registry row for it (a 404, not a graceful

+// wake) or that could never get it a matching cert — see inScope.

func meshFromEnv(hc *hibernateConfig) (*meshSvc, error) {

self := strings.TrimSpace(os.Getenv("LCHAT_MESH_SELF"))

if self == "" {

@@ -126,6 +144,10 @@ func meshFromEnv(hc *hibernateConfig) (*meshSvc, error) {

if hc == nil {

return nil, errors.New("LCHAT_MESH_SELF set but LCHAT_HIBERNATE_BUCKET missing — the mesh rides on the hibernation bucket and only places hibernating instances")

}

+ domains := splitDomains(os.Getenv("LCHAT_MESH_DOMAINS"))

+ if len(domains) == 0 {

+ return nil, errors.New("LCHAT_MESH_SELF set but LCHAT_MESH_DOMAINS missing — mesh placement needs an explicit domain allowlist so HRW never assigns a host to a box that has no registry row for it or can't obtain it a matching cert; set it to the domains every mesh member can serve (typically the same value as LCHAT_WILDCARD_DOMAINS)")

+ }

self = strings.TrimRight(self, "/")

u, err := url.Parse(self)

if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") || u.Path != "" {

@@ -135,7 +157,7 @@ func meshFromEnv(hc *hibernateConfig) (*meshSvc, error) {

if err != nil {

return nil, fmt.Errorf("mesh heartbeat store: %w", err)

}

- return newMesh(self, store), nil

+ return newMesh(self, store, domains), nil

}

// start registers this box (one synchronous beat, so the first sweep and the

@@ -236,17 +258,48 @@ func (m *meshSvc) liveBoxes() []string {

return boxes

}

+// inScope reports whether host participates in HRW placement at all. Only

+// hosts under a configured LCHAT_MESH_DOMAINS domain are ever hashed across

+// boxes; everything else always resolves to whichever box is asked — correct

+// because registerInstance only ever replicates a route to the boxes covered

+// by those same domains (control.go), so no other box has a row for it

+// anyway, and this box (which was asked) is the only one that could possibly

+// serve it. An empty domain list is unrestricted — a test-construction

+// convenience (direct newMesh callers exercising bare HRW); meshFromEnv

+// always supplies a non-empty list in production.

+func (m *meshSvc) inScope(host string) bool {

+ if len(m.domains) == 0 {

+ return true

+ }

+ host = strings.ToLower(host)

+ for _, d := range m.domains {

+ if host == d || strings.HasSuffix(host, "."+d) {

+ return true

+ }

+ return false

// owns reports whether this box is host's HRW owner in the live member set —

-// the due-time sweeper's gate: only the owner due-wakes a host.

+// the due-time sweeper's gate: only the owner due-wakes a host. Out-of-scope

+// hosts always report true: this box is the only one ever asked about them

+// (see inScope), so there is nothing to arbitrate.

func (m *meshSvc) owns(host string) bool {

+ if !m.inScope(host) {

+ return true

+ }

return hrwOwner(host, m.liveBoxes()) == m.self

}

// remoteOwner returns the peer to proxy host to when its HRW owner is another

-// live box. ok=false means serve locally: we are the owner, or the previous

-// owner is dead/stale (it's out of the member set, so the hash has already

-// reassigned — possibly to us — and a local wake is the failover).

+// live box. ok=false means serve locally: host is out of mesh scope, we are

+// the owner, or the previous owner is dead/stale (it's out of the member set,

+// so the hash has already reassigned — possibly to us — and a local wake is

+// the failover).

func (m *meshSvc) remoteOwner(host string) (peer string, ok bool) {

+ if !m.inScope(host) {

+ return "", false

+ }

if o := hrwOwner(host, m.liveBoxes()); o != m.self {

return o, true

}

diff --git a/mesh_live_test.go b/mesh_live_test.go

index c3ade11..646fb6c 100644

--- a/mesh_live_test.go

+++ b/mesh_live_test.go

@@ -146,7 +146,7 @@ func setupMeshLive(t *testing.T, host string) (boxA, boxB *liveMeshBox, adminTok

if err != nil {

t.Fatalf("heartbeat store: %v", err)

}

- box.mesh = newMesh(box.self, hb)

+ box.mesh = newMesh(box.self, hb, nil)

box.mesh.interval = time.Second

box.mesh.staleAfter = 3 * time.Second

box.act.mesh = box.mesh

diff --git a/mesh_test.go b/mesh_test.go

index 2929265..eaf6d8f 100644

--- a/mesh_test.go

+++ b/mesh_test.go

@@ -120,7 +120,7 @@ type meshTestBox struct {

func (box *meshTestBox) self() string { return box.front.URL }

-func newMeshTestBox(t *testing.T, name string, store meshStore, clk *fakeClock) *meshTestBox {

+func newMeshTestBox(t *testing.T, name string, store meshStore, clk *fakeClock, domains []string) *meshTestBox {

t.Helper()

box := &meshTestBox{name: name, b: &fakeBackend{}}

@@ -154,7 +154,7 @@ func newMeshTestBox(t *testing.T, name string, store meshStore, clk *fakeClock)

box.front = httptest.NewServer(box.rt)

t.Cleanup(box.front.Close)

- box.mesh = newMesh(box.front.URL, store)

+ box.mesh = newMesh(box.front.URL, store, domains)

box.mesh.now = clk.now

box.act.mesh = box.mesh

return box

@@ -281,7 +281,7 @@ func TestHRWRedistributionMinimal(t *testing.T) {

func TestMeshMembershipStaleness(t *testing.T) {

clk := &fakeClock{t: time.Unix(1_700_000_000, 0)}

store := newFakeMeshStore(clk.now)

- m := newMesh("http://a", store)

+ m := newMesh("http://a", store, nil)

m.now = clk.now

// Before any beat: membership of one — the degenerate single box.

@@ -314,7 +314,7 @@ func TestMeshMembershipStaleness(t *testing.T) {

func TestMeshRoutingDecision(t *testing.T) {

clk := &fakeClock{t: time.Unix(1_700_000_000, 0)}

store := newFakeMeshStore(clk.now)

- m := newMesh("http://a", store)

+ m := newMesh("http://a", store, nil)

m.now = clk.now

store.setBeat("http://b", clk.now())

m.beat()

@@ -343,6 +343,102 @@ func TestMeshRoutingDecision(t *testing.T) {

}

+// --- domain scoping ------------------------------------------------------------

+// inScope's matching rule: exact domain or a proper subdomain of one, case-

+// insensitive (hosts and configured domains alike); nothing else.

+func TestMeshInScope(t *testing.T) {

+ m := newMesh("http://a", newFakeMeshStore(time.Now), []string{"11ch.at", "11relay.com"})

+ cases := []struct {

+ host string

+ want bool

+ }{

+ {"11ch.at", true}, // the bare domain itself

+ {"abc.11ch.at", true}, // a subdomain

+ {"ABC.11CH.AT", true}, // case-insensitive

+ {"11relay.com", true},

+ {"abc.11relay.com", true},

+ {"family.luan.chat", false}, // a different domain entirely

+ {"evil11ch.at", false}, // must not match on a bare suffix

+ {"11ch.at.evil.com", false}, // must not match as a prefix

+ }

+ for _, c := range cases {

+ if got := m.inScope(c.host); got != c.want {

+ t.Errorf("inScope(%q) = %v, want %v", c.host, got, c.want)

+ }

+// An out-of-scope host always resolves locally, regardless of what HRW would

+// otherwise say — it's never proxied away, and this box always reports owning

+// it (the sweep's due-wake gate must never skip it). Domain scoping exists

+// because a registry row for an out-of-scope host is only ever replicated to

+// the box(es) it already lives on (control.go's registerInstance): a host in

+// scope legitimately has rows on every mesh member and needs real HRW

+// placement; a host out of scope does not, and hashing it elsewhere would

+// just 404 there.

+func TestMeshDomainScopingBypassesHRW(t *testing.T) {

+ clk := &fakeClock{t: time.Unix(1_700_000_000, 0)}

+ store := newFakeMeshStore(clk.now)

+ m := newMesh("http://a", store, []string{"11ch.at"})

+ m.now = clk.now

+ store.setBeat("http://b", clk.now())

+ m.beat()

+ boxes := []string{"http://a", "http://b"}

+ // A host under the in-scope domain that genuinely hashes to the peer.

+ // hostOwnedBy's "space-N.test" hosts fall under no configured domain, so

+ // build one under 11ch.at directly.

+ var inScopeTheirs string

+ for i := 0; ; i++ {

+ h := fmt.Sprintf("space-%d.11ch.at", i)

+ if hrwOwner(h, boxes) == "http://b" {

+ inScopeTheirs = h

+ break

+ }

+ if peer, remote := m.remoteOwner(inScopeTheirs); !remote || peer != "http://b" {

+ t.Fatalf("in-scope host: remoteOwner = %q, %v; want b, true", peer, remote)

+ }

+ // The same host name, but under a domain the mesh isn't scoped to. Find

+ // one that HRW would (incorrectly, if honored) assign to the peer.

+ var outOfScope string

+ for i := 0; ; i++ {

+ h := fmt.Sprintf("family-%d.luan.chat", i)

+ if hrwOwner(h, boxes) == "http://b" {

+ outOfScope = h

+ break

+ }

+ if _, remote := m.remoteOwner(outOfScope); remote {

+ t.Fatalf("out-of-scope host %q was proxied away — domain scoping did not apply", outOfScope)

+ }

+ if !m.owns(outOfScope) {

+ t.Fatalf("out-of-scope host %q: owns() = false — the due-wake gate would wrongly skip it", outOfScope)

+ }

+// meshFromEnv refuses to start the mesh without an explicit domain allowlist —

+// failing loudly beats silently letting HRW assign a host to a box that has

+// no registry row for it (see the inScope doc comment).

+func TestMeshFromEnvRequiresDomains(t *testing.T) {

+ t.Setenv("LCHAT_MESH_SELF", "https://box-a.internal")

+ t.Setenv("LCHAT_MESH_DOMAINS", "")

+ if _, err := meshFromEnv(&hibernateConfig{Bucket: "b", Region: "eu-west-1"}); err == nil {

+ t.Fatal("mesh without LCHAT_MESH_DOMAINS should be a config error")

+ }

+ t.Setenv("LCHAT_MESH_DOMAINS", "11ch.at, 11relay.com")

+ m, err := meshFromEnv(&hibernateConfig{Bucket: "b", Region: "eu-west-1"})

+ if err != nil || m == nil {

+ t.Fatalf("valid domains: %v, %v", m, err)

+ }

+ if len(m.domains) != 2 || m.domains[0] != "11ch.at" || m.domains[1] != "11relay.com" {

+ t.Fatalf("domains = %v, want the parsed, trimmed, lowercased list", m.domains)

+ }

// --- the proxy hop -------------------------------------------------------------

// Scheme/Host/SNI semantics of the peer hop: the instance hostname survives

@@ -396,8 +492,8 @@ func TestPeerProxyRewrite(t *testing.T) {

func TestMeshProxyToOwner(t *testing.T) {

clk := &fakeClock{t: time.Unix(1_700_000_000, 0)}

store := newFakeMeshStore(clk.now)

- box1 := newMeshTestBox(t, "box1", store, clk)

- box2 := newMeshTestBox(t, "box2", store, clk)

+ box1 := newMeshTestBox(t, "box1", store, clk, nil)

+ box2 := newMeshTestBox(t, "box2", store, clk, nil)

// Beat both (twice for box1, whose first beat predates box2's PUT) so

// each box sees the full membership.

box1.mesh.beat()

@@ -474,8 +570,8 @@ func TestMeshProxyToOwner(t *testing.T) {

func TestMeshOffTripwire(t *testing.T) {

clk := &fakeClock{t: time.Unix(1_700_000_000, 0)}

store := newFakeMeshStore(clk.now)

- box1 := newMeshTestBox(t, "box1", store, clk)

- box2 := newMeshTestBox(t, "box2", store, clk)

+ box1 := newMeshTestBox(t, "box1", store, clk, nil)

+ box2 := newMeshTestBox(t, "box2", store, clk, nil)

box1.act.mesh = nil // the tripwire: box1 is mesh-off

box2.mesh.beat() // box2 participates; its heartbeat is in the store

@@ -520,6 +616,10 @@ func TestMeshOffTripwire(t *testing.T) {

// meshFromEnv: unset = off (nil, nil — the zero-change default); set without

// the hibernation bucket = a loud config error; malformed URLs rejected.

func TestMeshFromEnv(t *testing.T) {

+ // A valid domain allowlist throughout — TestMeshFromEnvRequiresDomains

+ // covers the missing-domains case on its own.

+ t.Setenv("LCHAT_MESH_DOMAINS", "11ch.at")

t.Setenv("LCHAT_MESH_SELF", "")

if m, err := meshFromEnv(&hibernateConfig{Bucket: "b"}); err != nil || m != nil {

t.Fatalf("unset: got %v, %v; want nil, nil (mesh off)", m, err)

@@ -557,7 +657,7 @@ func TestMeshDrainDeletesHeartbeatFirst(t *testing.T) {

a, b, clk, _ := testActivator(t, time.Hour)

store := newFakeMeshStore(clk.now)

store.rec = b.record // interleave heartbeat ops with the backend's order log

- m := newMesh("http://box-a", store)

+ m := newMesh("http://box-a", store, nil)

m.now = clk.now

m.interval = time.Hour // the loop must never beat on its own mid-test

a.mesh = m

@@ -599,7 +699,7 @@ func TestMeshDrainDeletesHeartbeatFirst(t *testing.T) {

func TestMeshDueSweepOnlyOwner(t *testing.T) {

a, b, clk, reg := testActivator(t, 10*time.Second)

store := newFakeMeshStore(clk.now)

- m := newMesh("http://box-a", store)

+ m := newMesh("http://box-a", store, nil)

m.now = clk.now

a.mesh = m

store.setBeat("http://box-b", clk.now())

diff --git a/router.go b/router.go

index 98c4abd..76a8173 100644

--- a/router.go

+++ b/router.go

@@ -257,8 +257,8 @@ func runRouter(args []string) {

// membership view.

act.mesh = mesh

mesh.start()

- log.Printf("router: activator mesh on — this box is %s (heartbeats s3://%s/%s every %s, peers stale after %s)",

- mesh.self, hc.Bucket, hc.meshPrefix(), mesh.interval, mesh.staleAfter)

+ log.Printf("router: activator mesh on — this box is %s (heartbeats s3://%s/%s every %s, peers stale after %s, placement scoped to %v)",

+ mesh.self, hc.Bucket, hc.meshPrefix(), mesh.interval, mesh.staleAfter, mesh.domains)

}

act.start()

log.Printf("router: hibernation active (bucket %s, prefix %q, idle %s) — flagged instances wake on demand",