eleven / messenger Public

status: partition each box's Fleet stats by HRW ownership, add peer region (#851)

Each mesh box previously reported on every hibernating route it happened to
know about (including ones replicated to it but actually running/verified
asleep on a different box), so the "Serving hosts" panel showed the same
instance as "asleep" on more than one host with no way to tell they overlap
— confusing even though the top-level fleet-wide sum was already correct.
computeBoxStats now filters to routes this box currently HRW-owns
(mesh.owns), making every box's own awake/asleep a genuine, non-overlapping
slice of the fleet.

Also add an explicit LCHAT_BOX_REGION env (read by meshstats.go's boxRegion,
mirroring status-publish.sh's own STATUS_BOX_REGION override) so a mesh
peer can publish its own coarse region — previously structurally impossible,
since BoxStats carried no region field at all. status-publish.sh's merge is
already a raw passthrough of each peer's published object, so no bash change
was needed for the value itself to ride through.

Co-authored-by: Claude <noreply@anthropic.com>

James Adam 3 months ago cfa72d87c561ad17a576cea948305a74c36d7f8f parent a1747a0

7 files changed, +164 −15

deploy/status-publish.sh +10 −4
internal/activator/heartbeat.go +6 −0
meshstats.go +36 −6
meshstats_test.go +108 −1
router.go +1 −1
test/status-contract.test.mjs +2 −2
test/status-fixtures/fleet-multi-host.json +1 −1

diff --git a/deploy/status-publish.sh b/deploy/status-publish.sh

index bc984da..d5b04d3 100755

--- a/deploy/status-publish.sh

+++ b/deploy/status-publish.sh

@@ -546,9 +546,12 @@ build_fleet() {

# is deliberately this box's own view, not the fleet total.) The optional

# region is the operator-set coarse continent + country word pair (see

# PRIVACY) — absent env, absent key, never guessed; escaped like the version

- # strings so a quote can't break out of the SQL literal. Peers don't carry

- # always_on/region (both stay local-box-only concepts for now — an honest

- # limitation, not a bug).

+ # strings so a quote can't break out of the SQL literal. A peer's region

+ # (if its own LCHAT_BOX_REGION is set — meshstats.go's boxRegion) rides

+ # through automatically: mesh_peer_boxes inserts each peer's whole

+ # self-published object verbatim, so an extra key needs no bash change

+ # here. `always_on` stays a genuinely local-box-only concept, though — no

+ # peer publishes one.

local boxes peer_boxes region_kv=""

# Resolved here, not at file top: DRY_RUN/argv must be parsed first so the

# dry-run's no-live-probes guard actually guards (and the dry-run fixture's

@@ -1328,8 +1331,11 @@ EOF

# (echo + foxtrot, 2 of them) — otherwise the fleet-wide awake/asleep sum

# below goes negative, which would mean the fixture models an impossible

# state, not a real one.

+ # region present here (unlike an earlier version of this fixture) —

+ # exercises a peer that HAS set its own LCHAT_BOX_REGION, proving the

+ # merge's raw passthrough carries an extra key with no bash-side change.

cat > "$FIXDIR/mesh-peers.json" <<'EOF'

-[{"label":"turing-orienteering","awake":1,"asleep":1,"load":"busy"}]

+[{"label":"turing-orienteering","awake":1,"asleep":1,"load":"busy","region":"UK · London"}]

EOF

diff --git a/internal/activator/heartbeat.go b/internal/activator/heartbeat.go

index b609276..281479d 100644

--- a/internal/activator/heartbeat.go

+++ b/internal/activator/heartbeat.go

@@ -109,6 +109,12 @@ type BoxStats struct {

Awake *int `json:"awake"`

Asleep *int `json:"asleep"`

Load string `json:"load,omitempty"`

+ // Region is this box's own coarse continent+country string ("EU ·

+ // Finland" — see status.md's PRIVACY rule), set only via the operator's

+ // explicit LCHAT_BOX_REGION env (meshstats.go's boxRegion) — unlike the

+ // local box's own status-publish.sh reading, a peer has no cloud-metadata

+ // auto-detect here, so absent env means omitted, never guessed.

+ Region string `json:"region,omitempty"`

}

// PutStats writes this box's stats snapshot. Called on its own slower cadence

diff --git a/meshstats.go b/meshstats.go

index 359181a..0e7e91b 100644

--- a/meshstats.go

+++ b/meshstats.go

@@ -117,6 +117,20 @@ var boxLabelActions = []string{

"spelunking", "kayaking", "orienteering", "puppeteering", "tinkering",

}

+// boxRegion resolves this box's coarse region for peer publishing, straight

+// from LCHAT_BOX_REGION — an explicit operator override only. Unlike

+// status-publish.sh's own box_region() (which prefers auto-detecting via

+// Hetzner/AWS instance metadata for the LOCAL box, falling back to a hand-set

+// override), a mesh peer publishes here from the Go binary, which has no

+// metadata-probing code of its own — so this box's operator sets the string

+// once in its router.env, same coarse "continent · country" shape, and it

+// rides through status-publish.sh's mesh_peer_boxes unchanged (that merge is

+// already a raw passthrough of the whole published object). Empty env means

+// omitted, never guessed.

+func boxRegion() string {

+ return os.Getenv("LCHAT_BOX_REGION")

// machineIDPath is a seam for tests, matching meminfoPath's convention.

var machineIDPath = "/etc/machine-id"

@@ -145,7 +159,19 @@ func boxLabel() string {

// computeBoxStats builds this box's publishable snapshot: asleep is derived

// (every hibernating route not currently awake), never separately probed, so

// the two counts can never disagree with each other.

-func computeBoxStats(reg *Registry, act *activatorSvc) activator.BoxStats {

+//

+// hib only counts routes THIS box currently HRW-owns (mesh.owns — nil mesh,

+// e.g. in tests that don't exercise the mesh, counts everything). Without

+// this filter, a mesh-replicated route is known to (and counted by) every

+// peer it's been seeded onto, so two boxes would each report it — correctly,

+// from each one's own local blind spot — and the Fleet page's per-box rows

+// would visibly overlap (the same instance reads as "asleep" on more than one

+// host, with no way to tell it's the same instance). Filtering to the sole

+// current owner makes every box's row a genuine, non-overlapping partition of

+// the fleet: an out-of-scope route (mesh.owns always true for those, since

+// only one box was ever going to have its row anyway) still counts locally,

+// exactly as before mesh placement existed.

+func computeBoxStats(reg *Registry, act *activatorSvc, mesh *meshSvc) activator.BoxStats {

routes, err := reg.list()

if err != nil {

log.Printf("mesh: stats: list routes: %v", err)

@@ -153,15 +179,19 @@ func computeBoxStats(reg *Registry, act *activatorSvc) activator.BoxStats {

}

hib := 0

for _, r := range routes {

- if r.Kind == "instance" && r.Hibernate {

- hib++

+ if r.Kind != "instance" || !r.Hibernate {

+ continue

+ }

+ if mesh != nil && !mesh.owns(r.Host) {

+ continue // a live peer owns it — that box's own snapshot counts it, not this one

}

+ hib++

}

awake := act.awakeCount()

asleep := hib - awake

load, _ := memLoadWord() // "" (omitted) when unknown

- return activator.BoxStats{Label: boxLabel(), Awake: &awake, Asleep: &asleep, Load: load}

+ return activator.BoxStats{Label: boxLabel(), Awake: &awake, Asleep: &asleep, Load: load, Region: boxRegion()}

}

// startStatsReporter runs alongside the mesh (wired only when mesh != nil,

@@ -169,11 +199,11 @@ func computeBoxStats(reg *Registry, act *activatorSvc) activator.BoxStats {

// for as long as the process runs — like the router's other background

// loops (e.g. its registry-watcher poll), it has no graceful stop; the

// process exiting is what stops it.

-func startStatsReporter(store *activator.HeartbeatStore, self string, reg *Registry, act *activatorSvc) {

+func startStatsReporter(store *activator.HeartbeatStore, self string, reg *Registry, act *activatorSvc, mesh *meshSvc) {

report := func() {

ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)

defer cancel()

- if err := store.PutStats(ctx, self, computeBoxStats(reg, act)); err != nil {

+ if err := store.PutStats(ctx, self, computeBoxStats(reg, act, mesh)); err != nil {

log.Printf("mesh: stats: publish failed: %v", err)

}

diff --git a/meshstats_test.go b/meshstats_test.go

index 2e95d82..3f990f1 100644

--- a/meshstats_test.go

+++ b/meshstats_test.go

@@ -13,6 +13,7 @@ import (

"path/filepath"

"strconv"

"testing"

+ "time"

)

func TestPctBandThresholds(t *testing.T) {

@@ -113,7 +114,7 @@ func TestComputeBoxStats(t *testing.T) {

act.entries["a.11ch.at"] = &actEntry{host: "a.11ch.at", state: actAwake}

act.entries["b.11ch.at"] = &actEntry{host: "b.11ch.at", state: actWaking} // not yet awake

- stats := computeBoxStats(reg, act)

+ stats := computeBoxStats(reg, act, nil)

if stats.Label == "" {

t.Fatal("Label must be populated")

}

@@ -123,4 +124,110 @@ func TestComputeBoxStats(t *testing.T) {

if stats.Asleep == nil || *stats.Asleep != 2 {

t.Fatalf("Asleep = %v, want 2 (3 hibernating - 1 awake)", stats.Asleep)

}

+ if stats.Region != "" {

+ t.Fatalf("Region = %q, want empty (LCHAT_BOX_REGION unset)", stats.Region)

+ }

+// boxRegion is an explicit-override-only read of LCHAT_BOX_REGION — absent

+// env means empty (never guessed), and computeBoxStats must carry whatever

+// it resolves straight into BoxStats.Region.

+func TestBoxRegionExplicitOverride(t *testing.T) {

+ t.Setenv("LCHAT_BOX_REGION", "")

+ if got := boxRegion(); got != "" {

+ t.Fatalf("boxRegion() = %q, want empty when LCHAT_BOX_REGION is unset", got)

+ }

+ t.Setenv("LCHAT_BOX_REGION", "UK · London")

+ if got, want := boxRegion(), "UK · London"; got != want {

+ t.Fatalf("boxRegion() = %q, want %q", got, want)

+ }

+ reg, err := openRegistry(filepath.Join(t.TempDir(), "registry.db"))

+ if err != nil {

+ t.Fatalf("open registry: %v", err)

+ }

+ act := newActivator(hibernateConfig{Bucket: "test-bucket"}, &fakeBackend{}, reg)

+ if got := computeBoxStats(reg, act, nil).Region; got != "UK · London" {

+ t.Fatalf("computeBoxStats().Region = %q, want %q", got, "UK · London")

+ }

+// Two boxes that both know about the same mesh-replicated routes must never

+// both count the same route as hibernating in their own published snapshot —

+// that's exactly the "same instance reads as asleep on two hosts" confusion

+// the Fleet page must not show. computeBoxStats filters to routes THIS box

+// currently HRW-owns, so each box's own count is a genuine, non-overlapping

+// slice of the fleet: every in-scope route is owned by exactly one live box,

+// and an out-of-scope route (never replicated anywhere else) always counts

+// locally regardless of ownership.

+func TestComputeBoxStatsPartitionsOwnershipAcrossBoxes(t *testing.T) {

+ clk := &fakeClock{t: time.Now()}

+ store := newFakeMeshStore(clk.now)

+ domains := []string{"test"}

+ meshA := newMesh("http://box-a", store, domains)

+ meshB := newMesh("http://box-b", store, domains)

+ meshA.now, meshB.now = clk.now, clk.now

+ meshA.beat()

+ meshB.beat()

+ meshA.beat() // refresh A's view so it now also sees B as live

+ boxes := []string{meshA.self, meshB.self}

+ var ownedByA, ownedByB []string

+ for i := 0; len(ownedByA) < 2 || len(ownedByB) < 2; i++ {

+ h := "space-" + strconv.Itoa(i) + ".test"

+ switch hrwOwner(h, boxes) {

+ case meshA.self:

+ if len(ownedByA) < 2 {

+ ownedByA = append(ownedByA, h)

+ }

+ case meshB.self:

+ if len(ownedByB) < 2 {

+ ownedByB = append(ownedByB, h)

+ }

+ regA, err := openRegistry(filepath.Join(t.TempDir(), "a.db"))

+ if err != nil {

+ t.Fatalf("open registry A: %v", err)

+ }

+ regB, err := openRegistry(filepath.Join(t.TempDir(), "b.db"))

+ if err != nil {

+ t.Fatalf("open registry B: %v", err)

+ }

+ // Full replication: both boxes know about all 4 in-scope routes, exactly

+ // like mesh-seed/fanOutMeshRoute leave them today. Plus one out-of-scope

+ // route that only box A ever knew about (never replicated anywhere).

+ for _, reg := range []*Registry{regA, regB} {

+ for _, h := range append(append([]string{}, ownedByA...), ownedByB...) {

+ if err := reg.upsert(Route{Host: h, Kind: "instance", Socket: "/tmp/" + h + ".sock", Hibernate: true}); err != nil {

+ t.Fatal(err)

+ }

+ if err := regA.upsert(Route{Host: "local-only.example", Kind: "instance", Socket: "/tmp/local.sock", Hibernate: true}); err != nil {

+ t.Fatal(err)

+ }

+ actA := newActivator(hibernateConfig{Bucket: "test-bucket"}, &fakeBackend{}, regA)

+ actB := newActivator(hibernateConfig{Bucket: "test-bucket"}, &fakeBackend{}, regB)

+ statsA := computeBoxStats(regA, actA, meshA)

+ statsB := computeBoxStats(regB, actB, meshB)

+ // A owns its 2 + the 1 out-of-scope local-only route = 3; B owns its 2.

+ if statsA.Asleep == nil || *statsA.Asleep != 3 {

+ t.Fatalf("box A asleep = %v, want 3 (its 2 owned in-scope + 1 out-of-scope, never B's)", statsA.Asleep)

+ }

+ if statsB.Asleep == nil || *statsB.Asleep != 2 {

+ t.Fatalf("box B asleep = %v, want 2 (only its own 2 owned in-scope routes)", statsB.Asleep)

+ }

+ // The whole point: summed across both boxes' own reports, no route is

+ // ever double-counted, even though both registries physically hold all 4

+ // in-scope rows.

+ if got, want := *statsA.Asleep+*statsB.Asleep, 5; got != want {

+ t.Fatalf("fleet-wide total = %d, want %d (5 unique routes, no double count)", got, want)

+ }

}

diff --git a/router.go b/router.go

index 9eaebd9..f37dbea 100644

--- a/router.go

+++ b/router.go

@@ -277,7 +277,7 @@ func runRouter(args []string) {

if statsStore, err := activator.NewHeartbeatStore(context.Background(), hc.Bucket, hc.meshPrefix(), hc.Region); err != nil {

log.Printf("router: mesh stats reporter: %v — this box's capacity snapshot won't publish", err)

} else {

- startStatsReporter(statsStore, mesh.self, reg, act)

+ startStatsReporter(statsStore, mesh.self, reg, act, mesh)

}

act.start()

diff --git a/test/status-contract.test.mjs b/test/status-contract.test.mjs

index 39a1da5..3fd22cc 100644

--- a/test/status-contract.test.mjs

+++ b/test/status-contract.test.mjs

@@ -438,8 +438,8 @@ await test("multi-host fixture merges the local box with a live mesh peer", () =

assert.equal(fleetMH.hosts.boxes.length, 2);

const [local, peer] = fleetMH.hosts.boxes;

assert.ok("always_on" in local, "the local box still reports always_on");

- assert.ok(!("always_on" in peer), "a remote peer never reports always_on (a local-box-only concept)");

- assert.ok(!("region" in peer), "a remote peer never reports region in this fixture");

+ assert.ok(!("always_on" in peer), "a remote peer never reports always_on (a genuinely local-box-only concept)");

+ assert.equal(peer.region, "UK · London", "a peer's own LCHAT_BOX_REGION rides through the merge verbatim");

assert.notEqual(local.label, peer.label, "the two boxes must have distinct labels");

});

diff --git a/test/status-fixtures/fleet-multi-host.json b/test/status-fixtures/fleet-multi-host.json

index 401ec26..22c2521 100644

--- a/test/status-fixtures/fleet-multi-host.json

+++ b/test/status-fixtures/fleet-multi-host.json

@@ -1 +1 @@

-{"generated_at":"2026-07-13T22:39:45Z","instances":{"total":6,"systemd":2,"hibernating":4,"awake":3,"asleep":1},"hosts":{"count":2,"boxes":[{"label":"hypatia-sketching","always_on":2,"awake":2,"asleep":2,"load":"comfortable","region":"EU · Germany"},{"label":"turing-orienteering","awake":1,"asleep":1,"load":"busy"}]},"capacity":{"level":"plenty"},"versions":[{"version":"v100","count":3},{"version":"v101","count":2},{"version":"unknown","count":1}],"services":{"home":true,"push":true,"cloud":true,"connect":false},"replication":{"healthy":true,"lag_seconds":42}}

+{"generated_at":"2026-07-13T22:58:58Z","instances":{"total":6,"systemd":2,"hibernating":4,"awake":3,"asleep":1},"hosts":{"count":2,"boxes":[{"label":"hypatia-sketching","always_on":2,"awake":2,"asleep":2,"load":"comfortable","region":"EU · Germany"},{"label":"turing-orienteering","awake":1,"asleep":1,"load":"busy","region":"UK · London"}]},"capacity":{"level":"plenty"},"versions":[{"version":"v100","count":3},{"version":"v101","count":2},{"version":"unknown","count":1}],"services":{"home":true,"push":true,"cloud":true,"connect":false},"replication":{"healthy":true,"lag_seconds":42}}