amadan

eleven / messenger Public

status: fleet-wide awake/asleep counts, not just the local box's (#849)

Once the mesh actually places instances on a peer, that box's own `ss`
check can never see them, so they silently read as "asleep" in the
top-level instances.awake/asleep — while the per-box hosts.boxes entries
were already each correct. Sum every box's own verified awake count
instead (hib stays the fleet-wide total already, since a peer never
registers instances outside replication from this box).
Co-authored-by: Claude <noreply@anthropic.com>
James Adam a1747a0d6bc9788ba1fd7e9d9fc19c530325401a parent 0247d6c
3 files changed, +56 −11
  • deploy/status-publish.sh +41 −10
  • test/status-contract.test.mjs +14 −0
  • test/status-fixtures/fleet-multi-host.json +1 −1
diff --git a/deploy/status-publish.sh b/deploy/status-publish.sh
index 47ce8eb..bc984da 100755
--- a/deploy/status-publish.sh
+++ b/deploy/status-publish.sh
@@ -537,15 +537,18 @@ build_fleet() {
# ---- hosts: this box, plus every other live mesh member --------------------
# Labels are whimsical name-action pairs derived from a HASH of the machine
# id ("ada-stargazing") — stable per box, and never peer URLs, IPs,
- # hostnames, or the machine id itself. This box's own awake/asleep are the
- # fleet-wide counts (one box = the whole fleet when the mesh is off, exactly
- # as before); a peer's are its OWN self-published snapshot (mesh_peer_boxes
- # above), already banded/counted-only by the time it left that box — never
- # re-derived or second-guessed here. The optional region is the operator-set
- # coarse continent + country word pair (see PRIVACY) — absent env, absent
- # key, never guessed; escaped like the version strings so a quote can't
- # break out of the SQL literal. Peers don't carry always_on/region (both
- # stay local-box-only concepts for now — an honest limitation, not a bug).
+ # hostnames, or the machine id itself. Each box's awake/asleep entry here is
+ # its OWN locally-verified split — this box's is the `ss` check above, a
+ # peer's is its OWN self-published snapshot (mesh_peer_boxes above), already
+ # banded/counted-only by the time it left that box — never re-derived or
+ # second-guessed here. (Below, the top-level `instances.awake`/`asleep`
+ # fields are summed across every box in this array — a per-box figure here
+ # is deliberately this box's own view, not the fleet total.) The optional
+ # region is the operator-set coarse continent + country word pair (see
+ # PRIVACY) — absent env, absent key, never guessed; escaped like the version
+ # strings so a quote can't break out of the SQL literal. Peers don't carry
+ # always_on/region (both stay local-box-only concepts for now — an honest
+ # limitation, not a bug).
local boxes peer_boxes region_kv=""
# Resolved here, not at file top: DRY_RUN/argv must be parsed first so the
# dry-run's no-live-probes guard actually guards (and the dry-run fixture's
@@ -569,6 +572,28 @@ build_fleet() {
local host_count
host_count=$(sqj "SELECT json_array_length('$(printf '%s' "$boxes" | sed "s/'/''/g")')")
+ # ---- fleet-wide awake/asleep: sum every box's own verified count ----------
+ # `hib` above is already fleet-wide (this box's registry is the replicated
+ # superset — mesh-seed/fan-out only ever ADD rows to a peer, never remove
+ # them here), but `awake_json`/`asleep_json` up to this point are still only
+ # this box's own `ss` check — a host actually running on a peer has no local
+ # socket, so it silently read as "asleep" here even while genuinely awake.
+ # `boxes` now holds every live box's own verified split, so summing across
+ # it turns the local view into the real fleet-wide one. If any box's awake
+ # is unknown (null — e.g. `ss` missing there), the fleet split stays null
+ # too, rather than guessing: same "unknowns never guess" rule as the rest of
+ # this file.
+ local boxes_esc unknown_awake
+ boxes_esc=$(printf '%s' "$boxes" | sed "s/'/''/g")
+ unknown_awake=$(sqj "SELECT COUNT(*) FROM json_each('$boxes_esc') WHERE json_extract(value, '\$.awake') IS NULL")
+ if [ "$unknown_awake" = 0 ]; then
+ awake_json=$(sqj "SELECT SUM(json_extract(value, '\$.awake')) FROM json_each('$boxes_esc')")
+ asleep_json=$((hib - awake_json))
+ else
+ awake_json=null
+ asleep_json=null
+ fi
+
# Assembled by sqlite json_object: the only interpolated values are
# script-controlled numerals/booleans/band-words and the sqlite-built arrays.
sqj "SELECT json_object(
@@ -1297,8 +1322,14 @@ EOF
# (unset) keeps hosts.count at 1, byte-identical to the single-box shape —
# the mesh's own "absence changes nothing" convention, mirrored here.
if [ "${STATUS_DRY_MESH_PEER:-}" = 1 ]; then
+ # awake=1 here, not some arbitrary number: this peer only ever runs
+ # instances mesh-replicated FROM this box's own registry, so its awake
+ # count must fit within this box's own currently-"asleep" locally instances
+ # (echo + foxtrot, 2 of them) — otherwise the fleet-wide awake/asleep sum
+ # below goes negative, which would mean the fixture models an impossible
+ # state, not a real one.
cat > "$FIXDIR/mesh-peers.json" <<'EOF'
-[{"label":"turing-orienteering","awake":3,"asleep":1,"load":"busy"}]
+[{"label":"turing-orienteering","awake":1,"asleep":1,"load":"busy"}]
EOF
fi
diff --git a/test/status-contract.test.mjs b/test/status-contract.test.mjs
index f6b3a29..39a1da5 100644
--- a/test/status-contract.test.mjs
+++ b/test/status-contract.test.mjs
@@ -443,6 +443,20 @@ await test("multi-host fixture merges the local box with a live mesh peer", () =
assert.notEqual(local.label, peer.label, "the two boxes must have distinct labels");
});
+// A host genuinely running on a mesh peer has no local socket on this box, so
+// the top-level instances.awake/asleep must sum every box's own verified
+// split, not just this box's — otherwise a peer-hosted instance reads as
+// "asleep" fleet-wide even while it's actually awake there.
+await test("top-level awake/asleep are fleet-wide, not just the local box", () => {
+ const [local, peer] = fleetMH.hosts.boxes;
+ assert.equal(fleetMH.instances.awake, local.awake + peer.awake,
+ "instances.awake must sum every box's own awake count");
+ assert.equal(fleetMH.instances.asleep, fleetMH.instances.hibernating - fleetMH.instances.awake,
+ "instances.asleep must be hibernating minus the fleet-wide awake total");
+ assert.notEqual(fleetMH.instances.awake, local.awake,
+ "a regression would silently drop back to only the local box's awake count");
+});
+
// Each committed fixture set derives its band — pins the 70/90 thresholds and
// the disk-floor rule in the generator (see dry_run's capacity fixtures).
await test("capacity fixtures derive plenty / getting-full / at-capacity", () => {
diff --git a/test/status-fixtures/fleet-multi-host.json b/test/status-fixtures/fleet-multi-host.json
index ff1ff87..401ec26 100644
--- a/test/status-fixtures/fleet-multi-host.json
+++ b/test/status-fixtures/fleet-multi-host.json
@@ -1 +1 @@
-{"generated_at":"2026-07-13T13:37:03Z","instances":{"total":6,"systemd":2,"hibernating":4,"awake":2,"asleep":2},"hosts":{"count":2,"boxes":[{"label":"hypatia-sketching","always_on":2,"awake":2,"asleep":2,"load":"comfortable","region":"EU · Germany"},{"label":"turing-orienteering","awake":3,"asleep":1,"load":"busy"}]},"capacity":{"level":"plenty"},"versions":[{"version":"v100","count":3},{"version":"v101","count":2},{"version":"unknown","count":1}],"services":{"home":true,"push":true,"cloud":true,"connect":false},"replication":{"healthy":true,"lag_seconds":42}}
+{"generated_at":"2026-07-13T22:39:45Z","instances":{"total":6,"systemd":2,"hibernating":4,"awake":3,"asleep":1},"hosts":{"count":2,"boxes":[{"label":"hypatia-sketching","always_on":2,"awake":2,"asleep":2,"load":"comfortable","region":"EU · Germany"},{"label":"turing-orienteering","awake":1,"asleep":1,"load":"busy"}]},"capacity":{"level":"plenty"},"versions":[{"version":"v100","count":3},{"version":"v101","count":2},{"version":"unknown","count":1}],"services":{"home":true,"push":true,"cloud":true,"connect":false},"replication":{"healthy":true,"lag_seconds":42}}