amadan
server: multi-box Fleet status aggregation via the mesh's stats channel (#822)
fleet.json's hosts.boxes was hardcoded to the local box alone. Extends
the mesh's existing S3 substrate (never a new box-to-box port, per
mesh.go's own membership design) with a stats side-channel: each mesh
member self-publishes its own {label, awake, asleep, load} snapshot —
band words and counts only, matching status.md's disclosure rule word
for word, the underlying memory ratio dying in the same process that
read it — as a sibling S3 object next to its membership heartbeat
(internal/activator/heartbeat.go's PutStats, meshstats.go's reporter).
deploy/status-publish.sh's build_fleet reads every live peer's
snapshot (mesh_peer_boxes, via the AWS CLI — status-publish.sh already
shells out to it) and merges them into hosts.boxes alongside its own
already-local computation, with hosts.count reflecting both. Absent
mesh config keeps today's single-box output byte-identical (verified:
the existing fleet/fleet-getting-full/fleet-at-capacity fixtures
regenerate identically but for the timestamp).
The whimsical box-label algorithm is ported into Go (meshstats.go),
byte-for-byte matching status-publish.sh's box_label — cross-checked
against a real bash-computed example (TestBoxLabelMatchesBash) since
neither side can compute the OTHER box's label (it hashes a machine id
only that box has), so each box must self-report its own.
infra/prod/status.tf grants the status-publisher IAM user read-only
access to the hibernation bucket's "*-stats/" prefix only — never the
actual instance replicas next to it. NOT YET APPLIED (needs a human
`tofu apply` in infra/prod); until then mesh_peer_boxes finds nothing
to read and hosts stays single-box, same as today.
Co-authored-by: Claude <noreply@anthropic.com>
10 files changed,
+568
−20
activator.go+18 −0deploy/status-publish.sh+104 −14infra/prod/status.tf+26 −0internal/activator/heartbeat.go+44 −0internal/activator/heartbeat_test.go+25 −1meshstats.go+186 −0meshstats_test.go+126 −0router.go+10 −0test/status-contract.test.mjs+28 −5test/status-fixtures/fleet-multi-host.json+1 −0
diff --git a/activator.go b/activator.go| index 7883039..13f4411 100644 |
| --- a/activator.go |
| +++ b/activator.go |
| @@ -450,6 +450,24 @@ func (a *activatorSvc) hasLiveEntry(host string) bool { |
| return ok |
| } |
| +// awakeCount is this box's own count of genuinely serving hibernating |
| +// children — actAwake only, not actWaking/actStopping transients — for the |
| +// Fleet status page's per-box awake/asleep split (meshstats.go). Unlike |
| +// deploy/status-publish.sh's own `ss`-probing of the LOCAL box (an external |
| +// process with no other view into the router), this is the activator's |
| +// authoritative in-memory state — no socket probing needed. |
| +func (a *activatorSvc) awakeCount() int { |
| + a.mu.Lock() |
| + defer a.mu.Unlock() |
| + n := 0 |
| + for _, e := range a.entries { |
| + if e.state == actAwake { |
| + n++ |
| + } |
| + } |
| + return n |
| +} |
| + |
| func (a *activatorSvc) releaseFunc(e *actEntry) func() { |
| var once sync.Once |
| return func() { |
diff --git a/deploy/status-publish.sh b/deploy/status-publish.sh| index f3ef584..a0afb74 100755 |
| --- a/deploy/status-publish.sh |
| +++ b/deploy/status-publish.sh |
| @@ -121,6 +121,17 @@ CLOUD_ENV=${STATUS_CLOUD_ENV:-/data/cloud.env} |
| # root-owned/unreadable to the lchat user — then the code defaults below apply, |
| # which are exactly what controld itself defaults to when the vars are unset. |
| CONTROL_ENV=${STATUS_CONTROL_ENV:-/data/control.env} |
| +# The router's own env — read (same lchat user, same file the router loads) |
| +# only for the mesh's bucket/prefix, so this box can find its live peers' |
| +# self-published stats (mesh_peer_boxes below). Absent LCHAT_HIBERNATE_BUCKET/ |
| +# _PREFIX here just means the mesh isn't configured on this box — hosts stays |
| +# a single-box array, byte-identical to before multi-box existed. |
| +ROUTER_ENV=${STATUS_ROUTER_ENV:-/data/router.env} |
| +# Staleness cutoff for a peer's self-published stats object — three missed |
| +# beats, mirroring mesh.go's own meshStaleAfter convention, at the STATS |
| +# cadence (meshstats.go's meshStatsInterval, 60s) rather than the membership |
| +# heartbeat's faster one. |
| +MESH_STATS_STALE_SECONDS=${STATUS_MESH_STATS_STALE_SECONDS:-180} |
| # Box-load + disk inputs (STATUS_ overrides exist for testability; the dry-run |
| # uses fixture twins instead and never reads live box state). |
| MEMINFO=${STATUS_MEMINFO:-/proc/meminfo} |
| @@ -266,6 +277,12 @@ env_int() { |
| case "$v" in ''|*[!0-9]*) echo "$3" ;; *) echo "$v" ;; esac |
| } |
| +# env_str <file> <VAR> — like env_int but for string values (bucket names, |
| +# prefixes): no numeric coercion, empty when the file or var is absent. |
| +env_str() { |
| + grep -E "^$2=" "$1" 2>/dev/null | head -n1 | cut -d= -f2- || true |
| +} |
| + |
| # disk_free_gb — whole GB free on the data disk (`df -Pk`: POSIX, and avail |
| # matches controld's Statfs Bavail). Empty on failure — an unknown contributes |
| # nothing to the band, never a guess. |
| @@ -323,6 +340,60 @@ box_label() { |
| printf '%s-%s' "${names[$n]}" "${actions[$a]}" |
| } |
| +# mesh_peer_boxes prints a JSON array of every OTHER live mesh box's |
| +# self-published stats object — {label, awake, asleep, load} — read from the |
| +# hibernation bucket's stats side-channel (internal/activator/heartbeat.go's |
| +# PutStats, a sibling object next to the membership heartbeat every mesh |
| +# member already writes; meshstats.go computes each box's own snapshot on its |
| +# own slower cadence). Deliberately NOT a new box-to-box port: this rides the |
| +# same S3 substrate the mesh already uses for membership, exactly the seam |
| +# mesh.go's own doc comment anticipates ("a future AWS pool" that "may not |
| +# even share a network"). Empty array — not an error — whenever this box |
| +# isn't mesh-configured, has no live peers, or any AWS call fails; a missing |
| +# peer here just means the Fleet page undercounts by one box, never a |
| +# publish failure. |
| +mesh_peer_boxes() { |
| + if [ -n "$DRY_RUN" ]; then |
| + if [ -f "$FIXDIR/mesh-peers.json" ]; then cat "$FIXDIR/mesh-peers.json"; else echo '[]'; fi |
| + return |
| + fi |
| + |
| + local bucket region prefix |
| + bucket=$(env_str "$ROUTER_ENV" LCHAT_HIBERNATE_BUCKET) |
| + prefix=$(env_str "$ROUTER_ENV" LCHAT_HIBERNATE_PREFIX) |
| + if [ -z "$bucket" ] || [ -z "$prefix" ]; then echo '[]'; return; fi |
| + region=$(env_str "$ROUTER_ENV" LCHAT_HIBERNATE_REGION) |
| + region=${region:-eu-west-1} |
| + |
| + local listf="$TMP/mesh-stats-objects.json" |
| + if ! aws s3api list-objects-v2 --bucket "$bucket" --prefix "${prefix}-stats/" \ |
| + --region "$region" --output json > "$listf" 2>/dev/null; then |
| + echo '[]'; return |
| + fi |
| + |
| + # ISO8601 timestamps sort lexicographically, so a plain string compare |
| + # against the cutoff is exact — no date-library parsing needed, and a box |
| + # whose clock is off only ever misjudges its OWN staleness, same tradeoff |
| + # mesh.go's own reader-vs-S3-clock note accepts for membership. |
| + local cutoff peers="[]" key lastmod tmp body |
| + cutoff=$(date -u -d "-${MESH_STATS_STALE_SECONDS} seconds" +%Y-%m-%dT%H:%M:%S) |
| + |
| + while IFS=$'\t' read -r key lastmod; do |
| + [ -n "$key" ] || continue |
| + [ "$lastmod" '>' "$cutoff" ] || continue # stale — treat like a dead mesh member |
| + tmp="$TMP/mesh-stat-$(printf '%s' "$key" | tr '/' '_').json" |
| + aws s3 cp "s3://$bucket/$key" "$tmp" --region "$region" >/dev/null 2>&1 || continue |
| + body=$(cat "$tmp" 2>/dev/null) || continue |
| + [ -n "$body" ] || continue |
| + peers=$(sqj "SELECT json_insert( |
| + json('$(printf '%s' "$peers" | sed "s/'/''/g")'), '\$[#]', |
| + json('$(printf '%s' "$body" | sed "s/'/''/g")'))") |
| + done < <(sqj "SELECT json_extract(value, '\$.Key') || char(9) || json_extract(value, '\$.LastModified') |
| + FROM json_each(json_extract(readfile('$listf'), '\$.Contents'))" 2>/dev/null) |
| + |
| + printf '%s' "$peers" |
| +} |
| + |
| build_fleet() { |
| local out=$1 now total systemd_n hib awake_json asleep_json versions |
| now=$(now_iso) |
| @@ -421,19 +492,19 @@ build_fleet() { |
| local capacity_level |
| capacity_level=$(band_word "$band") |
| - # ---- hosts: v1 is the local box alone -------------------------------------- |
| - # Mesh membership isn't enabled on this box, and the status-publisher's S3 |
| - # key is scoped to the status bucket, so it couldn't list mesh heartbeats |
| - # anyway — multi-box joins here when the mesh goes live (it will need a read |
| - # grant on the mesh prefix). Labels are whimsical name-action pairs derived |
| - # from a HASH of the machine id ("ada-stargazing") — stable per box, and |
| - # never peer URLs, IPs, hostnames, or the machine id itself. Per-box |
| - # awake/asleep are the fleet-wide counts (one box = the whole fleet), |
| - # already public above. The optional region is the operator-set coarse |
| - # continent + country word pair (see PRIVACY) — absent env, absent key, |
| - # never guessed; escaped like the version strings so a quote can't break |
| - # out of the SQL literal. |
| - local boxes region_kv="" |
| + # ---- hosts: this box, plus every other live mesh member -------------------- |
| + # Labels are whimsical name-action pairs derived from a HASH of the machine |
| + # id ("ada-stargazing") — stable per box, and never peer URLs, IPs, |
| + # hostnames, or the machine id itself. This box's own awake/asleep are the |
| + # fleet-wide counts (one box = the whole fleet when the mesh is off, exactly |
| + # as before); a peer's are its OWN self-published snapshot (mesh_peer_boxes |
| + # above), already banded/counted-only by the time it left that box — never |
| + # re-derived or second-guessed here. The optional region is the operator-set |
| + # coarse continent + country word pair (see PRIVACY) — absent env, absent |
| + # key, never guessed; escaped like the version strings so a quote can't |
| + # break out of the SQL literal. Peers don't carry always_on/region (both |
| + # stay local-box-only concepts for now — an honest limitation, not a bug). |
| + local boxes peer_boxes region_kv="" |
| # Resolved here, not at file top: DRY_RUN/argv must be parsed first so the |
| # dry-run's no-live-probes guard actually guards (and the dry-run fixture's |
| # direct BOX_REGION= assignment still wins over a re-resolve). |
| @@ -448,6 +519,14 @@ build_fleet() { |
| 'asleep', json('$asleep_json'), |
| 'load', json('$load_json')$region_kv))") |
| + peer_boxes=$(mesh_peer_boxes) |
| + boxes=$(sqj "SELECT json_group_array(json(value)) FROM ( |
| + SELECT value FROM json_each('$(printf '%s' "$boxes" | sed "s/'/''/g")') |
| + UNION ALL |
| + SELECT value FROM json_each('$(printf '%s' "$peer_boxes" | sed "s/'/''/g")'))") |
| + local host_count |
| + host_count=$(sqj "SELECT json_array_length('$(printf '%s' "$boxes" | sed "s/'/''/g")')") |
| + |
| # Assembled by sqlite json_object: the only interpolated values are |
| # script-controlled numerals/booleans/band-words and the sqlite-built arrays. |
| sqj "SELECT json_object( |
| @@ -459,7 +538,7 @@ build_fleet() { |
| 'awake', json('$awake_json'), |
| 'asleep', json('$asleep_json')), |
| 'hosts', json_object( |
| - 'count', 1, |
| + 'count', $host_count, |
| 'boxes', json('$boxes')), |
| 'capacity', json_object( |
| 'level', '$capacity_level'), |
| @@ -1170,6 +1249,17 @@ EOF |
| BOX_REGION="EU · Germany" |
| echo 42 > "$FIXDIR/replication-lag.txt" |
| + # Multi-host fixture (STATUS_DRY_MESH_PEER=1): one other live mesh box's |
| + # self-published stats snapshot, exactly the shape internal/activator/ |
| + # heartbeat.go's PutStats writes and mesh_peer_boxes reads back. Default |
| + # (unset) keeps hosts.count at 1, byte-identical to the single-box shape — |
| + # the mesh's own "absence changes nothing" convention, mirrored here. |
| + if [ "${STATUS_DRY_MESH_PEER:-}" = 1 ]; then |
| + cat > "$FIXDIR/mesh-peers.json" <<'EOF' |
| +[{"label":"turing-orienteering","awake":3,"asleep":1,"load":"busy"}] |
| +EOF |
| + fi |
| + |
| # Capacity fixtures (control caps + disk + meminfo): one set per public band |
| # (default plenty; STATUS_DRY_CAPACITY=getting-full|at-capacity), so the |
| # abstract derivation is testable end-to-end at every level without touching |
diff --git a/infra/prod/status.tf b/infra/prod/status.tf| index a445b44..6f82c80 100644 |
| --- a/infra/prod/status.tf |
| +++ b/infra/prod/status.tf |
| @@ -107,6 +107,32 @@ data "aws_iam_policy_document" "status_publisher" { |
| resources = ["${aws_s3_bucket.status.arn}/*"] |
| } |
| + # Read-only access to the mesh's stats side-channel in the hibernation |
| + # bucket (hibernate.tf's prod_instances) — deploy/status-publish.sh's |
| + # mesh_peer_boxes reads every live mesh peer's self-published capacity |
| + # snapshot (band words + counts only — never raw ratios, meshstats.go) to |
| + # build fleet.json's multi-host hosts.boxes array. Scoped to the "*-stats/" |
| + # sibling prefix internal/activator/heartbeat.go's PutStats writes to, not |
| + # the whole bucket: the publisher must never be able to read or restore |
| + # actual (ciphertext) instance replicas, only the small stats objects next |
| + # to them. No s3:PutObject/DeleteObject here — this credential only reads. |
| + statement { |
| + sid = "ListMeshStatsPrefix" |
| + actions = ["s3:ListBucket"] |
| + resources = [aws_s3_bucket.prod_instances.arn] |
| + condition { |
| + test = "StringLike" |
| + variable = "s3:prefix" |
| + values = ["*-stats/*"] |
| + } |
| + } |
| + |
| + statement { |
| + sid = "ReadMeshStatsObjects" |
| + actions = ["s3:GetObject"] |
| + resources = ["${aws_s3_bucket.prod_instances.arn}/*-stats/*"] |
| + } |
| + |
| # The books page's expenses section: the publisher asks Cost Explorer for |
| # the account's month-to-date spend (deploy/status-publish.sh, cached — CE |
| # bills per request). ce:GetCostAndUsage has no resource-level scoping, so |
diff --git a/internal/activator/heartbeat.go b/internal/activator/heartbeat.go| index e433d9c..b609276 100644 |
| --- a/internal/activator/heartbeat.go |
| +++ b/internal/activator/heartbeat.go |
| @@ -82,6 +82,50 @@ func (s *HeartbeatStore) key(peer string) string { |
| return path.Join(s.prefix, encodeHeartbeatKey(peer)) |
| } |
| +// statsKey is a SIBLING object next to the membership heartbeat, under a |
| +// separate "<prefix>-stats/" path — never mixed into the heartbeat body |
| +// itself, so the hot membership path (List, polled every meshHeartbeatInterval |
| +// for HRW placement) never pays to write or parse a body it doesn't need. |
| +// Stats are read far less often (deploy/status-publish.sh's 5-minute Fleet |
| +// publish), by a different reader (bash, via the AWS CLI, not this package). |
| +func (s *HeartbeatStore) statsKey(peer string) string { |
| + return path.Join(s.prefix+"-stats", encodeHeartbeatKey(peer)) |
| +} |
| + |
| +// BoxStats is one box's self-published capacity snapshot: band words and |
| +// counts only, matching the Fleet dashboard's public disclosure rule |
| +// (status.md — "never remaining slot counts... percentages, or cap values"). |
| +// Awake/Asleep are nil when unknown (never guessed, same convention as |
| +// fleet.json's own awake/asleep). Load is a band word (comfortable/busy/ |
| +// critical) or "" when unknown — never a raw memory ratio. |
| +type BoxStats struct { |
| + // Label is this box's own whimsical, hash-derived name ("ada-stargazing", |
| + // meshstats.go's boxLabel — the exact same algorithm and word lists as |
| + // deploy/status-publish.sh's box_label, kept in sync by hand: neither |
| + // side can compute the OTHER box's label, since it hashes a machine id |
| + // only that box has, so each box must self-report the label, never the |
| + // id it was derived from. |
| + Label string `json:"label"` |
| + Awake *int `json:"awake"` |
| + Asleep *int `json:"asleep"` |
| + Load string `json:"load,omitempty"` |
| +} |
| + |
| +// PutStats writes this box's stats snapshot. Called on its own slower cadence |
| +// (meshstats.go), independent of the membership beat. |
| +func (s *HeartbeatStore) PutStats(ctx context.Context, peer string, stats BoxStats) error { |
| + body, err := json.Marshal(stats) |
| + if err != nil { |
| + return err |
| + } |
| + _, err = s.client.PutObject(ctx, &awss3.PutObjectInput{ |
| + Bucket: aws.String(s.bucket), |
| + Key: aws.String(s.statsKey(peer)), |
| + Body: bytes.NewReader(body), |
| + }) |
| + return err |
| +} |
| + |
| // Put writes (or refreshes) this box's heartbeat. The write itself is the |
| // timestamp — LastModified advances on every beat. |
| func (s *HeartbeatStore) Put(ctx context.Context, peer string) error { |
diff --git a/internal/activator/heartbeat_test.go b/internal/activator/heartbeat_test.go| index 25ff927..72ef1a2 100644 |
| --- a/internal/activator/heartbeat_test.go |
| +++ b/internal/activator/heartbeat_test.go |
| @@ -1,6 +1,9 @@ |
| package activator |
| -import "testing" |
| +import ( |
| + "strings" |
| + "testing" |
| +) |
| // The key IS the membership record (one LIST, no GETs), so the encoding must |
| // round-trip any peer URL and reject foreign objects. |
| @@ -21,3 +24,24 @@ func TestHeartbeatKeyRoundTrip(t *testing.T) { |
| t.Fatal("garbage key leaf decoded without error") |
| } |
| } |
| + |
| +// statsKey must live under a SIBLING prefix, never collide with (or nest |
| +// inside) the membership heartbeat's own key — the hot membership List path |
| +// must never accidentally pick up a stats object. |
| +func TestStatsKeySibling(t *testing.T) { |
| + s := &HeartbeatStore{prefix: "prod"} |
| + peer := "https://box-b.internal" |
| + |
| + hbKey := s.key(peer) |
| + statsKey := s.statsKey(peer) |
| + |
| + if statsKey == hbKey { |
| + t.Fatal("stats key must differ from the membership heartbeat key") |
| + } |
| + if strings.HasPrefix(statsKey, s.prefix+"/") { |
| + t.Fatalf("stats key %q must not live under the plain membership prefix %q", statsKey, s.prefix) |
| + } |
| + if !strings.HasPrefix(statsKey, s.prefix+"-stats/") { |
| + t.Fatalf("stats key %q should live under %q", statsKey, s.prefix+"-stats/") |
| + } |
| +} |
diff --git a/meshstats.go b/meshstats.go| new file mode 100644 |
| index 0000000..359181a |
| --- /dev/null |
| +++ b/meshstats.go |
| @@ -0,0 +1,186 @@ |
| +package main |
| + |
| +// Per-box capacity stats for the public Fleet status page's multi-host |
| +// aggregation (deploy/status-publish.sh). Each mesh member self-publishes its |
| +// own snapshot (internal/activator/heartbeat.go's PutStats, a sibling object |
| +// next to the membership heartbeat) on its own slower cadence; box A's |
| +// status-publish run reads every live peer's snapshot via the AWS CLI to |
| +// build fleet.json's `hosts.boxes` array, alongside its own already-local |
| +// computation. |
| +// |
| +// Deliberately narrow, matching status.md's disclosure rule word-for-word: |
| +// only awake/asleep counts and a banded load WORD ever leave this box — the |
| +// underlying memory ratio dies right here, in the same process that read it, |
| +// never serialized anywhere even over an authenticated internal channel. |
| + |
| +import ( |
| + "bufio" |
| + "context" |
| + "crypto/sha256" |
| + "encoding/hex" |
| + "log" |
| + "os" |
| + "strconv" |
| + "strings" |
| + "time" |
| + |
| + "lchat/internal/activator" |
| +) |
| + |
| +// meshStatsInterval paces the stats PutStats — far slower than the |
| +// membership heartbeat (meshHeartbeatInterval, 15s): capacity changes slowly, |
| +// and this is visibility, not coordination. |
| +const meshStatsInterval = 60 * time.Second |
| + |
| +// pctBand mirrors deploy/status-publish.sh's pct_band: 0 below 70%, 1 at |
| +// 70-90%, 2 above 90%. Both sides must band identically, or the same |
| +// utilization could publish a different word depending which box you asked. |
| +func pctBand(pct int) int { |
| + switch { |
| + case pct < 70: |
| + return 0 |
| + case pct <= 90: |
| + return 1 |
| + default: |
| + return 2 |
| + } |
| +} |
| + |
| +// loadWord mirrors status-publish.sh's load_word. |
| +func loadWord(band int) string { |
| + switch band { |
| + case 0: |
| + return "comfortable" |
| + case 1: |
| + return "busy" |
| + default: |
| + return "critical" |
| + } |
| +} |
| + |
| +// meminfoPath is a seam for tests — production always reads the real |
| +// /proc/meminfo, matching status-publish.sh's own $MEMINFO override convention. |
| +var meminfoPath = "/proc/meminfo" |
| + |
| +// memLoadWord reads /proc/meminfo and bands (MemTotal-MemAvailable)/MemTotal, |
| +// the same formula and thresholds as status-publish.sh's meminfo_kb/pct_band. |
| +// ok=false when unreadable (a non-Linux dev machine, a stripped-down |
| +// container) — unknowns publish as absent, never a guessed word. |
| +func memLoadWord() (word string, ok bool) { |
| + f, err := os.Open(meminfoPath) |
| + if err != nil { |
| + return "", false |
| + } |
| + defer f.Close() |
| + |
| + var total, avail int64 |
| + sc := bufio.NewScanner(f) |
| + for sc.Scan() { |
| + fields := strings.Fields(sc.Text()) |
| + if len(fields) < 2 { |
| + continue |
| + } |
| + switch strings.TrimSuffix(fields[0], ":") { |
| + case "MemTotal": |
| + total, _ = strconv.ParseInt(fields[1], 10, 64) |
| + case "MemAvailable": |
| + avail, _ = strconv.ParseInt(fields[1], 10, 64) |
| + } |
| + } |
| + if total <= 0 { |
| + return "", false |
| + } |
| + pct := int((total - avail) * 100 / total) |
| + return loadWord(pctBand(pct)), true |
| +} |
| + |
| +// boxLabelNames/boxLabelActions are IDENTICAL, in the SAME order, to |
| +// deploy/status-publish.sh's box_label names[]/actions[] arrays — the same |
| +// hash must produce the same whimsical label whichever side computes it. |
| +// Keep the two lists in sync by hand; TestBoxLabelMatchesBash pins one |
| +// cross-checked example against the real bash script's own output. |
| +var boxLabelNames = []string{ |
| + "ada", "alan", "grace", "marie", "nikola", "leonardo", "cleopatra", "homer", "sappho", |
| + "darwin", "einstein", "curie", "turing", "hopper", "tesla", "gandalf", "frodo", "sherlock", "watson", |
| + "ahab", "ishmael", "gatsby", "scout", "atticus", "hermione", "matilda", "mowgli", "merlin", "arthur", |
| + "morgana", "beowulf", "odysseus", "penelope", "achilles", "athena", "apollo", "freya", "thor", "loki", |
| + "amelia", "boudicca", "cyrano", "quixote", "sancho", "ophelia", "viola", "rosalind", "puck", "oberon", |
| + "titania", "darcy", "pip", "scrooge", "nemo", "phileas", "zorro", "heidi", "galileo", "kepler", "newton", |
| + "archimedes", "hypatia", "confucius", "rumi", "frida", "vincent", "wolfgang", "ludwig", "clara", |
| +} |
| + |
| +var boxLabelActions = []string{ |
| + "stargazing", "juggling", "whistling", "painting", "sailing", "gardening", |
| + "baking", "skating", "humming", "doodling", "fishing", "knitting", "strolling", "yodeling", |
| + "beekeeping", "fencing", "waltzing", "sketching", "brewing", "climbing", "cartwheeling", |
| + "birdwatching", "pondering", "inventing", "exploring", "composing", "conducting", |
| + "spelunking", "kayaking", "orienteering", "puppeteering", "tinkering", |
| +} |
| + |
| +// machineIDPath is a seam for tests, matching meminfoPath's convention. |
| +var machineIDPath = "/etc/machine-id" |
| + |
| +// boxLabel derives this box's stable, whimsical, anonymous name — the exact |
| +// port of status-publish.sh's box_label (sha256 of /etc/machine-id, first 16 |
| +// hex chars split into two 8-char slices indexing names/actions). Falls back |
| +// to the `hostname` command's output if the machine-id file is unreadable, |
| +// matching bash's `cat "$idfile" 2>/dev/null || hostname` byte-for-byte: a |
| +// machine-id file carries no trailing newline, but the `hostname` command's |
| +// output does, so only the fallback path appends one. |
| +func boxLabel() string { |
| + var seed []byte |
| + if data, err := os.ReadFile(machineIDPath); err == nil { |
| + seed = data |
| + } else { |
| + hn, _ := os.Hostname() |
| + seed = []byte(hn + "\n") |
| + } |
| + sum := sha256.Sum256(seed) |
| + hexSum := hex.EncodeToString(sum[:])[:16] |
| + n, _ := strconv.ParseUint(hexSum[:8], 16, 32) |
| + a, _ := strconv.ParseUint(hexSum[8:16], 16, 32) |
| + return boxLabelNames[int(n)%len(boxLabelNames)] + "-" + boxLabelActions[int(a)%len(boxLabelActions)] |
| +} |
| + |
| +// computeBoxStats builds this box's publishable snapshot: asleep is derived |
| +// (every hibernating route not currently awake), never separately probed, so |
| +// the two counts can never disagree with each other. |
| +func computeBoxStats(reg *Registry, act *activatorSvc) activator.BoxStats { |
| + routes, err := reg.list() |
| + if err != nil { |
| + log.Printf("mesh: stats: list routes: %v", err) |
| + return activator.BoxStats{} |
| + } |
| + hib := 0 |
| + for _, r := range routes { |
| + if r.Kind == "instance" && r.Hibernate { |
| + hib++ |
| + } |
| + } |
| + awake := act.awakeCount() |
| + asleep := hib - awake |
| + load, _ := memLoadWord() // "" (omitted) when unknown |
| + |
| + return activator.BoxStats{Label: boxLabel(), Awake: &awake, Asleep: &asleep, Load: load} |
| +} |
| + |
| +// startStatsReporter runs alongside the mesh (wired only when mesh != nil, |
| +// router.go): computes and PutStats this box's snapshot on meshStatsInterval, |
| +// for as long as the process runs — like the router's other background |
| +// loops (e.g. its registry-watcher poll), it has no graceful stop; the |
| +// process exiting is what stops it. |
| +func startStatsReporter(store *activator.HeartbeatStore, self string, reg *Registry, act *activatorSvc) { |
| + report := func() { |
| + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
| + defer cancel() |
| + if err := store.PutStats(ctx, self, computeBoxStats(reg, act)); err != nil { |
| + log.Printf("mesh: stats: publish failed: %v", err) |
| + } |
| + } |
| + report() // one synchronous publish so a box's very first status read isn't stale |
| + go func() { |
| + for range time.Tick(meshStatsInterval) { |
| + report() |
| + } |
| + }() |
| +} |
diff --git a/meshstats_test.go b/meshstats_test.go| new file mode 100644 |
| index 0000000..2e95d82 |
| --- /dev/null |
| +++ b/meshstats_test.go |
| @@ -0,0 +1,126 @@ |
| +package main |
| + |
| +// Tests for meshstats.go: the band thresholds must match |
| +// deploy/status-publish.sh's pct_band/load_word bit-for-bit (both sides |
| +// compute the same word for the same box, whichever process does it), the |
| +// whimsical label algorithm must match status-publish.sh's box_label |
| +// bit-for-bit too (TestBoxLabelMatchesBash pins a real bash-computed |
| +// example), and computeBoxStats must derive asleep from awake (never probe |
| +// them separately, so the two counts can't disagree). |
| + |
| +import ( |
| + "os" |
| + "path/filepath" |
| + "strconv" |
| + "testing" |
| +) |
| + |
| +func TestPctBandThresholds(t *testing.T) { |
| + cases := []struct { |
| + pct int |
| + want int |
| + }{ |
| + {0, 0}, {69, 0}, {70, 1}, {90, 1}, {91, 2}, {100, 2}, |
| + } |
| + for _, c := range cases { |
| + if got := pctBand(c.pct); got != c.want { |
| + t.Errorf("pctBand(%d) = %d, want %d", c.pct, got, c.want) |
| + } |
| + } |
| +} |
| + |
| +func TestLoadWord(t *testing.T) { |
| + if loadWord(0) != "comfortable" || loadWord(1) != "busy" || loadWord(2) != "critical" { |
| + t.Fatal("loadWord vocabulary must match status-publish.sh's load_word exactly") |
| + } |
| +} |
| + |
| +func TestMemLoadWord(t *testing.T) { |
| + dir := t.TempDir() |
| + write := func(total, avail int64) { |
| + content := "" |
| + if total > 0 { |
| + content += "MemTotal: " + strconv.FormatInt(total, 10) + " kB\n" |
| + } |
| + if avail > 0 { |
| + content += "MemAvailable: " + strconv.FormatInt(avail, 10) + " kB\n" |
| + } |
| + if err := os.WriteFile(filepath.Join(dir, "meminfo"), []byte(content), 0o644); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + orig := meminfoPath |
| + meminfoPath = filepath.Join(dir, "meminfo") |
| + defer func() { meminfoPath = orig }() |
| + |
| + write(1000, 500) // 50% used -> comfortable |
| + if w, ok := memLoadWord(); !ok || w != "comfortable" { |
| + t.Fatalf("50%% used: got %q, %v", w, ok) |
| + } |
| + write(1000, 250) // 75% used -> busy |
| + if w, ok := memLoadWord(); !ok || w != "busy" { |
| + t.Fatalf("75%% used: got %q, %v", w, ok) |
| + } |
| + write(1000, 50) // 95% used -> critical |
| + if w, ok := memLoadWord(); !ok || w != "critical" { |
| + t.Fatalf("95%% used: got %q, %v", w, ok) |
| + } |
| + |
| + meminfoPath = filepath.Join(dir, "does-not-exist") |
| + if _, ok := memLoadWord(); ok { |
| + t.Fatal("unreadable meminfo should report ok=false, never guess") |
| + } |
| +} |
| + |
| +// Cross-checked against the real bash script: running status-publish.sh's own |
| +// box_label() function against the same seed input produces the identical |
| +// "phileas-juggling" — pinning that the two word lists and hash-index math |
| +// haven't drifted apart. |
| +func TestBoxLabelMatchesBash(t *testing.T) { |
| + dir := t.TempDir() |
| + idFile := filepath.Join(dir, "machine-id") |
| + if err := os.WriteFile(idFile, []byte("test-machine-id-fixture-12345"), 0o644); err != nil { |
| + t.Fatal(err) |
| + } |
| + orig := machineIDPath |
| + machineIDPath = idFile |
| + defer func() { machineIDPath = orig }() |
| + |
| + if got, want := boxLabel(), "phileas-juggling"; got != want { |
| + t.Fatalf("boxLabel() = %q, want %q (bash's box_label for the same seed)", got, want) |
| + } |
| +} |
| + |
| +// computeBoxStats must derive asleep = hibernating total - awake, never |
| +// probe them independently, and count only actAwake (not a mid-wake/mid-stop |
| +// transient) as "awake". |
| +func TestComputeBoxStats(t *testing.T) { |
| + reg, err := openRegistry(filepath.Join(t.TempDir(), "registry.db")) |
| + if err != nil { |
| + t.Fatalf("open registry: %v", err) |
| + } |
| + for _, h := range []string{"a.11ch.at", "b.11ch.at", "c.11ch.at"} { |
| + if err := reg.upsert(Route{Host: h, Kind: "instance", Socket: "/tmp/" + h + ".sock", Hibernate: true}); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + // A non-hibernating route must not count toward the hibernating total. |
| + if err := reg.upsert(Route{Host: "systemd.11ch.at", Kind: "instance", Socket: "/tmp/x.sock", Hibernate: false}); err != nil { |
| + t.Fatal(err) |
| + } |
| + |
| + act := newActivator(hibernateConfig{Bucket: "test-bucket"}, &fakeBackend{}, reg) |
| + act.entries["a.11ch.at"] = &actEntry{host: "a.11ch.at", state: actAwake} |
| + act.entries["b.11ch.at"] = &actEntry{host: "b.11ch.at", state: actWaking} // not yet awake |
| + |
| + stats := computeBoxStats(reg, act) |
| + if stats.Label == "" { |
| + t.Fatal("Label must be populated") |
| + } |
| + if stats.Awake == nil || *stats.Awake != 1 { |
| + t.Fatalf("Awake = %v, want 1 (only actAwake counts)", stats.Awake) |
| + } |
| + if stats.Asleep == nil || *stats.Asleep != 2 { |
| + t.Fatalf("Asleep = %v, want 2 (3 hibernating - 1 awake)", stats.Asleep) |
| + } |
| +} |
diff --git a/router.go b/router.go| index 89556b3..ba2404a 100644 |
| --- a/router.go |
| +++ b/router.go |
| @@ -19,6 +19,7 @@ import ( |
| "golang.org/x/crypto/acme/autocert" |
| + "lchat/internal/activator" |
| "lchat/internal/bot" |
| ) |
| @@ -259,6 +260,15 @@ func runRouter(args []string) { |
| mesh.start() |
| log.Printf("router: activator mesh on — this box is %s (heartbeats s3://%s/%s every %s, peers stale after %s, placement scoped to %v)", |
| mesh.self, hc.Bucket, hc.meshPrefix(), mesh.interval, mesh.staleAfter, mesh.domains) |
| + |
| + // This box's own capacity snapshot for the Fleet status page's |
| + // multi-host aggregation (meshstats.go) — a separate S3 client on |
| + // the same store, since meshSvc's is interface-typed for tests. |
| + if statsStore, err := activator.NewHeartbeatStore(context.Background(), hc.Bucket, hc.meshPrefix(), hc.Region); err != nil { |
| + log.Printf("router: mesh stats reporter: %v — this box's capacity snapshot won't publish", err) |
| + } else { |
| + startStatsReporter(statsStore, mesh.self, reg, act) |
| + } |
| } |
| act.start() |
| log.Printf("router: hibernation active (bucket %s, prefix %q, idle %s) — flagged instances wake on demand", |
diff --git a/test/status-contract.test.mjs b/test/status-contract.test.mjs| index 64fdb78..f6b3a29 100644 |
| --- a/test/status-contract.test.mjs |
| +++ b/test/status-contract.test.mjs |
| @@ -40,6 +40,9 @@ const fleet = JSON.parse(read("test/status-fixtures/fleet.json")); |
| // them) — the abstract derivation is pinned at every level. |
| const fleetGF = JSON.parse(read("test/status-fixtures/fleet-getting-full.json")); |
| const fleetAC = JSON.parse(read("test/status-fixtures/fleet-at-capacity.json")); |
| +// STATUS_DRY_MESH_PEER=1 regenerates this one — a second, live mesh peer's |
| +// self-published stats merged into hosts.boxes alongside the local box. |
| +const fleetMH = JSON.parse(read("test/status-fixtures/fleet-multi-host.json")); |
| const books = JSON.parse(read("test/status-fixtures/books.json")); |
| const booksDown = JSON.parse(read("test/status-fixtures/books-unavailable.json")); |
| const booksNoExp = JSON.parse(read("test/status-fixtures/books-no-expenses.json")); |
| @@ -388,7 +391,10 @@ await test("run fixture folds server_type + grid to the asserted watts→kg math |
| // capacity fails, so a precise number (remaining slots, GB, percentages, cap |
| // values) can't ship by accident, and the level/load words themselves must be |
| // digit-free vocabulary entries. |
| -const fleetShapes = [["fleet", fleet], ["fleet-getting-full", fleetGF], ["fleet-at-capacity", fleetAC]]; |
| +const fleetShapes = [ |
| + ["fleet", fleet], ["fleet-getting-full", fleetGF], ["fleet-at-capacity", fleetAC], |
| + ["fleet-multi-host", fleetMH], |
| +]; |
| await test("hosts + capacity carry only the allowed abstract schema", () => { |
| for (const [name, f] of fleetShapes) { |
| assert.deepEqual(Object.keys(f.capacity).sort(), ["level"], name + ": capacity keys"); |
| @@ -400,14 +406,19 @@ await test("hosts + capacity carry only the allowed abstract schema", () => { |
| // region is optional (omitted when the operator sets none). When present |
| // it must be a coarse public area — continent + country — and must never |
| // read like an IP or hostname (no dots-and-digits / label.tld shapes). |
| - const base = ["always_on", "asleep", "awake", "label", "load"]; |
| + // always_on is likewise optional: a REMOTE mesh peer's self-published |
| + // snapshot (mesh_peer_boxes) never carries it — that's a local-box-only |
| + // concept — so a peer's key set is the "required" set on its own, |
| + // while the LOCAL box always adds always_on. |
| + const required = ["asleep", "awake", "label", "load"]; |
| + const keys = Object.keys(b).sort(); |
| + const allowed = required.concat(["always_on", "region"]).sort(); |
| + assert.ok(required.every((k) => keys.includes(k)), name + ": box missing a required key: " + keys); |
| + assert.ok(keys.every((k) => allowed.includes(k)), name + ": box carries a disallowed key: " + keys); |
| if ("region" in b) { |
| - assert.deepEqual(Object.keys(b).sort(), base.concat("region"), name + ": box keys"); |
| assert.equal(typeof b.region, "string", name + ": region is a string"); |
| assert.ok(!/\d+\.\d+|[a-z0-9-]+\.[a-z]{2,}/i.test(b.region), |
| name + ": region reads like an IP/hostname: " + b.region); |
| - } else { |
| - assert.deepEqual(Object.keys(b).sort(), base, name + ": box keys"); |
| } |
| // Labels are anonymous ordinals — never a hostname, IP, or URL. |
| assert.match(b.label, /^[a-z]+-[a-z]+$/, name + ": box label must be a name-action pair"); |
| @@ -420,6 +431,18 @@ await test("hosts + capacity carry only the allowed abstract schema", () => { |
| } |
| }); |
| +// The multi-host fixture specifically: a second, live mesh peer merges into |
| +// hosts.boxes without displacing the local box, and count reflects both. |
| +await test("multi-host fixture merges the local box with a live mesh peer", () => { |
| + assert.equal(fleetMH.hosts.count, 2, "hosts.count reflects both boxes"); |
| + assert.equal(fleetMH.hosts.boxes.length, 2); |
| + const [local, peer] = fleetMH.hosts.boxes; |
| + assert.ok("always_on" in local, "the local box still reports always_on"); |
| + assert.ok(!("always_on" in peer), "a remote peer never reports always_on (a local-box-only concept)"); |
| + assert.ok(!("region" in peer), "a remote peer never reports region in this fixture"); |
| + assert.notEqual(local.label, peer.label, "the two boxes must have distinct labels"); |
| +}); |
| + |
| // Each committed fixture set derives its band — pins the 70/90 thresholds and |
| // the disk-floor rule in the generator (see dry_run's capacity fixtures). |
| await test("capacity fixtures derive plenty / getting-full / at-capacity", () => { |
diff --git a/test/status-fixtures/fleet-multi-host.json b/test/status-fixtures/fleet-multi-host.json| new file mode 100644 |
| index 0000000..ff1ff87 |
| --- /dev/null |
| +++ b/test/status-fixtures/fleet-multi-host.json |
| @@ -0,0 +1 @@ |
| +{"generated_at":"2026-07-13T13:37:03Z","instances":{"total":6,"systemd":2,"hibernating":4,"awake":2,"asleep":2},"hosts":{"count":2,"boxes":[{"label":"hypatia-sketching","always_on":2,"awake":2,"asleep":2,"load":"comfortable","region":"EU · Germany"},{"label":"turing-orienteering","awake":3,"asleep":1,"load":"busy"}]},"capacity":{"level":"plenty"},"versions":[{"version":"v100","count":3},{"version":"v101","count":2},{"version":"unknown","count":1}],"services":{"home":true,"push":true,"cloud":true,"connect":false},"replication":{"healthy":true,"lag_seconds":42}} |