| 1 | package main |
| 2 | |
| 3 | // Per-box capacity stats for the public Fleet status page's multi-host |
| 4 | // aggregation (deploy/status-publish.sh). Each mesh member self-publishes its |
| 5 | // own snapshot (internal/activator/heartbeat.go's PutStats, a sibling object |
| 6 | // next to the membership heartbeat) on its own slower cadence; box A's |
| 7 | // status-publish run reads every live peer's snapshot via the AWS CLI to |
| 8 | // build fleet.json's `hosts.boxes` array, alongside its own already-local |
| 9 | // computation. |
| 10 | // |
| 11 | // Deliberately narrow, matching status.md's disclosure rule word-for-word: |
| 12 | // only awake/asleep counts and a banded load WORD ever leave this box — the |
| 13 | // underlying memory ratio dies right here, in the same process that read it, |
| 14 | // never serialized anywhere even over an authenticated internal channel. |
| 15 | |
| 16 | import ( |
| 17 | "bufio" |
| 18 | "context" |
| 19 | "crypto/sha256" |
| 20 | "encoding/hex" |
| 21 | "log" |
| 22 | "os" |
| 23 | "strconv" |
| 24 | "strings" |
| 25 | "time" |
| 26 | |
| 27 | "lchat/internal/activator" |
| 28 | ) |
| 29 | |
| 30 | // meshStatsInterval paces the stats PutStats — far slower than the |
| 31 | // membership heartbeat (meshHeartbeatInterval, 15s): capacity changes slowly, |
| 32 | // and this is visibility, not coordination. |
| 33 | const meshStatsInterval = 60 * time.Second |
| 34 | |
| 35 | // pctBand mirrors deploy/status-publish.sh's pct_band: 0 below 70%, 1 at |
| 36 | // 70-90%, 2 above 90%. Both sides must band identically, or the same |
| 37 | // utilization could publish a different word depending which box you asked. |
| 38 | func pctBand(pct int) int { |
| 39 | switch { |
| 40 | case pct < 70: |
| 41 | return 0 |
| 42 | case pct <= 90: |
| 43 | return 1 |
| 44 | default: |
| 45 | return 2 |
| 46 | } |
| 47 | } |
| 48 | |
| 49 | // loadWord mirrors status-publish.sh's load_word. |
| 50 | func loadWord(band int) string { |
| 51 | switch band { |
| 52 | case 0: |
| 53 | return "comfortable" |
| 54 | case 1: |
| 55 | return "busy" |
| 56 | default: |
| 57 | return "critical" |
| 58 | } |
| 59 | } |
| 60 | |
| 61 | // meminfoPath is a seam for tests — production always reads the real |
| 62 | // /proc/meminfo, matching status-publish.sh's own $MEMINFO override convention. |
| 63 | var meminfoPath = "/proc/meminfo" |
| 64 | |
| 65 | // memLoadWord reads /proc/meminfo and bands (MemTotal-MemAvailable)/MemTotal, |
| 66 | // the same formula and thresholds as status-publish.sh's meminfo_kb/pct_band. |
| 67 | // ok=false when unreadable (a non-Linux dev machine, a stripped-down |
| 68 | // container) — unknowns publish as absent, never a guessed word. |
| 69 | func memLoadWord() (word string, ok bool) { |
| 70 | f, err := os.Open(meminfoPath) |
| 71 | if err != nil { |
| 72 | return "", false |
| 73 | } |
| 74 | defer f.Close() |
| 75 | |
| 76 | var total, avail int64 |
| 77 | sc := bufio.NewScanner(f) |
| 78 | for sc.Scan() { |
| 79 | fields := strings.Fields(sc.Text()) |
| 80 | if len(fields) < 2 { |
| 81 | continue |
| 82 | } |
| 83 | switch strings.TrimSuffix(fields[0], ":") { |
| 84 | case "MemTotal": |
| 85 | total, _ = strconv.ParseInt(fields[1], 10, 64) |
| 86 | case "MemAvailable": |
| 87 | avail, _ = strconv.ParseInt(fields[1], 10, 64) |
| 88 | } |
| 89 | } |
| 90 | if total <= 0 { |
| 91 | return "", false |
| 92 | } |
| 93 | pct := int((total - avail) * 100 / total) |
| 94 | return loadWord(pctBand(pct)), true |
| 95 | } |
| 96 | |
| 97 | // boxLabelNames/boxLabelActions are IDENTICAL, in the SAME order, to |
| 98 | // deploy/status-publish.sh's box_label names[]/actions[] arrays — the same |
| 99 | // hash must produce the same whimsical label whichever side computes it. |
| 100 | // Keep the two lists in sync by hand; TestBoxLabelMatchesBash pins one |
| 101 | // cross-checked example against the real bash script's own output. |
| 102 | var boxLabelNames = []string{ |
| 103 | "ada", "alan", "grace", "marie", "nikola", "leonardo", "cleopatra", "homer", "sappho", |
| 104 | "darwin", "einstein", "curie", "turing", "hopper", "tesla", "gandalf", "frodo", "sherlock", "watson", |
| 105 | "ahab", "ishmael", "gatsby", "scout", "atticus", "hermione", "matilda", "mowgli", "merlin", "arthur", |
| 106 | "morgana", "beowulf", "odysseus", "penelope", "achilles", "athena", "apollo", "freya", "thor", "loki", |
| 107 | "amelia", "boudicca", "cyrano", "quixote", "sancho", "ophelia", "viola", "rosalind", "puck", "oberon", |
| 108 | "titania", "darcy", "pip", "scrooge", "nemo", "phileas", "zorro", "heidi", "galileo", "kepler", "newton", |
| 109 | "archimedes", "hypatia", "confucius", "rumi", "frida", "vincent", "wolfgang", "ludwig", "clara", |
| 110 | } |
| 111 | |
| 112 | var boxLabelActions = []string{ |
| 113 | "stargazing", "juggling", "whistling", "painting", "sailing", "gardening", |
| 114 | "baking", "skating", "humming", "doodling", "fishing", "knitting", "strolling", "yodeling", |
| 115 | "beekeeping", "fencing", "waltzing", "sketching", "brewing", "climbing", "cartwheeling", |
| 116 | "birdwatching", "pondering", "inventing", "exploring", "composing", "conducting", |
| 117 | "spelunking", "kayaking", "orienteering", "puppeteering", "tinkering", |
| 118 | } |
| 119 | |
| 120 | // boxRegion resolves this box's coarse region for peer publishing, straight |
| 121 | // from LCHAT_BOX_REGION — an explicit operator override only. Unlike |
| 122 | // status-publish.sh's own box_region() (which prefers auto-detecting via |
| 123 | // Hetzner/AWS instance metadata for the LOCAL box, falling back to a hand-set |
| 124 | // override), a mesh peer publishes here from the Go binary, which has no |
| 125 | // metadata-probing code of its own — so this box's operator sets the string |
| 126 | // once in its router.env, same coarse "continent · country" shape, and it |
| 127 | // rides through status-publish.sh's mesh_peer_boxes unchanged (that merge is |
| 128 | // already a raw passthrough of the whole published object). Empty env means |
| 129 | // omitted, never guessed. |
| 130 | func boxRegion() string { |
| 131 | return os.Getenv("LCHAT_BOX_REGION") |
| 132 | } |
| 133 | |
| 134 | // machineIDPath is a seam for tests, matching meminfoPath's convention. |
| 135 | var machineIDPath = "/etc/machine-id" |
| 136 | |
| 137 | // boxLabel derives this box's stable, whimsical, anonymous name — the exact |
| 138 | // port of status-publish.sh's box_label (sha256 of /etc/machine-id, first 16 |
| 139 | // hex chars split into two 8-char slices indexing names/actions). Falls back |
| 140 | // to the `hostname` command's output if the machine-id file is unreadable, |
| 141 | // matching bash's `cat "$idfile" 2>/dev/null || hostname` byte-for-byte: a |
| 142 | // machine-id file carries no trailing newline, but the `hostname` command's |
| 143 | // output does, so only the fallback path appends one. |
| 144 | func boxLabel() string { |
| 145 | var seed []byte |
| 146 | if data, err := os.ReadFile(machineIDPath); err == nil { |
| 147 | seed = data |
| 148 | } else { |
| 149 | hn, _ := os.Hostname() |
| 150 | seed = []byte(hn + "\n") |
| 151 | } |
| 152 | sum := sha256.Sum256(seed) |
| 153 | hexSum := hex.EncodeToString(sum[:])[:16] |
| 154 | n, _ := strconv.ParseUint(hexSum[:8], 16, 32) |
| 155 | a, _ := strconv.ParseUint(hexSum[8:16], 16, 32) |
| 156 | return boxLabelNames[int(n)%len(boxLabelNames)] + "-" + boxLabelActions[int(a)%len(boxLabelActions)] |
| 157 | } |
| 158 | |
| 159 | // computeBoxStats builds this box's publishable snapshot: asleep is derived |
| 160 | // (every hibernating route not currently awake), never separately probed, so |
| 161 | // the two counts can never disagree with each other. |
| 162 | // |
| 163 | // hib only counts routes THIS box currently HRW-owns (mesh.owns — nil mesh, |
| 164 | // e.g. in tests that don't exercise the mesh, counts everything). Without |
| 165 | // this filter, a mesh-replicated route is known to (and counted by) every |
| 166 | // peer it's been seeded onto, so two boxes would each report it — correctly, |
| 167 | // from each one's own local blind spot — and the Fleet page's per-box rows |
| 168 | // would visibly overlap (the same instance reads as "asleep" on more than one |
| 169 | // host, with no way to tell it's the same instance). Filtering to the sole |
| 170 | // current owner makes every box's row a genuine, non-overlapping partition of |
| 171 | // the fleet: an out-of-scope route (mesh.owns always true for those, since |
| 172 | // only one box was ever going to have its row anyway) still counts locally, |
| 173 | // exactly as before mesh placement existed. |
| 174 | func computeBoxStats(reg *Registry, act *activatorSvc, mesh *meshSvc) activator.BoxStats { |
| 175 | routes, err := reg.list() |
| 176 | if err != nil { |
| 177 | log.Printf("mesh: stats: list routes: %v", err) |
| 178 | return activator.BoxStats{} |
| 179 | } |
| 180 | hib := 0 |
| 181 | for _, r := range routes { |
| 182 | if r.Kind != "instance" || !r.Hibernate { |
| 183 | continue |
| 184 | } |
| 185 | if mesh != nil && !mesh.owns(r.Host) { |
| 186 | continue // a live peer owns it — that box's own snapshot counts it, not this one |
| 187 | } |
| 188 | hib++ |
| 189 | } |
| 190 | awake := act.awakeCount() |
| 191 | asleep := hib - awake |
| 192 | load, _ := memLoadWord() // "" (omitted) when unknown |
| 193 | |
| 194 | return activator.BoxStats{Label: boxLabel(), Awake: &awake, Asleep: &asleep, Load: load, Region: boxRegion()} |
| 195 | } |
| 196 | |
| 197 | // startStatsReporter runs alongside the mesh (wired only when mesh != nil, |
| 198 | // router.go): computes and PutStats this box's snapshot on meshStatsInterval, |
| 199 | // for as long as the process runs — like the router's other background |
| 200 | // loops (e.g. its registry-watcher poll), it has no graceful stop; the |
| 201 | // process exiting is what stops it. |
| 202 | func startStatsReporter(store *activator.HeartbeatStore, self string, reg *Registry, act *activatorSvc, mesh *meshSvc) { |
| 203 | report := func() { |
| 204 | ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
| 205 | defer cancel() |
| 206 | if err := store.PutStats(ctx, self, computeBoxStats(reg, act, mesh)); err != nil { |
| 207 | log.Printf("mesh: stats: publish failed: %v", err) |
| 208 | } |
| 209 | } |
| 210 | report() // one synchronous publish so a box's very first status read isn't stale |
| 211 | go func() { |
| 212 | for range time.Tick(meshStatsInterval) { |
| 213 | report() |
| 214 | } |
| 215 | }() |
| 216 | } |
| 217 | |
amadan