| 1 | #!/usr/bin/env bash |
| 2 | # 🤖 Public status publish (issue #550 Parts 11/12): every 5 minutes, build two |
| 3 | # small world-readable JSONs — fleet.json (fleet state) and books.json (the |
| 4 | # books: Stripe income aggregates + itemized infrastructure expenses) — and |
| 5 | # upload them plus the static viewer pages to the public status bucket |
| 6 | # (infra/prod/status.tf). docs/status.md is the operator guide. |
| 7 | # |
| 8 | # ============================ PRIVACY — NON-NEGOTIABLE ======================== |
| 9 | # Everything this script uploads is PUBLIC. The rules (decision #19 in #550: |
| 10 | # income publishes AGGREGATED, never per-payer; invariant 12: no PII in our |
| 11 | # stores, so none can leak here — but we hold the line at the aggregation |
| 12 | # boundary anyway): |
| 13 | # |
| 14 | # fleet.json — AGGREGATES ONLY. Totals, counts, version distribution, |
| 15 | # service up/down booleans, replication health. NEVER a hostname (member |
| 16 | # spaces obviously, but also test-* canaries — the simple rule is NO |
| 17 | # hostnames at all), never per-space member/message counts, never anything |
| 18 | # keyed by space. Hostnames are read internally (to match sockets) and |
| 19 | # must never reach the JSON. |
| 20 | # |
| 21 | # Capacity + hosts (Paul's call: bucketed words, not needles): the box's |
| 22 | # room-for-new-spaces and per-box load publish as ABSTRACT BAND WORDS only |
| 23 | # (plenty / getting-full / at-capacity; comfortable / busy / critical). |
| 24 | # The caps, ratios, free GB, and percentages are read internally to pick |
| 25 | # the word and MUST NEVER reach the JSON — no remaining-slot counts, no GB, |
| 26 | # no percentages, no cap values. Host labels are whimsical name-action |
| 27 | # pairs hashed from the machine id ("ada-stargazing") — stable per box, |
| 28 | # never peer URLs, IPs, hostnames, or the machine id itself. |
| 29 | # |
| 30 | # A box's `region` is COARSE BY DESIGN: a continent + country pair |
| 31 | # ("EU · Finland") — never a city, datacenter, IP, or hostname. Resolved |
| 32 | # from the box's own cloud metadata (see box_region below); the |
| 33 | # STATUS_BOX_REGION env overrides; unknown → the field is omitted |
| 34 | # entirely. Detected, mapped, or absent — never guessed. |
| 35 | # |
| 36 | # books.json — AGGREGATED INCOME ONLY. Active-subscription count (by tier — |
| 37 | # tier names are the public price book), MRR total in EUR, gross collected |
| 38 | # this month, and the Connect aggregates (spaces' gross sales volume this |
| 39 | # month + our commission on it — Paul's call: if we know the GMV, it's |
| 40 | # public; totals only, like everything else here). NEVER customer ids, |
| 41 | # emails, subscription ids, connected-account ids, per-payer lines, or |
| 42 | # timestamps of individual payments. The Stripe key is read for the API |
| 43 | # calls and must never be echoed, logged, or uploaded. |
| 44 | # |
| 45 | # books.json expenses — the deliberate mirror image (decision #19's other |
| 46 | # half: income aggregates, EXPENSES PUBLISH ITEMIZED — the amounts are |
| 47 | # public on purpose). One line per provider: provider word + amount + its |
| 48 | # own billing currency (never FX-converted — no rate, no misstatement). |
| 49 | # NEVER account ids, invoice ids, workspace ids, resource-level lines, or |
| 50 | # usage breakdowns. The AWS creds and the Anthropic Admin key are read for |
| 51 | # the API calls and must never be echoed, logged, or uploaded. |
| 52 | # |
| 53 | # carbon.json — two halves. `built` is COMMITTED data (the AI-build carbon |
| 54 | # tally, web/status/carbon.json, accrued by tools/carbon-tally.mjs), staged |
| 55 | # with the pages and republished as-is — aggregate token counts + kg only, |
| 56 | # never per-session data. `run` is a LIVE operational estimate this script |
| 57 | # INJECTS at upload (build_carbon): AGGREGATES ONLY — total vCPUs, watts, |
| 58 | # grid intensity, banded kg, and the public instance count as the per-space |
| 59 | # denominator (NEVER per-member, never a hostname/IP). Absent inputs → run |
| 60 | # stays absent and the page shows "pending", never a fake zero. |
| 61 | # |
| 62 | # contributors.json — also COMMITTED data (tools/contributors.mjs), |
| 63 | # republished as-is. Public GitHub identities + commit counts only; |
| 64 | # contributors own their row (bio, hidden) in the committed JSON. |
| 65 | # |
| 66 | # security.json — also COMMITTED data (deploy/security-scan.sh, run at |
| 67 | # BUILD time where the Go toolchain lives — this box never computes a |
| 68 | # scan), republished as-is. It is REDACTED BY CONSTRUCTION: a status word, |
| 69 | # an advisory COUNT, a module count, the build sha, a timestamp — never a |
| 70 | # CVE/GHSA/OSV id, package, or code location of anything unpatched (that |
| 71 | # would be an attacker roadmap; see the scan script's disclosure header). |
| 72 | # Absent file → the page shows "scan pending", never a fake clean. |
| 73 | # |
| 74 | # Adding a field? It must be an aggregate over the whole fleet / all payers, |
| 75 | # or it doesn't ship. |
| 76 | # ============================================================================== |
| 77 | # |
| 78 | # Privilege: runs as the lchat user, not root — least privilege that works: |
| 79 | # - /data/registry.db and /data/cloud.env are lchat-owned on the hardened |
| 80 | # box (all units run as lchat since PR #186; /data is owned by it). |
| 81 | # - `systemctl is-active` and `ss -xlH` are unprivileged read-only queries. |
| 82 | # - the journal read (replication lag) needs the systemd-journal group, |
| 83 | # granted via SupplementaryGroups in the unit below. |
| 84 | # Only `install` (writing units, enabling the timer) needs root. |
| 85 | # |
| 86 | # Usage: |
| 87 | # status-publish.sh # build + upload (what the timer runs) |
| 88 | # status-publish.sh install # root: install unit + timer, enable |
| 89 | # status-publish.sh --dry-run DIR # build both JSONs into DIR from |
| 90 | # # self-constructed fake registry/Stripe |
| 91 | # # fixtures; no box state touched, no |
| 92 | # # upload — the committed contract test |
| 93 | # |
| 94 | # Prerequisites on the box: |
| 95 | # /data/status.env — AWS creds for the status-publisher IAM user |
| 96 | # (infra/prod/status.tf; Put/Get/List on the status |
| 97 | # bucket, no Delete, plus ce:GetCostAndUsage for the |
| 98 | # expenses section). chown lchat, chmod 0600. Optional |
| 99 | # expense vars ride here too: STATUS_HETZNER_TOKEN (a |
| 100 | # read-only Hetzner Cloud API token — computes the |
| 101 | # monthly server cost; never echoed) and/or |
| 102 | # STATUS_HETZNER_MONTHLY_CENTS (operator constant — |
| 103 | # overrides the token when both are set), |
| 104 | # STATUS_CLAUDE_TO_DATE_CENTS, STATUS_APPLE_YEARLY_CENTS, |
| 105 | # STATUS_ANTHROPIC_ADMIN_KEY (an sk-ant-admin… key with |
| 106 | # read-only usage scope — NOT the sidecars' completion |
| 107 | # key), and STATUS_DNSIMPLE_TOKEN + STATUS_DNSIMPLE_ACCOUNT |
| 108 | # (an account API token — read-only use here, never |
| 109 | # echoed — plus the numeric account id). |
| 110 | # /opt/lchat/status-pages/ — the viewer pages, copied from web/status/ |
| 111 | # (status.css, index.html, books.html, economics.html, |
| 112 | # economics.json, carbon.html, carbon.json, |
| 113 | # contributors.html, contributors.json, security.html, |
| 114 | # security.json, prompts.html, icon-192.png). status.css |
| 115 | # is the shared stylesheet every page links; it uploads |
| 116 | # before the pages (see publish()). |
| 117 | # aws CLI + sqlite3 (apt install awscli sqlite3). |
| 118 | set -euo pipefail |
| 119 | |
| 120 | BUCKET=${STATUS_BUCKET:-eleven-status-427880590669} |
| 121 | REGION=${STATUS_REGION:-eu-west-1} |
| 122 | REGISTRY_DB=${STATUS_REGISTRY_DB:-/data/registry.db} |
| 123 | CLOUD_ENV=${STATUS_CLOUD_ENV:-/data/cloud.env} |
| 124 | # The control daemon's env (capacity caps: LCHAT_MAX_INSTANCES etc.). May be |
| 125 | # root-owned/unreadable to the lchat user — then the code defaults below apply, |
| 126 | # which are exactly what controld itself defaults to when the vars are unset. |
| 127 | CONTROL_ENV=${STATUS_CONTROL_ENV:-/data/control.env} |
| 128 | # The router's own env — read (same lchat user, same file the router loads) |
| 129 | # only for the mesh's bucket/prefix, so this box can find its live peers' |
| 130 | # self-published stats (mesh_peer_boxes below). Absent LCHAT_HIBERNATE_BUCKET/ |
| 131 | # _PREFIX here just means the mesh isn't configured on this box — hosts stays |
| 132 | # a single-box array, byte-identical to before multi-box existed. |
| 133 | ROUTER_ENV=${STATUS_ROUTER_ENV:-/data/router.env} |
| 134 | # Staleness cutoff for a peer's self-published stats object — three missed |
| 135 | # beats, mirroring mesh.go's own meshStaleAfter convention, at the STATS |
| 136 | # cadence (meshstats.go's meshStatsInterval, 60s) rather than the membership |
| 137 | # heartbeat's faster one. |
| 138 | MESH_STATS_STALE_SECONDS=${STATUS_MESH_STATS_STALE_SECONDS:-180} |
| 139 | # Box-load + disk inputs (STATUS_ overrides exist for testability; the dry-run |
| 140 | # uses fixture twins instead and never reads live box state). |
| 141 | MEMINFO=${STATUS_MEMINFO:-/proc/meminfo} |
| 142 | DATA_DIR=${STATUS_DATA_DIR:-/data} |
| 143 | # Metered-expense cache (see cached_cost_cents): AWS Cost Explorer and the |
| 144 | # Anthropic cost report both bill/limit per request while this script runs |
| 145 | # every 5 minutes, so their results persist here between runs (one JSON object |
| 146 | # per provider: {fetched_at, month, month_to_date_cents}). Lives in /data so |
| 147 | # it survives reboots; the dry-run repoints it into its fixture dir. |
| 148 | COSTS_CACHE=${STATUS_COSTS_CACHE:-$DATA_DIR/status-costs-cache.json} |
| 149 | # Coarse public region of this box — continent + country ONLY ("EU · Germany"); |
| 150 | # see PRIVACY above. Optional: unset omits the field from fleet.json. Per-box |
| 151 | # regions will come from mesh heartbeats when multi-box lands. Resolution: |
| 152 | # STATUS_BOX_REGION env (explicit override wins) → the box's own cloud |
| 153 | # metadata service (Hetzner, then AWS IMDSv2), mapped zone→coarse region → |
| 154 | # omitted entirely when nothing answers. Hand-set values drift (the box was |
| 155 | # labelled Germany for a day; it lives in Helsinki) — asking the machine |
| 156 | # where it is can't. The map stays coarse by design: continent + country. |
| 157 | box_region() { |
| 158 | if [ -n "${STATUS_BOX_REGION:-}" ]; then printf '%s' "$STATUS_BOX_REGION"; return; fi |
| 159 | [ -n "$DRY_RUN" ] && return # dry-run never probes live box state |
| 160 | local zone |
| 161 | zone=$(curl -sm2 http://169.254.169.254/hetzner/v1/metadata/availability-zone 2>/dev/null || true) |
| 162 | case "$zone" in |
| 163 | hel*) printf 'EU · Finland'; return ;; |
| 164 | fsn*|nbg*) printf 'EU · Germany'; return ;; |
| 165 | ash*|hil*) printf 'NA · USA'; return ;; |
| 166 | sin*) printf 'AS · Singapore'; return ;; |
| 167 | esac |
| 168 | # AWS: IMDSv2 (token required; both calls fail closed in ~2s off-cloud) |
| 169 | local tok az |
| 170 | tok=$(curl -sm2 -X PUT http://169.254.169.254/latest/api/token -H "X-aws-ec2-metadata-token-ttl-seconds: 60" 2>/dev/null || true) |
| 171 | [ -n "$tok" ] && az=$(curl -sm2 -H "X-aws-ec2-metadata-token: $tok" http://169.254.169.254/latest/meta-data/placement/region 2>/dev/null || true) |
| 172 | case "$az" in |
| 173 | eu-west-1) printf 'EU · Ireland' ;; |
| 174 | eu-west-2) printf 'EU · UK' ;; |
| 175 | eu-central-1) printf 'EU · Germany' ;; |
| 176 | eu-north-1) printf 'EU · Sweden' ;; |
| 177 | us-east-*|us-west-*) printf 'NA · USA' ;; |
| 178 | *) : ;; # unknown zone: omit, never guess |
| 179 | esac |
| 180 | } |
| 181 | STATUS_ENV=${STATUS_ENV_FILE:-/data/status.env} |
| 182 | PAGES_DIR=${STATUS_PAGES_DIR:-/opt/lchat/status-pages} |
| 183 | STRIPE_API=${STATUS_STRIPE_API:-https://api.stripe.com} |
| 184 | |
| 185 | # Carbon "run" block coefficients (see build_carbon + docs/comparison-carbon.md |
| 186 | # § The run methodology; snapshot July 2026, all BANDED low/central/high). The |
| 187 | # fleet hibernates and runs at a fraction of the box, so the attributable draw |
| 188 | # is a low-single-digit-to-~15W slice of a lightly loaded server — modelled here |
| 189 | # as per-vCPU average watts (marginal → whole-box-idle-inclusive): |
| 190 | CARBON_W_LO=${STATUS_CARBON_W_LO:-1.5} # W/vCPU, marginal (load-proportional) |
| 191 | CARBON_W_CE=${STATUS_CARBON_W_CE:-3.5} # W/vCPU, central |
| 192 | CARBON_W_HI=${STATUS_CARBON_W_HI:-7.5} # W/vCPU, whole-box idle-inclusive share |
| 193 | CARBON_PUE=${STATUS_CARBON_PUE:-1.15} # AWS-published fleet-PUE ballpark |
| 194 | # AWS storage+requests: near-zero but NOT zero — a small explicit banded term |
| 195 | # (~a gram a day: ~0.1 TB of our ciphertext × ~1 Wh/TB-h × 24h × a grid). |
| 196 | CARBON_AWS_LO=${STATUS_CARBON_AWS_LO:-0.0002} |
| 197 | CARBON_AWS_CE=${STATUS_CARBON_AWS_CE:-0.001} |
| 198 | CARBON_AWS_HI=${STATUS_CARBON_AWS_HI:-0.003} |
| 199 | # Bot inference (the sidecars' ongoing Anthropic API use): small today, bounded |
| 200 | # wide — the high end allows for busy days. Precise token counts exist in the |
| 201 | # provider console; we publish a bounded estimate rather than couple to it. |
| 202 | CARBON_INF_LO=${STATUS_CARBON_INF_LO:-0.0005} |
| 203 | CARBON_INF_CE=${STATUS_CARBON_INF_CE:-0.005} |
| 204 | CARBON_INF_HI=${STATUS_CARBON_INF_HI:-0.05} |
| 205 | # The fleet's start-of-life (drives the cumulative-to-date figure). The dry-run |
| 206 | # pins STATUS_CARBON_DAYS instead so its cumulative is deterministic. |
| 207 | CARBON_SINCE=${STATUS_CARBON_SINCE:-2026-07-05} |
| 208 | SELF=/opt/lchat/status-publish.sh |
| 209 | UNIT=/etc/systemd/system/lchat-status.service |
| 210 | TIMER=/etc/systemd/system/lchat-status.timer |
| 211 | |
| 212 | DRY_RUN="" |
| 213 | FIXDIR="" |
| 214 | |
| 215 | TMP=$(mktemp -d) |
| 216 | trap 'rm -rf "$TMP"' EXIT |
| 217 | |
| 218 | # sq runs a query against an on-disk DB with a busy timeout (the router writes |
| 219 | # the registry concurrently); sqj evaluates JSON expressions in :memory:. All |
| 220 | # JSON this script emits is BUILT BY SQLITE's json functions, never by string |
| 221 | # interpolation in bash — DB/Stripe-sourced strings (versions, tier keys) get |
| 222 | # escaped correctly and can't break or inject into the output. |
| 223 | sq() { sqlite3 -cmd '.timeout 3000' "$REGISTRY_DB" "$1"; } |
| 224 | sqj() { sqlite3 :memory: "$1"; } |
| 225 | |
| 226 | now_iso() { date -u +%Y-%m-%dT%H:%M:%SZ; } |
| 227 | |
| 228 | # ---- box probes (each has a dry-run twin so --dry-run exercises the same |
| 229 | # ---- aggregation code with no live systemctl/ss/journalctl) ------------------ |
| 230 | |
| 231 | # svc_active <unit> — unprivileged read-only state query. |
| 232 | svc_active() { |
| 233 | if [ -n "$DRY_RUN" ]; then grep -qxF "$1" "$FIXDIR/active-units.txt"; return; fi |
| 234 | systemctl is-active --quiet "$1" |
| 235 | } |
| 236 | |
| 237 | # music_sockets — the kernel's list of LISTENING unix sockets. This is the |
| 238 | # awake test for hibernating hosts, and the choice matters: |
| 239 | # - Socket-file EXISTENCE is NOT sound: an instance unlinks a stale socket |
| 240 | # file on start (main.go listen()) but never unlinks on exit, so a slept |
| 241 | # instance leaves the file behind. Existence would count every host that |
| 242 | # has ever woken as awake forever. |
| 243 | # - Dialing the socket (curl --unix-socket) IS sound but touches the |
| 244 | # instance — a probe every 5 minutes must not risk interacting with the |
| 245 | # activator's idle accounting or show up in instance logs. |
| 246 | # - `ss -xlH` reads the socket table: an entry exists iff a process has the |
| 247 | # path BOUND AND LISTENING right now (the activator's children bind the |
| 248 | # same /run/lchat/<host>.sock path systemd units use). Zero side effects. |
| 249 | music_sockets() { |
| 250 | if [ -n "$DRY_RUN" ]; then cat "$FIXDIR/music.txt"; return; fi |
| 251 | if command -v ss >/dev/null 2>&1; then ss -xlH 2>/dev/null || true; fi |
| 252 | } |
| 253 | |
| 254 | # replication_lag_json — seconds since the fleet replicator (lchat-litestream, |
| 255 | # deploy/fleet-backup.sh) last logged anything: an approximation of replication |
| 256 | # lag (litestream logs each sync; a healthy replicator is chatty, a wedged one |
| 257 | # goes silent). "null" when the journal is unreadable or empty — published as |
| 258 | # null, never faked. |
| 259 | replication_lag_json() { |
| 260 | if [ -n "$DRY_RUN" ]; then cat "$FIXDIR/replication-lag.txt"; return; fi |
| 261 | local line ts now |
| 262 | line=$(journalctl -u lchat-litestream -n1 -o json 2>/dev/null || true) |
| 263 | if [ -z "$line" ]; then echo "null"; return; fi |
| 264 | printf '%s' "$line" > "$TMP/journal.json" |
| 265 | ts=$(sqj "SELECT CAST(json_extract(readfile('$TMP/journal.json'), '\$.__REALTIME_TIMESTAMP') AS INTEGER) / 1000000" || echo "") |
| 266 | if [ -z "$ts" ] || [ "$ts" = "0" ]; then echo "null"; return; fi |
| 267 | now=$(date -u +%s) |
| 268 | echo $((now - ts)) |
| 269 | } |
| 270 | |
| 271 | # ---- capacity + hosts probes (ABSTRACT BANDS ONLY — see PRIVACY above) ------- |
| 272 | |
| 273 | # env_int <file> <var> <default> — one integer from an env file (same grep/cut |
| 274 | # shape as the Stripe key read). Missing file/var or a non-numeric value falls |
| 275 | # back to the default — for the capacity caps the defaults mirror controld.go's |
| 276 | # code defaults (#663), so an absent/unreadable control.env yields the same |
| 277 | # numbers controld itself enforces. |
| 278 | env_int() { |
| 279 | local v |
| 280 | v=$(grep -E "^$2=" "$1" 2>/dev/null | head -n1 | cut -d= -f2- || true) |
| 281 | case "$v" in ''|*[!0-9]*) echo "$3" ;; *) echo "$v" ;; esac |
| 282 | } |
| 283 | |
| 284 | # env_str <file> <VAR> — like env_int but for string values (bucket names, |
| 285 | # prefixes): no numeric coercion, empty when the file or var is absent. |
| 286 | env_str() { |
| 287 | grep -E "^$2=" "$1" 2>/dev/null | head -n1 | cut -d= -f2- || true |
| 288 | } |
| 289 | |
| 290 | # b64url_decode reverses encodeHeartbeatKey (mesh.go): url-safe alphabet, |
| 291 | # padding stripped — re-add it (len mod 4 tells you how much) before handing |
| 292 | # off to the standard decoder. Used by mesh_peer_boxes to recognize (and |
| 293 | # skip) THIS box's own entry in the stats prefix. |
| 294 | b64url_decode() { |
| 295 | local s |
| 296 | s=$(printf '%s' "$1" | tr -- '-_' '+/') |
| 297 | case $(( ${#s} % 4 )) in |
| 298 | 2) s="${s}==" ;; |
| 299 | 3) s="${s}=" ;; |
| 300 | esac |
| 301 | printf '%s' "$s" | base64 -d 2>/dev/null |
| 302 | } |
| 303 | |
| 304 | # disk_free_gb — whole GB free on the data disk (`df -Pk`: POSIX, and avail |
| 305 | # matches controld's Statfs Bavail). Empty on failure — an unknown contributes |
| 306 | # nothing to the band, never a guess. |
| 307 | disk_free_gb() { |
| 308 | if [ -n "$DRY_RUN" ]; then cat "$FIXDIR/disk-free-gb.txt"; return; fi |
| 309 | df -Pk "$DATA_DIR" 2>/dev/null | awk 'NR==2 && $4 ~ /^[0-9]+$/ { printf "%d", $4 / 1048576 }' || true |
| 310 | } |
| 311 | |
| 312 | # meminfo_kb <Field> — one kB figure from /proc/meminfo (MemTotal / |
| 313 | # MemAvailable drive the box-load band). Empty when unreadable (e.g. a |
| 314 | # non-Linux dev machine) — box load then publishes null, never a guess. |
| 315 | meminfo_kb() { |
| 316 | local src="$MEMINFO" |
| 317 | if [ -n "$DRY_RUN" ]; then src="$FIXDIR/meminfo.txt"; fi |
| 318 | awk -v f="$1:" '$1 == f { print $2; exit }' "$src" 2>/dev/null || true |
| 319 | } |
| 320 | |
| 321 | # pct_band <pct> — the three public bands as an index: 0 below 70%, 1 at |
| 322 | # 70–90%, 2 above 90%. These thresholds ARE the abstraction: only the band |
| 323 | # word ships, the percentage dies here. |
| 324 | pct_band() { if [ "$1" -lt 70 ]; then echo 0; elif [ "$1" -le 90 ]; then echo 1; else echo 2; fi; } |
| 325 | |
| 326 | # The two public vocabularies, one per surface: capacity (room for new spaces) |
| 327 | # and per-box load. Same bands, different words. |
| 328 | band_word() { case "$1" in 0) echo plenty ;; 1) echo getting-full ;; *) echo at-capacity ;; esac; } |
| 329 | load_word() { case "$1" in 0) echo comfortable ;; 1) echo busy ;; *) echo critical ;; esac; } |
| 330 | |
| 331 | # ---- fleet.json -------------------------------------------------------------- |
| 332 | |
| 333 | # box_label derives a stable, whimsical, anonymous name for this box — |
| 334 | # "ada-stargazing" — from a hash of the machine id. Notable names, real and |
| 335 | # fictional (history, literature, myth, science, pop culture), paired with an |
| 336 | # action. The hash direction only (id -> label) is published, so the label |
| 337 | # leaks nothing; the action list deliberately avoids the dashboard's own |
| 338 | # status vocabulary (no sleeping/waking/resting — a box is not an instance). |
| 339 | box_label() { |
| 340 | local names=(ada alan grace marie nikola leonardo cleopatra homer sappho |
| 341 | darwin einstein curie turing hopper tesla gandalf frodo sherlock watson |
| 342 | ahab ishmael gatsby scout atticus hermione matilda mowgli merlin arthur |
| 343 | morgana beowulf odysseus penelope achilles athena apollo freya thor loki |
| 344 | amelia boudicca cyrano quixote sancho ophelia viola rosalind puck oberon |
| 345 | titania darcy pip scrooge nemo phileas zorro heidi galileo kepler newton |
| 346 | archimedes hypatia confucius rumi frida vincent wolfgang ludwig clara) |
| 347 | local actions=(stargazing juggling whistling painting sailing gardening |
| 348 | baking skating humming doodling fishing knitting strolling yodeling |
| 349 | beekeeping fencing waltzing sketching brewing climbing cartwheeling |
| 350 | birdwatching pondering inventing exploring composing conducting |
| 351 | spelunking kayaking orienteering puppeteering tinkering) |
| 352 | local idfile=${STATUS_MACHINE_ID_FILE:-/etc/machine-id} |
| 353 | local seed |
| 354 | seed=$( { cat "$idfile" 2>/dev/null || hostname; } | sha256sum | cut -c1-16) |
| 355 | # two independent hex slices -> indices; bash arithmetic handles hex via 16# |
| 356 | local n=$(( 16#${seed:0:8} % ${#names[@]} )) |
| 357 | local a=$(( 16#${seed:8:8} % ${#actions[@]} )) |
| 358 | printf '%s-%s' "${names[$n]}" "${actions[$a]}" |
| 359 | } |
| 360 | |
| 361 | # mesh_peer_boxes prints a JSON array of every OTHER live mesh box's |
| 362 | # self-published stats object — {label, awake, asleep, load} — read from the |
| 363 | # hibernation bucket's stats side-channel (internal/activator/heartbeat.go's |
| 364 | # PutStats, a sibling object next to the membership heartbeat every mesh |
| 365 | # member already writes; meshstats.go computes each box's own snapshot on its |
| 366 | # own slower cadence). Deliberately NOT a new box-to-box port: this rides the |
| 367 | # same S3 substrate the mesh already uses for membership, exactly the seam |
| 368 | # mesh.go's own doc comment anticipates ("a future AWS pool" that "may not |
| 369 | # even share a network"). Empty array — not an error — whenever this box |
| 370 | # isn't mesh-configured, has no live peers, or any AWS call fails; a missing |
| 371 | # peer here just means the Fleet page undercounts by one box, never a |
| 372 | # publish failure. |
| 373 | # |
| 374 | # As a side effect (one listing, not two), also writes THIS box's own |
| 375 | # self-published stats body to $TMP/mesh-self-stats-result.json (removed if |
| 376 | # unpublished/stale/mesh off) — computeBoxStats (meshstats.go) already |
| 377 | # ownership-partitions this box's own hib count the same way it does for a |
| 378 | # peer, so build_fleet can prefer this over its own local `ss` sweep instead |
| 379 | # of re-deriving an unfiltered view that double-counts against a peer's row. |
| 380 | # A file, not a global var: the caller captures this function's stdout with |
| 381 | # `$(...)`, which runs it in a subshell — a global assignment made in there |
| 382 | # never survives back to the parent shell, but a file write does. |
| 383 | mesh_peer_boxes() { |
| 384 | local self_out="$TMP/mesh-self-stats-result.json" |
| 385 | rm -f "$self_out" |
| 386 | if [ -n "$DRY_RUN" ]; then |
| 387 | if [ -f "$FIXDIR/mesh-self-stats.json" ]; then cp "$FIXDIR/mesh-self-stats.json" "$self_out"; fi |
| 388 | if [ -f "$FIXDIR/mesh-peers.json" ]; then cat "$FIXDIR/mesh-peers.json"; else echo '[]'; fi |
| 389 | return |
| 390 | fi |
| 391 | |
| 392 | local bucket region prefix mesh_prefix self |
| 393 | bucket=$(env_str "$ROUTER_ENV" LCHAT_HIBERNATE_BUCKET) |
| 394 | prefix=$(env_str "$ROUTER_ENV" LCHAT_HIBERNATE_PREFIX) |
| 395 | if [ -z "$bucket" ] || [ -z "$prefix" ]; then echo '[]'; return; fi |
| 396 | region=$(env_str "$ROUTER_ENV" LCHAT_HIBERNATE_REGION) |
| 397 | region=${region:-eu-west-1} |
| 398 | # This box's own mesh identity, if its OWN mesh is on (LCHAT_MESH_SELF set) |
| 399 | # — skipped below. Without this, a box whose mesh is on double-counts |
| 400 | # itself the moment it starts self-publishing to this same stats channel: |
| 401 | # once as "the local box" (this script's own OS reads, elsewhere in |
| 402 | # build_fleet) and again as a "peer" read back from S3. Found live |
| 403 | # 2026-07-13, box A's flip: the Fleet page briefly showed hosts.count 3 |
| 404 | # (one real duplicate) instead of 2. |
| 405 | self=$(env_str "$ROUTER_ENV" LCHAT_MESH_SELF) |
| 406 | # Must match hc.meshPrefix() (activator.go) exactly — the membership |
| 407 | # heartbeat lives at "<prefix>/mesh/members/<peer>", and statsKey |
| 408 | # (internal/activator/heartbeat.go) appends "-stats" to that WHOLE path, |
| 409 | # not to the bare hibernation prefix: "<prefix>/mesh/members-stats/<peer>". |
| 410 | # (Caught 2026-07-13 before the flip: an earlier version of this function |
| 411 | # listed "<prefix>-stats/" instead — empty either way today since nothing |
| 412 | # has ever published there, so it silently proved nothing.) |
| 413 | mesh_prefix="${prefix}/mesh/members" |
| 414 | |
| 415 | # `aws s3 ls` (the high-level command), never `aws s3api list-objects*` — |
| 416 | # the latter crashes on this fleet's aws-cli/Python combination (a Python |
| 417 | # 3.13+ argparse regression in botocore's generated help text: ValueError |
| 418 | # "badly formed help string", thrown while building the argument parser, |
| 419 | # before any request is even sent — found live on box A, 2026-07-13, |
| 420 | # confirmed against both list-objects-v2 and the older list-objects; `aws |
| 421 | # s3 ls`/`aws s3 cp` are unaffected, and are what the rest of this script |
| 422 | # already relies on). `s3 ls` hits the same ListObjectsV2 API underneath; |
| 423 | # its plain-text output already has everything needed (date, time, key). |
| 424 | local listing |
| 425 | listing=$(aws s3 ls "s3://$bucket/${mesh_prefix}-stats/" --region "$region" 2>/dev/null) || true |
| 426 | [ -n "$listing" ] || { echo '[]'; return; } |
| 427 | |
| 428 | # A plain string compare against the cutoff is exact as long as both sides |
| 429 | # use the same "YYYY-MM-DD HH:MM:SS" shape (space-, not T-separated — |
| 430 | # `aws s3 ls`'s own format) — no date-library parsing needed, and a box |
| 431 | # whose clock is off only ever misjudges its OWN staleness, same tradeoff |
| 432 | # mesh.go's own reader-vs-S3-clock note accepts for membership. |
| 433 | local cutoff peers="[]" line date_part time_part key tmp body |
| 434 | cutoff=$(date -u -d "-${MESH_STATS_STALE_SECONDS} seconds" '+%Y-%m-%d %H:%M:%S') |
| 435 | |
| 436 | while IFS= read -r line; do |
| 437 | [ -n "$line" ] || continue |
| 438 | date_part=$(printf '%s' "$line" | awk '{print $1}') |
| 439 | time_part=$(printf '%s' "$line" | awk '{print $2}') |
| 440 | key=$(printf '%s' "$line" | awk '{print $4}') |
| 441 | [ -n "$key" ] || continue |
| 442 | [ "$date_part $time_part" '>' "$cutoff" ] || continue # stale — treat like a dead mesh member |
| 443 | tmp="$TMP/mesh-stat-$(printf '%s' "$key" | tr '/' '_').json" |
| 444 | aws s3 cp "s3://$bucket/${mesh_prefix}-stats/$key" "$tmp" --region "$region" >/dev/null 2>&1 || continue |
| 445 | body=$(cat "$tmp" 2>/dev/null) || continue |
| 446 | [ -n "$body" ] || continue |
| 447 | if [ -n "$self" ] && [ "$(b64url_decode "$key")" = "$self" ]; then |
| 448 | printf '%s' "$body" > "$self_out" |
| 449 | continue |
| 450 | fi |
| 451 | peers=$(sqj "SELECT json_insert( |
| 452 | json('$(printf '%s' "$peers" | sed "s/'/''/g")'), '\$[#]', |
| 453 | json('$(printf '%s' "$body" | sed "s/'/''/g")'))") |
| 454 | done <<< "$listing" |
| 455 | |
| 456 | printf '%s' "$peers" |
| 457 | } |
| 458 | |
| 459 | build_fleet() { |
| 460 | local out=$1 now total systemd_n hib awake_json asleep_json versions |
| 461 | now=$(now_iso) |
| 462 | |
| 463 | total=$(sq "SELECT COUNT(*) FROM routes WHERE kind='instance'") |
| 464 | hib=$(sq "SELECT COUNT(*) FROM routes WHERE kind='instance' AND hibernate=1") |
| 465 | systemd_n=$((total - hib)) |
| 466 | |
| 467 | # Awake = hibernating hosts whose socket is currently bound+music (see |
| 468 | # music_sockets above). Hostnames stay inside this loop — only the |
| 469 | # counts leave it. If `ss` is unavailable the split is published as null |
| 470 | # (unknown), never guessed. |
| 471 | if [ -n "$DRY_RUN" ] || command -v ss >/dev/null 2>&1; then |
| 472 | local music awake=0 sock |
| 473 | music=$(music_sockets) |
| 474 | while IFS= read -r sock; do |
| 475 | [ -z "$sock" ] && continue |
| 476 | # -F fixed-string match: every socket path starts /run/lchat/, so one |
| 477 | # path can never be a substring of another host's line. |
| 478 | if printf '%s\n' "$music" | grep -qF "$sock"; then awake=$((awake+1)); fi |
| 479 | done < <(sq "SELECT socket FROM routes WHERE kind='instance' AND hibernate=1") |
| 480 | awake_json=$awake |
| 481 | asleep_json=$((hib - awake)) |
| 482 | else |
| 483 | awake_json=null |
| 484 | asleep_json=null |
| 485 | fi |
| 486 | |
| 487 | # Version distribution: version strings and counts only (git SHAs / |
| 488 | # release versions — already public via the signed release feed). No |
| 489 | # hostnames, ever. |
| 490 | versions=$(sq "SELECT COALESCE(json_group_array(json_object('version', v, 'count', c)), json('[]')) |
| 491 | FROM (SELECT CASE WHEN version='' THEN 'unknown' ELSE version END AS v, COUNT(*) AS c |
| 492 | FROM routes WHERE kind='instance' GROUP BY v ORDER BY c DESC, v)") |
| 493 | |
| 494 | local home_up=false push_up=false cloud_up=false connect_up=false rep_ok=false |
| 495 | svc_active lchat-home && home_up=true |
| 496 | svc_active lchat-push && push_up=true |
| 497 | svc_active lchat-cloud && cloud_up=true |
| 498 | svc_active lchat-connect && connect_up=true |
| 499 | svc_active lchat-litestream && rep_ok=true |
| 500 | local lag |
| 501 | lag=$(replication_lag_json) |
| 502 | |
| 503 | # ---- capacity: the abstract room-for-new-spaces band ------------------------ |
| 504 | # Worst-of over every input controld's atCapacity() enforces (controld.go |
| 505 | # #663) plus memory pressure, each folded through pct_band's 70/90 lines. |
| 506 | # Only the resulting WORD is published; every number below dies in this |
| 507 | # block. Caps come from the control daemon's env, defaulting to controld's |
| 508 | # own code defaults: 40 instances, 5 GB disk floor, registered ceiling |
| 509 | # 0 → 10x instances. |
| 510 | local max_inst min_free max_reg hib_creates band=0 b |
| 511 | max_inst=$(env_int "$CONTROL_ENV" LCHAT_MAX_INSTANCES 40) |
| 512 | min_free=$(env_int "$CONTROL_ENV" LCHAT_MIN_FREE_GB 5) |
| 513 | max_reg=$(env_int "$CONTROL_ENV" LCHAT_MAX_REGISTERED 0) |
| 514 | hib_creates=$(env_int "$CONTROL_ENV" LCHAT_CONTROL_HIBERNATE_CREATES 0) |
| 515 | # A zero cap would be a config typo, and we must not divide by it. |
| 516 | [ "$max_inst" -gt 0 ] || max_inst=40 |
| 517 | |
| 518 | if [ "$hib_creates" = 1 ]; then |
| 519 | # hibernate-creates on: the instance cap guards what's MATERIALIZED |
| 520 | # (systemd-managed rows — hibernating rows hold no process); total |
| 521 | # registrations get the higher sanity ceiling. Mirrors atCapacity(). |
| 522 | b=$(pct_band $((systemd_n * 100 / max_inst))); if [ "$b" -gt "$band" ]; then band=$b; fi |
| 523 | [ "$max_reg" -gt 0 ] || max_reg=$((10 * max_inst)) |
| 524 | b=$(pct_band $((total * 100 / max_reg))); if [ "$b" -gt "$band" ]; then band=$b; fi |
| 525 | else |
| 526 | b=$(pct_band $((total * 100 / max_inst))); if [ "$b" -gt "$band" ]; then band=$b; fi |
| 527 | fi |
| 528 | |
| 529 | # Disk: the floor is controld's hard refusal line, so at/below it is |
| 530 | # at-capacity outright. Above it, band the PROXIMITY to the floor |
| 531 | # (floor/free): free at 10x the floor reads 10% (plenty), at 1.25x it reads |
| 532 | # 80% (getting-full). |
| 533 | local free_gb |
| 534 | free_gb=$(disk_free_gb) |
| 535 | if [ -n "$free_gb" ]; then |
| 536 | if [ "$free_gb" -le "$min_free" ]; then |
| 537 | band=2 |
| 538 | else |
| 539 | b=$(pct_band $((min_free * 100 / free_gb))); if [ "$b" -gt "$band" ]; then band=$b; fi |
| 540 | fi |
| 541 | fi |
| 542 | |
| 543 | # Memory: utilization (MemTotal - MemAvailable)/MemTotal feeds both the |
| 544 | # capacity band and this box's load word. Unreadable meminfo → load is |
| 545 | # null and memory contributes nothing (unknowns never guess). |
| 546 | local mem_total mem_avail mem_pct load_json=null |
| 547 | mem_total=$(meminfo_kb MemTotal) |
| 548 | mem_avail=$(meminfo_kb MemAvailable) |
| 549 | if [ -n "$mem_total" ] && [ -n "$mem_avail" ] && [ "$mem_total" -gt 0 ]; then |
| 550 | mem_pct=$(((mem_total - mem_avail) * 100 / mem_total)) |
| 551 | b=$(pct_band "$mem_pct"); if [ "$b" -gt "$band" ]; then band=$b; fi |
| 552 | load_json="\"$(load_word "$b")\"" |
| 553 | fi |
| 554 | local capacity_level |
| 555 | capacity_level=$(band_word "$band") |
| 556 | |
| 557 | # ---- hosts: this box, plus every other live mesh member -------------------- |
| 558 | # Labels are whimsical name-action pairs derived from a HASH of the machine |
| 559 | # id ("ada-stargazing") — stable per box, and never peer URLs, IPs, |
| 560 | # hostnames, or the machine id itself. Each box's awake/asleep entry here is |
| 561 | # its OWN locally-verified split. mesh_peer_boxes must run FIRST: besides |
| 562 | # every peer's own snapshot, it also writes THIS box's own self-published |
| 563 | # one to $TMP/mesh-self-stats-result.json — computeBoxStats (meshstats.go) |
| 564 | # already filters that to routes this box currently HRW-owns, so preferring |
| 565 | # it here |
| 566 | # (over the raw local `ss` sweep above) keeps this box's row from |
| 567 | # double-reporting a mesh-replicated route a peer now owns. Falls back to |
| 568 | # the local sweep when mesh is off, unpublished yet, or stale — same |
| 569 | # "unknowns degrade gracefully" rule as the rest of this file. (Below, the |
| 570 | # top-level `instances.awake`/`asleep` fields are summed across every box in |
| 571 | # this array — a per-box figure here is this box's own ownership-scoped |
| 572 | # view, not the fleet total.) The optional region is the operator-set coarse |
| 573 | # continent + country word pair (see PRIVACY) — absent env, absent key, |
| 574 | # never guessed; escaped like the version strings so a quote can't break out |
| 575 | # of the SQL literal. A peer's region (if its own LCHAT_BOX_REGION is set — |
| 576 | # meshstats.go's boxRegion) rides through automatically: mesh_peer_boxes |
| 577 | # inserts each peer's whole self-published object verbatim, so an extra key |
| 578 | # needs no bash change here. `always_on` stays a genuinely local-box-only |
| 579 | # concept, though — no peer publishes one. |
| 580 | local boxes peer_boxes region_kv="" |
| 581 | peer_boxes=$(mesh_peer_boxes) |
| 582 | local self_stats_file="$TMP/mesh-self-stats-result.json" |
| 583 | if [ -s "$self_stats_file" ]; then |
| 584 | local self_esc self_awake |
| 585 | self_esc=$(cat "$self_stats_file" | sed "s/'/''/g") |
| 586 | self_awake=$(sqj "SELECT json_extract('$self_esc', '\$.awake')") |
| 587 | if [ "$self_awake" != null ] && [ -n "$self_awake" ]; then |
| 588 | awake_json=$self_awake |
| 589 | asleep_json=$(sqj "SELECT json_extract('$self_esc', '\$.asleep')") |
| 590 | fi |
| 591 | fi |
| 592 | # Resolved here, not at file top: DRY_RUN/argv must be parsed first so the |
| 593 | # dry-run's no-live-probes guard actually guards (and the dry-run fixture's |
| 594 | # direct BOX_REGION= assignment still wins over a re-resolve). |
| 595 | [ -z "${BOX_REGION:-}" ] && BOX_REGION=$(box_region) |
| 596 | if [ -n "$BOX_REGION" ]; then |
| 597 | region_kv=", 'region', '$(printf '%s' "$BOX_REGION" | sed "s/'/''/g")'" |
| 598 | fi |
| 599 | boxes=$(sqj "SELECT json_array(json_object( |
| 600 | 'label', '$(box_label)', |
| 601 | 'always_on', $systemd_n, |
| 602 | 'awake', json('$awake_json'), |
| 603 | 'asleep', json('$asleep_json'), |
| 604 | 'load', json('$load_json')$region_kv))") |
| 605 | |
| 606 | boxes=$(sqj "SELECT json_group_array(json(value)) FROM ( |
| 607 | SELECT value FROM json_each('$(printf '%s' "$boxes" | sed "s/'/''/g")') |
| 608 | UNION ALL |
| 609 | SELECT value FROM json_each('$(printf '%s' "$peer_boxes" | sed "s/'/''/g")'))") |
| 610 | local host_count |
| 611 | host_count=$(sqj "SELECT json_array_length('$(printf '%s' "$boxes" | sed "s/'/''/g")')") |
| 612 | |
| 613 | # ---- fleet-wide awake/asleep: sum every box's own verified count ---------- |
| 614 | # `hib` above is already fleet-wide (this box's registry is the replicated |
| 615 | # superset — mesh-seed/fan-out only ever ADD rows to a peer, never remove |
| 616 | # them here), but `awake_json`/`asleep_json` up to this point are still only |
| 617 | # this box's own `ss` check — a host actually running on a peer has no local |
| 618 | # socket, so it silently read as "asleep" here even while genuinely awake. |
| 619 | # `boxes` now holds every live box's own verified split, so summing across |
| 620 | # it turns the local view into the real fleet-wide one. If any box's awake |
| 621 | # is unknown (null — e.g. `ss` missing there), the fleet split stays null |
| 622 | # too, rather than guessing: same "unknowns never guess" rule as the rest of |
| 623 | # this file. |
| 624 | local boxes_esc unknown_awake |
| 625 | boxes_esc=$(printf '%s' "$boxes" | sed "s/'/''/g") |
| 626 | unknown_awake=$(sqj "SELECT COUNT(*) FROM json_each('$boxes_esc') WHERE json_extract(value, '\$.awake') IS NULL") |
| 627 | if [ "$unknown_awake" = 0 ]; then |
| 628 | awake_json=$(sqj "SELECT SUM(json_extract(value, '\$.awake')) FROM json_each('$boxes_esc')") |
| 629 | asleep_json=$((hib - awake_json)) |
| 630 | else |
| 631 | awake_json=null |
| 632 | asleep_json=null |
| 633 | fi |
| 634 | |
| 635 | # Assembled by sqlite json_object: the only interpolated values are |
| 636 | # script-controlled numerals/booleans/band-words and the sqlite-built arrays. |
| 637 | sqj "SELECT json_object( |
| 638 | 'generated_at', '$now', |
| 639 | 'instances', json_object( |
| 640 | 'total', $total, |
| 641 | 'systemd', $systemd_n, |
| 642 | 'hibernating', $hib, |
| 643 | 'awake', json('$awake_json'), |
| 644 | 'asleep', json('$asleep_json')), |
| 645 | 'hosts', json_object( |
| 646 | 'count', $host_count, |
| 647 | 'boxes', json('$boxes')), |
| 648 | 'capacity', json_object( |
| 649 | 'level', '$capacity_level'), |
| 650 | 'versions', json('$(printf '%s' "$versions" | sed "s/'/''/g")'), |
| 651 | 'services', json_object( |
| 652 | 'home', json('$home_up'), |
| 653 | 'push', json('$push_up'), |
| 654 | 'cloud', json('$cloud_up'), |
| 655 | 'connect', json('$connect_up')), |
| 656 | 'replication', json_object( |
| 657 | 'healthy', json('$rep_ok'), |
| 658 | 'lag_seconds', json('$lag')))" > "$out" |
| 659 | } |
| 660 | |
| 661 | # ---- books.json -------------------------------------------------------------- |
| 662 | |
| 663 | # stripe_get <path-and-query> <outfile> — one authenticated GET. The key rides |
| 664 | # only in the header; -sS keeps curl from ever printing it. Dry-run reads |
| 665 | # numbered fixture files instead (same downstream aggregation code). |
| 666 | STRIPE_FIXTURE_N=0 |
| 667 | stripe_get() { |
| 668 | if [ -n "$DRY_RUN" ]; then |
| 669 | STRIPE_FIXTURE_N=$((STRIPE_FIXTURE_N + 1)) |
| 670 | local kind=subs |
| 671 | case "$1" in |
| 672 | /v1/balance_transactions*) kind=bal ;; |
| 673 | /v1/application_fees*) kind=fees ;; |
| 674 | esac |
| 675 | cp "$FIXDIR/stripe-$kind-$STRIPE_FIXTURE_N.json" "$2" |
| 676 | return |
| 677 | fi |
| 678 | curl -sSg --fail --max-time 30 -H "Authorization: Bearer $STRIPE_KEY" \ |
| 679 | "$STRIPE_API$1" > "$2" |
| 680 | } |
| 681 | |
| 682 | # fetch_pages <path> <query> <prefix> — follow Stripe's cursor pagination |
| 683 | # (has_more + starting_after) into $TMP/<prefix>-N.json. Hard cap of 50 pages |
| 684 | # (5000 objects) keeps a pathological account from wedging the run. Every |
| 685 | # failure is an explicit `return 1` — callers run this inside an `if !`, where |
| 686 | # set -e is suspended, and a half-fetched month must degrade to "unavailable", |
| 687 | # never aggregate to zeros that read as "no income". |
| 688 | fetch_pages() { |
| 689 | local path=$1 query=$2 prefix=$3 after="" n=0 f has_more |
| 690 | while :; do |
| 691 | n=$((n + 1)) |
| 692 | f="$TMP/$prefix-$n.json" |
| 693 | stripe_get "$path?limit=100${query:+&$query}${after:+&starting_after=$after}" "$f" || return 1 |
| 694 | # A page that isn't a valid list envelope is a failed/garbled fetch. |
| 695 | [ "$(sqj "SELECT json_valid(readfile('$f')) AND json_type(readfile('$f'), '\$.data')='array'")" = "1" ] || return 1 |
| 696 | has_more=$(sqj "SELECT json_extract(readfile('$f'), '\$.has_more')") |
| 697 | [ "$has_more" = "1" ] && [ "$n" -lt 50 ] || break |
| 698 | after=$(sqj "SELECT json_extract(readfile('$f'), '\$.data[#-1].id')") |
| 699 | [ -n "$after" ] || return 1 |
| 700 | done |
| 701 | # Reset the fixture counter so the next endpoint's dry-run pages start at 1. |
| 702 | STRIPE_FIXTURE_N=0 |
| 703 | } |
| 704 | |
| 705 | books_unavailable() { # $1=out $2=reason (fixed, script-controlled) $3=expenses json ("" = omit) |
| 706 | local exp_kv="" |
| 707 | [ -n "${3:-}" ] && exp_kv=", 'expenses', json('$3')" |
| 708 | sqj "SELECT json_object('generated_at', '$(now_iso)', 'available', json('false'), 'reason', '$2'$exp_kv)" > "$1" |
| 709 | } |
| 710 | |
| 711 | # ---- expenses (infrastructure spend) ----------------------------------------- |
| 712 | # Decision #19's other half: income aggregates, EXPENSES PUBLISH ITEMIZED — |
| 713 | # one line per provider, real amount, that provider's own billing currency |
| 714 | # (never FX-converted: publishing USD as USD and EUR as EUR means there is no |
| 715 | # exchange rate to misstate). A provider that can't be honestly resolved is |
| 716 | # OMITTED — never zero, never a guess — and with no providers at all the whole |
| 717 | # expenses object is omitted, matching books_unavailable's degrade discipline. |
| 718 | |
| 719 | # env_cents <value> — validate an operator-set integer-cents constant; echoes |
| 720 | # nothing for anything that isn't a plain non-negative integer. |
| 721 | env_cents() { case "${1:-}" in ''|*[!0-9]*) ;; *) printf '%s' "$1" ;; esac; } |
| 722 | |
| 723 | # fetch_aws_cents <month> — one live Cost Explorer query: month start → |
| 724 | # tomorrow (so today's partial day is included), MONTHLY granularity, |
| 725 | # UnblendedCost. Cost Explorer is a GLOBAL endpoint homed in us-east-1 — the |
| 726 | # --region below is the CE API's home, unrelated to STATUS_REGION (the status |
| 727 | # bucket's region). Integrity before publishing: the Amount must be a plain |
| 728 | # decimal and the Unit USD, or the fetch counts as failed — a garbled or |
| 729 | # negative figure (credits) is omitted, never misstated. Dry-run twin reads a |
| 730 | # fixture instead of calling AWS. |
| 731 | fetch_aws_cents() { |
| 732 | local month=$1 f="$TMP/aws-ce.json" end amount unit |
| 733 | if [ -n "$DRY_RUN" ]; then |
| 734 | [ -f "$FIXDIR/aws-ce.json" ] || return 1 |
| 735 | cp "$FIXDIR/aws-ce.json" "$f" |
| 736 | else |
| 737 | command -v aws >/dev/null 2>&1 || return 1 |
| 738 | end=$(sqj "SELECT date('now','+1 day')") |
| 739 | aws ce get-cost-and-usage --region us-east-1 \ |
| 740 | --time-period "Start=$month-01,End=$end" \ |
| 741 | --granularity MONTHLY --metrics UnblendedCost \ |
| 742 | --cli-connect-timeout 10 --cli-read-timeout 30 \ |
| 743 | --output json > "$f" 2>/dev/null || return 1 |
| 744 | fi |
| 745 | [ "$(sqj "SELECT json_valid(readfile('$f'))")" = "1" ] || return 1 |
| 746 | amount=$(sqj "SELECT json_extract(readfile('$f'), '\$.ResultsByTime[0].Total.UnblendedCost.Amount')") |
| 747 | unit=$(sqj "SELECT json_extract(readfile('$f'), '\$.ResultsByTime[0].Total.UnblendedCost.Unit')") |
| 748 | [ "$unit" = "USD" ] || return 1 |
| 749 | case "$amount" in ''|.|*[!0-9.]*|*.*.*) return 1 ;; esac |
| 750 | sqj "SELECT CAST(ROUND($amount * 100) AS INTEGER)" |
| 751 | } |
| 752 | |
| 753 | # hetzner_get_page <page> <outfile> — one authenticated GET of the Hetzner |
| 754 | # Cloud servers list (Bearer auth; the token rides only in the header — -sS |
| 755 | # keeps curl from ever printing it). Dry-run reads numbered fixture pages |
| 756 | # instead, so the caller's pagination loop is exercised by the twin. |
| 757 | hetzner_get_page() { |
| 758 | if [ -n "$DRY_RUN" ]; then |
| 759 | [ -f "$FIXDIR/hetzner-servers-$1.json" ] || return 1 |
| 760 | cp "$FIXDIR/hetzner-servers-$1.json" "$2" |
| 761 | return |
| 762 | fi |
| 763 | [ -n "${STATUS_HETZNER_TOKEN:-}" ] || return 1 |
| 764 | curl -sS --fail --max-time 30 \ |
| 765 | -H "Authorization: Bearer $STATUS_HETZNER_TOKEN" \ |
| 766 | "https://api.hetzner.cloud/v1/servers?page=$1&per_page=50" > "$2" |
| 767 | } |
| 768 | |
| 769 | # fetch_hetzner_cents <month> — the account's monthly server cost, computed |
| 770 | # from the Cloud API: GET /v1/servers, pages followed via |
| 771 | # meta.pagination.next_page (hard cap 10 pages / 500 servers — a partial sum |
| 772 | # must never publish). Each server embeds its type's per-location price list |
| 773 | # (server_type.prices[]); the entry whose location matches the server's own |
| 774 | # location.name prices it — price_monthly.gross, a decimal string, VAT |
| 775 | # included. EXACTLY one entry may match per server or the fetch counts as |
| 776 | # failed: no match means the server can't be honestly priced, and a duplicate |
| 777 | # would double-count. The prices carry no currency field, so the published |
| 778 | # `eur` is the documented constant (Hetzner bills in EUR), not parsed. The |
| 779 | # figure is SERVER COMPUTE ONLY, deliberately: primary IPs, backups, volumes, |
| 780 | # and traffic overages price via the separate /pricing endpoint, not |
| 781 | # per-server — revisit if that delta ever matters. The <month> arg is unused |
| 782 | # (a flat price list, not month-to-date); it exists for cached_cost_cents' |
| 783 | # shared interface. STATUS_HETZNER_TOKEN is a READ-ONLY Cloud API token and |
| 784 | # must never be echoed or logged; the operator constant |
| 785 | # STATUS_HETZNER_MONTHLY_CENTS overrides this fetch entirely (expenses_object). |
| 786 | fetch_hetzner_cents() { |
| 787 | local page=1 n=0 f next amt load="CREATE TABLE srv(j TEXT);" |
| 788 | local match="json_extract(p.value,'\$.location') = json_extract(s.j,'\$.location.name')" |
| 789 | while :; do |
| 790 | n=$((n + 1)) |
| 791 | f="$TMP/hetzner-servers-$n.json" |
| 792 | hetzner_get_page "$page" "$f" || return 1 |
| 793 | [ "$(sqj "SELECT json_valid(readfile('$f')) AND json_type(readfile('$f'), '\$.servers')='array' |
| 794 | AND json_type(readfile('$f'), '\$.meta.pagination')='object'")" = "1" ] || return 1 |
| 795 | load="$load INSERT INTO srv SELECT value FROM json_each(readfile('$f'), '\$.servers');" |
| 796 | next=$(sqj "SELECT COALESCE(json_extract(readfile('$f'), '\$.meta.pagination.next_page'), '')") |
| 797 | [ -n "$next" ] || break |
| 798 | case "$next" in *[!0-9]*) return 1 ;; esac |
| 799 | [ "$n" -lt 10 ] || return 1 |
| 800 | page=$next |
| 801 | done |
| 802 | # Exactly one matching price row per server (see above), and every matched |
| 803 | # amount a plain decimal — or the figure is omitted, never misstated. |
| 804 | [ "$(sqj "$load SELECT COUNT(*) FROM srv s |
| 805 | WHERE (SELECT COUNT(*) FROM json_each(s.j, '\$.server_type.prices') p WHERE $match) <> 1")" = "0" ] || return 1 |
| 806 | while IFS= read -r amt; do |
| 807 | case "$amt" in ''|.|*[!0-9.]*|*.*.*) return 1 ;; esac |
| 808 | done < <(sqj "$load SELECT json_extract(p.value, '\$.price_monthly.gross') |
| 809 | FROM srv s, json_each(s.j, '\$.server_type.prices') p WHERE $match") |
| 810 | sqj "$load SELECT CAST(ROUND(COALESCE(SUM(json_extract(p.value, '\$.price_monthly.gross')), 0) * 100) AS INTEGER) |
| 811 | FROM srv s, json_each(s.j, '\$.server_type.prices') p WHERE $match" |
| 812 | } |
| 813 | |
| 814 | # fetch_anthropic_cents <month> — one live Admin API cost-report call: |
| 815 | # GET /v1/organizations/cost_report, daily buckets from the month start |
| 816 | # (limit=31 covers a full month in one page; no ending_at → runs to now, so |
| 817 | # today's partial day is included). Amounts come back as decimal strings of |
| 818 | # CENTS ("123.45" = $1.23), currency documented as always USD — a non-USD row |
| 819 | # or a has_more page we didn't fetch counts as failure (never publish a |
| 820 | # partial or unverified figure). STATUS_ANTHROPIC_ADMIN_KEY is an ADMIN key |
| 821 | # (sk-ant-admin…) with read-only usage scope — NOT the sidecars' completion |
| 822 | # key: it can read org cost/usage metadata and nothing else. It rides only in |
| 823 | # the x-api-key header (-sS keeps curl from printing it) and must never be |
| 824 | # echoed or logged. Dry-run twin reads a fixture instead. |
| 825 | fetch_anthropic_cents() { |
| 826 | local month=$1 f="$TMP/anthropic-cost.json" |
| 827 | if [ -n "$DRY_RUN" ]; then |
| 828 | [ -f "$FIXDIR/anthropic-cost.json" ] || return 1 |
| 829 | cp "$FIXDIR/anthropic-cost.json" "$f" |
| 830 | else |
| 831 | [ -n "${STATUS_ANTHROPIC_ADMIN_KEY:-}" ] || return 1 |
| 832 | curl -sS --fail --max-time 30 \ |
| 833 | -H "x-api-key: $STATUS_ANTHROPIC_ADMIN_KEY" \ |
| 834 | -H "anthropic-version: 2023-06-01" \ |
| 835 | "https://api.anthropic.com/v1/organizations/cost_report?starting_at=${month}-01T00:00:00Z&bucket_width=1d&limit=31" \ |
| 836 | > "$f" || return 1 |
| 837 | fi |
| 838 | [ "$(sqj "SELECT json_valid(readfile('$f')) AND json_type(readfile('$f'), '\$.data')='array'")" = "1" ] || return 1 |
| 839 | [ "$(sqj "SELECT COALESCE(json_extract(readfile('$f'), '\$.has_more'), 0)")" != "1" ] || return 1 |
| 840 | [ "$(sqj "SELECT COUNT(*) FROM json_each(readfile('$f'), '\$.data') d, json_each(d.value, '\$.results') r |
| 841 | WHERE json_extract(r.value, '\$.currency') <> 'USD'")" = "0" ] || return 1 |
| 842 | sqj "SELECT CAST(ROUND(COALESCE(SUM(json_extract(r.value, '\$.amount')), 0)) AS INTEGER) |
| 843 | FROM json_each(readfile('$f'), '\$.data') d, json_each(d.value, '\$.results') r" |
| 844 | } |
| 845 | |
| 846 | # fetch_dnsimple_cents <month> — one billing-charges page from DNSimple's API |
| 847 | # (the product domains live there): GET /v2/{account}/billing/charges with |
| 848 | # start_date=<month>-01 / end_date=<tomorrow> (ISO dates; tomorrow so today's |
| 849 | # charges are included), per_page=100. A month of domain charges is a handful — |
| 850 | # pagination.total_pages > 1 counts as failure (a partial sum must never |
| 851 | # publish). Amounts are decimal strings of DOLLARS ("14.50"); the response |
| 852 | # carries NO currency field, so the published currency is the documented |
| 853 | # constant (DNSimple's price book is USD), not parsed. Only `collected` |
| 854 | # charges sum — a refunded charge is not spend — and every summed amount must |
| 855 | # be a plain decimal or the fetch counts as failed. STATUS_DNSIMPLE_TOKEN is |
| 856 | # an account API token, read-only use here: it rides only in the |
| 857 | # Authorization header (-sS keeps curl from printing it) and must never be |
| 858 | # echoed or logged. Dry-run twin reads a fixture instead. |
| 859 | fetch_dnsimple_cents() { |
| 860 | local month=$1 f="$TMP/dnsimple-charges.json" end amt |
| 861 | if [ -n "$DRY_RUN" ]; then |
| 862 | [ -f "$FIXDIR/dnsimple-charges.json" ] || return 1 |
| 863 | cp "$FIXDIR/dnsimple-charges.json" "$f" |
| 864 | else |
| 865 | [ -n "${STATUS_DNSIMPLE_TOKEN:-}" ] && [ -n "${STATUS_DNSIMPLE_ACCOUNT:-}" ] || return 1 |
| 866 | end=$(sqj "SELECT date('now','+1 day')") |
| 867 | curl -sS --fail --max-time 30 \ |
| 868 | -H "Authorization: Bearer $STATUS_DNSIMPLE_TOKEN" \ |
| 869 | "https://api.dnsimple.com/v2/$STATUS_DNSIMPLE_ACCOUNT/billing/charges?start_date=${month}-01&end_date=$end&per_page=100" \ |
| 870 | > "$f" || return 1 |
| 871 | fi |
| 872 | [ "$(sqj "SELECT json_valid(readfile('$f')) AND json_type(readfile('$f'), '\$.data')='array'")" = "1" ] || return 1 |
| 873 | [ "$(sqj "SELECT COALESCE(json_extract(readfile('$f'), '\$.pagination.total_pages'), 1) <= 1")" = "1" ] || return 1 |
| 874 | while IFS= read -r amt; do |
| 875 | case "$amt" in ''|.|*[!0-9.]*|*.*.*) return 1 ;; esac |
| 876 | done < <(sqj "SELECT json_extract(c.value, '\$.total_amount') |
| 877 | FROM json_each(readfile('$f'), '\$.data') c |
| 878 | WHERE json_extract(c.value, '\$.state')='collected'") |
| 879 | sqj "SELECT CAST(ROUND(COALESCE(SUM(json_extract(c.value, '\$.total_amount')), 0) * 100) AS INTEGER) |
| 880 | FROM json_each(readfile('$f'), '\$.data') c |
| 881 | WHERE json_extract(c.value, '\$.state')='collected'" |
| 882 | } |
| 883 | |
| 884 | # cached_cost_cents <provider> <month> [default-age-hours] — the shared |
| 885 | # fetch-through-cache for the API-computed providers (aws, hetzner, anthropic, |
| 886 | # dnsimple — fetch_<provider>_cents above). These APIs charge or rate-limit |
| 887 | # per request (Cost Explorer bills ~$0.01 per query) and this script runs |
| 888 | # every 5 minutes — an uncached publisher would spend real money asking how |
| 889 | # much it spends. So each provider's result is cached in $COSTS_CACHE and |
| 890 | # refetched only when its entry is older than STATUS_COSTS_MAX_AGE_HOURS |
| 891 | # (default: the caller's default-age-hours arg, else 6 — hetzner passes 24, |
| 892 | # its flat price list changes ~never) or from a previous month (last month's |
| 893 | # total is not this month's spend — a cross-month entry never publishes; for |
| 894 | # hetzner that just means one refetch at the month turn). A failed refetch |
| 895 | # falls back to the valid-but-stale same-month entry (stale beats absent); |
| 896 | # no usable entry + failure → echoes nothing and the provider is omitted. |
| 897 | # The cache field name month_to_date_cents is generic storage — for hetzner |
| 898 | # it holds the flat monthly figure. |
| 899 | cached_cost_cents() { |
| 900 | local provider=$1 month=$2 now have_cache="" cache_ts=0 cache_month="" cache_cents="" max_age_h cents |
| 901 | now=$(date -u +%s) |
| 902 | max_age_h=${STATUS_COSTS_MAX_AGE_HOURS:-${3:-6}} |
| 903 | case "$max_age_h" in ''|*[!0-9]*) max_age_h=6 ;; esac |
| 904 | |
| 905 | if [ -f "$COSTS_CACHE" ] && [ "$(sqj "SELECT json_valid(readfile('$COSTS_CACHE'))" 2>/dev/null)" = "1" ]; then |
| 906 | have_cache=1 |
| 907 | cache_ts=$(sqj "SELECT COALESCE(json_extract(readfile('$COSTS_CACHE'), '\$.$provider.fetched_at'), 0)") |
| 908 | cache_month=$(sqj "SELECT COALESCE(json_extract(readfile('$COSTS_CACHE'), '\$.$provider.month'), '')") |
| 909 | cache_cents=$(sqj "SELECT COALESCE(json_extract(readfile('$COSTS_CACHE'), '\$.$provider.month_to_date_cents'), '')") |
| 910 | fi |
| 911 | case "$cache_ts" in ''|*[!0-9]*) cache_ts=0 ;; esac |
| 912 | case "$cache_cents" in ''|*[!0-9]*) cache_cents="" ;; esac |
| 913 | [ "$cache_month" = "$month" ] || cache_cents="" # cross-month cache never publishes |
| 914 | |
| 915 | if [ -n "$cache_cents" ] && [ $((now - cache_ts)) -lt $((max_age_h * 3600)) ]; then |
| 916 | echo "$cache_cents" |
| 917 | return 0 |
| 918 | fi |
| 919 | |
| 920 | cents=$("fetch_${provider}_cents" "$month") || cents="" |
| 921 | case "$cents" in ''|*[!0-9]*) cents="" ;; esac |
| 922 | if [ -n "$cents" ]; then |
| 923 | # Merge this provider's entry into the cache (other providers' entries survive). |
| 924 | local base="'{}'" |
| 925 | [ -n "$have_cache" ] && base="readfile('$COSTS_CACHE')" |
| 926 | sqj "SELECT json_set($base, '\$.$provider', |
| 927 | json_object('fetched_at', $now, 'month', '$month', 'month_to_date_cents', $cents))" \ |
| 928 | > "$COSTS_CACHE.tmp" && mv "$COSTS_CACHE.tmp" "$COSTS_CACHE" |
| 929 | echo "$cents" |
| 930 | return 0 |
| 931 | fi |
| 932 | [ -n "$cache_cents" ] && echo "$cache_cents" |
| 933 | return 0 |
| 934 | } |
| 935 | |
| 936 | # expenses_object — the expenses JSON object, or nothing when no provider |
| 937 | # resolves. Providers, in publish order: |
| 938 | # aws — Cost Explorer month-to-date (metered; cached above), USD. |
| 939 | # hetzner — the flat-rate Hetzner Cloud servers. No billing API exists, |
| 940 | # but the servers API prices each server (its type's embedded |
| 941 | # per-location price list), so STATUS_HETZNER_TOKEN computes |
| 942 | # the account's monthly server cost (fetch_hetzner_cents — |
| 943 | # server compute only; cached, 24h default freshness). The |
| 944 | # operator-set constant STATUS_HETZNER_MONTHLY_CENTS overrides |
| 945 | # the API when set — operator truth wins. EUR either way |
| 946 | # (Hetzner bills in EUR). Neither set → omitted. |
| 947 | # anthropic — Admin API cost report month-to-date (metered; cached), USD — |
| 948 | # the bot sidecars' API spend. |
| 949 | # claude.ai — the claude.ai subscriptions: no API for them, and the honest |
| 950 | # figure Paul can attest is LIFETIME spend, not a monthly — so |
| 951 | # the operator-set constant STATUS_CLAUDE_TO_DATE_CENTS (EUR) |
| 952 | # publishes as to_date_cents, the cumulative shape. It's an |
| 953 | # operator-attested estimate by nature, which is why the page |
| 954 | # labels it "(approx.)". |
| 955 | # dnsimple — the product domains (elevenmessenger.com, 11ch.at, |
| 956 | # 11factor.org): billing-charges month-to-date (metered; |
| 957 | # cached), USD. |
| 958 | # apple — the Apple Developer Program membership: fixed YEARLY, no |
| 959 | # billing API, so the operator-set constant |
| 960 | # STATUS_APPLE_YEARLY_CENTS (EUR). Published as billed — |
| 961 | # yearly_cents, never divided into a fake monthly. |
| 962 | expenses_object() { |
| 963 | local month providers="" cents |
| 964 | month=$(sqj "SELECT strftime('%Y-%m','now')") |
| 965 | |
| 966 | cents=$(cached_cost_cents aws "$month") |
| 967 | [ -n "$cents" ] && providers=$(sqj "SELECT json_object('provider','aws','month_to_date_cents',$cents,'currency','usd')") |
| 968 | |
| 969 | # Operator constant first — when set, the API is never called (or cached). |
| 970 | cents=$(env_cents "${STATUS_HETZNER_MONTHLY_CENTS:-}") |
| 971 | [ -n "$cents" ] || cents=$(cached_cost_cents hetzner "$month" 24) |
| 972 | [ -n "$cents" ] && providers="${providers:+$providers,}$(sqj "SELECT json_object('provider','hetzner','monthly_cents',$cents,'currency','eur')")" |
| 973 | |
| 974 | cents=$(cached_cost_cents anthropic "$month") |
| 975 | [ -n "$cents" ] && providers="${providers:+$providers,}$(sqj "SELECT json_object('provider','anthropic','month_to_date_cents',$cents,'currency','usd')")" |
| 976 | |
| 977 | # Cumulative, not monthly: the operator attests lifetime spend to date (an |
| 978 | # approximation by nature — the page carries the "(approx.)" marker). |
| 979 | cents=$(env_cents "${STATUS_CLAUDE_TO_DATE_CENTS:-}") |
| 980 | [ -n "$cents" ] && providers="${providers:+$providers,}$(sqj "SELECT json_object('provider','claude.ai','to_date_cents',$cents,'currency','eur')")" |
| 981 | |
| 982 | cents=$(cached_cost_cents dnsimple "$month") |
| 983 | [ -n "$cents" ] && providers="${providers:+$providers,}$(sqj "SELECT json_object('provider','dnsimple','month_to_date_cents',$cents,'currency','usd')")" |
| 984 | |
| 985 | cents=$(env_cents "${STATUS_APPLE_YEARLY_CENTS:-}") |
| 986 | [ -n "$cents" ] && providers="${providers:+$providers,}$(sqj "SELECT json_object('provider','apple','yearly_cents',$cents,'currency','eur')")" |
| 987 | |
| 988 | [ -n "$providers" ] || return 0 |
| 989 | sqj "SELECT json_object('month', '$month', 'providers', json('[$providers]'))" |
| 990 | } |
| 991 | |
| 992 | build_books() { |
| 993 | local out=$1 now month month_start expenses_json expenses_kv="" |
| 994 | now=$(now_iso) |
| 995 | |
| 996 | # Expenses resolve independently of Stripe — income being unavailable must |
| 997 | # not hide infrastructure spend we can honestly state, so the expenses |
| 998 | # object rides the degraded shape too (and is simply absent when no |
| 999 | # provider resolves). |
| 1000 | expenses_json=$(expenses_object) |
| 1001 | [ -n "$expenses_json" ] && expenses_kv=", |
| 1002 | 'expenses', json('$expenses_json')" |
| 1003 | |
| 1004 | # The Stripe key comes from the cloud provisioner's env (billing.go reads the |
| 1005 | # same var). Absent → billing isn't configured here; publish an honest |
| 1006 | # "unavailable" rather than zeros that would read as "no income". |
| 1007 | STRIPE_KEY=$(grep -E '^LCHAT_STRIPE_SECRET_KEY=' "$CLOUD_ENV" 2>/dev/null | head -n1 | cut -d= -f2- || true) |
| 1008 | if [ -z "${STRIPE_KEY:-}" ]; then |
| 1009 | books_unavailable "$out" "billing not configured" "$expenses_json" |
| 1010 | return |
| 1011 | fi |
| 1012 | |
| 1013 | # Month boundary via sqlite (portable — GNU/BSD date flags differ). |
| 1014 | month=$(sqj "SELECT strftime('%Y-%m','now')") |
| 1015 | month_start=$(sqj "SELECT strftime('%s', datetime('now','start of month'))") |
| 1016 | |
| 1017 | # The three aggregation sources (decision #19 — these are the ONLY Stripe |
| 1018 | # reads, all fleet-wide): |
| 1019 | # GET /v1/subscriptions?status=active&limit=100 (+ cursor pages) |
| 1020 | # → active count, per-tier counts (metadata.tier = public price-book |
| 1021 | # key), MRR from each item's price amount normalized to monthly. |
| 1022 | # GET /v1/balance_transactions?created[gte]=<month start>&limit=100 |
| 1023 | # → gross collected this month (positive charge/payment amounts). |
| 1024 | # GET /v1/application_fees?created[gte]=<month start>&expand[]=data.charge |
| 1025 | # → spaces selling to their own members via Stripe Connect |
| 1026 | # (subscriptions.go's stripeConnect: direct charges on the space |
| 1027 | # admin's connected account, an application fee to the platform). |
| 1028 | # Those charges live on the CONNECTED accounts, so the two reads |
| 1029 | # above never see them; the fee list is the platform's one honest |
| 1030 | # window: each fee's amount is our commission, and its expanded |
| 1031 | # charge carries the gross sale amount → GMV. |
| 1032 | # Degrade to "unavailable" on any fetch failure — a Stripe outage must not |
| 1033 | # publish half-numbers. |
| 1034 | if ! fetch_pages /v1/subscriptions "status=active" subs || |
| 1035 | ! fetch_pages /v1/balance_transactions "created[gte]=$month_start" bal || |
| 1036 | ! fetch_pages /v1/application_fees "created[gte]=$month_start&expand[]=data.charge" fees; then |
| 1037 | books_unavailable "$out" "stripe unreachable" "$expenses_json" |
| 1038 | return |
| 1039 | fi |
| 1040 | |
| 1041 | # Aggregate with sqlite json_each — never per-object output, only SUM/COUNT/ |
| 1042 | # GROUP BY. Per-payer fields (ids, customer, created) die inside this query. |
| 1043 | { |
| 1044 | echo "CREATE TABLE subs(j TEXT);" |
| 1045 | echo "CREATE TABLE bal(j TEXT);" |
| 1046 | echo "CREATE TABLE fees(j TEXT);" |
| 1047 | local f |
| 1048 | for f in "$TMP"/subs-*.json; do |
| 1049 | echo "INSERT INTO subs SELECT value FROM json_each(readfile('$f'), '\$.data');" |
| 1050 | done |
| 1051 | for f in "$TMP"/bal-*.json; do |
| 1052 | echo "INSERT INTO bal SELECT value FROM json_each(readfile('$f'), '\$.data');" |
| 1053 | done |
| 1054 | for f in "$TMP"/fees-*.json; do |
| 1055 | echo "INSERT INTO fees SELECT value FROM json_each(readfile('$f'), '\$.data');" |
| 1056 | done |
| 1057 | # MRR: unit_amount × quantity per item, normalized to a month. EUR items |
| 1058 | # only (the price book is EUR; mixing currencies without FX would lie). |
| 1059 | # Unknown intervals contribute 0 to MRR but still count as subscribers. |
| 1060 | # Connect: commission = the fee amounts, GMV = each fee's expanded charge |
| 1061 | # amount — same gross semantics as collected_month_cents (positive EUR |
| 1062 | # amounts; refunds don't net). GMV is the spaces' money, not ours: our |
| 1063 | # income from these sales is exactly the commission line, which is why it |
| 1064 | # ships as its own field and never folds into collected_month_cents. |
| 1065 | cat <<SQL |
| 1066 | SELECT json_object( |
| 1067 | 'generated_at', '$now', |
| 1068 | 'available', json('true'), |
| 1069 | 'currency', 'eur', |
| 1070 | 'month', '$month', |
| 1071 | 'subscriptions', json_object( |
| 1072 | 'active', (SELECT COUNT(*) FROM subs), |
| 1073 | 'by_tier', (SELECT COALESCE(json_group_array(json_object('tier', tier, 'count', c)), json('[]')) |
| 1074 | FROM (SELECT COALESCE(NULLIF(json_extract(j,'\$.metadata.tier'),''),'other') AS tier, |
| 1075 | COUNT(*) AS c |
| 1076 | FROM subs GROUP BY tier ORDER BY c DESC, tier))), |
| 1077 | 'mrr_cents', CAST(ROUND(COALESCE(( |
| 1078 | SELECT SUM( |
| 1079 | json_extract(i.value,'\$.price.unit_amount') |
| 1080 | * COALESCE(json_extract(i.value,'\$.quantity'),1) |
| 1081 | * (CASE json_extract(i.value,'\$.price.recurring.interval') |
| 1082 | WHEN 'month' THEN 1.0 |
| 1083 | WHEN 'year' THEN 1.0/12.0 |
| 1084 | WHEN 'week' THEN 52.0/12.0 |
| 1085 | WHEN 'day' THEN 365.0/12.0 |
| 1086 | ELSE 0 END) |
| 1087 | / COALESCE(json_extract(i.value,'\$.price.recurring.interval_count'),1)) |
| 1088 | FROM subs s, json_each(s.j, '\$.items.data') i |
| 1089 | WHERE json_extract(i.value,'\$.price.currency')='eur'), 0)) AS INTEGER), |
| 1090 | 'collected_month_cents', CAST(COALESCE(( |
| 1091 | SELECT SUM(json_extract(b.j,'\$.amount')) FROM bal b |
| 1092 | WHERE json_extract(b.j,'\$.type') IN ('charge','payment') |
| 1093 | AND json_extract(b.j,'\$.currency')='eur' |
| 1094 | AND json_extract(b.j,'\$.amount') > 0), 0) AS INTEGER), |
| 1095 | 'connect', json_object( |
| 1096 | 'gmv_month_cents', CAST(COALESCE(( |
| 1097 | SELECT SUM(json_extract(f.j,'\$.charge.amount')) FROM fees f |
| 1098 | WHERE json_extract(f.j,'\$.charge.currency')='eur' |
| 1099 | AND json_extract(f.j,'\$.charge.amount') > 0), 0) AS INTEGER), |
| 1100 | 'commission_month_cents', CAST(COALESCE(( |
| 1101 | SELECT SUM(json_extract(f.j,'\$.amount')) FROM fees f |
| 1102 | WHERE json_extract(f.j,'\$.currency')='eur' |
| 1103 | AND json_extract(f.j,'\$.amount') > 0), 0) AS INTEGER))$expenses_kv); |
| 1104 | SQL |
| 1105 | } | sqlite3 :memory: > "$out" |
| 1106 | } |
| 1107 | |
| 1108 | # ---- carbon.json: republish the committed built ledger + inject a live run block |
| 1109 | |
| 1110 | # grid_band — the box region's grid carbon intensity as "<low> <central> <high>" |
| 1111 | # g CO₂e/kWh. low is always ~0 (a market-based renewable contract — Hetzner's |
| 1112 | # EU sites run on renewable electricity); central+high are the LOCATION grid, |
| 1113 | # banded. Region resolves from box_region() (coarse continent+country; the prod |
| 1114 | # box is Helsinki → Finland's low-carbon grid). Operator overrides: |
| 1115 | # STATUS_CARBON_GRID_CENTRAL / _HIGH. Cited in docs/comparison-carbon.md. |
| 1116 | grid_band() { |
| 1117 | local gce ghi region |
| 1118 | region=${BOX_REGION:-$(box_region)} |
| 1119 | case "$region" in |
| 1120 | "EU · Finland") gce=100; ghi=130 ;; # Finnish grid ~80–130 (Fingrid/ElectricityMaps 2026) |
| 1121 | "EU · Sweden") gce=40; ghi=60 ;; |
| 1122 | "EU · Germany") gce=350; ghi=400 ;; |
| 1123 | "EU · Ireland") gce=300; ghi=380 ;; |
| 1124 | "EU · UK") gce=210; ghi=280 ;; |
| 1125 | "NA · USA") gce=370; ghi=480 ;; |
| 1126 | *) gce=350; ghi=480 ;; # unknown: IEA-ish global band |
| 1127 | esac |
| 1128 | [ -n "${STATUS_CARBON_GRID_CENTRAL:-}" ] && gce=$STATUS_CARBON_GRID_CENTRAL |
| 1129 | [ -n "${STATUS_CARBON_GRID_HIGH:-}" ] && ghi=$STATUS_CARBON_GRID_HIGH |
| 1130 | printf '0 %s %s' "$gce" "$ghi" |
| 1131 | } |
| 1132 | |
| 1133 | # hetzner_vcpus — total vCPUs across every server in the Hetzner account (the |
| 1134 | # hardware the whole fleet runs on; sums so it scales as boxes are added). Same |
| 1135 | # paginated servers-API read as fetch_hetzner_cents (dry-run twin included); |
| 1136 | # each server_type.cores must be an integer or we can't honestly total. Prints |
| 1137 | # the sum, or returns 1 (no token / bad page) so the run block stays absent. |
| 1138 | hetzner_vcpus() { |
| 1139 | local page=1 n=0 f next load="CREATE TABLE srv(j TEXT);" |
| 1140 | while :; do |
| 1141 | n=$((n + 1)) |
| 1142 | f="$TMP/hetzner-vcpus-$n.json" |
| 1143 | hetzner_get_page "$page" "$f" || return 1 |
| 1144 | [ "$(sqj "SELECT json_valid(readfile('$f')) AND json_type(readfile('$f'), '\$.servers')='array' |
| 1145 | AND json_type(readfile('$f'), '\$.meta.pagination')='object'")" = "1" ] || return 1 |
| 1146 | load="$load INSERT INTO srv SELECT value FROM json_each(readfile('$f'), '\$.servers');" |
| 1147 | next=$(sqj "SELECT COALESCE(json_extract(readfile('$f'), '\$.meta.pagination.next_page'), '')") |
| 1148 | [ -n "$next" ] || break |
| 1149 | case "$next" in *[!0-9]*) return 1 ;; esac |
| 1150 | [ "$n" -lt 10 ] || return 1 |
| 1151 | page=$next |
| 1152 | done |
| 1153 | [ "$(sqj "$load SELECT COUNT(*) FROM srv WHERE json_type(j, '\$.server_type.cores') <> 'integer'")" = "0" ] || return 1 |
| 1154 | sqj "$load SELECT COALESCE(SUM(json_extract(j, '\$.server_type.cores')), 0) FROM srv" |
| 1155 | } |
| 1156 | |
| 1157 | # carbon_run_object — the live operational estimate as a JSON object, or nothing |
| 1158 | # printed when the router term can't be resolved (no Hetzner token / fetch fail) |
| 1159 | # so the published `run` stays absent and the page shows "pending", never a fake |
| 1160 | # zero. Math (all BANDED): daily kg = vCPUs × W/vCPU × 24h × grid(g/kWh) × PUE |
| 1161 | # / 1e6, plus small explicit AWS-storage + bot-inference terms. Reported as a |
| 1162 | # daily rate, a cumulative-to-date (× days since CARBON_SINCE, or a pinned |
| 1163 | # STATUS_CARBON_DAYS), and normalised per active space (denominator = the public |
| 1164 | # instance count from the registry — NEVER per-member). Aggregates only: vCPU |
| 1165 | # total, watts, grid, kg, space count — no hostnames/IPs/per-space needles. |
| 1166 | carbon_run_object() { |
| 1167 | local vcpus glo gce ghi spaces days since now |
| 1168 | vcpus=$(hetzner_vcpus) || vcpus="" |
| 1169 | case "$vcpus" in ''|*[!0-9]*|0) return 0 ;; esac # no router term → run absent |
| 1170 | set -- $(grid_band); glo=$1 gce=$2 ghi=$3 |
| 1171 | spaces=$(sq "SELECT COUNT(*) FROM routes WHERE kind='instance'" 2>/dev/null || echo "") |
| 1172 | case "$spaces" in ''|*[!0-9]*|0) spaces=1 ;; esac |
| 1173 | since=$CARBON_SINCE |
| 1174 | days=${STATUS_CARBON_DAYS:-} |
| 1175 | [ -n "$days" ] || days=$(sqj "SELECT MAX(1, CAST(julianday('now') - julianday('$since') AS INTEGER))") |
| 1176 | case "$days" in ''|*[!0-9]*) days=1 ;; esac |
| 1177 | now=$(now_iso) |
| 1178 | # Daily bands live in the CTE d(lo,ce,hi); cumulative/per-space derive from the |
| 1179 | # UNROUNDED daily so rounding never compounds. Watts round to whole; kg to 6dp. |
| 1180 | sqj "WITH d(lo, ce, hi) AS (VALUES( |
| 1181 | $vcpus*$CARBON_W_LO*24*$glo*$CARBON_PUE/1e6 + $CARBON_AWS_LO + $CARBON_INF_LO, |
| 1182 | $vcpus*$CARBON_W_CE*24*$gce*$CARBON_PUE/1e6 + $CARBON_AWS_CE + $CARBON_INF_CE, |
| 1183 | $vcpus*$CARBON_W_HI*24*$ghi*$CARBON_PUE/1e6 + $CARBON_AWS_HI + $CARBON_INF_HI)) |
| 1184 | SELECT json_object( |
| 1185 | 'generated_at', '$now', |
| 1186 | 'since', '$since', |
| 1187 | 'spaces', $spaces, |
| 1188 | 'pue', $CARBON_PUE, |
| 1189 | 'vcpus', $vcpus, |
| 1190 | 'watts', json_object('low', CAST(ROUND($vcpus*$CARBON_W_LO) AS INTEGER), |
| 1191 | 'central', CAST(ROUND($vcpus*$CARBON_W_CE) AS INTEGER), |
| 1192 | 'high', CAST(ROUND($vcpus*$CARBON_W_HI) AS INTEGER)), |
| 1193 | 'grid_g_per_kwh', json_object('low', $glo, 'central', $gce, 'high', $ghi), |
| 1194 | 'daily_kg', json_object('low', ROUND(lo,6), 'central', ROUND(ce,6), 'high', ROUND(hi,6)), |
| 1195 | 'cumulative_kg', json_object('low', ROUND(lo*$days,6), 'central', ROUND(ce*$days,6), 'high', ROUND(hi*$days,6)), |
| 1196 | 'per_space_daily_kg', json_object('low', ROUND(lo/$spaces,6), 'central', ROUND(ce/$spaces,6), 'high', ROUND(hi/$spaces,6)), |
| 1197 | 'services', json_array( |
| 1198 | json_object('name', 'router + instances', 'daily_kg_central', ROUND($vcpus*$CARBON_W_CE*24*$gce*$CARBON_PUE/1e6, 6)), |
| 1199 | json_object('name', 'object storage (AWS)', 'daily_kg_central', $CARBON_AWS_CE), |
| 1200 | json_object('name', 'bot inference', 'daily_kg_central', $CARBON_INF_CE))) |
| 1201 | FROM d" |
| 1202 | } |
| 1203 | |
| 1204 | # build_carbon <out> <src> — republish the committed carbon.json (the built |
| 1205 | # ledger, tools/carbon-tally.mjs) and inject a live `run` block. We nest HERE |
| 1206 | # rather than in the tool: the committed file stays exactly the built ledger |
| 1207 | # ({ built:{…} }), and the only transform is this one merge — so "carbon is |
| 1208 | # committed data" holds and the run block never touches git. A malformed source |
| 1209 | # or no live inputs → republished verbatim (run stays absent → page = pending). |
| 1210 | build_carbon() { |
| 1211 | local out=$1 src=$2 |
| 1212 | if [ "$(sqj "SELECT json_valid(readfile('$src')) AND json_type(readfile('$src'), '\$.built')='object'")" != "1" ]; then |
| 1213 | cp "$src" "$out"; return |
| 1214 | fi |
| 1215 | carbon_run_object > "$TMP/carbon-run.json" || true |
| 1216 | if [ -s "$TMP/carbon-run.json" ]; then |
| 1217 | sqj "SELECT json_set(readfile('$src'), '\$.run', json(readfile('$TMP/carbon-run.json')))" > "$out" |
| 1218 | else |
| 1219 | cp "$src" "$out" |
| 1220 | fi |
| 1221 | } |
| 1222 | |
| 1223 | # ---- upload ------------------------------------------------------------------ |
| 1224 | |
| 1225 | upload() { # file key content-type cache-control |
| 1226 | aws s3 cp "$1" "s3://$BUCKET/$2" --region "$REGION" \ |
| 1227 | --content-type "$3" --cache-control "$4" --no-progress >/dev/null |
| 1228 | } |
| 1229 | |
| 1230 | publish() { |
| 1231 | command -v aws >/dev/null 2>&1 || { echo "aws CLI missing (apt install awscli)" >&2; exit 1; } |
| 1232 | # Creds for the scoped status-publisher IAM user (never in tofu state; mint |
| 1233 | # by hand per infra/prod/status.tf). The unit injects them via |
| 1234 | # EnvironmentFile; a manual run sources the same file. |
| 1235 | if [ -z "${AWS_ACCESS_KEY_ID:-}" ] && [ -f "$STATUS_ENV" ]; then |
| 1236 | set -a; . "$STATUS_ENV"; set +a |
| 1237 | fi |
| 1238 | |
| 1239 | build_fleet "$TMP/fleet.json" |
| 1240 | build_books "$TMP/books.json" |
| 1241 | |
| 1242 | # JSONs: short cache so "as of" stays honest at the 5-minute cadence. |
| 1243 | upload "$TMP/fleet.json" fleet.json "application/json" "max-age=60" |
| 1244 | upload "$TMP/books.json" books.json "application/json" "max-age=60" |
| 1245 | |
| 1246 | # The viewer pages + brand mark (from web/status/, staged on the box). |
| 1247 | # Re-uploading tiny idempotent files every run self-heals a missed copy. |
| 1248 | if [ -d "$PAGES_DIR" ]; then |
| 1249 | # The shared stylesheet goes up FIRST, before any page that <link>s it, so a |
| 1250 | # freshly-uploaded page is never served ahead of its styles (a page fetched |
| 1251 | # in the gap would render unstyled until it revalidates). Same-origin in the |
| 1252 | # status bucket — no third-party request, so the privacy line holds. |
| 1253 | [ -f "$PAGES_DIR/status.css" ] && upload "$PAGES_DIR/status.css" status.css "text/css; charset=utf-8" "max-age=300" |
| 1254 | # The shared masthead+nav builder rides up with the stylesheet, before any |
| 1255 | # page that mounts it, for the same never-served-ahead reason. |
| 1256 | [ -f "$PAGES_DIR/status-nav.js" ] && upload "$PAGES_DIR/status-nav.js" status-nav.js "text/javascript; charset=utf-8" "max-age=300" |
| 1257 | [ -f "$PAGES_DIR/index.html" ] && upload "$PAGES_DIR/index.html" index.html "text/html; charset=utf-8" "max-age=300" |
| 1258 | [ -f "$PAGES_DIR/books.html" ] && upload "$PAGES_DIR/books.html" books.html "text/html; charset=utf-8" "max-age=300" |
| 1259 | # Tier economics: page + committed snapshot JSON (web/status/economics.json, |
| 1260 | # re-derived by hand when prices or input rates change — docs/status.md |
| 1261 | # § Keeping the tier-economics page current). Republished as-is. |
| 1262 | [ -f "$PAGES_DIR/economics.html" ] && upload "$PAGES_DIR/economics.html" economics.html "text/html; charset=utf-8" "max-age=300" |
| 1263 | [ -f "$PAGES_DIR/economics.json" ] && upload "$PAGES_DIR/economics.json" economics.json "application/json" "max-age=300" |
| 1264 | # Investing: page + committed funding ledger + the waterfall reference |
| 1265 | # implementation (docs/funding.md). waterfall.js is the NORMATIVE |
| 1266 | # algorithm module — the page imports and runs the same file it serves, |
| 1267 | # so it must publish with an explicit JS content-type (module imports |
| 1268 | # fail on a guessed one). Republished as-is, like economics. |
| 1269 | [ -f "$PAGES_DIR/funding.html" ] && upload "$PAGES_DIR/funding.html" funding.html "text/html; charset=utf-8" "max-age=300" |
| 1270 | [ -f "$PAGES_DIR/funding.json" ] && upload "$PAGES_DIR/funding.json" funding.json "application/json" "max-age=300" |
| 1271 | [ -f "$PAGES_DIR/waterfall.js" ] && upload "$PAGES_DIR/waterfall.js" waterfall.js "text/javascript; charset=utf-8" "max-age=300" |
| 1272 | [ -f "$PAGES_DIR/carbon.html" ] && upload "$PAGES_DIR/carbon.html" carbon.html "text/html; charset=utf-8" "max-age=300" |
| 1273 | # The carbon tally is committed data (the built ledger) — republished as-is |
| 1274 | # EXCEPT for a live `run` block injected here (build_carbon). Built untouched, |
| 1275 | # run recomputed each publish; absent inputs → run stays absent (page pending). |
| 1276 | if [ -f "$PAGES_DIR/carbon.json" ]; then |
| 1277 | build_carbon "$TMP/carbon.json" "$PAGES_DIR/carbon.json" |
| 1278 | upload "$TMP/carbon.json" carbon.json "application/json" "max-age=300" |
| 1279 | fi |
| 1280 | [ -f "$PAGES_DIR/contributors.html" ] && upload "$PAGES_DIR/contributors.html" contributors.html "text/html; charset=utf-8" "max-age=300" |
| 1281 | [ -f "$PAGES_DIR/contributors.json" ] && upload "$PAGES_DIR/contributors.json" contributors.json "application/json" "max-age=300" |
| 1282 | # The security ledger: page + committed, build-time-scanned, REDACTED JSON |
| 1283 | # (deploy/security-scan.sh — see the header). A missing security.json is |
| 1284 | # honest: the page renders "scan pending" rather than a fake clean, so the |
| 1285 | # page uploads even when the JSON hasn't been staged. |
| 1286 | [ -f "$PAGES_DIR/security.html" ] && upload "$PAGES_DIR/security.html" security.html "text/html; charset=utf-8" "max-age=300" |
| 1287 | [ -f "$PAGES_DIR/security.json" ] && upload "$PAGES_DIR/security.json" security.json "application/json" "max-age=300" |
| 1288 | # The prompt registry viewer. Its corpus (prompts/…) is NOT uploaded here: |
| 1289 | # a contributor accrues + syncs it straight to the bucket — far too large |
| 1290 | # for a 5-minute republish (docs/status.md § The prompt registry). |
| 1291 | [ -f "$PAGES_DIR/prompts.html" ] && upload "$PAGES_DIR/prompts.html" prompts.html "text/html; charset=utf-8" "max-age=300" |
| 1292 | # The music ledger: page + committed index + month shards (accrued per |
| 1293 | # session by tools/music-ledger.mjs — art rides inside the shards as |
| 1294 | # data URIs, so the page stays external-request-free). Republished as-is, |
| 1295 | # like economics; the glob picks up new months automatically. |
| 1296 | [ -f "$PAGES_DIR/music.html" ] && upload "$PAGES_DIR/music.html" music.html "text/html; charset=utf-8" "max-age=300" |
| 1297 | for f in "$PAGES_DIR"/music*.json; do |
| 1298 | [ -f "$f" ] && upload "$f" "$(basename "$f")" "application/json" "max-age=300" |
| 1299 | done |
| 1300 | # Milestones: reconciled from the box's running copy 2026-07-18 — the line |
| 1301 | # was hand-added there (pre-dating this file's funding additions) and the |
| 1302 | # repo owed it back. |
| 1303 | [ -f "$PAGES_DIR/milestones.html" ] && upload "$PAGES_DIR/milestones.html" milestones.html "text/html; charset=utf-8" "max-age=300" |
| 1304 | # Roadmap: the forward companion to milestones — the plan for the road to V1. |
| 1305 | # Static like milestones (no JSON feed, no contract-test coverage). |
| 1306 | [ -f "$PAGES_DIR/roadmap.html" ] && upload "$PAGES_DIR/roadmap.html" roadmap.html "text/html; charset=utf-8" "max-age=300" |
| 1307 | [ -f "$PAGES_DIR/icon-192.png" ] && upload "$PAGES_DIR/icon-192.png" icon-192.png "image/png" "max-age=86400" |
| 1308 | else |
| 1309 | echo "warning: $PAGES_DIR missing — JSONs published, pages skipped (copy web/status/* there)" >&2 |
| 1310 | fi |
| 1311 | echo "published to s3://$BUCKET (fleet + books + pages, $(now_iso))" |
| 1312 | } |
| 1313 | |
| 1314 | # ---- install (root) ---------------------------------------------------------- |
| 1315 | |
| 1316 | install_units() { |
| 1317 | [ "$(id -u)" = 0 ] || { echo "install needs root (writes systemd units)" >&2; exit 1; } |
| 1318 | [ -f "$STATUS_ENV" ] || { echo "missing $STATUS_ENV (AWS creds for status-publisher) — mint per infra/prod/status.tf" >&2; exit 1; } |
| 1319 | |
| 1320 | if [ "$(readlink -f "$0" 2>/dev/null || echo "$0")" != "$SELF" ]; then |
| 1321 | cp "$0" "$SELF" && chmod 0755 "$SELF" |
| 1322 | fi |
| 1323 | mkdir -p "$PAGES_DIR" |
| 1324 | |
| 1325 | # Same hardening posture as the litestream unit. ReadWritePaths=/data is |
| 1326 | # required even though we only SELECT: WAL readers take shared-memory locks |
| 1327 | # (writes to the -shm file), so a read-only /data breaks registry reads. |
| 1328 | cat > "$UNIT" <<'EOF' |
| 1329 | [Unit] |
| 1330 | Description=Eleven public status publish (fleet + books aggregates) |
| 1331 | Wants=network-online.target |
| 1332 | After=network-online.target |
| 1333 | |
| 1334 | [Service] |
| 1335 | Type=oneshot |
| 1336 | User=lchat |
| 1337 | Group=lchat |
| 1338 | SupplementaryGroups=systemd-journal |
| 1339 | EnvironmentFile=/data/status.env |
| 1340 | ExecStart=/opt/lchat/status-publish.sh |
| 1341 | NoNewPrivileges=true |
| 1342 | ProtectSystem=strict |
| 1343 | ReadWritePaths=/data |
| 1344 | ProtectHome=true |
| 1345 | PrivateTmp=true |
| 1346 | MemoryMax=256M |
| 1347 | EOF |
| 1348 | |
| 1349 | # Persistent=false on purpose: a missed run's output is superseded by the |
| 1350 | # next one five minutes later — catch-up publishes are pure noise. |
| 1351 | cat > "$TIMER" <<'EOF' |
| 1352 | [Unit] |
| 1353 | Description=Eleven public status publish — every 5 minutes |
| 1354 | |
| 1355 | [Timer] |
| 1356 | OnBootSec=2min |
| 1357 | OnUnitActiveSec=5min |
| 1358 | RandomizedDelaySec=20 |
| 1359 | Persistent=false |
| 1360 | |
| 1361 | [Install] |
| 1362 | WantedBy=timers.target |
| 1363 | EOF |
| 1364 | |
| 1365 | systemctl daemon-reload |
| 1366 | systemctl enable --now lchat-status.timer |
| 1367 | echo "installed — verify with: systemctl list-timers lchat-status.timer; journalctl -u lchat-status -n 20" |
| 1368 | echo "remember: copy web/status/* (pages, committed JSONs, icon-192.png) to $PAGES_DIR" |
| 1369 | } |
| 1370 | |
| 1371 | # ---- dry-run: self-constructed fixtures, real aggregation code ---------------- |
| 1372 | |
| 1373 | dry_run() { |
| 1374 | local outdir=$1 |
| 1375 | mkdir -p "$outdir" |
| 1376 | DRY_RUN=1 |
| 1377 | FIXDIR="$TMP/fixtures" |
| 1378 | mkdir -p "$FIXDIR" |
| 1379 | REGISTRY_DB="$FIXDIR/registry.db" |
| 1380 | CLOUD_ENV="$FIXDIR/cloud.env" |
| 1381 | CONTROL_ENV="$FIXDIR/control.env" |
| 1382 | |
| 1383 | # A fake registry: 6 instances (2 systemd-managed, 4 hibernating of which 2 |
| 1384 | # are awake), 2 service rows (must not count as instances), 3 versions. |
| 1385 | sqlite3 "$REGISTRY_DB" <<'SQL' |
| 1386 | CREATE TABLE routes ( |
| 1387 | host TEXT PRIMARY KEY, kind TEXT NOT NULL, socket TEXT NOT NULL, |
| 1388 | version TEXT NOT NULL DEFAULT '', hibernate INTEGER NOT NULL DEFAULT 0 |
| 1389 | ); |
| 1390 | INSERT INTO routes VALUES |
| 1391 | ('alpha.example', 'instance', '/run/lchat/alpha.example.sock', 'v100', 0), |
| 1392 | ('bravo.example', 'instance', '/run/lchat/bravo.example.sock', 'v100', 0), |
| 1393 | ('charlie.example', 'instance', '/run/lchat/charlie.example.sock', 'v100', 1), |
| 1394 | ('delta.example', 'instance', '/run/lchat/delta.example.sock', 'v101', 1), |
| 1395 | ('echo.example', 'instance', '/run/lchat/echo.example.sock', 'v101', 1), |
| 1396 | ('foxtrot.example', 'instance', '/run/lchat/foxtrot.example.sock', '', 1), |
| 1397 | ('home.example', 'service', '/run/lchat/home.sock', '', 0), |
| 1398 | ('push.example', 'service', '/run/lchat/push.sock', '', 0); |
| 1399 | SQL |
| 1400 | |
| 1401 | # ss -xlH shape: charlie + delta music (awake), echo + foxtrot not. |
| 1402 | cat > "$FIXDIR/music.txt" <<'EOF' |
| 1403 | u_str LISTEN 0 4096 /run/lchat/alpha.example.sock 12001 * 0 |
| 1404 | u_str LISTEN 0 4096 /run/lchat/bravo.example.sock 12002 * 0 |
| 1405 | u_str LISTEN 0 4096 /run/lchat/charlie.example.sock 12003 * 0 |
| 1406 | u_str LISTEN 0 4096 /run/lchat/delta.example.sock 12004 * 0 |
| 1407 | EOF |
| 1408 | printf '%s\n' lchat-home lchat-push lchat-cloud lchat-litestream > "$FIXDIR/active-units.txt" # connect down on purpose |
| 1409 | # Fixed machine-id fixture so the dry-run label is deterministic everywhere |
| 1410 | # (a dev Mac has no /etc/machine-id; hostname fallback would vary per machine). |
| 1411 | echo dry-run-fixture-machine-id > "$FIXDIR/machine-id" |
| 1412 | export STATUS_MACHINE_ID_FILE="$FIXDIR/machine-id" |
| 1413 | # Fixed region fixture: exercises the STATUS_BOX_REGION seam end-to-end (the |
| 1414 | # contract test pins both the value and its coarse continent+country shape). |
| 1415 | BOX_REGION="EU · Germany" |
| 1416 | echo 42 > "$FIXDIR/replication-lag.txt" |
| 1417 | |
| 1418 | # Multi-host fixture (STATUS_DRY_MESH_PEER=1): one other live mesh box's |
| 1419 | # self-published stats snapshot, exactly the shape internal/activator/ |
| 1420 | # heartbeat.go's PutStats writes and mesh_peer_boxes reads back. Default |
| 1421 | # (unset) keeps hosts.count at 1, byte-identical to the single-box shape — |
| 1422 | # the mesh's own "absence changes nothing" convention, mirrored here. This |
| 1423 | # also seeds THIS box's own self-published stats fixture |
| 1424 | # (mesh-self-stats.json), gated the same way: when mesh is off, this box |
| 1425 | # never self-publishes either, so the single-box shape must stay untouched. |
| 1426 | if [ "${STATUS_DRY_MESH_PEER:-}" = 1 ]; then |
| 1427 | # awake=1 here, not some arbitrary number: this peer only ever runs |
| 1428 | # instances mesh-replicated FROM this box's own registry, so its awake |
| 1429 | # count must fit within this box's own currently-"asleep" locally instances |
| 1430 | # (echo + foxtrot, 2 of them) — otherwise the fleet-wide awake/asleep sum |
| 1431 | # below goes negative, which would mean the fixture models an impossible |
| 1432 | # state, not a real one. |
| 1433 | # region present here (unlike an earlier version of this fixture) — |
| 1434 | # exercises a peer that HAS set its own LCHAT_BOX_REGION, proving the |
| 1435 | # merge's raw passthrough carries an extra key with no bash-side change. |
| 1436 | cat > "$FIXDIR/mesh-peers.json" <<'EOF' |
| 1437 | [{"label":"turing-orienteering","awake":1,"asleep":1,"load":"busy","region":"UK · London"}] |
| 1438 | EOF |
| 1439 | # awake=1/asleep=2 (3 total) here, one fewer than the local `ss` sweep's |
| 1440 | # own awake=2/asleep=2 (4 total) above — proves build_fleet actually |
| 1441 | # prefers this ownership-filtered snapshot over its own unfiltered local |
| 1442 | # sweep, not just reads it and ignores it. |
| 1443 | cat > "$FIXDIR/mesh-self-stats.json" <<'EOF' |
| 1444 | {"label":"hypatia-sketching","awake":1,"asleep":2,"load":"comfortable","region":"EU · Germany"} |
| 1445 | EOF |
| 1446 | fi |
| 1447 | |
| 1448 | # Capacity fixtures (control caps + disk + meminfo): one set per public band |
| 1449 | # (default plenty; STATUS_DRY_CAPACITY=getting-full|at-capacity), so the |
| 1450 | # abstract derivation is testable end-to-end at every level without touching |
| 1451 | # box state. hibernate-creates is on in all sets (the live posture the bands |
| 1452 | # mirror); the caps are otherwise left unset so the default set exercises |
| 1453 | # controld.go's code defaults (40 / 5 GB floor / 10x registered). |
| 1454 | echo "LCHAT_CONTROL_HIBERNATE_CREATES=1" > "$FIXDIR/control.env" |
| 1455 | case "${STATUS_DRY_CAPACITY:-}" in |
| 1456 | getting-full) |
| 1457 | # Registered ratio 6/8 = 75% → getting-full, while disk (100 GB) and |
| 1458 | # memory (25% used) stay roomy — capacity and box load are independent |
| 1459 | # signals, so load reads comfortable here. |
| 1460 | echo "LCHAT_MAX_REGISTERED=8" >> "$FIXDIR/control.env" |
| 1461 | echo 100 > "$FIXDIR/disk-free-gb.txt" |
| 1462 | printf 'MemTotal: 4014980 kB\nMemAvailable: 3011235 kB\n' > "$FIXDIR/meminfo.txt" |
| 1463 | ;; |
| 1464 | at-capacity) |
| 1465 | # Disk below the 5 GB floor → at-capacity outright; memory at 95% used |
| 1466 | # → load critical. |
| 1467 | echo 3 > "$FIXDIR/disk-free-gb.txt" |
| 1468 | printf 'MemTotal: 4014980 kB\nMemAvailable: 200749 kB\n' > "$FIXDIR/meminfo.txt" |
| 1469 | ;; |
| 1470 | *) |
| 1471 | # Everything roomy: ratios ~5%/1.5%, floor proximity 5%, memory 25% |
| 1472 | # used → plenty / comfortable. |
| 1473 | echo 100 > "$FIXDIR/disk-free-gb.txt" |
| 1474 | printf 'MemTotal: 4014980 kB\nMemAvailable: 3011235 kB\n' > "$FIXDIR/meminfo.txt" |
| 1475 | ;; |
| 1476 | esac |
| 1477 | |
| 1478 | # STATUS_DRY_BOOKS=unavailable exercises the degraded books path (no key). |
| 1479 | if [ "${STATUS_DRY_BOOKS:-}" = unavailable ]; then |
| 1480 | : > "$FIXDIR/cloud.env" |
| 1481 | else |
| 1482 | echo "LCHAT_STRIPE_SECRET_KEY=sk_test_dryrun_not_a_real_key" > "$FIXDIR/cloud.env" |
| 1483 | fi |
| 1484 | |
| 1485 | # Expense fixtures (STATUS_DRY_EXPENSES=omit|stale-cache|hetzner-constant; |
| 1486 | # default = all six providers resolve, hetzner from its API fixture). The |
| 1487 | # cost cache is repointed into FIXDIR — the dry run never touches $DATA_DIR. |
| 1488 | # The API fixtures carry fixture-only ids (request id, workspace id, charge |
| 1489 | # references/domains, server names): only the parsed integers may survive |
| 1490 | # into books.json — the leak grep below is the proof. |
| 1491 | COSTS_CACHE="$FIXDIR/costs-cache.json" |
| 1492 | case "${STATUS_DRY_EXPENSES:-}" in |
| 1493 | omit) |
| 1494 | # No API fixtures, no cache, no operator constants → every provider, |
| 1495 | # and then the whole expenses object, is omitted. |
| 1496 | STATUS_HETZNER_MONTHLY_CENTS="" |
| 1497 | STATUS_CLAUDE_TO_DATE_CENTS="" |
| 1498 | STATUS_APPLE_YEARLY_CENTS="" |
| 1499 | ;; |
| 1500 | stale-cache) |
| 1501 | # No API fixtures (all four "fetches" fail) but valid same-month cache |
| 1502 | # entries well past the max age → the stale figures serve rather than |
| 1503 | # vanishing (stale beats absent for month-to-date). |
| 1504 | STATUS_HETZNER_MONTHLY_CENTS="" |
| 1505 | STATUS_CLAUDE_TO_DATE_CENTS="" |
| 1506 | STATUS_APPLE_YEARLY_CENTS="" |
| 1507 | sqj "SELECT json_object( |
| 1508 | 'aws', json_object('fetched_at', strftime('%s','now') - 999999, 'month', strftime('%Y-%m','now'), 'month_to_date_cents', 999), |
| 1509 | 'hetzner', json_object('fetched_at', strftime('%s','now') - 999999, 'month', strftime('%Y-%m','now'), 'month_to_date_cents', 1399), |
| 1510 | 'anthropic', json_object('fetched_at', strftime('%s','now') - 999999, 'month', strftime('%Y-%m','now'), 'month_to_date_cents', 888), |
| 1511 | 'dnsimple', json_object('fetched_at', strftime('%s','now') - 999999, 'month', strftime('%Y-%m','now'), 'month_to_date_cents', 777))" \ |
| 1512 | > "$COSTS_CACHE" |
| 1513 | ;; |
| 1514 | *) |
| 1515 | # Default: hetzner's constant is cleared so its API fixture drives the |
| 1516 | # figure (and an ambient operator env can't leak in); |
| 1517 | # STATUS_DRY_EXPENSES=hetzner-constant sets it WITH the fixture still |
| 1518 | # present, asserting the constant wins over a resolvable API path. |
| 1519 | STATUS_HETZNER_MONTHLY_CENTS="" |
| 1520 | [ "${STATUS_DRY_EXPENSES:-}" = hetzner-constant ] && STATUS_HETZNER_MONTHLY_CENTS=1400 |
| 1521 | STATUS_CLAUDE_TO_DATE_CENTS=229000 |
| 1522 | STATUS_APPLE_YEARLY_CENTS=9900 |
| 1523 | # Cost Explorer shape; Amount 12.3399999 USD exercises the cents rounding (→ 1234). |
| 1524 | cat > "$FIXDIR/aws-ce.json" <<'EOF' |
| 1525 | {"ce_fixture_request_id":"req_fixture_1","ResultsByTime":[{"TimePeriod":{"Start":"2026-07-01","End":"2026-07-12"},"Total":{"UnblendedCost":{"Amount":"12.3399999","Unit":"USD"}},"Groups":[],"Estimated":true}]} |
| 1526 | EOF |
| 1527 | # Hetzner servers shape, TWO pages (exercises meta.pagination.next_page): |
| 1528 | # each server carries its type's per-location price list, and only the |
| 1529 | # entry matching the server's own location.name may price it — the |
| 1530 | # decoy entries for the other location differ on purpose. Gross monthly |
| 1531 | # EUR strings: 16.4288 (cpx31 @ hel1; NOT the fsn1 16.1840) + 4.5100 |
| 1532 | # (cx22 @ fsn1; NOT the hel1 4.6900) = 20.9388 → rounds to 2094 cents. |
| 1533 | # Server names are fixture markers that must not survive aggregation. |
| 1534 | cat > "$FIXDIR/hetzner-servers-1.json" <<'EOF' |
| 1535 | {"servers":[ |
| 1536 | {"id":101,"name":"hzfix-alpha","status":"running", |
| 1537 | "location":{"id":3,"name":"hel1","description":"Helsinki DC Park 1","country":"FI","city":"Helsinki","network_zone":"eu-central"}, |
| 1538 | "server_type":{"id":104,"name":"cpx31","cores":4,"memory":8,"disk":160,"deprecated":false, |
| 1539 | "prices":[ |
| 1540 | {"location":"fsn1","price_hourly":{"net":"0.0225","gross":"0.0268"},"price_monthly":{"net":"13.6000","gross":"16.1840"}}, |
| 1541 | {"location":"hel1","price_hourly":{"net":"0.0229","gross":"0.0273"},"price_monthly":{"net":"13.8057","gross":"16.4288"}} |
| 1542 | ]}} |
| 1543 | ],"meta":{"pagination":{"page":1,"per_page":1,"previous_page":null,"next_page":2,"last_page":2,"total_entries":2}}} |
| 1544 | EOF |
| 1545 | cat > "$FIXDIR/hetzner-servers-2.json" <<'EOF' |
| 1546 | {"servers":[ |
| 1547 | {"id":102,"name":"hzfix-bravo","status":"running", |
| 1548 | "location":{"id":1,"name":"fsn1","description":"Falkenstein DC Park 1","country":"DE","city":"Falkenstein","network_zone":"eu-central"}, |
| 1549 | "server_type":{"id":22,"name":"cx22","cores":2,"memory":4,"disk":40,"deprecated":false, |
| 1550 | "prices":[ |
| 1551 | {"location":"fsn1","price_hourly":{"net":"0.0063","gross":"0.0075"},"price_monthly":{"net":"3.7900","gross":"4.5100"}}, |
| 1552 | {"location":"hel1","price_hourly":{"net":"0.0066","gross":"0.0079"},"price_monthly":{"net":"3.9400","gross":"4.6900"}} |
| 1553 | ]}} |
| 1554 | ],"meta":{"pagination":{"page":2,"per_page":1,"previous_page":1,"next_page":null,"last_page":2,"total_entries":2}}} |
| 1555 | EOF |
| 1556 | # Anthropic cost-report shape: two daily buckets, amounts are decimal |
| 1557 | # strings of CENTS (150.25 + 99.5 → rounds to 250). The workspace id is |
| 1558 | # a fixture marker that must not survive aggregation. |
| 1559 | cat > "$FIXDIR/anthropic-cost.json" <<'EOF' |
| 1560 | {"data":[ |
| 1561 | {"starting_at":"2026-07-01T00:00:00Z","ending_at":"2026-07-02T00:00:00Z", |
| 1562 | "results":[{"amount":"150.25","currency":"USD","workspace_id":"wrkspc_fixture_1"}]}, |
| 1563 | {"starting_at":"2026-07-02T00:00:00Z","ending_at":"2026-07-03T00:00:00Z", |
| 1564 | "results":[{"amount":"99.5","currency":"USD","workspace_id":"wrkspc_fixture_1"}]} |
| 1565 | ],"has_more":false,"next_page":null} |
| 1566 | EOF |
| 1567 | # DNSimple billing-charges shape (amounts = decimal DOLLAR strings, no |
| 1568 | # currency field, pagination envelope): two collected charges |
| 1569 | # (14.50 + 10.00 → 2450 cents) plus a refunded one that must NOT count — |
| 1570 | # the state filter is part of the contract. References and domains are |
| 1571 | # fixture markers that must not survive aggregation. |
| 1572 | cat > "$FIXDIR/dnsimple-charges.json" <<'EOF' |
| 1573 | {"data":[ |
| 1574 | {"invoiced_at":"2026-07-02T09:00:00Z","total_amount":"14.50","balance_amount":"0.00","reference":"dnsfix-1001","state":"collected", |
| 1575 | "items":[{"description":"Renewal dnsfix-one.example","amount":"14.50","product_id":11,"product_type":"domain-renewal","product_reference":"dnsfix-one.example"}]}, |
| 1576 | {"invoiced_at":"2026-07-03T09:00:00Z","total_amount":"10.00","balance_amount":"0.00","reference":"dnsfix-1002","state":"collected", |
| 1577 | "items":[{"description":"Renewal dnsfix-two.example","amount":"10.00","product_id":12,"product_type":"domain-renewal","product_reference":"dnsfix-two.example"}]}, |
| 1578 | {"invoiced_at":"2026-07-04T09:00:00Z","total_amount":"14.50","balance_amount":"0.00","reference":"dnsfix-1003","state":"refunded", |
| 1579 | "items":[{"description":"Register dnsfix-three.example","amount":"14.50","product_id":13,"product_type":"domain-registration","product_reference":"dnsfix-three.example"}]} |
| 1580 | ],"pagination":{"current_page":1,"per_page":100,"total_entries":3,"total_pages":1}} |
| 1581 | EOF |
| 1582 | ;; |
| 1583 | esac |
| 1584 | |
| 1585 | # Two subscription pages (exercises pagination) + one balance page. Shapes |
| 1586 | # follow Stripe's list envelope; per-payer fields present in the fixture |
| 1587 | # (ids, customer) must NOT survive into books.json — that's the test. |
| 1588 | cat > "$FIXDIR/stripe-subs-1.json" <<'EOF' |
| 1589 | {"object":"list","has_more":true,"data":[ |
| 1590 | {"id":"sub_fixture_a","customer":"cus_fixture_a","status":"active","metadata":{"tier":"friends","space_id":"7"}, |
| 1591 | "items":{"data":[{"quantity":1,"price":{"unit_amount":400,"currency":"eur","recurring":{"interval":"month","interval_count":1}}}]}}, |
| 1592 | {"id":"sub_fixture_b","customer":"cus_fixture_b","status":"active","metadata":{"tier":"family","space_id":"9"}, |
| 1593 | "items":{"data":[{"quantity":1,"price":{"unit_amount":800,"currency":"eur","recurring":{"interval":"month","interval_count":1}}}]}} |
| 1594 | ]} |
| 1595 | EOF |
| 1596 | cat > "$FIXDIR/stripe-subs-2.json" <<'EOF' |
| 1597 | {"object":"list","has_more":false,"data":[ |
| 1598 | {"id":"sub_fixture_c","customer":"cus_fixture_c","status":"active","metadata":{"tier":"friends","space_id":"12"}, |
| 1599 | "items":{"data":[{"quantity":1,"price":{"unit_amount":4800,"currency":"eur","recurring":{"interval":"year","interval_count":1}}}]}} |
| 1600 | ]} |
| 1601 | EOF |
| 1602 | cat > "$FIXDIR/stripe-bal-1.json" <<'EOF' |
| 1603 | {"object":"list","has_more":false,"data":[ |
| 1604 | {"id":"txn_fixture_1","type":"charge","amount":400,"currency":"eur","created":1751328000}, |
| 1605 | {"id":"txn_fixture_2","type":"charge","amount":800,"currency":"eur","created":1751414400}, |
| 1606 | {"id":"txn_fixture_3","type":"refund","amount":-400,"currency":"eur","created":1751500800}, |
| 1607 | {"id":"txn_fixture_4","type":"payout","amount":-1200,"currency":"eur","created":1751587200} |
| 1608 | ]} |
| 1609 | EOF |
| 1610 | # Application fees with the charge expanded (Connect: spaces selling to |
| 1611 | # their members). Two EUR sales — a €9.00 PWYW sale with its 90c commission |
| 1612 | # (the real first-sale story) plus a €5.50 one — and a USD decoy that must |
| 1613 | # not count on either side of the currency filter. GMV 900+550=1450, |
| 1614 | # commission 90+55=145. The fee/charge/account ids are fixture markers that |
| 1615 | # must not survive aggregation. |
| 1616 | cat > "$FIXDIR/stripe-fees-1.json" <<'EOF' |
| 1617 | {"object":"list","has_more":false,"data":[ |
| 1618 | {"id":"fee_fixture_1","account":"acct_fixture_a","amount":90,"currency":"eur","refunded":false, |
| 1619 | "charge":{"id":"ch_fixture_1","object":"charge","amount":900,"currency":"eur","paid":true}}, |
| 1620 | {"id":"fee_fixture_2","account":"acct_fixture_a","amount":55,"currency":"eur","refunded":false, |
| 1621 | "charge":{"id":"ch_fixture_2","object":"charge","amount":550,"currency":"eur","paid":true}}, |
| 1622 | {"id":"fee_fixture_3","account":"acct_fixture_b","amount":120,"currency":"usd","refunded":false, |
| 1623 | "charge":{"id":"ch_fixture_3","object":"charge","amount":1200,"currency":"usd","paid":true}} |
| 1624 | ]} |
| 1625 | EOF |
| 1626 | |
| 1627 | build_fleet "$outdir/fleet.json" |
| 1628 | build_books "$outdir/books.json" |
| 1629 | |
| 1630 | # Self-checks: valid JSON, and no per-payer fixture value leaked through. |
| 1631 | sqj "SELECT CASE WHEN json_valid(readfile('$outdir/fleet.json')) AND json_valid(readfile('$outdir/books.json')) THEN 'ok' ELSE 'INVALID' END" \ |
| 1632 | | grep -qx ok || { echo "dry-run: output is not valid JSON" >&2; exit 1; } |
| 1633 | if grep -qE 'sub_fixture|cus_fixture|txn_fixture|fee_fixture|ch_fixture|acct_fixture|req_fixture|ce_fixture|wrkspc_fixture|dnsfix|hzfix|\.example' "$outdir/fleet.json" "$outdir/books.json"; then |
| 1634 | echo "dry-run: PRIVACY LEAK — a per-payer/provider-account id or hostname reached the output" >&2 |
| 1635 | exit 1 |
| 1636 | fi |
| 1637 | # Connect aggregates: GMV from the expanded charges, commission from the |
| 1638 | # fee amounts, both EUR-only (the USD decoy proves the filter). Skipped in |
| 1639 | # the degraded-books mode, where no income fields exist at all. |
| 1640 | if [ "${STATUS_DRY_BOOKS:-}" != unavailable ]; then |
| 1641 | [ "$(sqj "SELECT json_extract(readfile('$outdir/books.json'), '\$.connect.gmv_month_cents')")" = "1450" ] || |
| 1642 | { echo "dry-run: connect GMV must sum the expanded charges, EUR only (want 1450)" >&2; exit 1; } |
| 1643 | [ "$(sqj "SELECT json_extract(readfile('$outdir/books.json'), '\$.connect.commission_month_cents')")" = "145" ] || |
| 1644 | { echo "dry-run: connect commission must sum the fee amounts, EUR only (want 145)" >&2; exit 1; } |
| 1645 | fi |
| 1646 | # Capacity must publish as an abstract band word — a known word, and not a |
| 1647 | # single digit anywhere in the capacity subtree (no slots, GB, percentages). |
| 1648 | local lvl cap |
| 1649 | lvl=$(sqj "SELECT json_extract(readfile('$outdir/fleet.json'), '\$.capacity.level')") |
| 1650 | case "$lvl" in plenty|getting-full|at-capacity) ;; *) |
| 1651 | echo "dry-run: capacity.level '$lvl' is not one of the three band words" >&2 |
| 1652 | exit 1 ;; |
| 1653 | esac |
| 1654 | cap=$(sqj "SELECT json_extract(readfile('$outdir/fleet.json'), '\$.capacity')") |
| 1655 | if printf '%s' "$cap" | grep -q '[0-9]'; then |
| 1656 | echo "dry-run: PRIVACY — capacity subtree carries a number: $cap" >&2 |
| 1657 | exit 1 |
| 1658 | fi |
| 1659 | # Expenses: itemized by design, but the shape is still pinned — and the |
| 1660 | # cache seams must behave (cached_cost_cents). Default: all six providers |
| 1661 | # from fixtures/env (hetzner computed from its servers-API fixture), with |
| 1662 | # the API results freshly cached; hetzner-constant: same fixtures but the |
| 1663 | # operator constant wins; omit: no expenses key at all; stale-cache: all |
| 1664 | # four API fetches failed and the seeded same-month cache served |
| 1665 | # (aws 999 / hetzner 1399 / anthropic 888 / dnsimple 777). |
| 1666 | local aws_c hz_c an_c dn_c |
| 1667 | aws_c=$(sqj "SELECT json_extract(readfile('$outdir/books.json'), '\$.expenses.providers[0].month_to_date_cents')") |
| 1668 | case "${STATUS_DRY_EXPENSES:-}" in |
| 1669 | omit) |
| 1670 | [ "$(sqj "SELECT json_type(readfile('$outdir/books.json'), '\$.expenses') IS NULL")" = "1" ] || |
| 1671 | { echo "dry-run: expenses must be omitted entirely when no provider resolves" >&2; exit 1; } |
| 1672 | ;; |
| 1673 | stale-cache) |
| 1674 | hz_c=$(sqj "SELECT json_extract(readfile('$outdir/books.json'), '\$.expenses.providers[1].monthly_cents')") |
| 1675 | an_c=$(sqj "SELECT json_extract(readfile('$outdir/books.json'), '\$.expenses.providers[2].month_to_date_cents')") |
| 1676 | dn_c=$(sqj "SELECT json_extract(readfile('$outdir/books.json'), '\$.expenses.providers[3].month_to_date_cents')") |
| 1677 | [ "$aws_c" = "999" ] && [ "$hz_c" = "1399" ] && [ "$an_c" = "888" ] && [ "$dn_c" = "777" ] || |
| 1678 | { echo "dry-run: stale same-month cache must serve on fetch failure (got aws='$aws_c' hetzner='$hz_c' anthropic='$an_c' dnsimple='$dn_c')" >&2; exit 1; } |
| 1679 | ;; |
| 1680 | *) |
| 1681 | an_c=$(sqj "SELECT json_extract(readfile('$outdir/books.json'), '\$.expenses.providers[2].month_to_date_cents')") |
| 1682 | dn_c=$(sqj "SELECT json_extract(readfile('$outdir/books.json'), '\$.expenses.providers[4].month_to_date_cents')") |
| 1683 | [ "$aws_c" = "1234" ] || { echo "dry-run: CE fixture must yield 1234 cents (got '$aws_c')" >&2; exit 1; } |
| 1684 | [ "$an_c" = "250" ] || { echo "dry-run: anthropic fixture must yield 250 cents (got '$an_c')" >&2; exit 1; } |
| 1685 | [ "$dn_c" = "2450" ] || { echo "dry-run: dnsimple fixture must yield 2450 cents — collected only (got '$dn_c')" >&2; exit 1; } |
| 1686 | hz_c=$(sqj "SELECT json_extract(readfile('$outdir/books.json'), '\$.expenses.providers[1].monthly_cents')") |
| 1687 | if [ "${STATUS_DRY_EXPENSES:-}" = hetzner-constant ]; then |
| 1688 | [ "$hz_c" = "1400" ] || |
| 1689 | { echo "dry-run: hetzner constant must override the API fixture — 1400 cents (got '$hz_c')" >&2; exit 1; } |
| 1690 | # The constant must short-circuit the API entirely: nothing cached. |
| 1691 | [ "$(sqj "SELECT json_extract(readfile('$COSTS_CACHE'), '\$.hetzner') IS NULL")" = "1" ] || |
| 1692 | { echo "dry-run: hetzner constant must skip the API (a hetzner cache entry exists)" >&2; exit 1; } |
| 1693 | else |
| 1694 | [ "$hz_c" = "2094" ] || |
| 1695 | { echo "dry-run: hetzner servers fixture must yield 2094 cents — location-matched gross, rounded (got '$hz_c')" >&2; exit 1; } |
| 1696 | [ "$(sqj "SELECT json_extract(readfile('$COSTS_CACHE'), '\$.hetzner.month_to_date_cents')")" = "2094" ] || |
| 1697 | { echo "dry-run: hetzner API result must be cached" >&2; exit 1; } |
| 1698 | fi |
| 1699 | [ "$(sqj "SELECT json_extract(readfile('$outdir/books.json'), '\$.expenses.providers[3].to_date_cents')")" = "229000" ] || |
| 1700 | { echo "dry-run: claude.ai constant must yield 229000 cents, cumulative (to_date_cents)" >&2; exit 1; } |
| 1701 | [ "$(sqj "SELECT json_extract(readfile('$outdir/books.json'), '\$.expenses.providers[5].yearly_cents')")" = "9900" ] || |
| 1702 | { echo "dry-run: apple constant must yield 9900 cents, as billed (yearly)" >&2; exit 1; } |
| 1703 | [ -f "$COSTS_CACHE" ] && |
| 1704 | [ "$(sqj "SELECT json_extract(readfile('$COSTS_CACHE'), '\$.aws.month_to_date_cents')")" = "1234" ] && |
| 1705 | [ "$(sqj "SELECT json_extract(readfile('$COSTS_CACHE'), '\$.anthropic.month_to_date_cents')")" = "250" ] && |
| 1706 | [ "$(sqj "SELECT json_extract(readfile('$COSTS_CACHE'), '\$.dnsimple.month_to_date_cents')")" = "2450" ] || |
| 1707 | { echo "dry-run: metered results must be cached per provider" >&2; exit 1; } |
| 1708 | ;; |
| 1709 | esac |
| 1710 | |
| 1711 | # ---- carbon.json: the committed BUILT ledger (a fixed fixture here, so the |
| 1712 | # run fixture stays stable across real accruals) + a live-computed RUN block. |
| 1713 | # The run block folds from the Hetzner servers fixture (cpx31=4 + cx22=2 → 6 |
| 1714 | # vCPUs), the box region grid (Germany → 0/350/400 g/kWh), PUE 1.15, a pinned |
| 1715 | # 10-day span and the fixture registry's 6 instances. Only the expense modes |
| 1716 | # that stage the Hetzner servers fixture (default + hetzner-constant) give the |
| 1717 | # run its vCPU input; omit/stale-cache legitimately leave run absent (pending). |
| 1718 | cat > "$FIXDIR/carbon-built.json" <<'EOF' |
| 1719 | { |
| 1720 | "updated_at": "2026-07-11T00:00:00.000Z", |
| 1721 | "methodology": "https://github.com/elevenmessenger/messenger/blob/main/docs/comparison-carbon.md", |
| 1722 | "built": { |
| 1723 | "unit": "kg CO2e", |
| 1724 | "factors_mg_co2e_per_output_token": { "low": 0.1, "central": 0.55, "high": 3 }, |
| 1725 | "not_counted": "cache reads, cloud-side agent runs, and contributor machines not yet accrued", |
| 1726 | "entries": [ |
| 1727 | { "period_start": "2026-07-01T00:00:00.000Z", "period_end": "2026-07-11T00:00:00.000Z", |
| 1728 | "source": "ada-sketching", "basis": "measured", "sessions": 10, "output_tokens": 1000000, |
| 1729 | "fresh_input_tokens": 5000000, "cache_read_tokens": 90000000, |
| 1730 | "kg_low": 0.1, "kg_central": 0.6, "kg_high": 3, "note": "dry-run fixture" } |
| 1731 | ], |
| 1732 | "totals": { "output_tokens": 1000000, "kg_low": 0.1, "kg_central": 0.6, "kg_high": 3 } |
| 1733 | } |
| 1734 | } |
| 1735 | EOF |
| 1736 | local cvc cwt cgr cda ccu cps |
| 1737 | CARBON_SINCE=2026-07-02 |
| 1738 | STATUS_CARBON_DAYS=10 |
| 1739 | build_carbon "$outdir/carbon.json" "$FIXDIR/carbon-built.json" |
| 1740 | [ "$(sqj "SELECT json_valid(readfile('$outdir/carbon.json'))")" = "1" ] || |
| 1741 | { echo "dry-run: carbon.json is not valid JSON" >&2; exit 1; } |
| 1742 | [ "$(sqj "SELECT json_type(readfile('$outdir/carbon.json'), '\$.built')='object'")" = "1" ] || |
| 1743 | { echo "dry-run: carbon built ledger missing (built must be republished verbatim)" >&2; exit 1; } |
| 1744 | case "${STATUS_DRY_EXPENSES:-}" in |
| 1745 | omit|stale-cache) |
| 1746 | [ "$(sqj "SELECT json_type(readfile('$outdir/carbon.json'), '\$.run') IS NULL")" = "1" ] || |
| 1747 | { echo "dry-run: carbon run must be absent when no server inputs resolve (pending, never a fake zero)" >&2; exit 1; } |
| 1748 | ;; |
| 1749 | *) |
| 1750 | cvc=$(sqj "SELECT json_extract(readfile('$outdir/carbon.json'), '\$.run.vcpus')") |
| 1751 | cwt=$(sqj "SELECT json_extract(readfile('$outdir/carbon.json'), '\$.run.watts.central')") |
| 1752 | cgr=$(sqj "SELECT json_extract(readfile('$outdir/carbon.json'), '\$.run.grid_g_per_kwh.central')") |
| 1753 | cda=$(sqj "SELECT json_extract(readfile('$outdir/carbon.json'), '\$.run.daily_kg.central')") |
| 1754 | ccu=$(sqj "SELECT json_extract(readfile('$outdir/carbon.json'), '\$.run.cumulative_kg.central')") |
| 1755 | cps=$(sqj "SELECT json_extract(readfile('$outdir/carbon.json'), '\$.run.per_space_daily_kg.central')") |
| 1756 | # daily central = 6 × 3.5 W × 24h × 350 g/kWh × 1.15 / 1e6 + 0.001 + 0.005 |
| 1757 | # = 0.20886 kg; cumulative = ×10 days = 2.0886; per space = ÷6. |
| 1758 | [ "$cvc" = "6" ] && [ "$cwt" = "21" ] && [ "$cgr" = "350" ] && |
| 1759 | [ "$cda" = "0.20886" ] && [ "$ccu" = "2.0886" ] && [ "$cps" = "0.03481" ] || |
| 1760 | { echo "dry-run: carbon run math (got vcpus=$cvc watts=$cwt grid=$cgr daily=$cda cumulative=$ccu per_space=$cps)" >&2; exit 1; } |
| 1761 | ;; |
| 1762 | esac |
| 1763 | # Privacy: no fixture marker, hostname, socket path, or IP may reach carbon.json. |
| 1764 | if grep -qE 'hzfix|ada-sketching@|\.example|\.sock|[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' "$outdir/carbon.json"; then |
| 1765 | echo "dry-run: PRIVACY LEAK — a hostname/IP reached carbon.json" >&2 |
| 1766 | exit 1 |
| 1767 | fi |
| 1768 | |
| 1769 | echo "--- fleet.json ---"; cat "$outdir/fleet.json" |
| 1770 | echo "--- books.json ---"; cat "$outdir/books.json" |
| 1771 | echo "--- carbon.json ---"; cat "$outdir/carbon.json" |
| 1772 | echo "dry-run ok: valid JSON, no hostnames, no per-payer fields, capacity abstract ($lvl), expenses ${STATUS_DRY_EXPENSES:-itemized}, carbon run ${cda:-absent}" |
| 1773 | } |
| 1774 | |
| 1775 | case "${1:-}" in |
| 1776 | install) install_units ;; |
| 1777 | --dry-run) [ -n "${2:-}" ] || { echo "usage: $0 --dry-run <outdir>" >&2; exit 1; } |
| 1778 | dry_run "$2" ;; |
| 1779 | "") publish ;; |
| 1780 | *) echo "usage: $0 [install | --dry-run <outdir>]" >&2; exit 1 ;; |
| 1781 | esac |
| 1782 | |
amadan