| 1 | #!/usr/bin/env sh |
| 2 | # 🤖 ops canary — per-session test instances (CLAUDE.md § Per-session canaries), |
| 3 | # with the rules that used to live only in prose encoded as behaviour: |
| 4 | # |
| 5 | # ./deploy/ops canary create <slug> # test-<slug>.11ch.at, flagged --canary |
| 6 | # ./deploy/ops canary deploy <slug> # build THIS branch -> stage -> deploy --pin |
| 7 | # ./deploy/ops canary reset <slug> # wipe back to first-run (iterate onboarding) |
| 8 | # ./deploy/ops canary destroy <slug> # reset (archive DB) + remove --stop |
| 9 | # ./deploy/ops canary list # every canary + pinned row on the box |
| 10 | # |
| 11 | # The rules, and where each is enforced: |
| 12 | # - Canaries are test-<slug> on the wildcard domain — no DNS step, no cert |
| 13 | # wait, visually distinct from real member spaces (this script only ever |
| 14 | # forms test-<slug>.<domain> hostnames). |
| 15 | # - Creation passes --canary: the registry flag is a DELIBERATE signal, never |
| 16 | # inferred from the hostname — it is what lets `lchat reset` wipe the DB at |
| 17 | # destroy time while refusing to touch any real instance. |
| 18 | # - Deploys always --pin. An unpinned canary is moved onto the channel build |
| 19 | # by the hourly auto-updater — out from under the session, silently. |
| 20 | # - Destroy wipes the data. `lchat remove` deliberately leaves /data behind |
| 21 | # (right for real instances, wrong for a canary), and the hardened sudoers |
| 22 | # allowlist has no `rm` — so the encoded path is `lchat reset` (archives |
| 23 | # the DB to /data/trash, canary-gated) then `remove --stop` (which also |
| 24 | # disables any bot/hooks sidecar units). |
| 25 | # - Admin tokens for canaries are stored NOWHERE (not 1Password) on purpose: |
| 26 | # everything about a canary dies with its session. |
| 27 | # - Fleet-wide operations skip AUTO=pinned rows — each is someone's live |
| 28 | # canary. `list` shows them so a sweep can check first. |
| 29 | set -eu |
| 30 | |
| 31 | here="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" |
| 32 | root="$(CDPATH= cd -- "$here/.." && pwd)" |
| 33 | # The box canaries live on — an ssh alias; user/hostname come from |
| 34 | # ~/.ssh/config (box specifics in CLAUDE.local.md). Canaries live on the DEV |
| 35 | # MESH by default (Paul, 2026-07-22): throwaway test instances don't belong on |
| 36 | # the box real members live on, and they'd eat its instance cap. Helsinki is |
| 37 | # the dev default because build-linux.sh emits amd64 (Dublin is arm64). Alias |
| 38 | # to add once per machine: |
| 39 | # Host elevendev-fi |
| 40 | # HostName 157.180.123.183 |
| 41 | # User root |
| 42 | # IdentityFile ~/.ssh/eleven-dev-box |
| 43 | # Prod (LCHAT_CANARY_BOX=elevenbox LCHAT_CANARY_DOMAIN=11ch.at) is the |
| 44 | # deliberate exception, not the default. |
| 45 | # NB dev has no wildcard cert BY DESIGN (the DNSimple token never goes on a |
| 46 | # dev box), so each dev canary host burns one per-host HTTP-01 cert from the |
| 47 | # shared 50/week 11ch.at LE budget — fine at canary cadence, not for bursts. |
| 48 | BOX="${LCHAT_CANARY_BOX:-elevendev-fi}" |
| 49 | DOMAIN="${LCHAT_CANARY_DOMAIN:-fi.dev.11ch.at}" |
| 50 | # The home server a canary's instance points at (LCHAT_HOME_ORIGIN — the |
| 51 | # sign-in/enrol origin baked into its pages). Dev canaries must ride the DEV |
| 52 | # home: without this, add-instance leaves the env empty and the binary falls |
| 53 | # back to the compiled PRODUCTION home — a dev canary whose sign-in round |
| 54 | # trips hit prod (old code, real accounts). Keyed off the domain so the |
| 55 | # deliberate prod exception (LCHAT_CANARY_DOMAIN=11ch.at) keeps the compiled |
| 56 | # default; LCHAT_CANARY_HOME_ORIGIN overrides either way. |
| 57 | case "$DOMAIN" in |
| 58 | *.dev.*) HOME_ORIGIN="${LCHAT_CANARY_HOME_ORIGIN:-https://home.dev.elevenmessenger.com}" ;; |
| 59 | *) HOME_ORIGIN="${LCHAT_CANARY_HOME_ORIGIN:-}" ;; |
| 60 | esac |
| 61 | LCHAT=/opt/lchat/lchat |
| 62 | |
| 63 | usage() { |
| 64 | echo "usage: ops canary create|deploy|reset|destroy <slug> | ops canary list" >&2 |
| 65 | echo " slug: lowercase letters/digits/hyphens — your branch or feature name" >&2 |
| 66 | exit 2 |
| 67 | } |
| 68 | |
| 69 | # Canary links (the admin link's token, the setup/onboarding link's one-time |
| 70 | # secret) must never land in a session transcript — transcripts are public. So |
| 71 | # every verb that mints one writes the full output to ~/<host>.link ON THE BOX |
| 72 | # and this filter redacts the URL fragments/tokens from what echoes locally. |
| 73 | # Fetch the real links from your own terminal: ssh <box> cat <host>.link |
| 74 | redact_secrets() { sed -E 's/#[^[:space:]]+/#<redacted>/g'; } |
| 75 | |
| 76 | cmd="${1:-}"; slug="${2:-}" |
| 77 | |
| 78 | host_for() { |
| 79 | # Everything below derives the hostname from here, so this script is |
| 80 | # structurally incapable of touching a non-test- host. |
| 81 | case "$1" in |
| 82 | *[!a-z0-9-]*|-*|"") echo "ops canary: bad slug '$1' (lowercase letters/digits/hyphens)" >&2; exit 2 ;; |
| 83 | esac |
| 84 | echo "test-$1.$DOMAIN" |
| 85 | } |
| 86 | |
| 87 | case "$cmd" in |
| 88 | |
| 89 | create) |
| 90 | host="$(host_for "$slug")" |
| 91 | echo "ops canary: creating $host on $BOX (flagged --canary)" |
| 92 | # add-instance prints the admin link + the one-time setup link — both carry |
| 93 | # secrets, so the full output is staged box-side and echoed here redacted. |
| 94 | homeflag="" |
| 95 | [ -n "$HOME_ORIGIN" ] && homeflag="--home-origin '$HOME_ORIGIN'" |
| 96 | out="$(ssh "$BOX" "sudo -n $LCHAT add-instance --canary $homeflag '$host' > \$HOME/$host.link 2>&1; st=\$?; cat \$HOME/$host.link; exit \$st")" || { |
| 97 | printf '%s\n' "$out" | redact_secrets >&2 |
| 98 | exit 1 |
| 99 | } |
| 100 | printf '%s\n' "$out" | redact_secrets |
| 101 | cat <<EOF |
| 102 | |
| 103 | Canary up. Its links live in $host.link on the box, nowhere else — canary |
| 104 | tokens deliberately die with the session (don't vault them). |
| 105 | fetch links: ssh $BOX cat $host.link # run in YOUR terminal |
| 106 | next: ./deploy/ops canary deploy $slug # put THIS branch's build on it |
| 107 | reset: ./deploy/ops canary reset $slug # back to first-run, same token |
| 108 | tear down: ./deploy/ops canary destroy $slug # when the work merges |
| 109 | EOF |
| 110 | ;; |
| 111 | |
| 112 | deploy) |
| 113 | host="$(host_for "$slug")" |
| 114 | ver="$(git -C "$root" rev-parse --short HEAD)" |
| 115 | if ! git -C "$root" diff --quiet || ! git -C "$root" diff --cached --quiet; then |
| 116 | ver="${ver}-dirty" |
| 117 | echo "ops canary: working tree has uncommitted changes — building anyway ($ver)" >&2 |
| 118 | fi |
| 119 | |
| 120 | echo "ops canary: building $ver for linux/amd64" |
| 121 | out="$(mktemp -t lchat-canary.XXXXXX)" |
| 122 | trap 'rm -f "$out"' EXIT |
| 123 | cd "$root" # build-linux.sh compiles the package at the current directory |
| 124 | OUT="$out" "$here/build-linux.sh" "$ver" |
| 125 | |
| 126 | # A per-slug upload name so parallel sessions' uploads never trample each |
| 127 | # other in the shared incoming dir (a fixed name is a race). |
| 128 | up="/opt/lchat/incoming/lchat-upload-$slug" |
| 129 | ssh "$BOX" 'mkdir -p /opt/lchat/incoming' |
| 130 | scp -q "$out" "$BOX:$up" |
| 131 | staged="$(ssh "$BOX" "sudo -n $LCHAT stage -file '$up' && rm -f '$up'")" |
| 132 | echo " $staged" |
| 133 | staged_ver="$(echo "$staged" | awk '{print $2}')" |
| 134 | [ "$staged_ver" = "$ver" ] || { |
| 135 | echo "ops canary: staged '$staged_ver' != built '$ver' — aborting" >&2; exit 1; } |
| 136 | |
| 137 | # --pin is not optional: without it the hourly auto-updater moves the canary |
| 138 | # back onto its channel build, out from under the session. deploy itself |
| 139 | # health-checks /api/version and auto-rolls back a bad build. |
| 140 | ssh "$BOX" "sudo -n $LCHAT deploy '$host' --version '$ver' --pin" |
| 141 | cat <<EOF |
| 142 | |
| 143 | $host is running $ver (pinned — the auto-updater will leave it alone). |
| 144 | iterate: commit, then ./deploy/ops canary deploy $slug |
| 145 | step back: ssh $BOX sudo -n $LCHAT rollback $host |
| 146 | EOF |
| 147 | ;; |
| 148 | |
| 149 | reset) |
| 150 | host="$(host_for "$slug")" |
| 151 | echo "ops canary: resetting $host in place (DB archived to /data/trash; env/token kept)" |
| 152 | # `lchat reset` is canary-gated (a real instance refuses), restarts the |
| 153 | # instance on an empty DB, and mints a fresh one-time onboarding link — the |
| 154 | # loop for iterating setup/onboarding scripts without a new host (each new |
| 155 | # dev host would burn another cert from the shared 50/week LE budget). |
| 156 | out="$(ssh "$BOX" "sudo -n $LCHAT reset '$host' --yes > \$HOME/$host.link 2>&1; st=\$?; cat \$HOME/$host.link; exit \$st")" || { |
| 157 | printf '%s\n' "$out" | redact_secrets >&2 |
| 158 | echo "ops canary: reset refused — if $host is genuinely a throwaway, flag it:" >&2 |
| 159 | echo " ssh $BOX sudo -n $LCHAT canary $host --on # then re-run reset" >&2 |
| 160 | exit 1 |
| 161 | } |
| 162 | printf '%s\n' "$out" | redact_secrets |
| 163 | cat <<EOF |
| 164 | |
| 165 | $host is back at first-run (same admin token, same pinned build). A bot |
| 166 | sidecar, if the canary had one, must be re-provisioned — its identities died |
| 167 | with the DB (add-bot again after the new admin claims the space). |
| 168 | fetch the fresh onboarding link: ssh $BOX cat $host.link # YOUR terminal |
| 169 | EOF |
| 170 | ;; |
| 171 | |
| 172 | destroy) |
| 173 | host="$(host_for "$slug")" |
| 174 | echo "ops canary: destroying $host on $BOX" |
| 175 | # reset first (it needs the registry row remove deletes): archives the DB + |
| 176 | # blobs to /data/trash — canary-gated, so a host that was never flagged |
| 177 | # --canary refuses here rather than wiping. That refusal is the safety |
| 178 | # model working; flag it by hand only if it is genuinely a throwaway. |
| 179 | out="$(ssh "$BOX" "sudo -n $LCHAT reset '$host' --yes")" || { |
| 180 | printf '%s\n' "$out" | redact_secrets >&2 |
| 181 | echo "ops canary: reset refused — if $host is genuinely a throwaway, flag it:" >&2 |
| 182 | echo " ssh $BOX sudo -n $LCHAT canary $host --on # then re-run destroy" >&2 |
| 183 | exit 1 |
| 184 | } |
| 185 | printf '%s\n' "$out" | redact_secrets |
| 186 | # remove --stop: drops the route (and in.<host>), stops + disables the |
| 187 | # instance unit and any bot/hooks sidecar units. The staged link file goes |
| 188 | # with it — its secrets are dead, and a stale file would mislead the next |
| 189 | # session to reuse this slug. |
| 190 | ssh "$BOX" "sudo -n $LCHAT remove '$host' --stop; rm -f \$HOME/$host.link" |
| 191 | cat <<EOF |
| 192 | |
| 193 | $host destroyed. Its DB is archived in /data/trash (the reaper's trash purge |
| 194 | deletes it for good); the leftover env file is harmless and is overwritten if |
| 195 | the slug is ever recreated. |
| 196 | Remember the session's why-entry if you haven't yet: ./deploy/ops ledger "..." |
| 197 | EOF |
| 198 | ;; |
| 199 | |
| 200 | list) |
| 201 | echo "canaries + pinned rows on $BOX (a pinned row is someone's live canary —" |
| 202 | echo "fleet-wide operations must leave it alone):" |
| 203 | echo |
| 204 | ssh "$BOX" "sudo -n $LCHAT instances" | awk 'NR==1 || /test-|canary/' |
| 205 | echo |
| 206 | ssh "$BOX" "sudo -n $LCHAT update status" | awk 'NR==1 || /pinned/' |
| 207 | ;; |
| 208 | |
| 209 | *) usage ;; |
| 210 | esac |
| 211 | |
amadan