| 1 | #!/usr/bin/env sh |
| 2 | # 🤖 doctor — one read-only preflight for every credential and access path an |
| 3 | # operator needs. Answers "can I actually deploy right now?" in one command, |
| 4 | # instead of six documents. Verb one of the ops surface (`./deploy/ops`) — as |
| 5 | # verbs land there, their prerequisites get a check here, so the preflight |
| 6 | # stays honest. |
| 7 | # |
| 8 | # ./deploy/ops doctor # everything (doctor.sh runs standalone too) |
| 9 | # ./deploy/ops doctor release # one section: tools|release|fleet|dev |
| 10 | # ./deploy/ops doctor --no-ssh # skip the box checks (see the note below) |
| 11 | # ./deploy/ops doctor --no-unlock # never prompt (it prompts to unlock by |
| 12 | # # default, so the key check is real) |
| 13 | # |
| 14 | # Why this exists: the knowledge for this was spread across CLAUDE.md, |
| 15 | # CLAUDE.local.md, docs/server-releases.md, docs/multi-edge-dev-cutover.md, the |
| 16 | # vault's RUNBOOK.md, and raw tofu source — and drifted from reality in four |
| 17 | # places at once. Prose drifts silently; a script that runs drifts loudly. Every |
| 18 | # check below is a fact about *this machine right now*, not a claim in a doc. |
| 19 | # |
| 20 | # Two rules this script holds to, because it touches real credentials: |
| 21 | # 1. It NEVER prints a secret. Checks assert reachability/permission and |
| 22 | # discard output — a fetched credential is tested for existence, never |
| 23 | # echoed. The one interactive moment (the Bitwarden unlock) captures the |
| 24 | # session token into a variable and never displays it. |
| 25 | # 2. It NEVER mutates anything — no deploys, no writes, no unlocks. Every |
| 26 | # remote command is a read (`lchat instances`, `id`, `groups`). |
| 27 | # |
| 28 | # NOTE: the `fleet` section opens real operator SSH sessions, which incur a |
| 29 | # why-entry in the public access ledger (web/status/ssh/, RUNBOOK.md § Box |
| 30 | # access). Use --no-ssh when you only want the local picture. |
| 31 | set -u |
| 32 | |
| 33 | section="${1:-all}" |
| 34 | no_ssh=no |
| 35 | unlock=yes |
| 36 | for a in "$@"; do |
| 37 | [ "$a" = "--no-ssh" ] && no_ssh=yes |
| 38 | [ "$a" = "--no-unlock" ] && unlock=no |
| 39 | done |
| 40 | case "$section" in --*) section=all ;; esac |
| 41 | |
| 42 | root="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" |
| 43 | # The Bitwarden item names checked below — the vault contract. |
| 44 | # shellcheck source=deploy/vault-items.sh |
| 45 | . "$root/deploy/vault-items.sh" |
| 46 | # shellcheck source=deploy/bw-lib.sh |
| 47 | . "$root/deploy/bw-lib.sh" |
| 48 | # Only used to point at gen-fleet-conf.sh when a tunnel is down. |
| 49 | VAULT="${LCHAT_VAULT_DIR:-$HOME/elevenmessenger/secrets-vault}" |
| 50 | # Box aliases from ~/.ssh/config; `ssh -G` resolves user+hostname, so the ssh |
| 51 | # config stays the single source of truth for how to reach a box. |
| 52 | BOXES="${LCHAT_BOXES:-elevenbox kepler}" |
| 53 | RELEASES_REPO="${LCHAT_RELEASES_REPO:-elevenmessenger/releases}" |
| 54 | |
| 55 | fails=0 |
| 56 | warns=0 |
| 57 | |
| 58 | # --- output helpers ------------------------------------------------------- |
| 59 | # Two columns: a fixed-width label, then a verdict. Fix hints are indented |
| 60 | # under the failing line so the next action is always adjacent to the problem. |
| 61 | ok() { printf ' \033[32mok\033[0m %s\n' "$1"; } |
| 62 | bad() { printf ' \033[31mFAIL\033[0m %s\n' "$1"; fails=$((fails + 1)); } |
| 63 | warn() { printf ' \033[33mwarn\033[0m %s\n' "$1"; warns=$((warns + 1)); } |
| 64 | fix() { printf ' → %s\n' "$1"; } |
| 65 | head_() { printf '\n\033[1m%s\033[0m\n' "$1"; } |
| 66 | have() { command -v "$1" >/dev/null 2>&1; } |
| 67 | |
| 68 | want() { [ "$section" = all ] || [ "$section" = "$1" ]; } |
| 69 | |
| 70 | # --- tools ---------------------------------------------------------------- |
| 71 | check_tools() { |
| 72 | head_ "Local tooling" |
| 73 | # Each tool is annotated with what breaks without it, so a missing one is |
| 74 | # self-explanatory rather than a bare name. |
| 75 | for pair in \ |
| 76 | "bw:release signing key (Bitwarden)" \ |
| 77 | "gh:publishing to the releases repo" \ |
| 78 | "jq:parsing manifests + bw/gh output" \ |
| 79 | "go:building the release binaries" \ |
| 80 | "tofu:the dev-mesh infrastructure" \ |
| 81 | "aws:the dev-mesh + release S3 mirror" \ |
| 82 | "ssh:reaching every box" |
| 83 | do |
| 84 | t="${pair%%:*}"; why="${pair#*:}" |
| 85 | if have "$t"; then ok "$t"; else |
| 86 | bad "$t — needed for $why" |
| 87 | fix "brew install $t" |
| 88 | fi |
| 89 | done |
| 90 | } |
| 91 | |
| 92 | # --- release publishing --------------------------------------------------- |
| 93 | # Confirm the item is reachable WITHOUT printing it: we only look at whether |
| 94 | # the fetch succeeded, never at the value. |
| 95 | check_signing_item() { |
| 96 | if bw_run 30 get item "$BW_ITEM_SIGNING_KEY" \ |
| 97 | >/dev/null 2>&1; then |
| 98 | ok "signing key item is present" |
| 99 | else |
| 100 | bad "signing key item not found in this vault" |
| 101 | fix "check it is shared to your org collection (BITWARDEN-MIGRATION.md)" |
| 102 | fi |
| 103 | } |
| 104 | |
| 105 | check_release() { |
| 106 | head_ "Publishing a release" |
| 107 | |
| 108 | if ! have bw; then |
| 109 | bad "bw not installed — cannot reach the signing key" |
| 110 | fix "brew install bitwarden-cli" |
| 111 | else |
| 112 | st="$(bw_run 20 status 2>/dev/null | tr ',' '\n' | sed -n 's/.*"status":"\([a-z]*\)".*/\1/p')" |
| 113 | case "$st" in |
| 114 | unlocked) |
| 115 | ok "bitwarden unlocked" |
| 116 | check_signing_item ;; |
| 117 | locked) |
| 118 | # Unlock by default: "locked, so I can't tell you" is a non-answer, and |
| 119 | # the whole point is to find out whether the key is actually reachable. |
| 120 | # The session is this process's only — a child cannot export BW_SESSION |
| 121 | # back to the calling shell — so this deepens the check and nothing |
| 122 | # more. Without a tty there is nobody to prompt, so degrade instead of |
| 123 | # hanging; --no-unlock forces that path deliberately. |
| 124 | if [ "$unlock" = yes ] && [ -t 0 ]; then |
| 125 | printf ' unlocking bitwarden to verify the signing key\n' |
| 126 | # Our prompt, never bw's; watchdogged (bw-lib.sh) — bw's own |
| 127 | # prompt inside $(...) is invisible and reads as a hang. |
| 128 | BW_SESSION="$(bw_unlock_session)" || BW_SESSION="" |
| 129 | export BW_SESSION |
| 130 | if [ -n "${BW_SESSION:-}" ]; then |
| 131 | ok "bitwarden unlocked (this run only)" |
| 132 | check_signing_item |
| 133 | else |
| 134 | bad "unlock failed — signing key not verified" |
| 135 | fix "wrong master password, or run 'bw unlock' by hand to see why" |
| 136 | fi |
| 137 | else |
| 138 | warn "bitwarden is locked — signing key not verified" |
| 139 | fix "bw unlock # then paste the 'export BW_SESSION=...' line it prints, and re-run" |
| 140 | fi ;; |
| 141 | *) |
| 142 | bad "bitwarden not logged in" |
| 143 | fix "bw login # EU vault: https://vault.bitwarden.eu" ;; |
| 144 | esac |
| 145 | fi |
| 146 | |
| 147 | if have gh && gh auth status >/dev/null 2>&1; then |
| 148 | ok "gh authenticated" |
| 149 | push="$(gh api "repos/$RELEASES_REPO" --jq .permissions.push 2>/dev/null)" |
| 150 | if [ "$push" = "true" ]; then ok "write access to $RELEASES_REPO"; else |
| 151 | bad "no write access to $RELEASES_REPO" |
| 152 | fix "ask for push rights on the releases repo" |
| 153 | fi |
| 154 | else |
| 155 | bad "gh not authenticated" |
| 156 | fix "gh auth login" |
| 157 | fi |
| 158 | |
| 159 | # release-server.sh refuses to build from a dirty tree; surface that here |
| 160 | # rather than 20 minutes into a release. |
| 161 | if [ -n "$(git -C "$root" status --porcelain 2>/dev/null)" ]; then |
| 162 | warn "working tree is dirty — release-server.sh will refuse to publish" |
| 163 | fix "commit or stash, and publish from a clean main" |
| 164 | else |
| 165 | ok "working tree is clean" |
| 166 | fi |
| 167 | } |
| 168 | |
| 169 | # --- fleet boxes ---------------------------------------------------------- |
| 170 | check_fleet() { |
| 171 | head_ "Fleet boxes (WireGuard + SSH + sudo)" |
| 172 | |
| 173 | if [ "$no_ssh" = yes ]; then |
| 174 | warn "skipped (--no-ssh)" |
| 175 | return |
| 176 | fi |
| 177 | |
| 178 | for box in $BOXES; do |
| 179 | # ssh -G resolves the config the way ssh itself will, so we test what will |
| 180 | # actually happen rather than re-parsing ~/.ssh/config by hand. |
| 181 | hostname="$(ssh -G "$box" 2>/dev/null | sed -n 's/^hostname //p' | head -1)" |
| 182 | user="$(ssh -G "$box" 2>/dev/null | sed -n 's/^user //p' | head -1)" |
| 183 | # One connection attempt tells us both things. A tunnel that is down and a |
| 184 | # key that is refused look identical from a bare exit code, so classify the |
| 185 | # error text instead — and skip ping, whose -W flag means milliseconds on |
| 186 | # macOS and seconds on Linux. |
| 187 | err="$(ssh -o ConnectTimeout=8 -o BatchMode=yes "$box" true 2>&1)" |
| 188 | if [ -z "$err" ]; then |
| 189 | ok "$box — tunnel up, ssh as '$user' ($hostname)" |
| 190 | else |
| 191 | case "$err" in |
| 192 | *"Permission denied"*) |
| 193 | bad "$box — reachable, but ssh as '$user' was refused" |
| 194 | fix "confirm which key the box authorises; a key not at a default" |
| 195 | fix "path needs IdentityFile + IdentitiesOnly in ~/.ssh/config" ;; |
| 196 | *"Host key verification failed"*) |
| 197 | # The box answered — BatchMode just can't accept a first-seen host |
| 198 | # key. Distinct from unreachable/refused, and self-serve fixable. |
| 199 | bad "$box — reachable, but its host key isn't in known_hosts yet" |
| 200 | fix "one-off interactive: ssh $box # accept the host key, then re-run" ;; |
| 201 | *"Could not resolve hostname"*) |
| 202 | # ssh -G echoes an unknown alias back rather than failing, so an |
| 203 | # absent Host block surfaces here as a resolve error. |
| 204 | bad "$box — no Host block in ~/.ssh/config (or DNS failure)" |
| 205 | fix "add a Host block for $box, or set LCHAT_BOXES" ;; |
| 206 | *"timed out"*|*"No route"*|*"Network is unreachable"*) |
| 207 | bad "$box — $hostname unreachable (WireGuard down?)" |
| 208 | fix "bring the tunnel up, then re-run; regenerate it with" |
| 209 | fix "cd $VAULT && ./gen-fleet-conf.sh > ~/Downloads/fleet.conf" ;; |
| 210 | *) |
| 211 | bad "$box — $(printf '%s' "$err" | head -1)" ;; |
| 212 | esac |
| 213 | continue |
| 214 | fi |
| 215 | |
| 216 | # lchat-operators is what the sudoers allowlist keys off; without it the |
| 217 | # ssh login succeeds but every privileged command fails. |
| 218 | if ssh -o BatchMode=yes "$box" 'id -nG' 2>/dev/null \ |
| 219 | | tr ' ' '\n' | grep -qx lchat-operators; then |
| 220 | ok "$box — in lchat-operators" |
| 221 | elif [ "$user" = root ]; then |
| 222 | ok "$box — root (allowlist not applicable)" |
| 223 | else |
| 224 | bad "$box — not in lchat-operators" |
| 225 | fix "ask an operator to add you to the group" |
| 226 | fi |
| 227 | |
| 228 | if ssh -o BatchMode=yes "$box" \ |
| 229 | 'sudo -n /opt/lchat/lchat instances' >/dev/null 2>&1; then |
| 230 | ok "$box — sudo allowlist works (can deploy)" |
| 231 | else |
| 232 | bad "$box — 'sudo -n lchat' denied" |
| 233 | fix "check /etc/sudoers.d/lchat-operators (deploy/sudoers-operators)" |
| 234 | fi |
| 235 | done |
| 236 | } |
| 237 | |
| 238 | # --- dev mesh ------------------------------------------------------------- |
| 239 | check_dev() { |
| 240 | head_ "Dev mesh (Dublin + Helsinki)" |
| 241 | |
| 242 | # The Dublin box authorises exactly one key, committed in compute.tf: the |
| 243 | # SHARED box key (its comment is the box's name, not a person's) — created |
| 244 | # at provisioning, never vaulted, so copies exist only on operator machines. |
| 245 | keyfile="$HOME/.ssh/eleven-dev-box" |
| 246 | if [ -f "$keyfile" ]; then |
| 247 | ok "eleven-dev-box key present" |
| 248 | else |
| 249 | bad "no $keyfile" |
| 250 | fix "the shared dev-box key — not in any vault; ask an operator who holds it" |
| 251 | fi |
| 252 | |
| 253 | # What `ops dev deploy` actually needs: a working SSH session to each box. |
| 254 | # Same one-attempt classification as the fleet section — and here a timeout |
| 255 | # is USUALLY the single-/32 security group (named by the CIDR check below), |
| 256 | # not a dead box. |
| 257 | if [ "$no_ssh" = yes ]; then |
| 258 | warn "dev box ssh checks skipped (--no-ssh)" |
| 259 | else |
| 260 | for spec in "ie:${LCHAT_DEV_BOX_IE:-ubuntu@54.154.95.205}" \ |
| 261 | "fi:${LCHAT_DEV_BOX_FI:-root@157.180.123.183}"; do |
| 262 | node="${spec%%:*}"; dest="${spec#*:}" |
| 263 | if [ "$node" = ie ] && [ -f "$keyfile" ]; then |
| 264 | err="$(ssh -i "$keyfile" -o ConnectTimeout=8 -o BatchMode=yes "$dest" true 2>&1)" |
| 265 | else |
| 266 | err="$(ssh -o ConnectTimeout=8 -o BatchMode=yes "$dest" true 2>&1)" |
| 267 | fi |
| 268 | if [ -z "$err" ]; then |
| 269 | ok "dev $node — ssh works ($dest)" |
| 270 | else |
| 271 | case "$err" in |
| 272 | *"Permission denied"*) |
| 273 | bad "dev $node — reachable, but ssh as '$dest' was refused" |
| 274 | fix "ie wants ~/.ssh/eleven-dev-box; fi wants the 1Password hetzner key" ;; |
| 275 | *"Host key verification failed"*) |
| 276 | bad "dev $node — reachable, but its host key isn't in known_hosts yet" |
| 277 | fix "one-off interactive: ssh $dest # accept the host key, then re-run" ;; |
| 278 | *"timed out"*|*"No route"*|*"Network is unreachable"*) |
| 279 | bad "dev $node — unreachable; if the IP check below fails, that is why" |
| 280 | fix "your IP must be in the security group's allowlist (ssh_ingress_cidr)" ;; |
| 281 | *) |
| 282 | bad "dev $node — $(printf '%s' "$err" | head -1)" ;; |
| 283 | esac |
| 284 | fi |
| 285 | done |
| 286 | fi |
| 287 | |
| 288 | # ops dev pulls AWS_*/DNSIMPLE_* from the per-operator dev agent item in |
| 289 | # Bitwarden itself, so a standing profile is a convenience, not a |
| 290 | # prerequisite — only "no route to any credential" is a failure. |
| 291 | if [ -n "${AWS_ACCESS_KEY_ID:-}" ]; then |
| 292 | ok "AWS credentials in the environment" |
| 293 | elif aws configure list-profiles 2>/dev/null | grep -qx eleven-dev; then |
| 294 | ok "aws profile 'eleven-dev' configured" |
| 295 | elif have bw; then |
| 296 | warn "no standing AWS auth — ops dev will pull the dev agent creds from Bitwarden per run" |
| 297 | else |
| 298 | bad "no AWS auth at all (no env creds, no 'eleven-dev' profile, no bw)" |
| 299 | fix "install bw, or seed the profile from your dev agent item (docs/secrets.md § AWS)" |
| 300 | fi |
| 301 | |
| 302 | # SSH ingress is a short per-operator /32 allowlist (compute.tf, |
| 303 | # ssh_ingress_cidr), so being outside it looks exactly like the box being |
| 304 | # down. Say which it is. Anchored to the variable block — every a.b.c.d/nn |
| 305 | # inside it counts, so keep literal IPs out of its description text. |
| 306 | cidrs="$(awk '/variable "ssh_ingress_cidr"/,/^}/' \ |
| 307 | "$root/infra/dev-mesh/compute.tf" 2>/dev/null \ |
| 308 | | grep -oE '[0-9]+(\.[0-9]+){3}/[0-9]+')" |
| 309 | if [ -n "$cidrs" ]; then |
| 310 | mine="$(curl -s -m5 https://api.ipify.org 2>/dev/null)" |
| 311 | if [ -n "$mine" ] && printf '%s\n' "$cidrs" | sed 's|/.*||' | grep -qxF "$mine"; then |
| 312 | ok "your IP ($mine) is in the dev security group's allowlist" |
| 313 | else |
| 314 | bad "your IP (${mine:-unknown}) is not in [$(printf '%s' "$cidrs" | tr '\n' ' ')] — SSH will hang, not refuse" |
| 315 | fix "add your /32 to var.ssh_ingress_cidr (a PR), then: ops dev apply" |
| 316 | fix "removing someone else's /32 revokes their access — only with their say-so" |
| 317 | fi |
| 318 | fi |
| 319 | } |
| 320 | |
| 321 | # --- run ------------------------------------------------------------------ |
| 322 | printf '\033[1mlchat operator preflight\033[0m — read-only; no secrets printed\n' |
| 323 | |
| 324 | want tools && check_tools |
| 325 | want release && check_release |
| 326 | want fleet && check_fleet |
| 327 | want dev && check_dev |
| 328 | |
| 329 | printf '\n' |
| 330 | if [ "$fails" -eq 0 ] && [ "$warns" -eq 0 ]; then |
| 331 | printf '\033[32mAll checks passed.\033[0m\n' |
| 332 | elif [ "$fails" -eq 0 ]; then |
| 333 | printf '\033[33m%s warning(s), no failures.\033[0m\n' "$warns" |
| 334 | else |
| 335 | printf '\033[31m%s failure(s), %s warning(s).\033[0m\n' "$fails" "$warns" |
| 336 | fi |
| 337 | # Non-zero on hard failures only, so this is usable as a gate in other scripts |
| 338 | # while warnings (a locked vault, a dirty tree) stay informational. |
| 339 | [ "$fails" -eq 0 ] |
| 340 | |
amadan