| 1 | #!/usr/bin/env sh |
| 2 | # 🤖 ops provision — refresh a fleet box's systemd units, and declare which box |
| 3 | # is the short-link apex. |
| 4 | # |
| 5 | # ./deploy/ops provision units [box...] # refresh unit files (lchat bootstrap) |
| 6 | # ./deploy/ops provision apex <host> [box] # ...and make that box the apex for <host> |
| 7 | # ./deploy/ops provision status [box...] # read-only: who runs what |
| 8 | # ./deploy/ops provision operator <name> <box> (--github <handle> | --keys-file <path>) |
| 9 | # # create/converge an operator account |
| 10 | # |
| 11 | # Why this verb exists: **no new systemd unit ever reaches an existing box on |
| 12 | # its own.** `deploy` and `ship` swap binaries and restart units that already |
| 13 | # exist; the hourly auto-updater does the same. Only `lchat bootstrap` writes |
| 14 | # unit files, and nothing re-runs it after the first provision. So a unit added |
| 15 | # to the repo lands on fresh boxes and silently never appears on the fleet — |
| 16 | # which is exactly how the short-link redirector shipped in the binary and ran |
| 17 | # on no box at all (issue #905). |
| 18 | # |
| 19 | # `provision units` is therefore the missing fleet-side counterpart to |
| 20 | # `ops dev provision`, and the thing to run after merging any change that adds |
| 21 | # or edits a unit file. |
| 22 | # |
| 23 | # What it does NOT do: touch /data, instances, or versions. `bootstrap` is |
| 24 | # idempotent and data-safe — it writes units, ensures directories and the |
| 25 | # service user, and restarts the ROUTER. That router restart is the one |
| 26 | # member-visible effect (a sub-second blip on a retrying router), which is why |
| 27 | # this is a deliberate verb you run, never something folded into a deploy. |
| 28 | # |
| 29 | # NEVER point this at a dev-mesh box. Plain `bootstrap` ENABLES the hourly |
| 30 | # auto-updater, which would replace the dev mesh's unmerged code with the |
| 31 | # current stable build — `ops dev provision` exists for those boxes and passes |
| 32 | # --no-update-timer. The only guardrail here is the $LCHAT_BOXES default, so |
| 33 | # be deliberate when overriding it or naming boxes explicitly. |
| 34 | # |
| 35 | # `provision apex` additionally passes --links-apex, which starts the |
| 36 | # redirector and registers <host> as a route pointing at its socket. Exactly |
| 37 | # one box in a fleet is the apex; every other box keeps the unit written and |
| 38 | # disabled. After the first time, the registry route is the source of truth — |
| 39 | # plain `provision units` converges to it, so the apex survives later runs |
| 40 | # without anyone remembering a flag. |
| 41 | set -eu |
| 42 | |
| 43 | # Same default fleet as doctor/ship, same override. |
| 44 | BOXES="${LCHAT_BOXES:-elevenbox kepler}" |
| 45 | # The box that fronts the short-link apex, when `apex` is given no box. |
| 46 | APEX_BOX="${LCHAT_APEX_BOX:-elevenbox}" |
| 47 | |
| 48 | usage() { |
| 49 | echo "usage: ops provision units [box...] | ops provision apex <host> [box] |" >&2 |
| 50 | echo " ops provision status [box...] |" >&2 |
| 51 | echo " ops provision operator <name> <box> (--github <handle> | --keys-file <path>)" >&2 |
| 52 | echo >&2 |
| 53 | echo " units re-run 'lchat bootstrap' so newly-added systemd units land (restarts the router)" >&2 |
| 54 | echo " apex the same, declaring that box the short-link apex for <host>" >&2 |
| 55 | echo " status read-only: unit + route state, no changes" >&2 |
| 56 | echo " operator create/converge a named operator account on <box> (bootstrap --add-operator)" >&2 |
| 57 | exit 2 |
| 58 | } |
| 59 | |
| 60 | # on <box> <command>: privileged box command through the committed sudo |
| 61 | # allowlist (`/opt/lchat/lchat *`, deploy/sudoers-operators). Operator SSH |
| 62 | # rides WireGuard and the username comes from ~/.ssh/config — root SSH is |
| 63 | # bootstrap-only, so no user is baked in here. |
| 64 | on() { box="$1"; shift; ssh -o ConnectTimeout=10 "$box" "$@"; } |
| 65 | |
| 66 | # report <box>: the read-only picture — is the redirector running here, and |
| 67 | # which host (if any) routes to it. |
| 68 | report() { |
| 69 | box="$1" |
| 70 | printf ' %s:\n' "$box" |
| 71 | # is-enabled/is-active exit non-zero for the states we most want to SEE |
| 72 | # (disabled, inactive, not-found), and is-enabled adds a stderr explanation |
| 73 | # for a missing unit — so discard stderr, keep the first line, and only |
| 74 | # substitute a word when there was genuinely no output. |
| 75 | enabled="$(on "$box" 'systemctl is-enabled lchat-links 2>/dev/null | head -1' || true)" |
| 76 | active="$(on "$box" 'systemctl is-active lchat-links 2>/dev/null | head -1' || true)" |
| 77 | printf ' lchat-links %s / %s\n' "${enabled:-not-found}" "${active:-inactive}" |
| 78 | route="$(on "$box" "sudo /opt/lchat/lchat instances 2>/dev/null | grep -F '/run/lchat/links.sock' || true")" |
| 79 | if [ -n "$route" ]; then |
| 80 | printf ' apex route %s\n' "$(printf '%s' "$route" | awk '{print $1}')" |
| 81 | else |
| 82 | printf ' apex route (none — not the short-link apex)\n' |
| 83 | fi |
| 84 | } |
| 85 | |
| 86 | cmd="${1:-}" |
| 87 | [ $# -gt 0 ] && shift |
| 88 | case "$cmd" in |
| 89 | |
| 90 | units) |
| 91 | boxes="${*:-$BOXES}" |
| 92 | echo "ops provision: refreshing systemd units on: $boxes" |
| 93 | echo " (this restarts the router on each — a sub-second blip, not a data operation)" |
| 94 | for box in $boxes; do |
| 95 | echo "ops provision: $box" |
| 96 | on "$box" 'sudo /opt/lchat/lchat bootstrap' |
| 97 | report "$box" |
| 98 | done |
| 99 | echo |
| 100 | echo "note: this opened box SSH sessions — if this session hasn't written its" |
| 101 | echo " access-ledger why-entry yet: ./deploy/ops ledger \"...\"" |
| 102 | ;; |
| 103 | |
| 104 | apex) |
| 105 | host="${1:-}"; [ -n "$host" ] || usage |
| 106 | shift |
| 107 | box="${1:-$APEX_BOX}" |
| 108 | |
| 109 | # The apex is a real, public hostname whose DNS must already point at this |
| 110 | # box — the router issues its certificate lazily on first handshake, and an |
| 111 | # apex is NOT covered by a *.domain wildcard, so this is the one part the |
| 112 | # script can't do for you. Say so plainly rather than failing later at TLS. |
| 113 | echo "ops provision: making $box the short-link apex for $host" |
| 114 | echo " prerequisite: $host must already resolve to $box, with its OWN A record." |
| 115 | # Only worth spelling out where a covering wildcard plausibly exists: a host |
| 116 | # under a parent domain (dev.11ch.at), not a registrable apex (11ch.at), |
| 117 | # where "a *.ch.at wildcard" would be nonsense. |
| 118 | case "$host" in |
| 119 | *.*.*) echo " A *.${host#*.} wildcard does NOT cover $host — an apex never is." ;; |
| 120 | esac |
| 121 | on "$box" "sudo /opt/lchat/lchat bootstrap --links-apex $host" |
| 122 | report "$box" |
| 123 | |
| 124 | echo "ops provision: checking https://$host/ ..." |
| 125 | # The certificate is issued on the first handshake, so a cold apex can take a |
| 126 | # few seconds. A TLS failure here (rather than an HTTP status) almost always |
| 127 | # means DNS doesn't point at this box, or the route didn't register. |
| 128 | apex="" |
| 129 | for _ in 1 2 3 4 5 6; do |
| 130 | apex="$(curl -sS -o /dev/null -m 15 -w '%{http_code} -> %{redirect_url}' "https://$host/" 2>&1 || true)" |
| 131 | case "$apex" in 30*) break ;; esac |
| 132 | sleep 5 |
| 133 | done |
| 134 | echo " apex: $apex" |
| 135 | case "$apex" in |
| 136 | 30*) echo " (a 30x to the marketing site is the redirector answering — short links are live)" ;; |
| 137 | *) echo "ops provision: $host did not redirect." >&2 |
| 138 | echo " TLS error -> DNS probably doesn't point at $box, or the route didn't register" >&2 |
| 139 | echo " 404/timeout -> check: ssh $box journalctl -u lchat-links -n 40 --no-pager" >&2 |
| 140 | exit 1 ;; |
| 141 | esac |
| 142 | echo |
| 143 | echo "note: this opened box SSH sessions — if this session hasn't written its" |
| 144 | echo " access-ledger why-entry yet: ./deploy/ops ledger \"...\"" |
| 145 | ;; |
| 146 | |
| 147 | status) |
| 148 | boxes="${*:-$BOXES}" |
| 149 | echo "ops provision: short-link state" |
| 150 | for box in $boxes; do report "$box"; done |
| 151 | ;; |
| 152 | |
| 153 | # The later-joiner half of operator setup (docs/instance-router.md § |
| 154 | # Hardening): the first operators were created in the box's one root |
| 155 | # session, and a box hardened before an operator joined has no root shell |
| 156 | # left — kepler had James's WireGuard peer but no account. This drives the |
| 157 | # allowlist-reachable `bootstrap --add-operator`, keys piped on stdin. |
| 158 | # The account is only HALF of access: the operator's WireGuard peer must |
| 159 | # already be on the box's wg0 (see RUNBOOK § Box access), or nothing routes. |
| 160 | operator) |
| 161 | name="${1:-}"; box="${2:-}"; src="${3:-}"; arg="${4:-}" |
| 162 | [ -n "$name" ] && [ -n "$box" ] && [ -n "$arg" ] || usage |
| 163 | case "$name" in |
| 164 | *[!a-z0-9-]*|-*) echo "ops provision: operator name must be lowercase [a-z0-9-]" >&2; exit 1 ;; |
| 165 | esac |
| 166 | case "$src" in |
| 167 | --github) keys="$(curl -fsS "https://github.com/$arg.keys")" ;; |
| 168 | --keys-file) keys="$(cat "$arg")" ;; |
| 169 | *) usage ;; |
| 170 | esac |
| 171 | [ -n "$keys" ] || { echo "ops provision: no keys from $src $arg" >&2; exit 1; } |
| 172 | echo "ops provision: installing for '$name' on $box:" |
| 173 | printf '%s\n' "$keys" | awk '{print " " $1 " ..." substr($2, length($2)-7) " " $3}' |
| 174 | # Public keys fetched from GitHub are an INTAKE, not an identity proof — |
| 175 | # same caveat as the vault recipients: confirm over the messenger that the |
| 176 | # keys are currently theirs. |
| 177 | [ "$src" = "--github" ] && echo " (from github.com/$arg.keys — confirm with them over the messenger that these are current)" |
| 178 | printf '%s\n' "$keys" | on "$box" "sudo /opt/lchat/lchat bootstrap --add-operator $name" |
| 179 | echo "ops provision: verifying:" |
| 180 | on "$box" "id $name" |
| 181 | echo |
| 182 | echo "note: their WireGuard peer must already be on $box's wg0 for SSH to route" |
| 183 | echo " (sudo wg show wg0 — see RUNBOOK § Box access to add one)." |
| 184 | echo "note: this opened box SSH sessions — if this session hasn't written its" |
| 185 | echo " access-ledger why-entry yet: ./deploy/ops ledger \"...\"" |
| 186 | ;; |
| 187 | |
| 188 | -h|--help|help|"") usage ;; |
| 189 | *) echo "ops provision: unknown subcommand '$cmd'" >&2; usage ;; |
| 190 | esac |
| 191 | |
amadan