| 1 | #!/usr/bin/env sh |
| 2 | # 🤖 ops dev — the two-node dev mesh (Dublin + Helsinki; docs/multi-edge.md, |
| 3 | # runbook docs/multi-edge-dev-cutover.md). |
| 4 | # |
| 5 | # ./deploy/ops dev plan # tofu plan, with the AMI-replacement guard |
| 6 | # ./deploy/ops dev deploy [ie|fi] # build -> scp -> swap -> restart (both by default) |
| 7 | # ./deploy/ops dev deploy --stage-only # stop after scp (pair the swap with env edits) |
| 8 | # ./deploy/ops dev deploy --home # ALSO swap the home service binary (lchat-home) |
| 9 | # ./deploy/ops dev provision [ie|fi] # refresh systemd units + route the short-link apex |
| 10 | # ./deploy/ops dev rebuild --yes # destroy + recreate the Dublin box (new keys) |
| 11 | # ./deploy/ops dev rebuild --yes --adopt # provision a Dublin box something ELSE already |
| 12 | # # replaced (no tofu touch) — same bootstrap |
| 13 | # |
| 14 | # What this encodes that prose used to carry: |
| 15 | # - The dev mesh is NOT a release channel. Nothing here touches the signed |
| 16 | # feed; deploying is scp + stage + swap, by hand, on purpose (the mesh runs |
| 17 | # unmerged code the fleet updater must never see). |
| 18 | # - The verb sorts its own credentials out (the release-server.sh shape): |
| 19 | # values already in the environment win; otherwise Bitwarden fills the |
| 20 | # gaps — AWS_* from the per-operator dev agent item (docs/secrets.md |
| 21 | # § AWS), and DNSIMPLE_* from that item too when it carries them, else |
| 22 | # from the standalone DNSimple token item the migration split out. |
| 23 | # Unlocks interactively when the vault is locked; no tty -> instructions, |
| 24 | # never a hang; values are exported for this process only, never printed. |
| 25 | # - Dublin is arm64 (t4g), Helsinki amd64 — build both, ship the right one. |
| 26 | # - The binary is staged BESIDE the live one and swapped in a separate step, |
| 27 | # so router+cloud restart exactly once, and the previous binary is parked |
| 28 | # as a .bak for one-command rollback. |
| 29 | # - Dublin runs router + cloud + control (restart all three); Helsinki runs |
| 30 | # only the router. |
| 31 | # - `deploy` swaps the BINARY; `provision` refreshes the UNITS. They are |
| 32 | # separate because deploy restarts only units that already exist, so a |
| 33 | # newly-added unit never appears on a dev box however often you deploy — |
| 34 | # only `lchat bootstrap` writes unit files. provision runs it with |
| 35 | # --no-update-timer and then asserts the timer really is off, because a |
| 36 | # plain bootstrap would enable the hourly updater and quietly replace the |
| 37 | # unmerged code under test with the current stable build. |
| 38 | # - compute.tf's aws_ami is `most_recent = true` with no lifecycle guard: the |
| 39 | # day Canonical publishes a newer image, a plain `tofu apply` DESTROYS AND |
| 40 | # RECREATES the Dublin box (its /data dies with it; only the EIP survives). |
| 41 | # `plan` refuses to look green when that is pending; `rebuild` is the |
| 42 | # deliberate version of the same event. |
| 43 | # - The dev security group admits SSH only from the per-operator /32 |
| 44 | # allowlist (compute.tf ssh_ingress_cidr), so being outside it looks |
| 45 | # exactly like a dead box (`ops doctor dev` names which it is). Adding |
| 46 | # yourself is a PR + `ops dev apply` (an in-place SG update); removing an |
| 47 | # entry revokes that operator — only with their say-so. |
| 48 | # - EC2 injects the SSH keypair ONLY at first boot — which is why `rebuild` |
| 49 | # exists: swapping the key pair in compute.tf does nothing to a running |
| 50 | # box, and replacing the box is the honest way to rotate a key you don't |
| 51 | # hold. Helsinki is hand-provisioned Hetzner and cannot be rebuilt from |
| 52 | # here (only its scoped S3 IAM user lives in the tofu). |
| 53 | set -eu |
| 54 | |
| 55 | here="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" |
| 56 | root="$(CDPATH= cd -- "$here/.." && pwd)" |
| 57 | # The Bitwarden item names + value locations this script reads — a contract, |
| 58 | # not a search (deploy/vault-items.sh). |
| 59 | # shellcheck source=deploy/vault-items.sh |
| 60 | . "$here/vault-items.sh" |
| 61 | # shellcheck source=deploy/bw-lib.sh |
| 62 | . "$here/bw-lib.sh" |
| 63 | |
| 64 | # The boxes, as committed in docs/multi-edge-dev-cutover.md. The Dublin key is |
| 65 | # the SHARED box key (comment `eleven-dev-box` — box-named, not personal), |
| 66 | # never vaulted: copies live only on operator machines. ssh falls back to |
| 67 | # ~/.ssh/config if it's absent. |
| 68 | BOX_IE="${LCHAT_DEV_BOX_IE:-ubuntu@54.154.95.205}" |
| 69 | BOX_FI="${LCHAT_DEV_BOX_FI:-root@157.180.123.183}" |
| 70 | KEY_IE="$HOME/.ssh/eleven-dev-box" |
| 71 | AWS_PROFILE="${LCHAT_DEV_AWS_PROFILE:-eleven-dev}" |
| 72 | # The dev mesh's short-link apex — the dev-side stand-in for 11ch.at. It is the |
| 73 | # APEX of the dev subtree, not a host under it, because that is what production |
| 74 | # is: the wildcard cert that covers every space does not cover this name, so |
| 75 | # rehearsing short links here exercises the same apex DNS + certificate path |
| 76 | # that production needs. Its A record is aws_route53_record.dev_apex. |
| 77 | DEV_LINKS_HOST="${LCHAT_DEV_LINKS_HOST:-dev.11ch.at}" |
| 78 | |
| 79 | ssh_ie() { if [ -f "$KEY_IE" ]; then ssh -i "$KEY_IE" -o ConnectTimeout=10 "$@"; else ssh -o ConnectTimeout=10 "$@"; fi; } |
| 80 | scp_ie() { if [ -f "$KEY_IE" ]; then scp -i "$KEY_IE" "$@"; else scp "$@"; fi; } |
| 81 | |
| 82 | usage() { |
| 83 | echo "usage: ops dev plan [tofu args] | ops dev apply [tofu args] |" >&2 |
| 84 | echo " ops dev deploy [ie|fi] [--stage-only] [--home] | ops dev provision [ie|fi] |" >&2 |
| 85 | echo " ops dev rebuild [--yes]" >&2 |
| 86 | exit 2 |
| 87 | } |
| 88 | |
| 89 | # plan_guard <planfile|-> [tofu args...]: run a plan, show it, and hard-stop |
| 90 | # when it would DESTROY anything. Shared by `plan` (whose whole job this is) |
| 91 | # and `apply` (so an interactive "yes" can't sleepwalk into the AMI rebuild). |
| 92 | # Extra args pass through, so a targeted apply guards exactly the graph it |
| 93 | # will touch. With a planfile (not "-"), the plan is also SAVED so apply can |
| 94 | # execute exactly what was inspected — nothing can change between the guarded |
| 95 | # plan and the applied one. |
| 96 | # |
| 97 | # The stop condition is the plan summary's own destroy COUNT, not resource- |
| 98 | # header wording: the original grep for "must be replaced" demonstrably missed |
| 99 | # a live AMI replacement on 2026-07-23 (the operator got a bare approve prompt |
| 100 | # and the Dublin box was destroyed), and a count can't be missed by phrasing |
| 101 | # or formatting drift. The wording grep stays as a second net. |
| 102 | plan_guard() { |
| 103 | pf="$1"; shift |
| 104 | planout="$(mktemp -t lchat-dev-plan.XXXXXX)" |
| 105 | outflag="" |
| 106 | [ "$pf" = - ] || outflag="-out=$pf" |
| 107 | # -detailed-exitcode: 0 no changes, 2 changes present, 1 error. Changes are |
| 108 | # fine; what we refuse to shrug at is anything pending DESTRUCTION. |
| 109 | # -no-color so the detection greps see the same bytes a human does. |
| 110 | # (Captured to a file, not piped through tee — a pipeline would report tee's |
| 111 | # exit status and silently lose tofu's.) |
| 112 | rc=0 |
| 113 | # shellcheck disable=SC2086 # outflag is deliberately word-split (empty or one flag) |
| 114 | run_tofu plan -input=false -no-color -detailed-exitcode $outflag "$@" >"$planout" 2>&1 || rc=$? |
| 115 | cat "$planout" |
| 116 | if [ "$rc" -eq 1 ]; then rm -f "$planout"; exit 1; fi |
| 117 | destroys="$(sed -n 's/^Plan: .* \([0-9][0-9]*\) to destroy.*/\1/p' "$planout" | tail -1)" |
| 118 | if [ "${destroys:-0}" -gt 0 ] || grep -Eq "must be replaced|forces replacement" "$planout"; then |
| 119 | rm -f "$planout" |
| 120 | cat >&2 <<'EOF' |
| 121 | |
| 122 | ██ STOP — this plan DESTROYS or REPLACES something (see the plan above). |
| 123 | If it is aws_instance.ie_node, the cause is usually a newer Canonical AMI — |
| 124 | a plain apply would DESTROY the Dublin box; its /data dies with it, only |
| 125 | the EIP (and DNS) survive. Your options: |
| 126 | - apply just the safe resource: ops dev apply -target=<addr> |
| 127 | - make the rebuild deliberate: ops dev rebuild (new keys, guided) |
| 128 | - a genuinely wanted destroy: run tofu by hand in infra/dev-mesh, |
| 129 | eyes open — this verb will not do it for you. |
| 130 | EOF |
| 131 | exit 1 |
| 132 | fi |
| 133 | rm -f "$planout" |
| 134 | } |
| 135 | |
| 136 | # --- credentials ---------------------------------------------------------- |
| 137 | # Only these five ever leave the Bitwarden item for the environment — an |
| 138 | # allowlist, so a stray line in the item can never export something else. |
| 139 | DEV_CRED_VARS='AWS_ACCESS_KEY_ID|AWS_SECRET_ACCESS_KEY|AWS_SESSION_TOKEN|DNSIMPLE_TOKEN|DNSIMPLE_ACCOUNT' |
| 140 | |
| 141 | have_aws_auth() { |
| 142 | [ -n "${AWS_ACCESS_KEY_ID:-}" ] && return 0 |
| 143 | aws configure list-profiles 2>/dev/null | grep -qx "$AWS_PROFILE" |
| 144 | } |
| 145 | |
| 146 | # Run tofu with whichever auth we ended up with. Explicit env credentials win; |
| 147 | # AWS_PROFILE is only set when that profile really exists (setting it to an |
| 148 | # absent profile would itself be the error). |
| 149 | run_tofu() { |
| 150 | if [ -n "${AWS_ACCESS_KEY_ID:-}" ]; then tofu "$@"; else AWS_PROFILE="$AWS_PROFILE" tofu "$@"; fi |
| 151 | } |
| 152 | |
| 153 | # find_item <search> <pinned-name> <hint> — resolve exactly one vault item, |
| 154 | # echoing its name. bw reads a LOCAL CACHE of the vault: an item added, |
| 155 | # removed, or deduplicated in another client is invisible here until |
| 156 | # `bw sync` — so a surprising result (none, or more than one) gets one sync |
| 157 | # and retry before it is believed. Two items SHARING a name is its own error: |
| 158 | # `bw get item <name>` can't disambiguate, so the vault needs a rename/delete. |
| 159 | find_item() { |
| 160 | search="$1"; pinned="$2"; hint="$3" |
| 161 | if [ -n "$pinned" ]; then printf '%s\n' "$pinned"; return 0; fi |
| 162 | synced=no |
| 163 | while :; do |
| 164 | names="$(bw_run 30 list items --search "$search" 2>/dev/null | jq -r '.[].name' || true)" |
| 165 | n="$(printf '%s\n' "$names" | grep -c . || true)" |
| 166 | if [ "$n" = 1 ]; then printf '%s\n' "$names"; return 0; fi |
| 167 | if [ "$synced" = no ]; then |
| 168 | echo "ops dev: $n items match '$search' — syncing bw's local cache and retrying" >&2 |
| 169 | bw_run 60 sync >/dev/null 2>&1 || true |
| 170 | synced=yes |
| 171 | continue |
| 172 | fi |
| 173 | if [ "$n" = 0 ]; then |
| 174 | echo "ops dev: no item matching '$search' in your vault ($hint)" >&2 |
| 175 | elif [ "$(printf '%s\n' "$names" | sort -u | grep -c .)" -lt "$n" ]; then |
| 176 | echo "ops dev: DUPLICATE items named alike match '$search' — bw can't fetch by an" >&2 |
| 177 | echo " ambiguous name; delete or rename one in the vault, then re-run:" >&2 |
| 178 | printf '%s\n' "$names" | sed 's/^/ /' >&2 |
| 179 | else |
| 180 | echo "ops dev: several items match '$search' — pin one ($hint):" >&2 |
| 181 | printf '%s\n' "$names" | sed 's/^/ /' >&2 |
| 182 | fi |
| 183 | return 1 |
| 184 | done |
| 185 | } |
| 186 | |
| 187 | load_dev_creds() { |
| 188 | command -v bw >/dev/null 2>&1 || { |
| 189 | echo "ops dev: bw is not installed, and the credentials aren't in the environment" >&2 |
| 190 | return 1; } |
| 191 | command -v jq >/dev/null 2>&1 || { echo "ops dev: jq is required" >&2; return 1; } |
| 192 | st="$(bw_run 20 status 2>/dev/null | jq -r '.status' 2>/dev/null || true)" |
| 193 | case "$st" in |
| 194 | unlocked) ;; |
| 195 | locked) |
| 196 | if [ -t 0 ]; then |
| 197 | echo "ops dev: unlocking bitwarden to read the dev agent creds" >&2 |
| 198 | # Our prompt, never bw's; watchdogged (bw-lib.sh) — bw's own prompt |
| 199 | # inside $(...) is invisible and reads as a hang. |
| 200 | BW_SESSION="$(bw_unlock_session)" || true |
| 201 | [ -n "${BW_SESSION:-}" ] || { echo "ops dev: unlock failed" >&2; return 1; } |
| 202 | export BW_SESSION |
| 203 | else |
| 204 | echo "ops dev: bitwarden is locked and there is no tty to prompt on" >&2 |
| 205 | echo " bw unlock # then paste the 'export BW_SESSION=...' line it prints, and re-run" >&2 |
| 206 | return 1 |
| 207 | fi ;; |
| 208 | *) |
| 209 | echo "ops dev: bw is not logged in (bw login — EU vault: https://vault.bitwarden.eu)" >&2 |
| 210 | return 1 ;; |
| 211 | esac |
| 212 | |
| 213 | # The dev agent item is the one contract entry that is a naming CONVENTION |
| 214 | # rather than an exact name — it is per-operator (each has a revocable |
| 215 | # aws-dev-agent-<name> IAM identity, docs/secrets.md § AWS), so resolve |
| 216 | # "exactly one item with the contract prefix"; LCHAT_DEV_CREDS_ITEM pins one. |
| 217 | item="$(find_item "$BW_ITEM_DEV_AGENT_PREFIX" "${LCHAT_DEV_CREDS_ITEM:-}" \ |
| 218 | "each operator gets their own dev agent creds, docs/secrets.md § AWS; LCHAT_DEV_CREDS_ITEM pins one")" \ |
| 219 | || return 1 |
| 220 | |
| 221 | # The item is the sealed env file: KEY=VALUE lines in its secure note (or |
| 222 | # custom fields). Pull only the allowlisted names; never echo a value. |
| 223 | lines="$(bw_run 30 get item "$item" 2>/dev/null | jq -r --arg re "^($DEV_CRED_VARS)=" ' |
| 224 | ((.notes // "" | split("\n")) + [.fields[]? | "\(.name)=\(.value)"])[] |
| 225 | | select(test($re))' || true)" |
| 226 | if [ -n "$lines" ]; then |
| 227 | while IFS= read -r line; do |
| 228 | k="${line%%=*}" |
| 229 | # Environment always wins — the item only fills gaps. ($k is allowlisted |
| 230 | # by the select() above, so the eval can't be steered by item content.) |
| 231 | eval "cur=\${$k:-}" |
| 232 | # shellcheck disable=SC2163 # the K=V-operand form of export is intended |
| 233 | [ -n "$cur" ] || export "$line" |
| 234 | done <<EOF |
| 235 | $lines |
| 236 | EOF |
| 237 | echo "ops dev: credentials loaded from bitwarden item '$item'" >&2 |
| 238 | else |
| 239 | echo "ops dev: note: '$item' carried no AWS_*/DNSIMPLE_* lines in its note or fields" >&2 |
| 240 | fi |
| 241 | |
| 242 | # The Bitwarden migration split the DNSimple token out of the sops-era |
| 243 | # bundle into its own item, so when the agent item didn't carry it, fetch it |
| 244 | # from there — by the contract's EXACT name and slot (the login password), |
| 245 | # no searching, no guessing. A failed fetch gets one cache sync + retry (a |
| 246 | # freshly shared item is invisible to bw's local cache until then), then a |
| 247 | # crisp contract error. The account id is public (committed in main.tf) — |
| 248 | # only the token is a secret. |
| 249 | if [ -z "${DNSIMPLE_TOKEN:-}" ]; then |
| 250 | ditem="${LCHAT_DNSIMPLE_ITEM:-$BW_ITEM_DNSIMPLE}" |
| 251 | json="$(bw_run 30 get item "$ditem" 2>/dev/null || true)" |
| 252 | if [ -z "$json" ]; then |
| 253 | echo "ops dev: item '$ditem' not in bw's local cache — syncing and retrying" >&2 |
| 254 | bw_run 60 sync >/dev/null 2>&1 || true |
| 255 | json="$(bw_run 30 get item "$ditem" 2>/dev/null || true)" |
| 256 | fi |
| 257 | if [ -z "$json" ]; then |
| 258 | echo "ops dev: no bitwarden item named exactly '$ditem'" >&2 |
| 259 | echo " — the vault contract (deploy/vault-items.sh); is it shared to your collection," >&2 |
| 260 | echo " or was it renamed without a matching PR?" >&2 |
| 261 | return 1 |
| 262 | fi |
| 263 | tok="$(printf '%s' "$json" | jq -r '.login.password // empty' 2>/dev/null || true)" |
| 264 | [ -n "$tok" ] || { |
| 265 | echo "ops dev: the vault contract expects the DNSimple token in the PASSWORD slot" >&2 |
| 266 | echo " of '$ditem' (deploy/vault-items.sh) — fix the item, or export DNSIMPLE_TOKEN" >&2 |
| 267 | return 1; } |
| 268 | export DNSIMPLE_TOKEN="$tok" |
| 269 | echo "ops dev: DNSIMPLE_TOKEN loaded from bitwarden item '$ditem'" >&2 |
| 270 | fi |
| 271 | } |
| 272 | |
| 273 | # ensure_creds: make DNSIMPLE_TOKEN + AWS auth real, from env or Bitwarden. |
| 274 | ensure_creds() { |
| 275 | # The account id is public (176940, committed in main.tf) — default it here |
| 276 | # rather than in the bw path, so a hand-exported DNSIMPLE_TOKEN (which skips |
| 277 | # Bitwarden entirely) still gets a working provider config. |
| 278 | [ -n "${DNSIMPLE_ACCOUNT:-}" ] || export DNSIMPLE_ACCOUNT=176940 |
| 279 | if [ -z "${DNSIMPLE_TOKEN:-}" ] || ! have_aws_auth; then |
| 280 | load_dev_creds || { |
| 281 | echo "ops dev: still missing credentials — export DNSIMPLE_TOKEN/DNSIMPLE_ACCOUNT and" >&2 |
| 282 | echo " AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY (or configure the '$AWS_PROFILE' aws" >&2 |
| 283 | echo " profile), or make your dev agent item reachable in bw and re-run" >&2 |
| 284 | exit 1 |
| 285 | } |
| 286 | fi |
| 287 | [ -n "${DNSIMPLE_TOKEN:-}" ] || { |
| 288 | echo "ops dev: DNSIMPLE_TOKEN still unset (the item didn't carry it?)" >&2; exit 1; } |
| 289 | have_aws_auth || { |
| 290 | echo "ops dev: no AWS credentials (env or '$AWS_PROFILE' profile)" >&2; exit 1; } |
| 291 | } |
| 292 | |
| 293 | # swap <ie|fi> <ver>: back up the live binary, move the staged one in, restart |
| 294 | # the units that box runs, and confirm the running stamp matches the build. |
| 295 | swap_box() { |
| 296 | which_box="$1"; ver="$2" |
| 297 | if [ "$which_box" = ie ]; then box="$BOX_IE"; units="lchat-router lchat-cloud lchat-control"; run() { ssh_ie "$box" "$@"; } |
| 298 | else box="$BOX_FI"; units="lchat-router"; run() { ssh -o ConnectTimeout=10 "$box" "$@"; } |
| 299 | fi |
| 300 | run "sudo cp /opt/lchat/lchat /opt/lchat/lchat.bak-pre-$ver \ |
| 301 | && sudo mv /opt/lchat/incoming/lchat-new /opt/lchat/lchat \ |
| 302 | && sudo chmod +x /opt/lchat/lchat \ |
| 303 | && sudo systemctl restart $units" |
| 304 | running="$(run '/opt/lchat/lchat version' 2>/dev/null || true)" |
| 305 | if [ "$running" = "$ver" ] && run 'systemctl is-active --quiet lchat-router'; then |
| 306 | echo " $which_box: running $running" |
| 307 | else |
| 308 | echo "ops dev: $which_box is NOT healthy after the swap (version '$running', wanted '$ver')" >&2 |
| 309 | echo " inspect: ssh $box journalctl -u lchat-router -n 60 --no-pager" >&2 |
| 310 | echo " rollback: ssh $box 'sudo mv /opt/lchat/lchat.bak-pre-$ver /opt/lchat/lchat && sudo systemctl restart $units'" >&2 |
| 311 | exit 1 |
| 312 | fi |
| 313 | } |
| 314 | |
| 315 | # swap_home <ie|fi> <ver>: the home service rides its own binary (releases |
| 316 | # ship it as lchat-home-linux-* via updaterservices.go; the dev mesh swaps it |
| 317 | # directly, same as the control binary). A box that doesn't RUN home (Helsinki |
| 318 | # today — home runs on the dev apex box only) is skipped, never failed. |
| 319 | # |
| 320 | # Enabled, not merely present: `lchat bootstrap` writes the unit file on every |
| 321 | # box it touches, so "the unit exists" says nothing about whether this box is |
| 322 | # a home server. Helsinki has the file, disabled, with no /data/home.env — and |
| 323 | # the old presence test walked straight into restarting it, which fails on the |
| 324 | # missing env ("Failed to load environment files") and took the whole deploy |
| 325 | # down with it after Dublin had already swapped fine. |
| 326 | swap_home() { |
| 327 | which_box="$1"; ver="$2" |
| 328 | if [ "$which_box" = ie ]; then box="$BOX_IE"; run() { ssh_ie "$box" "$@"; } |
| 329 | else box="$BOX_FI"; run() { ssh -o ConnectTimeout=10 "$box" "$@"; } |
| 330 | fi |
| 331 | case "$(run 'systemctl is-enabled lchat-home.service 2>/dev/null || true')" in |
| 332 | enabled|enabled-runtime|static) ;; |
| 333 | *) |
| 334 | echo " $which_box: lchat-home not enabled here — home skipped" |
| 335 | run 'rm -f /opt/lchat/incoming/home-new' || true |
| 336 | return 0 ;; |
| 337 | esac |
| 338 | run "sudo cp /opt/lchat/home /opt/lchat/home.prev-$ver 2>/dev/null || true; \ |
| 339 | sudo mv /opt/lchat/incoming/home-new /opt/lchat/home \ |
| 340 | && sudo chmod +x /opt/lchat/home \ |
| 341 | && sudo systemctl restart lchat-home" |
| 342 | if run 'systemctl is-active --quiet lchat-home'; then |
| 343 | echo " $which_box: lchat-home running ($ver)" |
| 344 | else |
| 345 | echo "ops dev: $which_box lchat-home is NOT active after the swap" >&2 |
| 346 | echo " inspect: ssh $box journalctl -u lchat-home -n 60 --no-pager" >&2 |
| 347 | echo " rollback: ssh $box 'sudo mv /opt/lchat/home.prev-$ver /opt/lchat/home && sudo systemctl restart lchat-home'" >&2 |
| 348 | exit 1 |
| 349 | fi |
| 350 | } |
| 351 | |
| 352 | cmd="${1:-}" |
| 353 | [ $# -gt 0 ] && shift |
| 354 | case "$cmd" in |
| 355 | |
| 356 | plan) |
| 357 | cd "$root/infra/dev-mesh" |
| 358 | ensure_creds |
| 359 | [ -d .terraform ] || run_tofu init -input=false |
| 360 | plan_guard - "$@" |
| 361 | ;; |
| 362 | |
| 363 | apply) |
| 364 | # The self-serve apply: same credential fetch, same destroy guard. What |
| 365 | # applies is the SAVED plan the guard inspected — never a fresh one — so a |
| 366 | # refresh between guard and apply can't smuggle a replacement in. The verb |
| 367 | # keeps its own explicit confirm (a saved-plan apply skips tofu's). Extra |
| 368 | # args pass through (e.g. `ops dev apply |
| 369 | # -target=aws_security_group.ie_node` to land a security-group change |
| 370 | # while something destructive is pending). |
| 371 | cd "$root/infra/dev-mesh" |
| 372 | ensure_creds |
| 373 | [ -d .terraform ] || run_tofu init -input=false |
| 374 | planfile="$(mktemp -t lchat-dev-planfile.XXXXXX)" |
| 375 | trap 'rm -f "$planfile"' EXIT |
| 376 | plan_guard "$planfile" "$@" |
| 377 | printf 'apply exactly this plan? (type yes): ' |
| 378 | read -r answer || answer="" # EOF (no tty) refuses below instead of dying silently |
| 379 | [ "$answer" = yes ] || { echo "ops dev: not applied"; exit 1; } |
| 380 | run_tofu apply -input=false "$planfile" |
| 381 | ;; |
| 382 | |
| 383 | deploy) |
| 384 | targets="ie fi"; stage_only=no; with_home=no |
| 385 | for a in "$@"; do |
| 386 | case "$a" in |
| 387 | ie|fi) targets="$a" ;; |
| 388 | --stage-only) stage_only=yes ;; |
| 389 | --home) with_home=yes ;; |
| 390 | *) usage ;; |
| 391 | esac |
| 392 | done |
| 393 | |
| 394 | # The invariant `provision` asserts, checked again HERE — before a build, a |
| 395 | # copy and a restart go to waste. A dev box with the auto-updater enabled |
| 396 | # takes your unmerged code back to the stable channel within the hour, and |
| 397 | # the deploy that put it there reports success on the way past: seen |
| 398 | # 2026-08-28, when an update pass landed 55 seconds after a swap and quietly |
| 399 | # restored the previous home binary. Cheap SSH, loud failure, named fix. |
| 400 | for t in $targets; do |
| 401 | if [ "$t" = ie ]; then probe_box="$BOX_IE"; timer="$(ssh_ie "$probe_box" 'systemctl is-enabled lchat-update.timer 2>/dev/null || true')" |
| 402 | else probe_box="$BOX_FI"; timer="$(ssh -o ConnectTimeout=10 "$probe_box" 'systemctl is-enabled lchat-update.timer 2>/dev/null || true')" |
| 403 | fi |
| 404 | if [ "$timer" = enabled ]; then |
| 405 | echo "ops dev: ABORT — $t's auto-update timer is ENABLED." >&2 |
| 406 | echo " It would replace whatever you deploy with the current stable build," >&2 |
| 407 | echo " within the hour, silently. Turn it off first:" >&2 |
| 408 | echo " ./deploy/ops dev provision $t" >&2 |
| 409 | exit 1 |
| 410 | fi |
| 411 | done |
| 412 | |
| 413 | ver="$(git -C "$root" rev-parse --short HEAD)" |
| 414 | if ! git -C "$root" diff --quiet || ! git -C "$root" diff --cached --quiet; then |
| 415 | ver="${ver}-dirty" |
| 416 | echo "ops dev: working tree has uncommitted changes — building anyway ($ver)" >&2 |
| 417 | fi |
| 418 | |
| 419 | tmp="$(mktemp -d -t lchat-dev.XXXXXX)" |
| 420 | trap 'rm -rf "$tmp"' EXIT |
| 421 | cd "$root" # build-linux.sh compiles the package at the current directory |
| 422 | for t in $targets; do |
| 423 | case "$t" in |
| 424 | ie) arch=arm64 ;; # Dublin is a Graviton box |
| 425 | fi) arch=amd64 ;; |
| 426 | esac |
| 427 | echo "ops dev: building $ver for linux/$arch ($t)" |
| 428 | OUT="$tmp/lchat-$t" ARCH="$arch" "$here/build-linux.sh" "$ver" |
| 429 | if [ "$with_home" = yes ]; then |
| 430 | # The home service is its own package/binary (like release-server.sh's |
| 431 | # service builds) — no version stamp, the unit check gates the swap. |
| 432 | echo "ops dev: building home for linux/$arch ($t)" |
| 433 | GOOS=linux GOARCH="$arch" CGO_ENABLED=0 go build -o "$tmp/home-$t" ./home |
| 434 | fi |
| 435 | done |
| 436 | |
| 437 | for t in $targets; do |
| 438 | if [ "$t" = ie ]; then box="$BOX_IE"; else box="$BOX_FI"; fi |
| 439 | echo "ops dev: staging on $t ($box)" |
| 440 | # Stage beside the live binary, never over it — the swap is its own step. |
| 441 | # An SSH hang here usually means you are outside the security group's |
| 442 | # /32 allowlist, not that the box is down: run `ops doctor dev`. |
| 443 | if [ "$t" = ie ]; then |
| 444 | # The hardening pass roots /opt/lchat — hand the staging dir back to |
| 445 | # the ssh user so plain scp can write it (the swap is sudo either way). |
| 446 | ssh_ie "$box" 'sudo mkdir -p /opt/lchat/incoming && sudo chown "$(id -un)" /opt/lchat/incoming' |
| 447 | scp_ie -q "$tmp/lchat-$t" "$box:/opt/lchat/incoming/lchat-new" |
| 448 | [ "$with_home" = yes ] && scp_ie -q "$tmp/home-$t" "$box:/opt/lchat/incoming/home-new" |
| 449 | else |
| 450 | ssh -o ConnectTimeout=10 "$box" 'sudo mkdir -p /opt/lchat/incoming && sudo chown "$(id -un)" /opt/lchat/incoming' |
| 451 | scp -q "$tmp/lchat-$t" "$box:/opt/lchat/incoming/lchat-new" |
| 452 | [ "$with_home" = yes ] && scp -q "$tmp/home-$t" "$box:/opt/lchat/incoming/home-new" |
| 453 | fi |
| 454 | done |
| 455 | |
| 456 | if [ "$stage_only" = yes ]; then |
| 457 | echo "ops dev: staged at /opt/lchat/incoming/lchat-new on: $targets (not swapped)" |
| 458 | [ "$with_home" = yes ] && echo " (home staged beside it as incoming/home-new)" |
| 459 | echo " swap by hand when ready (pairs with env edits — see the cutover runbook)" |
| 460 | exit 0 |
| 461 | fi |
| 462 | |
| 463 | for t in $targets; do |
| 464 | echo "ops dev: swapping + restarting $t" |
| 465 | swap_box "$t" "$ver" |
| 466 | [ "$with_home" = yes ] && swap_home "$t" "$ver" |
| 467 | done |
| 468 | echo |
| 469 | echo "note: this opened dev-box SSH sessions — if this session hasn't written its" |
| 470 | echo " access-ledger why-entry yet: ./deploy/ops ledger \"...\"" |
| 471 | ;; |
| 472 | |
| 473 | provision) |
| 474 | # Refresh the dev boxes' systemd units from the binary they are already |
| 475 | # running, and route the short-link redirector at the dev apex. |
| 476 | # |
| 477 | # Why this is a verb and not a documented ssh sequence: `deploy` only swaps |
| 478 | # the binary and restarts the units that already exist, so a NEW unit (like |
| 479 | # lchat-links) never appears on a dev box no matter how many times you |
| 480 | # deploy. Only `lchat bootstrap` writes units — and running plain bootstrap |
| 481 | # here would be a trap, because it enables the hourly auto-updater, which |
| 482 | # would replace this box's unmerged code with the current stable build. The |
| 483 | # whole point of the dev mesh is that it is NOT a release channel. So this |
| 484 | # passes --no-update-timer (which disables the timer outright rather than |
| 485 | # merely skipping it) and then VERIFIES the timer really is off before |
| 486 | # touching anything else. |
| 487 | targets="ie fi" |
| 488 | for a in "$@"; do |
| 489 | case "$a" in |
| 490 | ie|fi) targets="$a" ;; |
| 491 | *) usage ;; |
| 492 | esac |
| 493 | done |
| 494 | |
| 495 | for t in $targets; do |
| 496 | if [ "$t" = ie ]; then box="$BOX_IE"; run() { ssh_ie "$box" "$@"; } |
| 497 | else box="$BOX_FI"; run() { ssh -o ConnectTimeout=10 "$box" "$@"; } |
| 498 | fi |
| 499 | # Dublin holds the EIP that dev.11ch.at points at, so it is the dev apex |
| 500 | # and bootstraps AS it (one step: starts the redirector and registers the |
| 501 | # route). Helsinki is not the apex, so its bootstrap leaves the redirector |
| 502 | # disabled — which is the behaviour worth exercising here too. |
| 503 | if [ "$t" = ie ]; then apexflag="--links-apex $DEV_LINKS_HOST"; else apexflag=""; fi |
| 504 | |
| 505 | # Ordering guard: provision drives flags that only exist in a NEW binary, |
| 506 | # and `deploy` is what puts one on the box. Running these in the wrong |
| 507 | # order otherwise fails deep inside bootstrap's flag parsing with |
| 508 | # "flag provided but not defined", which reads like a bug in this script |
| 509 | # rather than a missing step. |
| 510 | if ! run 'sudo /opt/lchat/lchat bootstrap -h 2>&1' | grep -q 'links-apex'; then |
| 511 | echo "ops dev: ABORT — the binary on $t predates --links-apex." >&2 |
| 512 | echo " provision configures the box; deploy is what ships the code. Run:" >&2 |
| 513 | echo " ./deploy/ops dev deploy $t" >&2 |
| 514 | exit 1 |
| 515 | fi |
| 516 | |
| 517 | echo "ops dev: bootstrapping units on $t ($box)${apexflag:+ (as short-link apex $DEV_LINKS_HOST)}" |
| 518 | run "sudo /opt/lchat/lchat bootstrap --no-update-timer $apexflag" |
| 519 | |
| 520 | # The invariant, asserted rather than assumed: if bootstrap ever regains a |
| 521 | # path that leaves the updater on, this box starts silently overwriting the |
| 522 | # unmerged code under test and the mesh stops meaning anything. |
| 523 | timer="$(run 'systemctl is-enabled lchat-update.timer 2>/dev/null || true')" |
| 524 | case "$timer" in |
| 525 | enabled) |
| 526 | echo "ops dev: ABORT — $t's auto-update timer is ENABLED after bootstrap." >&2 |
| 527 | echo " The dev mesh runs unmerged code; an updater pass would replace it with" >&2 |
| 528 | echo " the current stable build. Disable it before using this box:" >&2 |
| 529 | echo " ssh $box 'sudo systemctl disable --now lchat-update.timer'" >&2 |
| 530 | exit 1 ;; |
| 531 | *) echo " $t: auto-update timer is '$timer' (not enabled — correct for dev)" ;; |
| 532 | esac |
| 533 | |
| 534 | links="$(run 'systemctl is-active lchat-links 2>/dev/null || true')" |
| 535 | if [ -n "$apexflag" ]; then |
| 536 | echo " $t: lchat-links is $links (apex — expected active)" |
| 537 | [ "$links" = active ] || { |
| 538 | echo "ops dev: $t is the apex but its links unit did not start — inspect:" >&2 |
| 539 | echo " ssh $box journalctl -u lchat-links -n 40 --no-pager" >&2 |
| 540 | exit 1 |
| 541 | } |
| 542 | else |
| 543 | echo " $t: lchat-links is $links (not the apex — expected inactive)" |
| 544 | [ "$links" != active ] || echo "ops dev: note — $t runs the redirector but isn't the apex" >&2 |
| 545 | fi |
| 546 | done |
| 547 | |
| 548 | # bootstrap --links-apex registered the route above, so there is no separate |
| 549 | # add-service step: prove the result instead. |
| 550 | case "$targets" in *ie*) |
| 551 | # Prove it end to end. The certificate is issued lazily on first handshake, |
| 552 | # so a cold apex can take a few seconds — and a TLS failure here is the |
| 553 | # exact symptom of the production bug this rehearses (an unrouted host gets |
| 554 | # no certificate at all, so it fails the handshake rather than 404ing). |
| 555 | echo "ops dev: checking https://$DEV_LINKS_HOST/ ..." |
| 556 | apex="" |
| 557 | for _ in 1 2 3 4 5 6; do |
| 558 | apex="$(curl -sS -o /dev/null -m 15 -w '%{http_code} -> %{redirect_url}' "https://$DEV_LINKS_HOST/" 2>&1 || true)" |
| 559 | case "$apex" in 30*) break ;; esac |
| 560 | sleep 5 |
| 561 | done |
| 562 | echo " apex: $apex" |
| 563 | echo " (expect a 30x to elevenmessenger.com — that is the redirector answering)" |
| 564 | case "$apex" in |
| 565 | 30*) ;; |
| 566 | *) echo "ops dev: the apex did not redirect. If this is a TLS error, check the" >&2 |
| 567 | echo " A record exists (ops dev apply lands aws_route53_record.dev_apex) —" >&2 |
| 568 | echo " a *.dev.11ch.at wildcard does NOT cover the bare apex." >&2 |
| 569 | exit 1 ;; |
| 570 | esac |
| 571 | |
| 572 | # /l/<host> only redirects to a host that verifiably runs lchat, so point it |
| 573 | # at a real dev space rather than a made-up name. |
| 574 | probe="${LCHAT_DEV_LINK_PROBE:-}" |
| 575 | if [ -n "$probe" ]; then |
| 576 | echo "ops dev: checking https://$DEV_LINKS_HOST/l/$probe ..." |
| 577 | echo " /l/: $(curl -sS -o /dev/null -m 20 -w '%{http_code} -> %{redirect_url}' "https://$DEV_LINKS_HOST/l/$probe" 2>&1 || true)" |
| 578 | echo " (expect a 30x to https://$probe — a 404 means the target failed verification)" |
| 579 | else |
| 580 | echo "ops dev: set LCHAT_DEV_LINK_PROBE=<a live dev space host> to also test /l/<host>" |
| 581 | fi |
| 582 | ;; esac |
| 583 | |
| 584 | echo |
| 585 | echo "note: this opened dev-box SSH sessions — if this session hasn't written its" |
| 586 | echo " access-ledger why-entry yet: ./deploy/ops ledger \"...\"" |
| 587 | ;; |
| 588 | |
| 589 | rebuild) |
| 590 | # Destroy + recreate the Dublin box — the deliberate key rotation. EC2 |
| 591 | # injects the keypair only at first boot, so this is how you take over a box |
| 592 | # whose key you don't hold: commit YOUR public key + /32 to compute.tf, |
| 593 | # replace the instance, and it boots trusting only you. The EIP, S3 bucket |
| 594 | # (hibernated spaces, mesh membership, the id ledger), IAM, and the Route53 |
| 595 | # zone are separate resources and survive; the box's /data does not. |
| 596 | confirm=no; adopt=no |
| 597 | for a in "$@"; do case "$a" in --yes) confirm=yes ;; --adopt) adopt=yes ;; *) usage ;; esac; done |
| 598 | ct="$root/infra/dev-mesh/compute.tf" |
| 599 | |
| 600 | # A fresh box trusts exactly one key — make sure it will be one you hold. |
| 601 | # Generating is safe (refuses to overwrite); committing it is the PR's job. |
| 602 | if [ ! -f "$KEY_IE" ]; then |
| 603 | echo "ops dev: no $KEY_IE — generating a fresh box key (never vaulted; the" |
| 604 | echo " rebuilt box trusts it alone, so share it with the other operators)" |
| 605 | ssh-keygen -q -t ed25519 -N '' -C eleven-dev-box -f "$KEY_IE" |
| 606 | fi |
| 607 | mypub="$(ssh-keygen -y -f "$KEY_IE" | awk '{print $1" "$2}')" |
| 608 | tfpub="$(awk '/resource "aws_key_pair" "ie_node"/,/^}/' "$ct" \ |
| 609 | | sed -n 's/.*public_key *= *"\([^"]*\)".*/\1/p' | awk '{print $1" "$2}')" |
| 610 | if [ "$mypub" != "$tfpub" ]; then |
| 611 | echo "ops dev: compute.tf's key pair is NOT your key — tofu applies what the file says," >&2 |
| 612 | echo " so the rebuilt box would trust someone else. Put your public key in" >&2 |
| 613 | echo " aws_key_pair.ie_node (a reviewed PR — the public key is committed on purpose):" >&2 |
| 614 | echo " public_key = \"$(ssh-keygen -y -f "$KEY_IE")\"" >&2 |
| 615 | exit 1 |
| 616 | fi |
| 617 | |
| 618 | # Same rule for the SSH allowlist: outside it, the post-rebuild SSH wait |
| 619 | # would hang forever and read as a dead box. TF_VAR_ssh_ingress_cidr (a |
| 620 | # JSON list) overrides the committed default, same as for tofu itself; |
| 621 | # either way, every a.b.c.d/nn token counts. |
| 622 | cidrs="$(printf '%s' "${TF_VAR_ssh_ingress_cidr:-}" \ |
| 623 | | grep -oE '[0-9]+(\.[0-9]+){3}/[0-9]+' || true)" |
| 624 | [ -n "$cidrs" ] || cidrs="$(awk '/variable "ssh_ingress_cidr"/,/^}/' "$ct" \ |
| 625 | | grep -oE '[0-9]+(\.[0-9]+){3}/[0-9]+' || true)" |
| 626 | mine="$(curl -s -m5 https://api.ipify.org 2>/dev/null || true)" |
| 627 | if [ -z "$mine" ] || ! printf '%s\n' "$cidrs" | sed 's|/.*||' | grep -qxF "$mine"; then |
| 628 | echo "ops dev: the security group would admit [$(printf '%s' "$cidrs" | tr '\n' ' ')] but your IP" >&2 |
| 629 | echo " is ${mine:-unknown} — the rebuilt box would be unreachable from here. Add your /32" >&2 |
| 630 | echo " to ssh_ingress_cidr in compute.tf (removing an entry revokes that operator)." >&2 |
| 631 | exit 1 |
| 632 | fi |
| 633 | |
| 634 | if [ "$confirm" != yes ]; then |
| 635 | cat <<EOF |
| 636 | ops dev rebuild — this DESTROYS and recreates the Dublin box. |
| 637 | dies: everything on its disk — /data (non-hibernating dev spaces, the |
| 638 | router registry, router/cloud/home env files and their tokens) |
| 639 | survives: the EIP (DNS keeps pointing right), the S3 bucket (hibernated |
| 640 | spaces, mesh membership, the id ledger), IAM, the Route53 zone |
| 641 | needs: LCHAT_MESH_TOKEN re-paired with Helsinki afterwards, and the env |
| 642 | files rebuilt (the runbook's Step 3 blocks) |
| 643 | With --adopt: no destroy — the box was ALREADY replaced (an AMI-drift apply, |
| 644 | a console rebuild); this just runs the same provisioning against it. |
| 645 | Coordinate first — this tears down the live half of the dev mesh until |
| 646 | re-provisioned. Re-run with --yes to proceed (tofu will still show its plan |
| 647 | and ask for approval). |
| 648 | EOF |
| 649 | exit 1 |
| 650 | fi |
| 651 | |
| 652 | if [ "$adopt" = yes ]; then |
| 653 | # Adopt: the instance was already replaced outside this verb, so there is |
| 654 | # nothing for tofu to do — which also means no AWS/DNSimple credentials |
| 655 | # are needed. The facts tofu would have output come from elsewhere: the |
| 656 | # EIP survives replacement (it IS the committed BOX_IE address), and the |
| 657 | # bucket is shared with Helsinki, which survived and knows it. |
| 658 | ip="${BOX_IE#*@}" |
| 659 | bucket="$(ssh -o ConnectTimeout=10 "$BOX_FI" \ |
| 660 | "sudo grep '^LCHAT_HIBERNATE_BUCKET=' /data/router.env | cut -d= -f2-" 2>/dev/null || true)" |
| 661 | [ -n "$bucket" ] || { echo "ops dev: couldn't read the mesh bucket from Helsinki's router.env — is $BOX_FI up?" >&2; exit 1; } |
| 662 | else |
| 663 | cd "$root/infra/dev-mesh" |
| 664 | ensure_creds |
| 665 | [ -d .terraform ] || run_tofu init -input=false |
| 666 | # tofu's own interactive approve is the second gate — read the plan: it |
| 667 | # should replace aws_instance.ie_node (and the key pair), nothing else |
| 668 | # destructive. (-replace overrides the instance's ignore_changes=[ami], |
| 669 | # so pinning the AMI does not break deliberate rebuilds.) |
| 670 | run_tofu apply -replace=aws_instance.ie_node |
| 671 | |
| 672 | ip="$(run_tofu output -raw ie_node_ip)" |
| 673 | bucket="$(run_tofu output -raw mesh_bucket)" |
| 674 | fi |
| 675 | # Same EIP, new host key — the stale known_hosts entry would make every ssh |
| 676 | # fail with "REMOTE HOST IDENTIFICATION HAS CHANGED", so drop it first. |
| 677 | ssh-keygen -R "$ip" >/dev/null 2>&1 || true |
| 678 | echo "ops dev: waiting for the new box to accept ssh ($ip)" |
| 679 | i=0 |
| 680 | until ssh -i "$KEY_IE" -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new \ |
| 681 | "ubuntu@$ip" true 2>/dev/null; do |
| 682 | i=$((i + 1)) |
| 683 | [ "$i" -lt 60 ] || { echo "ops dev: box not reachable after 5m — check the AWS console" >&2; exit 1; } |
| 684 | sleep 5 |
| 685 | done |
| 686 | |
| 687 | # First provision: fresh arm64 binary + `lchat bootstrap` (writes the |
| 688 | # router/control/cloud units and starts the router). After this, routine |
| 689 | # updates go through `ops dev deploy ie`. |
| 690 | ver="$(git -C "$root" rev-parse --short HEAD)" |
| 691 | tmp="$(mktemp -d -t lchat-rebuild.XXXXXX)" |
| 692 | trap 'rm -rf "$tmp"' EXIT |
| 693 | cd "$root" # we were in infra/dev-mesh; build-linux.sh compiles the cwd package |
| 694 | echo "ops dev: building $ver for linux/arm64 and bootstrapping" |
| 695 | OUT="$tmp/lchat" ARCH=arm64 "$here/build-linux.sh" "$ver" |
| 696 | scp_ie -q "$tmp/lchat" "ubuntu@$ip:/tmp/lchat" |
| 697 | ssh_ie "ubuntu@$ip" 'sudo mkdir -p /opt/lchat && sudo mv /tmp/lchat /opt/lchat/lchat \ |
| 698 | && sudo chmod +x /opt/lchat/lchat && sudo /opt/lchat/lchat bootstrap' |
| 699 | |
| 700 | # The env files are derived facts plus one secret — encoded here so no |
| 701 | # runbook is needed. The mesh token is READ from the surviving Helsinki box |
| 702 | # (the pair must match) and never printed; the cloud admin token is freshly |
| 703 | # minted, because a rebuild is a new cloud. |
| 704 | echo "ops dev: writing /data/router.env + /data/cloud.env" |
| 705 | mesh_token="$(ssh -o ConnectTimeout=10 "$BOX_FI" \ |
| 706 | "sudo grep '^LCHAT_MESH_TOKEN=' /data/router.env | cut -d= -f2-" 2>/dev/null || true)" |
| 707 | [ -n "$mesh_token" ] || echo "ops dev: WARNING: couldn't read Helsinki's mesh token —" \ |
| 708 | "router.env is written without LCHAT_MESH_TOKEN; the mesh stays unpaired until it is set on both boxes" >&2 |
| 709 | admin_token="$(head -c 24 /dev/urandom | od -An -tx1 | tr -d ' \n')" |
| 710 | { |
| 711 | printf 'LCHAT_MESH_DOMAINS=dev.11ch.at\n' |
| 712 | printf 'LCHAT_DNS_PROVIDER=route53\n' |
| 713 | printf 'LCHAT_WILDCARD_DOMAINS=dev.11ch.at\n' |
| 714 | printf 'AWS_REGION=eu-west-1\n' |
| 715 | printf 'LCHAT_HIBERNATE_BUCKET=%s\n' "$bucket" |
| 716 | printf 'LCHAT_MESH_REGION=ie\n' |
| 717 | printf 'LCHAT_MESH_SELF=https://one.ie.dev.11ch.at\n' |
| 718 | [ -z "$mesh_token" ] || printf 'LCHAT_MESH_TOKEN=%s\n' "$mesh_token" |
| 719 | } | ssh_ie "ubuntu@$ip" 'sudo tee /data/router.env >/dev/null && sudo chmod 600 /data/router.env' |
| 720 | { |
| 721 | printf 'LCHAT_CLOUD_BASE_DOMAIN=dev.11ch.at\n' |
| 722 | printf 'LCHAT_CLOUD_ID_LEDGER=1\n' |
| 723 | printf 'LCHAT_HIBERNATE_BUCKET=%s\n' "$bucket" |
| 724 | printf 'LCHAT_HIBERNATE_REGION=eu-west-1\n' |
| 725 | printf 'LCHAT_CLOUD_REGION=ie\n' |
| 726 | printf 'LCHAT_ADMIN_TOKEN=%s\n' "$admin_token" |
| 727 | } | ssh_ie "ubuntu@$ip" 'sudo tee /data/cloud.env >/dev/null && sudo chmod 600 /data/cloud.env' |
| 728 | ssh_ie "ubuntu@$ip" 'sudo systemctl restart lchat-router lchat-cloud lchat-control' |
| 729 | |
| 730 | # A fresh registry knows no routes: the surviving peer bulk-seeds every |
| 731 | # in-scope route into it (meshreplicate.go — mesh-seed exists exactly for a |
| 732 | # freshly joined box). sudo strips env, so the peer's mesh config is |
| 733 | # sourced explicitly (the old runbook's hardest-won gotcha). |
| 734 | if [ -n "$mesh_token" ]; then |
| 735 | echo "ops dev: seeding routes from Helsinki" |
| 736 | ssh -o ConnectTimeout=10 "$BOX_FI" \ |
| 737 | "sudo sh -c 'set -a; . /data/router.env; set +a; /opt/lchat/lchat mesh-seed https://one.ie.dev.11ch.at'" \ |
| 738 | || echo "ops dev: WARNING: mesh-seed failed — rerun it on Helsinki when the router is up" >&2 |
| 739 | fi |
| 740 | |
| 741 | # controld extras: without these a rebuilt Dublin silently provisions |
| 742 | # non-hibernating, uncapped spaces — the opposite of the dev mesh's point. |
| 743 | echo "ops dev: writing /data/control.env" |
| 744 | { |
| 745 | printf 'LCHAT_CONTROL_HIBERNATE_CREATES=1\n' |
| 746 | printf 'LCHAT_MAX_INSTANCES=40\n' |
| 747 | printf 'LCHAT_HIBERNATE_BUCKET=%s\n' "$bucket" |
| 748 | printf 'LCHAT_HIBERNATE_REGION=eu-west-1\n' |
| 749 | } | ssh_ie "ubuntu@$ip" 'sudo tee /data/control.env >/dev/null && sudo chmod 600 /data/control.env && sudo systemctl restart lchat-control' |
| 750 | |
| 751 | # The dev home server (encoded here per the old completion note). bootstrap |
| 752 | # already wrote its unit; home is its own binary. The env carries the |
| 753 | # WebAuthn RP origin — it MUST stay home.dev.elevenmessenger.com or every |
| 754 | # previously-enrolled dev passkey orphans — and one caller token per region, |
| 755 | # minted from each cloud's admin API over its local socket. Token values |
| 756 | # move through pipes and shell variables only; nothing prints them. |
| 757 | echo "ops dev: building + standing up the dev home server" |
| 758 | GOOS=linux GOARCH=arm64 CGO_ENABLED=0 go build -o "$tmp/home" ./home |
| 759 | scp_ie -q "$tmp/home" "ubuntu@$ip:/tmp/home" |
| 760 | ssh_ie "ubuntu@$ip" 'sudo mv /tmp/home /opt/lchat/home && sudo chmod +x /opt/lchat/home && sudo chown root:root /opt/lchat/home' |
| 761 | |
| 762 | tok_of() { sed -n 's/.*"token":"\([^"]*\)".*/\1/p'; } |
| 763 | ie_caller="$(ssh_ie "ubuntu@$ip" 'sudo sh' <<EOS | tok_of |
| 764 | curl -s --unix-socket /run/lchat/cloud.sock -X POST http://cloud/api/admin/tokens \ |
| 765 | -H "Authorization: Bearer $admin_token" -d '{"label":"home.dev"}' |
| 766 | EOS |
| 767 | )" |
| 768 | fi_caller="$(ssh -o ConnectTimeout=10 "$BOX_FI" 'sudo sh' <<'EOS' | tok_of |
| 769 | ADM="$(grep '^LCHAT_ADMIN_TOKEN=' /data/cloud.env | cut -d= -f2-)" |
| 770 | curl -s --unix-socket /run/lchat/cloud.sock -X POST http://cloud/api/admin/tokens \ |
| 771 | -H "Authorization: Bearer $ADM" -d '{"label":"home.dev"}' |
| 772 | EOS |
| 773 | )" |
| 774 | [ -n "$ie_caller" ] || echo "ops dev: WARNING: no ie caller token minted — home's ie region will 502 until one is set in /data/home.env" >&2 |
| 775 | [ -n "$fi_caller" ] || echo "ops dev: WARNING: no fi caller token minted — home's fi region will 502 until one is set in /data/home.env" >&2 |
| 776 | { |
| 777 | printf 'LCHAT_HOME_ORIGIN=https://home.dev.elevenmessenger.com\n' |
| 778 | printf 'LCHAT_CLOUD_REGIONS=[{"code":"ie","name":"Ireland","url":"https://cloud.ie.dev.11ch.at","token":"%s"},{"code":"fi","name":"Finland","url":"https://cloud.fi.dev.11ch.at","token":"%s"}]\n' "$ie_caller" "$fi_caller" |
| 779 | printf 'LCHAT_CLOUD_DEFAULT_REGION=ie\n' |
| 780 | # Sign in with Apple: the audiences are committed facts (web Services ID + |
| 781 | # iOS bundle id); the account-keying pepper is minted fresh — fine on a |
| 782 | # rebuilt (empty) anchor DB, where there are no siwa anchors to orphan. |
| 783 | # Without these the /siwa/return trampoline answers 404 and the Apple door |
| 784 | # dies silently (2026-07-23). Google/Microsoft doors need OAuth secrets and |
| 785 | # stay dark until set by hand. |
| 786 | printf 'LCHAT_HOME_SIWA_CLIENT_IDS=com.elevenmessenger.signin,com.elevenmessenger.messenger\n' |
| 787 | printf 'LCHAT_HOME_SIWA_PEPPER=%s\n' "$(head -c 24 /dev/urandom | od -An -tx1 | tr -d ' \n')" |
| 788 | } | ssh_ie "ubuntu@$ip" 'sudo tee /data/home.env >/dev/null && sudo chmod 600 /data/home.env' |
| 789 | ssh_ie "ubuntu@$ip" 'sudo systemctl enable --now lchat-home' |
| 790 | # Service routes are local registry rows mesh-seed doesn't carry; re-adding |
| 791 | # an existing row fails harmlessly on a re-run. |
| 792 | ssh_ie "ubuntu@$ip" 'sudo /opt/lchat/lchat add-service home.dev.elevenmessenger.com --socket /run/lchat/home.sock' \ |
| 793 | || echo "ops dev: (home route already present)" |
| 794 | ssh_ie "ubuntu@$ip" 'sudo /opt/lchat/lchat add-service cloud.ie.dev.11ch.at --socket /run/lchat/cloud.sock' \ |
| 795 | || echo "ops dev: (cloud route already present)" |
| 796 | |
| 797 | cat <<EOF |
| 798 | |
| 799 | Dublin rebuilt on $ip (same EIP — DNS untouched), running $ver, trusting only |
| 800 | your key: env files written (router/cloud/control/home), units restarted, |
| 801 | routes seeded from Helsinki, home + cloud service routes added. |
| 802 | Still not encoded here — EXTEND THIS VERB when next needed, don't reach for a |
| 803 | runbook: the dev connect broker (lchat-connect-dev needs the vault's sandbox |
| 804 | Stripe pair) and the cloud.env demo/pool/billing extras. |
| 805 | verify: ssh ubuntu@$ip journalctl -u lchat-router -n 40 --no-pager | grep -i wildcard |
| 806 | curl -s https://home.dev.elevenmessenger.com/api/features |
| 807 | then: land the compute.tf PR if it changed for this rebuild, and |
| 808 | ./deploy/ops ledger "rebuilt the dev-mesh Dublin box" |
| 809 | EOF |
| 810 | ;; |
| 811 | |
| 812 | *) usage ;; |
| 813 | esac |
| 814 | |
amadan