amadan
ops: dev deploy refuses a box with the updater on, and skips home where it isn't run (#1448)
Both found deploying today's create-flow fixes to the dev mesh. The updater timer was ENABLED on Dublin — a pass fired 55s after the swap and took the control binary and home back to stable, while the deploy reported success (the check only asserts the unit is active, not that it is yours). provision asserts that invariant; deploy now checks it too, before a build and restart go to waste. And --home failed the whole run on Helsinki: swap_home tested for the unit's presence, but bootstrap writes the unit file on every box. The "skipped, never failed" case needs is-enabled. Claude-Session: https://claude.ai/code/session_01GUA1EJFYwgr2nZNVBRFf8k
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
1 file changed,
+34
−5
deploy/ops-dev.sh+34 −5
diff --git a/deploy/ops-dev.sh b/deploy/ops-dev.sh| index b6f6094..28b7876 100755 |
| --- a/deploy/ops-dev.sh |
| +++ b/deploy/ops-dev.sh |
| @@ -314,17 +314,27 @@ swap_box() { |
| # swap_home <ie|fi> <ver>: the home service rides its own binary (releases |
| # ship it as lchat-home-linux-* via updaterservices.go; the dev mesh swaps it |
| -# directly, same as the control binary). A box without the unit (Helsinki |
| +# directly, same as the control binary). A box that doesn't RUN home (Helsinki |
| # today — home runs on the dev apex box only) is skipped, never failed. |
| +# |
| +# Enabled, not merely present: `lchat bootstrap` writes the unit file on every |
| +# box it touches, so "the unit exists" says nothing about whether this box is |
| +# a home server. Helsinki has the file, disabled, with no /data/home.env — and |
| +# the old presence test walked straight into restarting it, which fails on the |
| +# missing env ("Failed to load environment files") and took the whole deploy |
| +# down with it after Dublin had already swapped fine. |
| swap_home() { |
| which_box="$1"; ver="$2" |
| if [ "$which_box" = ie ]; then box="$BOX_IE"; run() { ssh_ie "$box" "$@"; } |
| else box="$BOX_FI"; run() { ssh -o ConnectTimeout=10 "$box" "$@"; } |
| fi |
| - if ! run 'systemctl list-unit-files lchat-home.service --no-legend 2>/dev/null' | grep -q lchat-home; then |
| - echo " $which_box: no lchat-home unit — home skipped" |
| - return 0 |
| - fi |
| + case "$(run 'systemctl is-enabled lchat-home.service 2>/dev/null || true')" in |
| + enabled|enabled-runtime|static) ;; |
| + *) |
| + echo " $which_box: lchat-home not enabled here — home skipped" |
| + run 'rm -f /opt/lchat/incoming/home-new' || true |
| + return 0 ;; |
| + esac |
| run "sudo cp /opt/lchat/home /opt/lchat/home.prev-$ver 2>/dev/null || true; \ |
| sudo mv /opt/lchat/incoming/home-new /opt/lchat/home \ |
| && sudo chmod +x /opt/lchat/home \ |
| @@ -381,6 +391,25 @@ deploy) |
| esac |
| done |
| + # The invariant `provision` asserts, checked again HERE — before a build, a |
| + # copy and a restart go to waste. A dev box with the auto-updater enabled |
| + # takes your unmerged code back to the stable channel within the hour, and |
| + # the deploy that put it there reports success on the way past: seen |
| + # 2026-08-28, when an update pass landed 55 seconds after a swap and quietly |
| + # restored the previous home binary. Cheap SSH, loud failure, named fix. |
| + for t in $targets; do |
| + if [ "$t" = ie ]; then probe_box="$BOX_IE"; timer="$(ssh_ie "$probe_box" 'systemctl is-enabled lchat-update.timer 2>/dev/null || true')" |
| + else probe_box="$BOX_FI"; timer="$(ssh -o ConnectTimeout=10 "$probe_box" 'systemctl is-enabled lchat-update.timer 2>/dev/null || true')" |
| + fi |
| + if [ "$timer" = enabled ]; then |
| + echo "ops dev: ABORT — $t's auto-update timer is ENABLED." >&2 |
| + echo " It would replace whatever you deploy with the current stable build," >&2 |
| + echo " within the hour, silently. Turn it off first:" >&2 |
| + echo " ./deploy/ops dev provision $t" >&2 |
| + exit 1 |
| + fi |
| + done |
| + |
| ver="$(git -C "$root" rev-parse --short HEAD)" |
| if ! git -C "$root" diff --quiet || ! git -C "$root" diff --cached --quiet; then |
| ver="${ver}-dirty" |