eleven / messenger Public

server+deploy: sibling-service binaries converge from the signed release feed (#1113)

connect/push/home deploy automatically now: publish uploads their linux
builds with sha256s pinned inside the signed manifest entry, and the hourly
root pass sha-compares each loose /opt/lchat/<name> against the channel
head — swap, bounce-if-active, roll back + quarantine on an unhealthy
restart. Same signature and bake discipline as the control binary's own
self-update. lchat-connect.service is repo-ified (verbatim from the box)
and bootstrap-installed like push/home.

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

Paul Campbell 3 months ago c8bdc1f91b5f7fee33b6fa6a707323e2593377c8 parent f0f7839

9 files changed, +599 −4

bootstrap.go +9 −0
deploy/release-server.sh +33 −2
deploy/systemd/lchat-connect.service +46 −0
docs/planet-scale-guide/deploys.html +1 −0
docs/server-releases.md +13 −2
release.go +53 −0
updater.go +3 −0
updaterservices.go +195 −0
updaterservices_test.go +246 −0

diff --git a/bootstrap.go b/bootstrap.go

index 0e1494b..03bb109 100644

--- a/bootstrap.go

+++ b/bootstrap.go

@@ -40,6 +40,14 @@ var unitPush string

//go:embed deploy/systemd/lchat-home.service

var unitHome string

+// The connect broker (OAuth relay, docs/integrations.md) is the third sibling

+// service; like push/home it installs everywhere and stays inert until an

+// operator stages /data/connect.env and enables it. This unit was hand-made on

+// the box before it lived here — keep the two in step.

+//

+//go:embed deploy/systemd/lchat-connect.service

+var unitConnect string

// The links redirector (11ch.at short links, issue #905) is also a sibling

// service, but unlike push/home it's a subcommand of this same binary, not a

// separate one, and holds no secrets or database — so it installs everywhere

@@ -174,6 +182,7 @@ func runBootstrap(args []string) {

write(filepath.Join(*unitDir, "lchat-hooks@.service"), unitHooks)

write(filepath.Join(*unitDir, "lchat-push.service"), unitPush)

write(filepath.Join(*unitDir, "lchat-home.service"), unitHome)

+ write(filepath.Join(*unitDir, "lchat-connect.service"), unitConnect)

write(filepath.Join(*unitDir, "lchat-links.service"), unitLinks)

write(filepath.Join(*unitDir, "lchat-control.service"), unitControl)

write(filepath.Join(*unitDir, "lchat-cloud.service"), unitCloud)

diff --git a/deploy/release-server.sh b/deploy/release-server.sh

index f35ddd4..33b2d2e 100755

--- a/deploy/release-server.sh

+++ b/deploy/release-server.sh

@@ -142,6 +142,32 @@ publish)

echo "$(shasum -a 256 "$TMP/$M" | cut -d' ' -f1) $M" > "$TMP/$M.sha256"

done

+ # Sibling-service binaries (connect broker, push relay, home server) ride the

+ # release too, with their sha256s pinned INSIDE the signed manifest entry —

+ # that's what lets the box's root update pass install them with no root SSH

+ # (updaterservices.go). name:builddir pairs; assets are lchat-<name>-linux-<arch>.

+ SERVICES="connect:broker push:push home:home"

+ SVC_JSON="{}"

+ SVC_ASSETS=""

+ for SPEC in $SERVICES; do

+ SNAME="${SPEC%%:*}"; SDIR="${SPEC##*:}"

+ SB="lchat-$SNAME-linux-amd64"; SBA="lchat-$SNAME-linux-arm64"

+ GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o "$TMP/$SB" "./$SDIR"

+ GOOS=linux GOARCH=arm64 CGO_ENABLED=0 go build -o "$TMP/$SBA" "./$SDIR"

+ echo "$(shasum -a 256 "$TMP/$SB" | cut -d' ' -f1) $SB" > "$TMP/$SB.sha256"

+ echo "$(shasum -a 256 "$TMP/$SBA" | cut -d' ' -f1) $SBA" > "$TMP/$SBA.sha256"

+ echo "built $VER (service $SNAME, amd64+arm64)"

+ SVC_JSON="$(printf '%s' "$SVC_JSON" | jq --arg n "$SNAME" \

+ --arg url "https://github.com/$REPO/releases/download/server-$VER/$SB" \

+ --arg sha "$(cut -d' ' -f1 < "$TMP/$SB.sha256")" \

+ --argjson size "$(wc -c < "$TMP/$SB" | tr -d ' ')" \

+ --arg aurl "https://github.com/$REPO/releases/download/server-$VER/$SBA" \

+ --arg asha "$(cut -d' ' -f1 < "$TMP/$SBA.sha256")" \

+ --argjson asize "$(wc -c < "$TMP/$SBA" | tr -d ' ')" \

+ '.[$n] = {url:$url, sha256:$sha, size:$size, arm64_url:$aurl, arm64_sha256:$asha, arm64_size:$asize}')"

+ SVC_ASSETS="$SVC_ASSETS $SB $SB.sha256 $SBA $SBA.sha256"

+ done

if gh release view "server-$VER" -R "$REPO" >/dev/null 2>&1; then

echo "release-server: server-$VER already exists on $REPO (promote it, or land a new commit)" >&2; exit 1

@@ -153,12 +179,16 @@ publish)

gh release create "server-$VER" -R "$REPO" --title "Server $VER" --notes "$NOTES" --latest=false \

"$TMP/$BIN" "$TMP/$BIN.sha256" "$TMP/$BIN_ARM" "$TMP/$BIN_ARM.sha256" \

"$TMP/$BIN_MAC" "$TMP/$BIN_MAC.sha256" "$TMP/$BIN_MAC_X64" "$TMP/$BIN_MAC_X64.sha256"

+ for A in $SVC_ASSETS; do

+ gh release upload "server-$VER" "$TMP/$A" -R "$REPO"

+ done

# Binaries mirror before the manifest that points at them, so the mirror

# never serves a feed whose mirror-side binaries don't exist yet. Keys copy

# the GitHub layout (server-<ver>/<asset>) — that's what the updater's

# mirror fallback derives from a primary binary URL.

for A in "$BIN" "$BIN.sha256" "$BIN_ARM" "$BIN_ARM.sha256" \

- "$BIN_MAC" "$BIN_MAC.sha256" "$BIN_MAC_X64" "$BIN_MAC_X64.sha256"; do

+ "$BIN_MAC" "$BIN_MAC.sha256" "$BIN_MAC_X64" "$BIN_MAC_X64.sha256" \

+ $SVC_ASSETS; do

mirror_upload "$TMP/$A" "server-$VER/$A"

done

@@ -174,9 +204,10 @@ publish)

--arg url "https://github.com/$REPO/releases/download/server-$VER/$BIN" \

--arg url_arm "https://github.com/$REPO/releases/download/server-$VER/$BIN_ARM" \

--arg now "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg semver "$SEMVER" \

+ --argjson svc "$SVC_JSON" \

'.channels.edge = ({version:$ver, url:$url, sha256:$sha, size:$size,

arm64_url:$url_arm, arm64_sha256:$sha_arm, arm64_size:$size_arm,

- notes:$notes, published_at:$now}

+ notes:$notes, published_at:$now, services:$svc}

+ (if $semver == "" then {} else {semver:$semver} end))' \

> "$TMP/payload.json"

sign_and_upload "$TMP/payload.json"

diff --git a/deploy/systemd/lchat-connect.service b/deploy/systemd/lchat-connect.service

new file mode 100644

index 0000000..2278137

--- /dev/null

+++ b/deploy/systemd/lchat-connect.service

@@ -0,0 +1,46 @@

+[Unit]

+Description=lchat connect broker (OAuth relay + token exchange)

+After=network-online.target

+Wants=network-online.target

+[Service]

+# /data/connect.env holds LCHAT_ADMIN_TOKEN + the DNSimple OAuth client_id/secret

+# + the registered redirect_uri. Required (no leading '-'): a misconfigured broker

+# breaks connecting services fleet-wide, so it must fail to boot rather than start

+# silently broken.

+EnvironmentFile=/data/connect.env

+ExecStart=/opt/lchat/connect --addr /run/lchat/connect.sock --db /data/connect.db

+Restart=always

+RestartSec=1

+# --- hardening (see docs/instance-router.md § Hardening) ---

+User=lchat

+Group=lchat

+CapabilityBoundingSet=

+NoNewPrivileges=yes

+ProtectSystem=strict

+ProtectHome=yes

+ReadWritePaths=/data /run/lchat

+PrivateTmp=yes

+PrivateDevices=yes

+ProtectClock=yes

+ProtectHostname=yes

+ProtectKernelTunables=yes

+ProtectKernelModules=yes

+ProtectKernelLogs=yes

+ProtectControlGroups=yes

+ProtectProc=invisible

+RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX

+RestrictNamespaces=yes

+RestrictRealtime=yes

+RestrictSUIDSGID=yes

+LockPersonality=yes

+MemoryDenyWriteExecute=yes

+SystemCallFilter=@system-service

+SystemCallErrorNumber=EPERM

+SystemCallArchitectures=native

+MemoryMax=256M

+TasksMax=256

+[Install]

+WantedBy=multi-user.target

diff --git a/docs/planet-scale-guide/deploys.html b/docs/planet-scale-guide/deploys.html

index d3467f8..6af3036 100644

--- a/docs/planet-scale-guide/deploys.html

+++ b/docs/planet-scale-guide/deploys.html

@@ -144,6 +144,7 @@ ssh box-1 sudo -n /opt/lchat/lchat stage --file /opt/lchat/incoming/home-<sha

ssh box-1 sudo -n /opt/lchat/lchat install home --version <ver>

ssh box-1 sudo -n /usr/local/sbin/lchat-svc restart lchat-home</pre>

<code>install</code> knows the sibling set (<code>home</code>, <code>push</code>, <code>connect</code>), backs the outgoing binary up beside itself as <code><name>.prev-<date></code> (the one-copy rollback), and deliberately does not restart the unit — the restart is the operator's visible act, so an install can be staged ahead of a maintenance window. The version store doesn't care what a binary is: <code>stage --file</code> records any build under its own version id, so sibling builds sit beside messenger builds. Extending the sibling set is a reviewed PR, never a box edit.

+running today Routine sibling deploys don't need even that: every publish also uploads <code>lchat-{connect,push,home}-linux-<arch></code> with their sha256s pinned inside the signed manifest entry, and the hourly pass converges the sibling binaries automatically right after the control binary's own self-update — sha-comparing each loose <code>/opt/lchat/<name></code> against the channel head, swapping any that drifted (a <code>.prev</code> kept beside it), bouncing the unit only if it was running, and rolling back + quarantining a build whose unit doesn't stay up. Same signature, same bake window, same channel (<code>LCHAT_CONTROL_CHANNEL</code>). A box that doesn't run a service has no file at its path and is skipped — nothing installs uninvited. The manual <code>install</code> verb stays as the ad-hoc/canary path.

running today The control/router binary <code>/opt/lchat/lchat</code> self-updates on the same hourly pass: after the instances, the updater installs its channel's head over its own path — same signed feed, same sha check, same bake window — keeping the outgoing build beside it as <code>lchat.prev-<build></code> (the one-rename rollback). The channel is <code>LCHAT_CONTROL_CHANNEL</code> in <code>/data/update.env</code> (<code>stable</code> by default — the trust plane takes new builds last; <code>off</code> disables self-update). After a swap the pass restarts <code>lchat-control</code> and <code>lchat-cloud</code> (backend sockets, invisible to members) but never the router — its front door must not blip on a timer's schedule, so <code>lchat-router</code> keeps running the old build until an operator restarts it, and the pass says so in its log. This is also how root-less operator boxes stay current: the manual root <code>cp</code> this replaced stopped being possible when root SSH went bootstrap-only.

<h2 id="verifying">Verifying what is deployed where</h2>

diff --git a/docs/server-releases.md b/docs/server-releases.md

index 1843262..edfc81f 100644

--- a/docs/server-releases.md

+++ b/docs/server-releases.md

@@ -18,8 +18,19 @@ app's Sparkle appcast uses, see `ios/release-mac.sh`):

`deploy/build-linux.sh` stamps into the binary as `main.buildVersion`),

carrying **both linux architectures** — `lchat-linux-amd64` and

`lchat-linux-arm64` — plus `lchat-darwin-{arm64,amd64}` (the website's

- `lchat try` download for Macs; never referenced by the manifest), each with

- its `.sha256`.

+ `lchat try` download for Macs; never referenced by the manifest) and the

+ **sibling-service binaries** `lchat-{connect,push,home}-linux-{amd64,arm64}`,

+ each with its `.sha256`.

+ The service binaries are how connect/push/home deploy at all now that the

+ box has no root SSH: their sha256s ride *inside* the signed manifest entry

+ (`services`), and the box's root update pass (`updaterservices.go`)

+ sha-compares each loose `/opt/lchat/<name>` against the channel head and

+ swaps + bounces any that drifted — same signature, same bake window, same

+ rollback-and-quarantine shape as the control binary's own self-update. A box

+ without a service's file is skipped, so nothing installs uninvited. (A

+ promote whose entry had to be rebuilt from release assets carries no

+ `services` block, and the converge simply skips until the next publish.)

- **One rolling release**: tag `server`, whose assets live at fixed URLs

(re-uploaded with `--clobber`, so the URLs never change):

- `manifest.json` — the **feed**, updated on every publish/promote:

diff --git a/release.go b/release.go

index 21b24cb..b06239c 100644

--- a/release.go

+++ b/release.go

@@ -117,6 +117,36 @@ type releaseEntry struct {

// pinning (update_major) evaluates, so a semver-less entry HOLDS a

// major-pinned instance rather than moving it (fail closed).

Semver string `json:"semver,omitempty"`

+ // Services are the sibling-service binaries (connect broker, push relay,

+ // home server) published beside this build. Their shas ride the SIGNED

+ // manifest — never the loose .sha256 assets — so the box's service

+ // converge (updaterservices.go) trusts nothing the signature doesn't

+ // cover. Additive: absent on pre-feature releases, ignored by old

+ // updaters, and a rebuilt promote entry (release-server.sh's asset-rebuild

+ // path) carries none — the converge then simply skips.

+ Services map[string]serviceBuild `json:"services,omitempty"`

+// serviceBuild is one sibling-service binary of a release, per arch — the

+// same flat-amd64 + optional-arm64 shape as releaseEntry, for the same

+// old-updater-compatibility reason.

+type serviceBuild struct {

+ URL string `json:"url"`

+ SHA256 string `json:"sha256"`

+ Size int64 `json:"size,omitempty"`

+ ARM64URL string `json:"arm64_url,omitempty"`

+ ARM64SHA256 string `json:"arm64_sha256,omitempty"`

+ ARM64Size int64 `json:"arm64_size,omitempty"`

+func (s serviceBuild) forArch(goarch string) (url, sha string, ok bool) {

+ switch goarch {

+ case "amd64":

+ return s.URL, s.SHA256, s.URL != ""

+ case "arm64":

+ return s.ARM64URL, s.ARM64SHA256, s.ARM64URL != ""

+ }

+ return "", "", false

}

// forArch resolves the binary for a GOARCH: amd64 is the flat fields, arm64

@@ -189,10 +219,33 @@ func validateManifest(m *releaseManifest) error {

return fmt.Errorf("channel %s: invalid semver %q", ch, e.Semver)

}

+ for name, sb := range e.Services {

+ if !serviceNameRe.MatchString(name) {

+ return fmt.Errorf("channel %s: invalid service name %q", ch, name)

+ }

+ if !strings.HasPrefix(sb.URL, "https://") && !isLoopbackURL(sb.URL) {

+ return fmt.Errorf("channel %s: service %s URL must be https (%q)", ch, name, sb.URL)

+ }

+ if raw, err := hex.DecodeString(sb.SHA256); err != nil || len(raw) != sha256.Size {

+ return fmt.Errorf("channel %s: service %s sha256 must be %d hex chars", ch, name, sha256.Size*2)

+ }

+ if sb.ARM64URL != "" || sb.ARM64SHA256 != "" {

+ if !strings.HasPrefix(sb.ARM64URL, "https://") && !isLoopbackURL(sb.ARM64URL) {

+ return fmt.Errorf("channel %s: service %s arm64 URL must be https (%q)", ch, name, sb.ARM64URL)

+ }

+ if raw, err := hex.DecodeString(sb.ARM64SHA256); err != nil || len(raw) != sha256.Size {

+ return fmt.Errorf("channel %s: service %s arm64 sha256 must be %d hex chars", ch, name, sha256.Size*2)

+ }

}

return nil

}

+// serviceNameRe pins service names to the boring lowercase words the box maps

+// to /opt/lchat/<name> — anything fancier is a manifest bug, not a feature.

+var serviceNameRe = regexp.MustCompile(`^[a-z][a-z0-9-]{0,31}$`)

// semverRe pins the plain MAJOR.MINOR.PATCH shape the release feed stamps —

// no pre-release/build suffixes until something actually needs them.

var semverRe = regexp.MustCompile(`^(\d{1,4})\.\d{1,4}\.\d{1,4}$`)

diff --git a/updater.go b/updater.go

index c66e977..455f107 100644

--- a/updater.go

+++ b/updater.go

@@ -562,6 +562,9 @@ func updatePass(reg *Registry, unitTmpl string, dryRun, ignoreBake bool) error {

if err := controlSelfUpdate(m, bake, poisoned, dryRun, ignoreBake); err != nil {

fmt.Printf("update: control self-update: %v\n", err)

}

+ // The sibling services (connect/push/home) converge from the same signed

+ // entry, after the control binary — see updaterservices.go.

+ serviceBinariesUpdate(m, bake, poisoned, dryRun, ignoreBake)

return nil

}

diff --git a/updaterservices.go b/updaterservices.go

new file mode 100644

index 0000000..6596d5f

--- /dev/null

+++ b/updaterservices.go

@@ -0,0 +1,195 @@

+package main

+// The sibling services — connect broker, push relay, home server — run from

+// loose /opt/lchat/<name> binaries that root used to scp onto the box, a path

+// the operator-allowlist hardening removed (docs/instance-router.md § Box

+// access). They now converge from the SAME signed manifest entry the control

+// binary follows (controlSelfUpdate): publish uploads lchat-<name>-linux-<arch>

+// beside the instance binaries and pins their sha256s INSIDE the signed

+// payload, and the already-root hourly pass swaps any binary whose on-disk

+// sha doesn't match — same signature, same bake window, same rollback shape.

+//

+// Convergence is sha-driven, not version-driven: the services carry no build

+// stamp, and comparing hashes also heals a tampered or half-written binary.

+// A box that doesn't run a service simply has no /opt/lchat/<name> file and

+// is skipped — nothing is ever installed onto a box that didn't opt in.

+import (

+ "crypto/sha256"

+ "encoding/hex"

+ "fmt"

+ "io"

+ "log"

+ "os"

+ "path/filepath"

+ "strings"

+ "time"

+// serviceBinaries maps a manifest `services` key to the loose binary path

+// component under optDir and the unit that execs it. Order is cosmetic.

+// lchat-links is deliberately absent: it's a subcommand of the control binary,

+// so it rides controlSelfUpdate.

+var serviceBinaries = []struct {

+ Name string // manifest key AND /opt/lchat/<Name>

+ Unit string

+}{

+ {"connect", "lchat-connect"},

+ {"push", "lchat-push"},

+ {"home", "lchat-home"},

+// serviceHealthSettle is how long a bounced unit gets to crash before we call

+// the swap good — Restart=always units flap fast, so a binary that can't boot

+// shows as inactive/activating within a beat. A var so tests shrink it.

+var serviceHealthSettle = 3 * time.Second

+func serviceFailedPath(name string) string {

+ return filepath.Join(optDir, ".service-failed-"+name)

+func readServiceFailed(name string) string {

+ b, err := os.ReadFile(serviceFailedPath(name))

+ if err != nil {

+ return ""

+ }

+ return strings.TrimSpace(string(b))

+// fileSHA256 hex-hashes a file; os.IsNotExist(err) is the caller's "this box

+// doesn't run the service" signal.

+func fileSHA256(path string) (string, error) {

+ f, err := os.Open(path)

+ if err != nil {

+ return "", err

+ }

+ defer f.Close()

+ h := sha256.New()

+ if _, err := io.Copy(h, f); err != nil {

+ return "", err

+ }

+ return hex.EncodeToString(h.Sum(nil)), nil

+// serviceBinariesUpdate converges the sibling-service binaries onto the

+// control channel's head. Runs from updatePass AFTER controlSelfUpdate — on

+// the pass that swaps the control binary this (still-old) process runs the

+// converge with old code, which is fine: the logic is data-driven from the

+// manifest, and the next pass runs it from the new build anyway. Like the

+// control swap, failures log and never fail the pass — the instances above

+// already converged.

+func serviceBinariesUpdate(m *releaseManifest, bake *firstSeenState, poisoned map[string]bool, dryRun, ignoreBake bool) {

+ channel := envOr("LCHAT_CONTROL_CHANNEL", "stable")

+ if channel == "off" {

+ return

+ }

+ e, ok := m.Channels[channel]

+ if !ok || len(e.Services) == 0 {

+ // Pre-feature releases (and promote's rebuilt-from-assets entries)

+ // carry no services block — nothing to converge on.

+ return

+ }

+ if poisoned[e.Version] {

+ fmt.Printf("update: skipping service binaries for %s — it failed a health check on an instance this pass\n", e.Version)

+ return

+ }

+ if hold := bakeHold(bake, channel, e.Version, ignoreBake, time.Now()); hold != "" {

+ fmt.Printf("update: service binaries -> %s: %s\n", e.Version, hold)

+ return

+ }

+ for _, svc := range serviceBinaries {

+ sb, ok := e.Services[svc.Name]

+ if !ok {

+ continue

+ }

+ path := filepath.Join(optDir, svc.Name)

+ cur, err := fileSHA256(path)

+ if err != nil {

+ // No file = this box doesn't run the service; anything else is

+ // worth a line but not a retry loop.

+ if !os.IsNotExist(err) {

+ fmt.Printf("update: service %s: %v\n", svc.Name, err)

+ }

+ continue

+ }

+ url, want, ok := sb.forArch(updaterArch)

+ if !ok {

+ continue // no build for this arch — same skip rule as instances

+ }

+ if cur == want {

+ continue

+ }

+ if failed := readServiceFailed(svc.Name); failed != "" {

+ if failed == e.Version {

+ fmt.Printf("update: service %s build %s is quarantined (failed its restart; pin or patch the channel to move)\n", svc.Name, e.Version)

+ continue

+ }

+ os.Remove(serviceFailedPath(svc.Name)) // channel moved on — retry

+ }

+ if dryRun {

+ fmt.Printf("update: service %s would update to %s (%s; dry run)\n", svc.Name, e.Version, channel)

+ continue

+ }

+ if err := swapServiceBinary(svc.Name, svc.Unit, path, url, want, e.Version); err != nil {

+ fmt.Printf("update: service %s: %v\n", svc.Name, err)

+ }

+// swapServiceBinary downloads (sha-gated, mirror fallback), atomically swaps

+// with a .prev kept beside it, and — only when the unit is actually running —

+// bounces it and confirms it stays up, rolling back and quarantining on

+// failure. An inactive unit gets the new file silently: the box staged the

+// binary but hasn't enabled the service, so there is nothing to health-check.

+func swapServiceBinary(name, unit, path, url, wantSha, ver string) error {

+ tmp, err := downloadBinaryToTemp(url, wantSha, ver)

+ if err != nil {

+ if releaseMirrorURL == "" {

+ return err

+ }

+ murl := mirrorBinaryURL(releaseMirrorURL, url)

+ log.Printf("update: service %s primary download failed (%v) — trying mirror %s", name, err, murl)

+ if tmp, err = downloadBinaryToTemp(murl, wantSha, ver); err != nil {

+ return fmt.Errorf("download: primary and mirror failed (%v)", err)

+ }

+ defer os.Remove(tmp)

+ if err := os.Chmod(tmp, 0o755); err != nil {

+ return fmt.Errorf("chmod: %v", err)

+ }

+ prev := path + ".prev"

+ if err := copyFileAtomic(path, prev, 0o755); err != nil {

+ return fmt.Errorf("keep rollback copy: %v", err)

+ }

+ // downloadBinaryToTemp lands in incomingDir() — same filesystem, atomic.

+ if err := os.Rename(tmp, path); err != nil {

+ return fmt.Errorf("install: %v", err)

+ }

+ if !isActive(unit) {

+ fmt.Printf("update: service %s -> %s (unit %s inactive — applies when it starts; rollback at %s)\n", name, ver, unit, prev)

+ return nil

+ }

+ if err := systemctl("try-restart", unit); err != nil {

+ fmt.Printf("update: service %s restart: %v (new build applies on its next start)\n", name, err)

+ return nil

+ }

+ time.Sleep(serviceHealthSettle)

+ if isActive(unit) {

+ os.Remove(serviceFailedPath(name))

+ fmt.Printf("update: service %s -> %s (%s healthy; rollback at %s)\n", name, ver, unit, prev)

+ return nil

+ }

+ // The bounced unit didn't stay up: restore the known-good binary, bounce it

+ // back, and quarantine the head so the hourly pass doesn't flap forever.

+ log.Printf("update: service %s -> %s is UNHEALTHY — rolling back", name, ver)

+ if rberr := os.Rename(prev, path); rberr != nil {

+ return fmt.Errorf("SERVICE %s BRICKED: new build unhealthy AND restoring %s failed: %v", name, prev, rberr)

+ }

+ if err := systemctl("try-restart", unit); err != nil {

+ log.Printf("update: service %s rollback restart: %v", name, err)

+ }

+ if werr := os.WriteFile(serviceFailedPath(name), []byte(ver+"\n"), 0o644); werr != nil {

+ log.Printf("update: service %s: persist quarantine: %v", name, werr)

+ }

+ return fmt.Errorf("build %s failed its restart — rolled back and quarantined", ver)

diff --git a/updaterservices_test.go b/updaterservices_test.go

new file mode 100644

index 0000000..0416c9d

--- /dev/null

+++ b/updaterservices_test.go

@@ -0,0 +1,246 @@

+package main

+import (

+ "crypto/sha256"

+ "encoding/hex"

+ "net/http"

+ "net/http/httptest"

+ "os"

+ "path/filepath"

+ "sync/atomic"

+ "testing"

+ "time"

+// serviceTestEnv points the service-converge seams at temp doubles: a temp

+// optDir holding (or not holding) the loose service binary, a recording

+// systemctl, a scriptable isActive, amd64 entries regardless of the dev

+// machine's arch, and a zero settle so unhealthy paths fail in a beat.

+func serviceTestEnv(t *testing.T) (calls *[]string, setActive func(func(string) bool)) {

+ t.Helper()

+ oldOpt, oldSysctl, oldActive := optDir, systemctl, isActive

+ oldArch, oldSettle := updaterArch, serviceHealthSettle

+ t.Cleanup(func() {

+ optDir, systemctl, isActive = oldOpt, oldSysctl, oldActive

+ updaterArch, serviceHealthSettle = oldArch, oldSettle

+ })

+ optDir = t.TempDir()

+ updaterArch = "amd64"

+ serviceHealthSettle = 0

+ t.Setenv("LCHAT_CONTROL_CHANNEL", "edge")

+ c := []string{}

+ calls = &c

+ systemctl = func(args ...string) error {

+ *calls = append(*calls, joinArgs(args))

+ return nil

+ }

+ setActive = func(f func(string) bool) { isActive = f }

+ setActive(func(string) bool { return true })

+ return calls, setActive

+func joinArgs(args []string) string {

+ out := ""

+ for i, a := range args {

+ if i > 0 {

+ out += " "

+ }

+ out += a

+ }

+ return out

+func shaOf(b []byte) string {

+ h := sha256.Sum256(b)

+ return hex.EncodeToString(h[:])

+// serviceManifest builds a one-channel manifest whose edge head carries one

+// "connect" service binary served by the given URL, plus a bake state that

+// has already seen the head (edge bakes for 0, so recorded = eligible).

+func serviceManifest(url string, bin []byte) (*releaseManifest, *firstSeenState) {

+ m := &releaseManifest{

+ Schema: 1,

+ GeneratedAt: time.Now(),

+ Channels: map[string]releaseEntry{

+ "edge": {

+ Version: "svcnew1",

+ URL: url, // instance binary URL — unused by the service pass

+ SHA256: shaOf([]byte("unused-instance-binary")),

+ Services: map[string]serviceBuild{

+ "connect": {URL: url, SHA256: shaOf(bin)},

+ },

+ }

+ bake := &firstSeenState{Channels: map[string]map[string]time.Time{

+ "edge": {"svcnew1": time.Now().Add(-time.Hour)},

+ }}

+ return m, bake

+func TestServiceUpdateSwapsAndRestarts(t *testing.T) {

+ calls, _ := serviceTestEnv(t)

+ newBin := []byte("NEW-CONNECT-BINARY")

+ var hits atomic.Int32

+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

+ hits.Add(1)

+ w.Write(newBin)

+ }))

+ defer ts.Close()

+ path := filepath.Join(optDir, "connect")

+ if err := os.WriteFile(path, []byte("OLD-CONNECT-BINARY"), 0o755); err != nil {

+ t.Fatal(err)

+ }

+ m, bake := serviceManifest(ts.URL+"/lchat-connect-linux-amd64", newBin)

+ serviceBinariesUpdate(m, bake, nil, false, false)

+ got, _ := os.ReadFile(path)

+ if string(got) != string(newBin) {

+ t.Fatalf("binary not swapped: %q", got)

+ }

+ prev, err := os.ReadFile(path + ".prev")

+ if err != nil || string(prev) != "OLD-CONNECT-BINARY" {

+ t.Fatalf("rollback copy wrong: %q err %v", prev, err)

+ }

+ if len(*calls) != 1 || (*calls)[0] != "try-restart lchat-connect" {

+ t.Fatalf("systemctl calls: %v", *calls)

+ }

+ if _, err := os.Stat(serviceFailedPath("connect")); !os.IsNotExist(err) {

+ t.Fatal("healthy swap must not quarantine")

+ }

+ if hits.Load() == 0 {

+ t.Fatal("nothing downloaded")

+ }

+func TestServiceUpdateSkipsAbsentBinary(t *testing.T) {

+ calls, _ := serviceTestEnv(t)

+ var hits atomic.Int32

+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

+ hits.Add(1)

+ }))

+ defer ts.Close()

+ m, bake := serviceManifest(ts.URL+"/x", []byte("NEW"))

+ serviceBinariesUpdate(m, bake, nil, false, false)

+ if hits.Load() != 0 || len(*calls) != 0 {

+ t.Fatalf("a box without the service must be untouched (hits %d, calls %v)", hits.Load(), *calls)

+ }

+func TestServiceUpdateNoopWhenCurrent(t *testing.T) {

+ calls, _ := serviceTestEnv(t)

+ cur := []byte("CURRENT-BINARY")

+ m, bake := serviceManifest("https://example.invalid/x", cur)

+ path := filepath.Join(optDir, "connect")

+ if err := os.WriteFile(path, cur, 0o755); err != nil {

+ t.Fatal(err)

+ }

+ serviceBinariesUpdate(m, bake, nil, false, false)

+ if len(*calls) != 0 {

+ t.Fatalf("sha match must be a no-op, got calls %v", *calls)

+ }

+ if _, err := os.Stat(path + ".prev"); !os.IsNotExist(err) {

+ t.Fatal("no-op must not write a rollback copy")

+ }

+func TestServiceUpdateRollsBackUnhealthyAndQuarantines(t *testing.T) {

+ calls, setActive := serviceTestEnv(t)

+ newBin := []byte("BAD-NEW-BINARY")

+ var hits atomic.Int32

+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

+ hits.Add(1)

+ w.Write(newBin)

+ }))

+ defer ts.Close()

+ // Active before the swap (so the bounce happens), dead ever after.

+ var checks atomic.Int32

+ setActive(func(string) bool { return checks.Add(1) == 1 })

+ path := filepath.Join(optDir, "connect")

+ if err := os.WriteFile(path, []byte("GOOD-OLD-BINARY"), 0o755); err != nil {

+ t.Fatal(err)

+ }

+ m, bake := serviceManifest(ts.URL+"/x", newBin)

+ serviceBinariesUpdate(m, bake, nil, false, false)

+ got, _ := os.ReadFile(path)

+ if string(got) != "GOOD-OLD-BINARY" {

+ t.Fatalf("unhealthy build must roll back, file is %q", got)

+ }

+ if readServiceFailed("connect") != "svcnew1" {

+ t.Fatalf("expected quarantine of svcnew1, got %q", readServiceFailed("connect"))

+ }

+ // try-restart onto the bad build + try-restart back = 2 calls.

+ if len(*calls) != 2 {

+ t.Fatalf("systemctl calls: %v", *calls)

+ }

+ // A second pass at the same head must short-circuit on the quarantine.

+ before := hits.Load()

+ serviceBinariesUpdate(m, bake, nil, false, false)

+ if hits.Load() != before {

+ t.Fatal("quarantined head was downloaded again")

+ }

+func TestServiceUpdateInactiveUnitSwapsQuietly(t *testing.T) {

+ calls, setActive := serviceTestEnv(t)

+ setActive(func(string) bool { return false })

+ newBin := []byte("NEW-BINARY")

+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

+ w.Write(newBin)

+ }))

+ defer ts.Close()

+ path := filepath.Join(optDir, "connect")

+ if err := os.WriteFile(path, []byte("OLD"), 0o755); err != nil {

+ t.Fatal(err)

+ }

+ m, bake := serviceManifest(ts.URL+"/x", newBin)

+ serviceBinariesUpdate(m, bake, nil, false, false)

+ got, _ := os.ReadFile(path)

+ if string(got) != string(newBin) {

+ t.Fatalf("inactive unit still gets the file, got %q", got)

+ }

+ if len(*calls) != 0 {

+ t.Fatalf("inactive unit must not be restarted: %v", *calls)

+ }

+func TestServiceUpdateFailsClosedOnUnbakedHead(t *testing.T) {

+ calls, _ := serviceTestEnv(t)

+ var hits atomic.Int32

+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

+ hits.Add(1)

+ }))

+ defer ts.Close()

+ path := filepath.Join(optDir, "connect")

+ if err := os.WriteFile(path, []byte("OLD"), 0o755); err != nil {

+ t.Fatal(err)

+ }

+ m, _ := serviceManifest(ts.URL+"/x", []byte("NEW"))

+ // A bake clock that has never seen the head: fail closed, nothing moves.

+ empty := &firstSeenState{Channels: map[string]map[string]time.Time{}}

+ serviceBinariesUpdate(m, empty, nil, false, false)

+ if hits.Load() != 0 || len(*calls) != 0 {

+ t.Fatalf("unbaked head must hold (hits %d, calls %v)", hits.Load(), *calls)

+ }

+func TestServiceUpdateIgnoresEntryWithoutServices(t *testing.T) {

+ calls, _ := serviceTestEnv(t)

+ path := filepath.Join(optDir, "connect")

+ if err := os.WriteFile(path, []byte("OLD"), 0o755); err != nil {

+ t.Fatal(err)

+ }

+ m := &releaseManifest{Schema: 1, GeneratedAt: time.Now(), Channels: map[string]releaseEntry{

+ "edge": {Version: "oldver12", URL: "https://example.invalid/x", SHA256: shaOf([]byte("y"))},

+ }}

+ bake := &firstSeenState{Channels: map[string]map[string]time.Time{"edge": {"oldver12": time.Now().Add(-time.Hour)}}}

+ serviceBinariesUpdate(m, bake, nil, false, false)

+ if len(*calls) != 0 {

+ t.Fatalf("pre-feature manifest must be a no-op, got %v", *calls)

+ }