eleven / ocho Public

server: the control binary self-updates from the signed release feed (#981)

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

Paul Campbell 3 months ago aa4d352d07137a301e4485d36987cae90178cb1a parent f2ab0cf

5 files changed, +230 −6

control_selfupdate_test.go +125 −0
docs/planet-scale-guide/deploys.html +2 −1
docs/secrets.md +7 −5
updater.go +89 −0
web/status/ssh/2026-07-15-managed-ai-deploys.json +7 −0

diff --git a/control_selfupdate_test.go b/control_selfupdate_test.go

new file mode 100644

index 0000000..e62ab2d

--- /dev/null

+++ b/control_selfupdate_test.go

@@ -0,0 +1,125 @@

+package main

+import (

+ "crypto/sha256"

+ "encoding/hex"

+ "net/http"

+ "net/http/httptest"

+ "os"

+ "path/filepath"

+ "strings"

+ "testing"

+ "time"

+// selfUpdateEnv points every seam at temp doubles: a fake control binary on

+// the same temp "filesystem" the download lands on, a fresh bake state, and a

+// recording systemctl.

+func selfUpdateEnv(t *testing.T, newBinary []byte) (self string, entry releaseEntry, calls *[]string) {

+ t.Helper()

+ oldOpt, oldSelf, oldSysctl, oldChannel := optDir, selfPath, systemctl, os.Getenv("LCHAT_CONTROL_CHANNEL")

+ oldBuild := buildVersion

+ buildVersion = "testold1" // the dev-build guard must not fire in these tests

+ t.Cleanup(func() {

+ optDir, selfPath, systemctl, buildVersion = oldOpt, oldSelf, oldSysctl, oldBuild

+ os.Setenv("LCHAT_CONTROL_CHANNEL", oldChannel)

+ })

+ optDir = t.TempDir()

+ self = filepath.Join(optDir, "lchat")

+ if err := os.WriteFile(self, []byte("OLD-CONTROL-BINARY"), 0o755); err != nil {

+ t.Fatal(err)

+ }

+ selfPath = func() (string, error) { return self, nil }

+ var c []string

+ calls = &c

+ systemctl = func(args ...string) error {

+ c = append(c, strings.Join(args, " "))

+ return nil

+ }

+ os.Setenv("LCHAT_CONTROL_CHANNEL", "stable")

+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

+ w.Write(newBinary)

+ }))

+ t.Cleanup(srv.Close)

+ sum := sha256.Sum256(newBinary)

+ entry = releaseEntry{Version: "abc1234", URL: srv.URL + "/lchat", SHA256: hex.EncodeToString(sum[:]), Size: int64(len(newBinary)),

+ ARM64URL: srv.URL + "/lchat-arm64", ARM64SHA256: hex.EncodeToString(sum[:]), ARM64Size: int64(len(newBinary))}

+ return self, entry, calls

+// The swap end to end: verified download, atomic install over the running

+// path, a .prev rollback beside it, and the backend services bounced —

+// router deliberately untouched.

+func TestControlSelfUpdateSwaps(t *testing.T) {

+ newBin := []byte("NEW-CONTROL-BINARY")

+ self, entry, calls := selfUpdateEnv(t, newBin)

+ m := &releaseManifest{Schema: 1, GeneratedAt: time.Now(), Channels: map[string]releaseEntry{"stable": entry}}

+ // ignoreBake=true is the deliberate override path; the bake-hold path is

+ // pinned separately below.

+ controlSelfUpdate(m, loadFirstSeen(), false, true)

+ got, err := os.ReadFile(self)

+ if err != nil || string(got) != string(newBin) {

+ t.Fatalf("control binary not swapped: %q %v", got, err)

+ }

+ prev, err := os.ReadFile(self + ".prev-" + buildVersion)

+ if err != nil || string(prev) != "OLD-CONTROL-BINARY" {

+ t.Fatalf("rollback copy missing/wrong: %q %v", prev, err)

+ }

+ joined := strings.Join(*calls, ";")

+ if !strings.Contains(joined, "try-restart lchat-control.service") || !strings.Contains(joined, "try-restart lchat-cloud.service") {

+ t.Fatalf("backend services not bounced: %v", *calls)

+ }

+ if strings.Contains(joined, "lchat-router") {

+ t.Fatalf("the router must never restart on the timer's schedule: %v", *calls)

+ }

+// A head still inside its bake window is held — the trust plane is the LAST

+// thing on a box that should take a new build.

+func TestControlSelfUpdateHonorsBake(t *testing.T) {

+ self, entry, calls := selfUpdateEnv(t, []byte("NEW"))

+ m := &releaseManifest{Schema: 1, GeneratedAt: time.Now(), Channels: map[string]releaseEntry{"stable": entry}}

+ bake := loadFirstSeen()

+ bake.observe(m, time.Now()) // first sighting is now → inside the window

+ controlSelfUpdate(m, bake, false, false)

+ if got, _ := os.ReadFile(self); string(got) != "OLD-CONTROL-BINARY" {

+ t.Fatalf("baking head must not install: %q", got)

+ }

+ if len(*calls) != 0 {

+ t.Fatalf("no restarts while held: %v", *calls)

+ }

+// "off" and already-at-head are silent no-ops; a sha mismatch leaves the

+// binary untouched (the verified-download contract, pinned here for the

+// control path specifically).

+func TestControlSelfUpdateNoOpsAndBadSha(t *testing.T) {

+ self, entry, _ := selfUpdateEnv(t, []byte("NEW"))

+ os.Setenv("LCHAT_CONTROL_CHANNEL", "off")

+ controlSelfUpdate(&releaseManifest{Channels: map[string]releaseEntry{"stable": entry}}, loadFirstSeen(), false, true)

+ if got, _ := os.ReadFile(self); string(got) != "OLD-CONTROL-BINARY" {

+ t.Fatal("off must be a no-op")

+ }

+ os.Setenv("LCHAT_CONTROL_CHANNEL", "stable")

+ same := entry

+ same.Version = buildVersion

+ controlSelfUpdate(&releaseManifest{Channels: map[string]releaseEntry{"stable": same}}, loadFirstSeen(), false, true)

+ if got, _ := os.ReadFile(self); string(got) != "OLD-CONTROL-BINARY" {

+ t.Fatal("at-head must be a no-op")

+ }

+ bad := entry

+ bad.SHA256 = strings.Repeat("00", 32)

+ bad.ARM64SHA256 = strings.Repeat("00", 32) // whichever arch runs the test

+ controlSelfUpdate(&releaseManifest{Channels: map[string]releaseEntry{"stable": bad}}, loadFirstSeen(), false, true)

+ if got, _ := os.ReadFile(self); string(got) != "OLD-CONTROL-BINARY" {

+ t.Fatal("a sha mismatch must never install")

+ }

diff --git a/docs/planet-scale-guide/deploys.html b/docs/planet-scale-guide/deploys.html

index 734a614..a75cb93 100644

--- a/docs/planet-scale-guide/deploys.html

+++ b/docs/planet-scale-guide/deploys.html

@@ -138,7 +138,8 @@ test.example.com stable pinned 4f0a2d19 9c21e77e</code></pre>

<code>lchat update run --dry-run</code> prints what a pass would do without doing it — useful before enabling anything new. Hibernating instances get updates too, by a quieter route: the updater reads their preferences straight from their on-disk database (read-only) and simply repoints their <code>live</code> link, so they come up on the new build at their next wake, with no health check (a bad build on a hibernator is recovered with <code>rollback</code>).

<h2 id="sibling-deploys">Deploying the sibling services and the control binary</h2>

-The home server and push relay are not versioned instances — they show <code>-</code> in the version column and deploy the plain way: build the <code>home</code> or <code>push</code> binary, copy it over <code>/opt/lchat/home</code> or <code>/opt/lchat/push</code>, and <code>systemctl restart lchat-home</code> (or <code>lchat-push</code>). The control/router binary <code>/opt/lchat/lchat</code> itself is refreshed by copying the new build over it atomically (copy to a temporary name in the same directory, then rename) and restarting <code>lchat-router</code>; the auto-updater does not manage it.

+The home server and push relay are not versioned instances — they show <code>-</code> in the version column and deploy the plain way: build the <code>home</code> or <code>push</code> binary, copy it over <code>/opt/lchat/home</code> or <code>/opt/lchat/push</code>, and <code>systemctl restart lchat-home</code> (or <code>lchat-push</code>).

+running today The control/router binary <code>/opt/lchat/lchat</code> self-updates on the same hourly pass: after the instances, the updater installs its channel's head over its own path — same signed feed, same sha check, same bake window — keeping the outgoing build beside it as <code>lchat.prev-<build></code> (the one-rename rollback). The channel is <code>LCHAT_CONTROL_CHANNEL</code> in <code>/data/update.env</code> (<code>stable</code> by default — the trust plane takes new builds last; <code>off</code> disables self-update). After a swap the pass restarts <code>lchat-control</code> and <code>lchat-cloud</code> (backend sockets, invisible to members) but never the router — its front door must not blip on a timer's schedule, so <code>lchat-router</code> keeps running the old build until an operator restarts it, and the pass says so in its log. This is also how root-less operator boxes stay current: the manual root <code>cp</code> this replaced stopped being possible when root SSH went bootstrap-only.

<h2 id="verifying">Verifying what is deployed where</h2>

diff --git a/docs/secrets.md b/docs/secrets.md

index 028cb20..b4c4774 100644

--- a/docs/secrets.md

+++ b/docs/secrets.md

@@ -84,11 +84,13 @@ private — is `docs/secrets-vault.md`.

## Bots & integrations

-- **`ANTHROPIC_API_KEY`** — sidecar env only; bot code never runs in the

- server process, so a server compromise doesn't reach it. *Blast radius*:

- spend on the Anthropic account. *Rotate*: mint at the Anthropic console,

- swap sidecar envs, restart sidecars (note: a server release alone never

- restarts sidecars).

+- **`ANTHROPIC_API_KEY`** — sidecar envs, plus (since the AI opt-in, #858)

+ `/data/control.env`: controld's attach-bot step writes it into each fresh

+ sidecar env it stages, so the provisioning daemon must hold it — bot code

+ still never runs in the server process, and no *instance* env ever carries

+ it. *Blast radius*: spend on the Anthropic account. *Rotate*: mint at the

+ Anthropic console, swap sidecar envs + control.env, restart sidecars and

+ `lchat-control` (note: a server release alone never restarts sidecars).

- **Bot identity keys** — the deliberate exception: real E2EE member keys,

held in sidecar session state, able to decrypt exactly the threads the bot

was added to. *Rotate*: there is no key rotation — remove the bot member

diff --git a/updater.go b/updater.go

index 4bf2ee7..104df14 100644

--- a/updater.go

+++ b/updater.go

@@ -550,9 +550,98 @@ func updatePass(reg *Registry, unitTmpl string, dryRun, ignoreBake bool) error {

if !dryRun {

gcVersions(reg, m, pinnedVers)

}

+ // Last, the box's own trust plane: the control binary rides the same

+ // signed feed (see controlSelfUpdate) — after instances, so a build that

+ // poisons a health check this pass never also becomes the control plane.

+ controlSelfUpdate(m, bake, dryRun, ignoreBake)

return nil

}

+// controlSelfUpdate keeps the CONTROL binary — /opt/lchat/lchat, the path

+// every privileged operation execs through (router, controld, cloud, this

+// very updater) — on its own release channel, from the same signed feed the

+// instance pass just verified. Before the operator-allowlist hardening this

+// was a manual root `cp`; with root SSH gone (docs/instance-router.md § Box

+// access), the already-root update pass is the one place the swap can live —

+// and it adds no trust the box didn't have: same signature, same sha, same

+// bake window as every instance binary.

+//

+// LCHAT_CONTROL_CHANNEL (in the unit's /data/update.env) picks the channel;

+// "stable" is the default — the trust plane should be the LAST thing on a

+// box to take a new build — and "off" disables self-update entirely. The

+// swap is an atomic same-filesystem rename over the existing path (running

+// services keep their old inode until restarted; sudo never sees a missing

+// path), with the outgoing build kept beside it as .prev-<build> — the

+// rollback is one root rename, or a re-pin of the channel. lchat-control and

+// lchat-cloud are bounced to converge immediately (backend sockets, invisible

+// to members); the ROUTER is deliberately NOT restarted here — its front door

+// should never blip on a timer's schedule, so it runs the old build until an

+// operator restarts it (`lchat-svc restart lchat-router`), loudly noted below.

+func controlSelfUpdate(m *releaseManifest, bake *firstSeenState, dryRun, ignoreBake bool) {

+ if buildVersion == "dev" {

+ // An unstamped binary (local build, test binary) has no place in the

+ // channel ordering — self-updating it would clobber whatever the

+ // operator is hand-running. Only release-stamped builds converge.

+ return

+ }

+ channel := envOr("LCHAT_CONTROL_CHANNEL", "stable")

+ if channel == "off" {

+ return

+ }

+ e, ok := m.Channels[channel]

+ if !ok || e.Version == buildVersion {

+ return

+ }

+ if hold := bakeHold(bake, channel, e.Version, ignoreBake, time.Now()); hold != "" {

+ fmt.Printf("update: control binary %s -> %s: %s\n", buildVersion, e.Version, hold)

+ return

+ }

+ if dryRun {

+ fmt.Printf("update: control binary would update %s -> %s (%s; dry run)\n", buildVersion, e.Version, channel)

+ return

+ }

+ self, err := selfPath()

+ if err != nil {

+ fmt.Printf("update: control self-update: %v\n", err)

+ return

+ }

+ tmp, err := downloadReleaseBinary(e)

+ if err != nil {

+ fmt.Printf("update: control self-update: %v\n", err)

+ return

+ }

+ defer os.Remove(tmp)

+ if err := os.Chmod(tmp, 0o755); err != nil {

+ fmt.Printf("update: control self-update: chmod: %v\n", err)

+ return

+ }

+ // One .prev at a time: the previous previous build is already two behind

+ // and reachable from the version store anyway.

+ if old, _ := filepath.Glob(self + ".prev-*"); len(old) > 0 {

+ for _, p := range old {

+ os.Remove(p)

+ }

+ prev := self + ".prev-" + buildVersion

+ if err := copyFileAtomic(self, prev, 0o755); err != nil {

+ fmt.Printf("update: control self-update: keep rollback copy: %v\n", err)

+ return

+ }

+ // downloadReleaseBinary lands in incomingDir() — same filesystem as the

+ // control binary, so this rename atomically replaces the path.

+ if err := os.Rename(tmp, self); err != nil {

+ fmt.Printf("update: control self-update: install: %v\n", err)

+ return

+ }

+ fmt.Printf("update: control binary %s -> %s (%s; rollback at %s)\n", buildVersion, e.Version, channel, prev)

+ for _, unit := range []string{"lchat-control.service", "lchat-cloud.service"} {

+ if err := systemctl("try-restart", unit); err != nil {

+ fmt.Printf("update: %s restart: %v (new build applies on its next start)\n", unit, err)

+ }

+ fmt.Printf("update: lchat-router still runs %s — restart it when convenient to converge\n", buildVersion)

// gcVersions prunes version-store dirs nothing references anymore: kept are

// every route's current + previous version, every channel head, the control

// binary's own build, every pinned version (keepExtra), and whatever any live

diff --git a/web/status/ssh/2026-07-15-managed-ai-deploys.json b/web/status/ssh/2026-07-15-managed-ai-deploys.json

new file mode 100644

index 0000000..892d3b4

--- /dev/null

+++ b/web/status/ssh/2026-07-15-managed-ai-deploys.json

@@ -0,0 +1,7 @@

+ "date": "2026-07-15",

+ "operator": "Paul Campbell",

+ "window": "07:00–10:00 UTC (approx)",

+ "why": "Managed-AI rollout (Claude session over the operator tunnel): verified box state after the WireGuard flip, staged the new release binary, installed the usage-pipe and AI env additions via lchat-env-install, and restarted the control/cloud services to pick them up.",

+ "refs": ["https://github.com/elevenmessenger/messenger/pull/867", "https://github.com/elevenmessenger/messenger/pull/877"]