eleven / ocho Public

server: bot sidecars track their instance's app version (#418)

Bots exec the control binary (/opt/lchat/lchat), decoupled from the
versioned per-instance binary, so a deploy/update moved the instance but
left its sidecar on stale code — the app gained a feature (e.g. member
apps) the bot never learned it could do.

Make the sidecar run the SAME versioned binary as its instance:

- lchat-bot@.service execs /opt/lchat/live/%i (the per-host live symlink
  the instance runs), so the bot's build == the app's build by
  construction. Reusing one symlink is the only design that stays correct
  when instances run different versions (e.g. a canary).
- `bot run` with no positional args now sources LCHAT_BASE + the
  LCHAT_*_SESSION paths from the EnvironmentFile, so the template is fully
  generic — a full trio or an Eleven-only box both work with no per-host
  args.
- deployHostVersion (shared by `deploy` and `update run`) and runRollback
  try-restart lchat-bot@<host> after the instance is healthy, keeping the
  bot in lock-step. try-restart is a no-op when no sidecar is active, so
  instances without a bot are untouched; a bot hiccup only logs and never
  fails an otherwise-healthy instance deploy. (A failed deploy that rolls
  back leaves the version unchanged, so it deliberately does not restart
  the bot.)
- The Go sidecar gets MemoryDenyWriteExecute (the Node/V8-JIT caveat that
  forced it off is gone).

Note: sync fires only on a version *change* — `deploy <host> --version
<same>` won't resync an already-current instance's stale bot.

Existing boxes may carry a hand-written
lchat-bot@<host>.service.d/override.conf pinning the old binary; a drop-in
ExecStart wins over the template, so remove it (+ daemon-reload + restart)
once for the versioned template to take effect. Migration runbook in
docs/instance-router.md § Upgrading an existing box to version-tracked bots.

Co-authored-by: Claude <noreply@anthropic.com>

James Adam 4 months ago 4adacb3ecaafad35789192f3dfa9d23d82c7d140 parent ff25e6d

8 files changed, +108 −41

bootstrap_test.go +4 −2
botcmd.go +13 −1
control.go +14 −0
deploy/README.md +16 −14
deploy/systemd/lchat-bot@.service +9 −7
docs/instance-router.md +39 −15
updater_test.go +12 −2
versions.go +1 −0

diff --git a/bootstrap_test.go b/bootstrap_test.go

index 6e1fad8..34d1521 100644

--- a/bootstrap_test.go

+++ b/bootstrap_test.go

@@ -18,8 +18,10 @@ func TestEmbeddedUnits(t *testing.T) {

if !strings.Contains(unitInstance, "ExecStart=/opt/lchat/live/%i --socket /run/lchat/%i.sock --db /data/%i.db") {

t.Error("instance unit missing its versioned ExecStart line")

}

- if !strings.Contains(unitBot, "ExecStart=/usr/bin/node /opt/lchat/agent/agent.mjs") {

- t.Error("bot unit missing its node ExecStart line")

+ // The bot execs the SAME per-instance live symlink as its instance, so its

+ // binary tracks the app version (deployHostVersion restarts it on a move).

+ if !strings.Contains(unitBot, "ExecStart=/opt/lchat/live/%i bot run") {

+ t.Error("bot unit missing its versioned `bot run` ExecStart line")

}

if !strings.Contains(unitBot, "EnvironmentFile=-/data/%i-bot.env") {

t.Error("bot unit missing its EnvironmentFile")

diff --git a/botcmd.go b/botcmd.go

index 013f470..39d2c50 100644

--- a/botcmd.go

+++ b/botcmd.go

@@ -111,7 +111,19 @@ func runBotRun(args []string) {

}

paths := fs.Args()

if len(paths) == 0 {

- paths = []string{envOr("LCHAT_SESSION", "session.json")}

+ // No sessions on the command line: source them from the env the sidecar

+ // unit provides, running whichever identities are set — so the generic

+ // lchat-bot@.service template needs no per-host args, and a full trio or an

+ // Eleven-only box both work. Falls back to a lone session.json for a

+ // hand-run bot with nothing set.

+ for _, ev := range []string{"LCHAT_SESSION", "LCHAT_ELEVEN_SESSION", "LCHAT_ANNOUNCER_SESSION"} {

+ if p := strings.TrimSpace(os.Getenv(ev)); p != "" {

+ paths = append(paths, p)

+ }

+ if len(paths) == 0 {

+ paths = []string{"session.json"}

+ }

}

for _, path := range paths {

sess, err := bot.LoadSession(path)

diff --git a/control.go b/control.go

index cc5d912..8fa36d0 100644

--- a/control.go

+++ b/control.go

@@ -6,6 +6,7 @@ import (

"crypto/tls"

"flag"

"fmt"

+ "log"

"net"

"net/http"

"os"

@@ -609,9 +610,22 @@ func deployHostVersion(reg *Registry, rt *Route, unit, ver string) (changed bool

if err := reg.upsert(*rt); err != nil {

return true, fmt.Errorf("record version: %v", err)

}

+ restartBotSidecar(host)

return true, nil

}

+// restartBotSidecar best-effort re-execs a host's bot sidecar so it picks up the

+// same versioned binary the instance now runs — the bot execs /opt/lchat/live/<host>

+// too (deploy/systemd/lchat-bot@.service), so restarting it after the live symlink

+// moves keeps the bot's code in lock-step with the app. `try-restart` is a no-op when

+// no sidecar is active, so instances without a bot are untouched; a bot hiccup only

+// logs and never fails an otherwise-healthy instance deploy.

+func restartBotSidecar(host string) {

+ if err := systemctl("try-restart", "lchat-bot@"+host); err != nil {

+ log.Printf("deploy: bot sidecar restart for %s failed (instance is healthy on the new version): %v", host, err)

+ }

// acquireDeployLock serialises everything that runs the repoint/restart/

// health-check dance — deploy, rollback, and the timer-fired `update run` —

// across processes, so an auto-update can never interleave with an operator's

diff --git a/deploy/README.md b/deploy/README.md

index d74a9fa..7ed1f4e 100644

--- a/deploy/README.md

+++ b/deploy/README.md

@@ -28,26 +28,28 @@ instead: `scp deploy/systemd/*.service` to `/etc/systemd/system/`, drop a

## Bots (sidecars)

-The sidecar is **Node**, not the Go binary (it reuses `web/static/crypto.js` for

-E2E — that's why it's Node), so a box that runs bots needs a Node runtime and the

-`agent/` code + its deps:

+The sidecar is the **same Go binary** as the instance — `lchat bot run` — so a box

+that runs the server runs its bots with **no extra install** (no Node runtime). The

+`lchat-bot@.service` template execs the instance's own versioned live symlink

+(`ExecStart=/opt/lchat/live/%i bot run`), so a bot always runs the **same version

+as its app**, and `deploy`/`update` restart it automatically when they move the

+instance (`docs/instance-router.md § Bots`, § Auto-updating).

```sh

-# one-time per box: Node 24 + the sidecar source + deps

-ssh root@<box> 'curl -fsSL https://deb.nodesource.com/setup_24.x | bash - && apt-get install -y nodejs'

-rsync -a agent web/static root@<box>:/opt/lchat/ # sidecar imports ../web/static/crypto.js

-ssh root@<box> 'cd /opt/lchat/agent && npm ci'

# per instance: mint the three agent invites + start the sidecar

# (API key via env, not a flag, so it doesn't leak into ps/shell history)

ssh root@<box> 'ANTHROPIC_API_KEY=sk-ant-... /opt/lchat/lchat add-bot chat.example.com'

```

`add-bot` mints an agent invite per identity (Claude / Eleven / Announcer) via the

-instance's admin API — running `agent/mint-invite.mjs`, which reuses `crypto.js` so

-the derived invite matches what the sidecar provisions with — writes

-`/data/<host>-bot.env` (invites + session paths + optional `ANTHROPIC_API_KEY`),

-and starts `lchat-bot@<host>`. Omit `ANTHROPIC_API_KEY` for a deterministic-only bot

-(Eleven still welcomes new members + answers slash commands). Update a bot's code

-with `systemctl restart lchat-bot@<host>` (it backfills on reconnect). Start the

+instance's admin API, writes `/data/<host>-bot.env` (invites + `LCHAT_BASE` +

+`LCHAT_*_SESSION` paths + optional `ANTHROPIC_API_KEY`), and enables

+`lchat-bot@<host>` — which reads all of that from the env file, so there's nothing

+per-host to configure in the unit. Omit `ANTHROPIC_API_KEY` for a deterministic-only

+bot (Eleven still welcomes new members + answers slash commands). Start the

**instance before** the sidecar so Eleven baselines the roster correctly.

+> The Node reference sidecar (`agent/`) is kept on purpose as a runnable example;

+> self-hosters who prefer it still need a Node runtime + `npm ci` in `/opt/lchat/agent`

+> and point the unit's `ExecStart` at `node /opt/lchat/agent/agent.mjs`. The default

+> deployment is Go.

diff --git a/deploy/systemd/lchat-bot@.service b/deploy/systemd/lchat-bot@.service

index fa35a3c..713b637 100644

--- a/deploy/systemd/lchat-bot@.service

+++ b/deploy/systemd/lchat-bot@.service

@@ -7,11 +7,14 @@ Wants=network-online.target

# add-bot writes /data/%i-bot.env (0600) with LCHAT_BASE, the three agent invites,

# session paths, and (optionally) ANTHROPIC_API_KEY.

EnvironmentFile=-/data/%i-bot.env

-WorkingDirectory=/opt/lchat/agent

-# The sidecar is Node (not the static Go binary): the box needs a node runtime and

-# `npm ci` in /opt/lchat/agent (see deploy/README.md). It connects to its instance

-# over the public URL, backfills on reconnect, so a deploy is just a restart.

-ExecStart=/usr/bin/node /opt/lchat/agent/agent.mjs

+# The sidecar is the SAME versioned Go binary as its instance: exec the per-host

+# live symlink (/opt/lchat/live/%i → versions/<ver>/lchat) that lchat@%i.service

+# runs, so the bot's code stays in lock-step with the app by construction. `bot run`

+# with no positional args reads LCHAT_BASE + the LCHAT_*_SESSION paths from the

+# EnvironmentFile above, running whichever identities are set (a full trio or an

+# Eleven-only box both work). A deploy/update repoints that symlink and restarts this

+# unit (deployHostVersion → restartBotSidecar), so the bot always tracks the version.

+ExecStart=/opt/lchat/live/%i bot run

Restart=always

RestartSec=2

@@ -41,8 +44,7 @@ RestrictNamespaces=yes

RestrictRealtime=yes

RestrictSUIDSGID=yes

LockPersonality=yes

-# NOTE: no MemoryDenyWriteExecute — V8's JIT maps writable+executable memory, so

-# enabling it (as the Go units do) would crash Node on boot.

+MemoryDenyWriteExecute=yes

SystemCallFilter=@system-service

SystemCallErrorNumber=EPERM

SystemCallArchitectures=native

diff --git a/docs/instance-router.md b/docs/instance-router.md

index 09e90ac..3f58c42 100644

--- a/docs/instance-router.md

+++ b/docs/instance-router.md

@@ -224,9 +224,16 @@ gets the conversational default (command floor + model; badges AI when

`ANTHROPIC_API_KEY` is set). A custom bot drops a `<session>.recipe.json`

alongside its session (`docs/bot-recipes.md`) — behaviour as data, no code.

-Supervise it as **one `lchat-bot@<host>.service` per instance**, restarted on

-deploy. Because the sidecar backfills on reconnect, a bot deploy is a plain

-restart — no socket-swap, no drain. One wrinkle: it reaches its instance at the

+Supervise it as **one `lchat-bot@<host>.service` per instance**. The unit execs

+the *same* per-instance live symlink as the instance itself

+(`ExecStart=/opt/lchat/live/%i bot run`), so the bot runs the **same version as its

+app by construction** — and `deploy`/`update` restart the bot right after they

+repoint that symlink and health-check the instance (`deployHostVersion →

+restartBotSidecar`), so the sidecar's code never lags the server. `bot run` with no

+positional args reads `LCHAT_BASE` + the `LCHAT_*_SESSION` paths from the unit's

+EnvironmentFile, so the template is fully generic — a full trio or an Eleven-only

+box both work with no per-host args. Because the sidecar backfills on reconnect,

+that restart is a plain reconnect — no socket-swap, no drain. One wrinkle: it reaches its instance at the

public URL through the router, so the box must resolve that public hostname **to

itself**. Pin every hostname the box hosts to loopback in `/etc/hosts`

(`127.0.0.1 chat.example.com …`) so on-box clients — sidecars, and an instance's

@@ -247,16 +254,11 @@ The session files are **byte-compatible** (`internal/bot/session.go` matches the

Node `session.json` shape), so a running Node sidecar swaps to Go in place with

no re-provisioning and no lost identity:

-1. Point the unit's `ExecStart` at the Go sidecar, keeping the same session

- directory and env file:

- ```ini

- ExecStart=/opt/lchat/lchat bot run -base https://%i \

- /data/%i-bot/session.json \

- /data/%i-bot/eleven-session.json \

- /data/%i-bot/announcer-session.json

- ```

- (The env file still carries `ANTHROPIC_API_KEY` and, if the bot does

- connected-services, `LCHAT_CONNECT_BROKER_URL`/`_TOKEN`.)

+1. The shipped `lchat-bot@.service` template already execs the versioned Go

+ sidecar — `ExecStart=/opt/lchat/live/%i bot run` — sourcing `LCHAT_BASE` and

+ the `LCHAT_*_SESSION` paths (and `ANTHROPIC_API_KEY`, and

+ `LCHAT_CONNECT_BROKER_URL`/`_TOKEN` for connected-services) from

+ `/data/%i-bot.env`. Nothing per-host to write; `add-bot` just enables it.

2. `systemctl daemon-reload && systemctl restart lchat-bot@<host>`. The bot

reconnects, re-declares its kind, and backfills — members see no gap.

3. The Go sidecar keeps the Node sidecar's `.state.json` (announce memory), so

@@ -266,6 +268,26 @@ Once **no** unit on the box still `ExecStart`s Node, the Node runtime and the

`agent/node_modules` install can be removed entirely — the box becomes a pure

Go deployment.

+### Upgrading an existing box to version-tracked bots

+Older boxes may carry a **hand-written drop-in** pinning the bot to a fixed

+binary — `/etc/systemd/system/lchat-bot@<host>.service.d/override.conf` with

+`ExecStart=/opt/lchat/lchat bot run …` (or an even older canary path). A drop-in

+`ExecStart` **wins over the template**, so those bots keep running one frozen

+build and never track their instance. Migrate each once:

+```sh

+# drop the pin so the versioned template (live/%i) takes effect, and reinstall

+# the current template in case the box predates it

+rm -f /etc/systemd/system/lchat-bot@<host>.service.d/override.conf

+scp deploy/systemd/lchat-bot@.service root@<box>:/etc/systemd/system/ # or re-run `lchat bootstrap`

+systemctl daemon-reload

+systemctl restart lchat-bot@<host>

+```

+After this the bot execs `/opt/lchat/live/<host>` and every subsequent

+`deploy`/`update` keeps it in lock-step with the app automatically.

## The registry (a small control DB)

The router owns one SQLite table. Status lives here (not a flat file) so an

@@ -431,7 +453,8 @@ described in `docs/server-releases.md`:

binary 404s, which reads as the defaults: auto-update on, stable channel),

then per instance: downloads + sha256-checks + stages the channel's binary

and applies it with the exact `deploy --version` dance (repoint → restart →

- health-check → auto-rollback). A version that fails its health check on one

+ health-check → auto-rollback → **restart the host's bot sidecar** so it tracks

+ the new version too). A version that fails its health check on one

host is not tried on the rest that pass ("poisoned" for the run); other

per-host failures log and continue, so one unreachable instance never

strands the fleet.

@@ -474,7 +497,8 @@ described in `docs/server-releases.md`:

give resource isolation; `Restart=always` gives crash isolation. Bump

`LimitNOFILE` (instances × WS connections × SQLite files).

- `lchat-bot@.service` — the optional per-instance sidecar (`lchat bot run`; see

- *Bots*). `Restart=always`; a deploy is just a restart (it backfills on reconnect).

+ *Bots*). `Restart=always`; execs the same `live/%i` binary as the instance, so a

+ deploy/update restarts it onto the matching version (it backfills on reconnect).

## Hardening

diff --git a/updater_test.go b/updater_test.go

index aa01bdb..4c82e61 100644

--- a/updater_test.go

+++ b/updater_test.go

@@ -221,9 +221,19 @@ func TestUpdatePassAppliesPerPrefs(t *testing.T) {

t.Fatalf("h2 registry = %s", rt.Version)

}

- // Exactly two restarts: h1 and h4.

+ // Two instances moved (h1, h4); each restarts the instance and then its bot

+ // sidecar so the bot tracks the new version (restartBotSidecar). Four calls,

+ // no more — h2 (download-only) and h3 (downloads off) never restart.

got := strings.Join(*calls, "; ")

- if !strings.Contains(got, "restart lchat@h1.example.com") || !strings.Contains(got, "restart lchat@h4.example.com") || len(*calls) != 2 {

+ for _, want := range []string{

+ "restart lchat@h1.example.com", "try-restart lchat-bot@h1.example.com",

+ "restart lchat@h4.example.com", "try-restart lchat-bot@h4.example.com",

+ } {

+ if !strings.Contains(got, want) {

+ t.Fatalf("missing %q in systemctl calls = %v", want, *calls)

+ }

+ if len(*calls) != 4 {

t.Fatalf("systemctl calls = %v", *calls)

}

diff --git a/versions.go b/versions.go

index d3981fc..6f71cc2 100644

--- a/versions.go

+++ b/versions.go

@@ -330,6 +330,7 @@ func runRollback(args []string) {

if err := reg.upsert(*rt); err != nil {

die("rollback %s: record: %v", host, err)

}

+ restartBotSidecar(host)

fmt.Printf("rolled back %s to %s\n", host, prev)

}