amadan
Instance router: self-hosted single-machine bootstrap (drop the orchestrator)
One static Go binary + systemd replaces the orchestrator for single-box multi-instance hosting: the lchat router (autocert HTTP-01 TLS + host routing over unix sockets), per-instance instances, add-instance / deploy (zero-downtime) / remove / instances / admin-link, add-service (home/push/third-party), add-bot (Node sidecar), and lchat bootstrap. Validated end-to-end on a real box, including a live instance migration. Docs: docs/instance-router.md, docs/instance-migration.md.
20 files changed,
+2301
−4
agent/mint-invite.mjs+52 −0bootstrap.go+69 −0bootstrap_test.go+26 −0bot.go+127 −0control.go+499 −0control_test.go+117 −0deploy/README.md+53 −0deploy/systemd/lchat-bot@.service+19 −0deploy/systemd/lchat-router.service+14 −0deploy/systemd/lchat@.service+15 −0docs/instance-migration.md+108 −0docs/instance-router.md+418 −0go.mod+3 −1go.sum+2 −0hub.go+21 −0hub_test.go+21 −0main.go+98 −3registry.go+149 −0router.go+215 −0router_test.go+275 −0
diff --git a/agent/mint-invite.mjs b/agent/mint-invite.mjs| new file mode 100644 |
| index 0000000..15b564b |
| --- /dev/null |
| +++ b/agent/mint-invite.mjs |
| @@ -0,0 +1,52 @@ |
| +// mint-invite.mjs — mint one agent invite via the admin API and print its secret. |
| +// |
| +// Used by `lchat add-bot` to provision a sidecar identity. It reuses |
| +// web/static/crypto.js so the id + claim_hash it derives from the secret T match |
| +// exactly what the sidecar (provision.mjs) computes from the same T — the crypto |
| +// never diverges (the whole reason the bot is Node). Reads LCHAT_BASE + |
| +// LCHAT_ADMIN_TOKEN from the env; takes --label / --description / --model / |
| +// --listen. Prints the bare secret T (the sidecar's LCHAT_*_INVITE) to stdout; |
| +// all diagnostics go to stderr. |
| +import { randomSecret, deriveInvite } from "../web/static/crypto.js"; |
| + |
| +const BASE = (process.env.LCHAT_BASE || "").replace(/\/$/, ""); |
| +const ADMIN = (process.env.LCHAT_ADMIN_TOKEN || "").trim(); |
| +if (!BASE || !ADMIN) { |
| + console.error("mint-invite: set LCHAT_BASE and LCHAT_ADMIN_TOKEN"); |
| + process.exit(1); |
| +} |
| + |
| +function arg(name, def = "") { |
| + const i = process.argv.indexOf(`--${name}`); |
| + return i >= 0 && i + 1 < process.argv.length ? process.argv[i + 1] : def; |
| +} |
| + |
| +const label = arg("label"); |
| +if (!label) { |
| + console.error("mint-invite: --label is required"); |
| + process.exit(1); |
| +} |
| + |
| +const T = randomSecret(); |
| +const { id, claimHash } = await deriveInvite(T); |
| +const res = await fetch(`${BASE}/api/admin/key-invites`, { |
| + method: "POST", |
| + headers: { "content-type": "application/json", authorization: `Bearer ${ADMIN}` }, |
| + body: JSON.stringify({ |
| + label, |
| + id, |
| + wrapped: "", |
| + claim_hash: claimHash, |
| + agent: true, |
| + available_to_members: true, // in the Helpers catalog, so Eleven's roster baseline sees members |
| + description: arg("description"), |
| + model: arg("model"), |
| + listen: arg("listen"), // "", "mention", or "name" |
| + }), |
| +}); |
| +if (!res.ok) { |
| + const detail = await res.text().catch(() => ""); |
| + console.error(`mint-invite: POST /api/admin/key-invites -> HTTP ${res.status} ${detail}`); |
| + process.exit(1); |
| +} |
| +process.stdout.write(T); // the bare secret; add-bot writes it as the sidecar's LCHAT_*_INVITE |
diff --git a/bootstrap.go b/bootstrap.go| new file mode 100644 |
| index 0000000..fbb9c89 |
| --- /dev/null |
| +++ b/bootstrap.go |
| @@ -0,0 +1,69 @@ |
| +package main |
| + |
| +import ( |
| + _ "embed" |
| + "flag" |
| + "fmt" |
| + "os" |
| + "path/filepath" |
| +) |
| + |
| +// The systemd units ship inside the binary (source of truth: deploy/systemd), so |
| +// `lchat bootstrap` can provision a fresh box with no separate files to copy. |
| +// |
| +//go:embed deploy/systemd/lchat-router.service |
| +var unitRouter string |
| + |
| +//go:embed deploy/systemd/lchat@.service |
| +var unitInstance string |
| + |
| +//go:embed deploy/systemd/lchat-bot@.service |
| +var unitBot string |
| + |
| +// tmpfilesConf keeps /run/lchat across reboots (/run is tmpfs); bootstrap also |
| +// creates it now. |
| +const tmpfilesConf = "d /run/lchat 0710 root root -\n" |
| + |
| +// runBootstrap installs the systemd units + runtime dir and starts the router, so |
| +// provisioning a fresh box is one command after the binary is copied to |
| +// /opt/lchat/lchat. Idempotent — safe to re-run. See docs/instance-router.md. |
| +func runBootstrap(args []string) { |
| + fs := flag.NewFlagSet("bootstrap", flag.ExitOnError) |
| + dataDir := fs.String("data-dir", "/data", "dir for per-instance DBs, env files, the registry, and the acme cache") |
| + unitDir := fs.String("unit-dir", "/etc/systemd/system", "systemd unit directory") |
| + noStart := fs.Bool("no-start", false, "install units but don't enable/start the router") |
| + _ = fs.Parse(args) |
| + |
| + for _, d := range []string{*dataDir, filepath.Join(*dataDir, "acme"), "/run/lchat"} { |
| + if err := os.MkdirAll(d, 0o750); err != nil { |
| + die("bootstrap: mkdir %s: %v", d, err) |
| + } |
| + } |
| + if err := os.Chmod("/run/lchat", 0o710); err != nil { |
| + die("bootstrap: chmod /run/lchat: %v", err) |
| + } |
| + |
| + write := func(path, content string) { |
| + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { |
| + die("bootstrap: write %s: %v", path, err) |
| + } |
| + fmt.Printf(" wrote %s\n", path) |
| + } |
| + write(filepath.Join(*unitDir, "lchat-router.service"), unitRouter) |
| + write(filepath.Join(*unitDir, "lchat@.service"), unitInstance) |
| + write(filepath.Join(*unitDir, "lchat-bot@.service"), unitBot) |
| + write("/etc/tmpfiles.d/lchat.conf", tmpfilesConf) |
| + |
| + if err := systemctl("daemon-reload"); err != nil { |
| + die("bootstrap: %v", err) |
| + } |
| + if *noStart { |
| + fmt.Println("bootstrap: units installed (router not started; --no-start)") |
| + return |
| + } |
| + if err := systemctl("enable", "--now", "lchat-router"); err != nil { |
| + die("bootstrap: start router: %v", err) |
| + } |
| + fmt.Println("bootstrap: router installed and started") |
| + fmt.Println(" next: lchat add-instance <host>") |
| +} |
diff --git a/bootstrap_test.go b/bootstrap_test.go| new file mode 100644 |
| index 0000000..054d65d |
| --- /dev/null |
| +++ b/bootstrap_test.go |
| @@ -0,0 +1,26 @@ |
| +package main |
| + |
| +import ( |
| + "strings" |
| + "testing" |
| +) |
| + |
| +// TestEmbeddedUnits guards the embedded systemd units against drift — the |
| +// instance template must carry the EnvironmentFile add-instance relies on. |
| +func TestEmbeddedUnits(t *testing.T) { |
| + if !strings.Contains(unitRouter, "ExecStart=/opt/lchat/lchat router") { |
| + t.Error("router unit missing its ExecStart line") |
| + } |
| + if !strings.Contains(unitInstance, "EnvironmentFile=-/data/%i.env") { |
| + t.Error("instance unit missing EnvironmentFile directive") |
| + } |
| + if !strings.Contains(unitInstance, "ExecStart=/opt/lchat/lchat --socket /run/lchat/%i.sock --db /data/%i.db") { |
| + t.Error("instance unit missing its ExecStart line") |
| + } |
| + if !strings.Contains(unitBot, "ExecStart=/usr/bin/node /opt/lchat/agent/agent.mjs") { |
| + t.Error("bot unit missing its node ExecStart line") |
| + } |
| + if !strings.Contains(unitBot, "EnvironmentFile=-/data/%i-bot.env") { |
| + t.Error("bot unit missing its EnvironmentFile") |
| + } |
| +} |
diff --git a/bot.go b/bot.go| new file mode 100644 |
| index 0000000..1e95551 |
| --- /dev/null |
| +++ b/bot.go |
| @@ -0,0 +1,127 @@ |
| +package main |
| + |
| +import ( |
| + "flag" |
| + "fmt" |
| + "os" |
| + "os/exec" |
| + "path/filepath" |
| + "strings" |
| +) |
| + |
| +// runAddBot: `lchat add-bot <host>` — provision the Node sidecar (Claude + Eleven + |
| +// Announcer) for an instance. It mints an agent invite per identity via the |
| +// instance's admin API (through agent/mint-invite.mjs, which reuses |
| +// web/static/crypto.js so the derived id/claim match what the sidecar computes), |
| +// writes the sidecar's EnvironmentFile, and starts lchat-bot@<host>. The instance |
| +// must be up first (Eleven baselines the roster on its first tick). See |
| +// docs/instance-router.md. |
| +func runAddBot(args []string) { |
| + fs := flag.NewFlagSet("add-bot", flag.ExitOnError) |
| + registryPath := fs.String("registry", envOr("LCHAT_REGISTRY", "/data/registry.db"), "router registry SQLite path") |
| + model := fs.String("model", "", "LCHAT_MODEL for the bots (optional; sidecar default otherwise)") |
| + agentDir := fs.String("agent-dir", "/opt/lchat/agent", "dir holding the Node sidecar (agent/)") |
| + nodeBin := fs.String("node", "/usr/bin/node", "path to the node binary") |
| + botUnit := fs.String("unit", envOr("LCHAT_BOT_UNIT", "lchat-bot@"), "systemd bot unit template prefix") |
| + _ = fs.Parse(reorderFlagsFirst(fs, args)) |
| + // The API key is a secret, so read it from the environment (a flag would leak it |
| + // into the process table + shell history). |
| + apiKey := strings.TrimSpace(os.Getenv("ANTHROPIC_API_KEY")) |
| + |
| + host := strings.ToLower(strings.TrimSpace(fs.Arg(0))) |
| + if !validHost(host) { |
| + die("add-bot: invalid host %q", host) |
| + } |
| + |
| + // The instance must exist (the bot connects to it), and we need its admin token |
| + // (from the instance's EnvironmentFile) to mint agent invites. |
| + reg, err := openRegistry(*registryPath) |
| + if err != nil { |
| + die("add-bot: open registry: %v", err) |
| + } |
| + rt, err := reg.get(host) |
| + if err != nil { |
| + die("add-bot: %v", err) |
| + } |
| + if rt == nil || rt.Kind != "instance" { |
| + die("add-bot: %q is not a messenger instance (run add-instance first)", host) |
| + } |
| + env, err := readEnvFile(filepath.Join(instanceDataDir, host+".env")) |
| + if err != nil { |
| + die("add-bot: read instance env: %v", err) |
| + } |
| + admin := env["LCHAT_ADMIN_TOKEN"] |
| + if admin == "" { |
| + die("add-bot: no LCHAT_ADMIN_TOKEN for %s", host) |
| + } |
| + |
| + base := "https://" + host |
| + mintScript := filepath.Join(*agentDir, "mint-invite.mjs") |
| + sessionDir := filepath.Join(instanceDataDir, host+"-bot") |
| + |
| + // One agent invite per identity. All three are catalog-available so Eleven's |
| + // roster baseline sees the joined members (docs/bots.md). |
| + identities := []struct{ envVar, label, desc string }{ |
| + {"LCHAT_INVITE", "Claude", "General assistant — DM me or @mention me in a group."}, |
| + {"LCHAT_ELEVEN_INVITE", "Eleven", "Your guide to the app — I welcome new members."}, |
| + {"LCHAT_ANNOUNCER_INVITE", "Announcer", "Posts what's new after each update."}, |
| + } |
| + botEnv := map[string]string{ |
| + "LCHAT_BASE": base, |
| + "LCHAT_SESSION": filepath.Join(sessionDir, "session.json"), |
| + "LCHAT_ELEVEN_SESSION": filepath.Join(sessionDir, "eleven-session.json"), |
| + "LCHAT_ANNOUNCER_SESSION": filepath.Join(sessionDir, "announcer-session.json"), |
| + } |
| + if apiKey != "" { |
| + botEnv["ANTHROPIC_API_KEY"] = apiKey |
| + } |
| + if *model != "" { |
| + botEnv["LCHAT_MODEL"] = *model |
| + } |
| + for _, id := range identities { |
| + t, err := mintAgentInvite(*nodeBin, mintScript, base, admin, id.label, id.desc, *model) |
| + if err != nil { |
| + die("add-bot: mint %s invite: %v", id.label, err) |
| + } |
| + botEnv[id.envVar] = t |
| + } |
| + |
| + // Persistent per-instance session dir + the sidecar's EnvironmentFile (0600 — |
| + // it holds the invites and the API key). |
| + if err := os.MkdirAll(sessionDir, 0o750); err != nil { |
| + die("add-bot: mkdir %s: %v", sessionDir, err) |
| + } |
| + botEnvPath := filepath.Join(instanceDataDir, host+"-bot.env") |
| + if err := writeEnvFile(botEnvPath, botEnv); err != nil { |
| + die("add-bot: write bot env: %v", err) |
| + } |
| + |
| + unit := *botUnit + host |
| + if err := systemctl("enable", "--now", unit); err != nil { |
| + die("add-bot: start %s: %v (invites minted; env written to %s)", unit, err, botEnvPath) |
| + } |
| + fmt.Printf("added bot sidecar for %s (%s)\n", host, unit) |
| + if apiKey == "" { |
| + fmt.Printf(" note: no ANTHROPIC_API_KEY in env — bots are deterministic-only (Eleven welcomes +\n"+ |
| + " slash commands); add ANTHROPIC_API_KEY to %s and restart %s for AI replies\n", botEnvPath, unit) |
| + } |
| + fmt.Printf(" logs: journalctl -u %s -f\n", unit) |
| +} |
| + |
| +// mintAgentInvite runs the Node minter (which reuses crypto.js) for one identity |
| +// and returns the bare invite secret it prints. |
| +func mintAgentInvite(nodeBin, script, base, admin, label, desc, model string) (string, error) { |
| + cmd := exec.Command(nodeBin, script, "--label", label, "--description", desc, "--model", model, "--listen", "mention") |
| + cmd.Env = append(os.Environ(), "LCHAT_BASE="+base, "LCHAT_ADMIN_TOKEN="+admin) |
| + var out, errb strings.Builder |
| + cmd.Stdout = &out |
| + cmd.Stderr = &errb |
| + if err := cmd.Run(); err != nil { |
| + return "", fmt.Errorf("%v: %s", err, strings.TrimSpace(errb.String())) |
| + } |
| + t := strings.TrimSpace(out.String()) |
| + if t == "" { |
| + return "", fmt.Errorf("minter returned no secret: %s", strings.TrimSpace(errb.String())) |
| + } |
| + return t, nil |
| +} |
diff --git a/control.go b/control.go| new file mode 100644 |
| index 0000000..62e6181 |
| --- /dev/null |
| +++ b/control.go |
| @@ -0,0 +1,499 @@ |
| +package main |
| + |
| +import ( |
| + "bufio" |
| + "context" |
| + "flag" |
| + "fmt" |
| + "net" |
| + "net/http" |
| + "os" |
| + "os/exec" |
| + "path/filepath" |
| + "regexp" |
| + "sort" |
| + "strings" |
| + "time" |
| +) |
| + |
| +// The control subcommands automate what an operator would otherwise do by hand to |
| +// stand up an instance behind the router: mint the admin token, write the unit's |
| +// EnvironmentFile, start the instance, register + reload the route. See |
| +// docs/instance-router.md. |
| + |
| +// Instance paths are fixed by the systemd units (deploy/systemd/lchat@.service), |
| +// so the control commands use these constants rather than per-invocation flags |
| +// that would silently diverge from where the instance actually listens + stores. |
| +const ( |
| + instanceDataDir = "/data" // <host>.db + <host>.env live here |
| + instanceSocketDir = "/run/lchat" // <host>.sock lives here |
| +) |
| + |
| +var hostnameRe = regexp.MustCompile(`^[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)+$`) |
| + |
| +// validHost reports whether h is a plausible DNS hostname. It's strict because |
| +// <host> becomes a filename, a systemd instance name, and a unix socket path. |
| +func validHost(h string) bool { |
| + return len(h) > 0 && len(h) <= 253 && hostnameRe.MatchString(h) |
| +} |
| + |
| +// checkSocketLen guards the unix sun_path limit (104 bytes on darwin/BSD, 108 on |
| +// Linux — stay under the smaller so a path is portable). A too-long path fails at |
| +// bind() with a cryptic error, so catch it early. |
| +func checkSocketLen(path string) error { |
| + if len(path) >= 104 { |
| + return fmt.Errorf("socket path %q is too long (%d bytes; unix limit ~104)", path, len(path)) |
| + } |
| + return nil |
| +} |
| + |
| +func die(format string, a ...any) { |
| + fmt.Fprintf(os.Stderr, format+"\n", a...) |
| + os.Exit(1) |
| +} |
| + |
| +// reorderFlagsFirst moves positional args after all flags so the stdlib flag |
| +// parser (which stops at the first positional) still sees every flag — letting |
| +// `add-instance <host> --flag v` work as naturally as `--flag v <host>`. |
| +func reorderFlagsFirst(fs *flag.FlagSet, args []string) []string { |
| + var flags, positional []string |
| + for i := 0; i < len(args); i++ { |
| + a := args[i] |
| + if a == "--" { |
| + positional = append(positional, args[i+1:]...) |
| + break |
| + } |
| + if len(a) > 1 && a[0] == '-' { |
| + flags = append(flags, a) |
| + name := strings.TrimLeft(a, "-") |
| + if strings.IndexByte(name, '=') >= 0 { |
| + continue // --flag=value carries its own value |
| + } |
| + f := fs.Lookup(name) |
| + if f == nil { |
| + continue // unknown flag; let Parse report it |
| + } |
| + if bf, ok := f.Value.(interface{ IsBoolFlag() bool }); ok && bf.IsBoolFlag() { |
| + continue // bool flag takes no value |
| + } |
| + // Pull the value along, unless the next token is itself a flag (a |
| + // missing value) — let Parse report that rather than silently eating it. |
| + if i+1 < len(args) && !(len(args[i+1]) > 1 && args[i+1][0] == '-') { |
| + i++ |
| + flags = append(flags, args[i]) |
| + } |
| + continue |
| + } |
| + positional = append(positional, a) |
| + } |
| + return append(flags, positional...) |
| +} |
| + |
| +// runAddInstance: `lchat add-instance <host>` — mint token + env file, start the |
| +// instance unit, register the route, reload the router. |
| +func runAddInstance(args []string) { |
| + fs := flag.NewFlagSet("add-instance", flag.ExitOnError) |
| + registryPath := fs.String("registry", envOr("LCHAT_REGISTRY", "/data/registry.db"), "router registry SQLite path") |
| + unitTmpl := fs.String("unit", envOr("LCHAT_UNIT", "lchat@"), "systemd instance unit template prefix") |
| + routerUnit := fs.String("router-unit", envOr("LCHAT_ROUTER_UNIT", "lchat-router"), "router systemd unit to reload") |
| + appStoreURL := fs.String("app-store-url", "", "per-instance LCHAT_APP_STORE_URL") |
| + homeOrigin := fs.String("home-origin", "", "per-instance LCHAT_HOME_ORIGIN") |
| + pushRelayURL := fs.String("push-relay-url", "", "per-instance LCHAT_PUSH_RELAY_URL") |
| + pushRelayToken := fs.String("push-relay-token", "", "per-instance LCHAT_PUSH_RELAY_TOKEN") |
| + _ = fs.Parse(reorderFlagsFirst(fs, args)) |
| + |
| + host := strings.ToLower(strings.TrimSpace(fs.Arg(0))) |
| + if !validHost(host) { |
| + die("add-instance: invalid host %q (want a hostname like chat.example.com)", host) |
| + } |
| + socket := filepath.Join(instanceSocketDir, host+".sock") |
| + if err := checkSocketLen(socket); err != nil { |
| + die("add-instance: %v", err) |
| + } |
| + |
| + reg, err := openRegistry(*registryPath) |
| + if err != nil { |
| + die("add-instance: open registry: %v", err) |
| + } |
| + if existing, err := reg.get(host); err != nil { |
| + die("add-instance: %v", err) |
| + } else if existing != nil && existing.Status == "ready" { |
| + die("add-instance: host %q already exists", host) |
| + } |
| + // A prior failed/creating attempt is overwritten on retry. |
| + |
| + // The instance stores only the token's hash, so this is the one chance to |
| + // capture the plaintext. Persist it (and per-instance config) to an |
| + // EnvironmentFile the unit reads — that keeps it recoverable via `admin-link`. |
| + token, err := randToken() |
| + if err != nil { |
| + die("add-instance: token: %v", err) |
| + } |
| + env := map[string]string{"LCHAT_ADMIN_TOKEN": token} |
| + for k, v := range map[string]string{ |
| + "LCHAT_APP_STORE_URL": *appStoreURL, |
| + "LCHAT_HOME_ORIGIN": *homeOrigin, |
| + "LCHAT_PUSH_RELAY_URL": *pushRelayURL, |
| + "LCHAT_PUSH_RELAY_TOKEN": *pushRelayToken, |
| + } { |
| + if v != "" { |
| + env[k] = v |
| + } |
| + } |
| + envPath := filepath.Join(instanceDataDir, host+".env") |
| + if err := writeEnvFile(envPath, env); err != nil { |
| + die("add-instance: write env file: %v", err) |
| + } |
| + |
| + dbPath := filepath.Join(instanceDataDir, host+".db") |
| + route := Route{Host: host, Kind: "instance", Socket: socket, DBPath: dbPath, Status: "creating", |
| + AppStoreURL: *appStoreURL, HomeOrigin: *homeOrigin, PushRelayURL: *pushRelayURL} |
| + if err := reg.upsert(route); err != nil { |
| + die("add-instance: register: %v", err) |
| + } |
| + |
| + unit := *unitTmpl + host |
| + if err := systemctl("enable", "--now", unit); err != nil { |
| + markInstanceFailed(reg, host, unit, "start: "+err.Error()) |
| + die("add-instance: start %s: %v", unit, err) |
| + } |
| + if err := waitHealthy(socket, 15*time.Second); err != nil { |
| + markInstanceFailed(reg, host, unit, "health: "+err.Error()) |
| + die("add-instance: %v", err) |
| + } |
| + |
| + route.Status = "ready" |
| + if err := reg.upsert(route); err != nil { |
| + die("add-instance: mark ready: %v", err) |
| + } |
| + if err := systemctl("reload", *routerUnit); err != nil { |
| + die("add-instance: reload %s: %v (route added; run: systemctl reload %s)", *routerUnit, err, *routerUnit) |
| + } |
| + |
| + fmt.Printf("added %s\n admin: https://%s/admin#token=%s\n (recover later: lchat admin-link %s)\n", host, host, token, host) |
| +} |
| + |
| +// runAddService: `lchat add-service <host> --socket <path>` — front an |
| +// externally-managed backend (the home server, push relay, or a third party's). |
| +// Unlike an instance, the router only proxies it; it doesn't start or supervise |
| +// the process. See docs/instance-router.md. |
| +func runAddService(args []string) { |
| + fs := flag.NewFlagSet("add-service", flag.ExitOnError) |
| + registryPath := fs.String("registry", envOr("LCHAT_REGISTRY", "/data/registry.db"), "router registry SQLite path") |
| + socket := fs.String("socket", "", "unix socket the service already listens on (required)") |
| + routerUnit := fs.String("router-unit", envOr("LCHAT_ROUTER_UNIT", "lchat-router"), "router systemd unit to reload") |
| + _ = fs.Parse(reorderFlagsFirst(fs, args)) |
| + |
| + host := strings.ToLower(strings.TrimSpace(fs.Arg(0))) |
| + if !validHost(host) { |
| + die("add-service: invalid host %q", host) |
| + } |
| + if *socket == "" { |
| + die("add-service: --socket <path> is required (where the service listens)") |
| + } |
| + if err := checkSocketLen(*socket); err != nil { |
| + die("add-service: %v", err) |
| + } |
| + reg, err := openRegistry(*registryPath) |
| + if err != nil { |
| + die("add-service: open registry: %v", err) |
| + } |
| + if existing, err := reg.get(host); err != nil { |
| + die("add-service: %v", err) |
| + } else if existing != nil { |
| + die("add-service: host %q already exists (status %s)", host, existing.Status) |
| + } |
| + // The backend must already be up — the router only proxies it. Use a lax |
| + // reachability check (any HTTP response, redirects not followed): a service's |
| + // "/" may legitimately 3xx/4xx/5xx, unlike a messenger instance. |
| + if err := waitReachable(*socket, 10*time.Second); err != nil { |
| + die("add-service: %s not answering on %s: %v (start the service first)", host, *socket, err) |
| + } |
| + if err := reg.upsert(Route{Host: host, Kind: "service", Socket: *socket, Status: "ready"}); err != nil { |
| + die("add-service: register: %v", err) |
| + } |
| + if err := systemctl("reload", *routerUnit); err != nil { |
| + die("add-service: reload %s: %v (route added; run: systemctl reload %s)", *routerUnit, err, *routerUnit) |
| + } |
| + fmt.Printf("added service %s -> %s\n", host, *socket) |
| +} |
| + |
| +// runRemove: `lchat remove <host>` — stop routing a host. Data-safe by default: |
| +// it deletes the route and reloads the router but leaves the instance running and |
| +// its DB/env in place; --stop also stops+disables an instance's unit. |
| +func runRemove(args []string) { |
| + fs := flag.NewFlagSet("remove", flag.ExitOnError) |
| + registryPath := fs.String("registry", envOr("LCHAT_REGISTRY", "/data/registry.db"), "router registry SQLite path") |
| + unitTmpl := fs.String("unit", envOr("LCHAT_UNIT", "lchat@"), "systemd instance unit template prefix") |
| + botUnit := fs.String("bot-unit", envOr("LCHAT_BOT_UNIT", "lchat-bot@"), "systemd bot unit template prefix") |
| + routerUnit := fs.String("router-unit", envOr("LCHAT_ROUTER_UNIT", "lchat-router"), "router systemd unit to reload") |
| + stop := fs.Bool("stop", false, "for an instance, also stop + disable its systemd unit (DB/env still kept)") |
| + _ = fs.Parse(reorderFlagsFirst(fs, args)) |
| + |
| + host := strings.ToLower(strings.TrimSpace(fs.Arg(0))) |
| + if !validHost(host) { |
| + die("remove: invalid host %q", host) |
| + } |
| + reg, err := openRegistry(*registryPath) |
| + if err != nil { |
| + die("remove: open registry: %v", err) |
| + } |
| + rt, err := reg.get(host) |
| + if err != nil { |
| + die("remove: %v", err) |
| + } |
| + if rt == nil { |
| + die("remove: unknown host %q", host) |
| + } |
| + if err := reg.delete(host); err != nil { |
| + die("remove: %v", err) |
| + } |
| + if err := systemctl("reload", *routerUnit); err != nil { |
| + die("remove: reload %s: %v", *routerUnit, err) |
| + } |
| + if *stop && rt.Kind == "instance" { |
| + _ = systemctl("disable", "--now", *botUnit+host) // best-effort: a sidecar may not exist |
| + if err := systemctl("disable", "--now", *unitTmpl+host); err != nil { |
| + fmt.Printf("removed %s (route deleted), but stopping the unit failed: %v\n", host, err) |
| + return |
| + } |
| + fmt.Printf("removed %s (route deleted, instance + sidecar stopped; DB/env kept in place)\n", host) |
| + return |
| + } |
| + fmt.Printf("removed %s (route deleted, router reloaded)\n", host) |
| + if rt.Kind == "instance" { |
| + fmt.Printf(" note: instance %s%s is still running and its DB/env remain; pass --stop to stop it\n", *unitTmpl, host) |
| + } |
| +} |
| + |
| +// runInstances: `lchat instances` — the route/status table. |
| +func runInstances(args []string) { |
| + fs := flag.NewFlagSet("instances", flag.ExitOnError) |
| + registryPath := fs.String("registry", envOr("LCHAT_REGISTRY", "/data/registry.db"), "router registry SQLite path") |
| + _ = fs.Parse(reorderFlagsFirst(fs, args)) |
| + reg, err := openRegistry(*registryPath) |
| + if err != nil { |
| + die("instances: %v", err) |
| + } |
| + routes, err := reg.list() |
| + if err != nil { |
| + die("instances: %v", err) |
| + } |
| + if len(routes) == 0 { |
| + fmt.Println("(no routes)") |
| + return |
| + } |
| + fmt.Printf("%-34s %-9s %-9s %s\n", "HOST", "KIND", "STATUS", "SOCKET") |
| + for _, r := range routes { |
| + fmt.Printf("%-34s %-9s %-9s %s\n", r.Host, r.Kind, r.Status, r.Socket) |
| + if r.LastError != "" { |
| + fmt.Printf(" last error: %s\n", r.LastError) |
| + } |
| + } |
| +} |
| + |
| +// runAdminLink: `lchat admin-link <host>` — recover the admin link from the |
| +// instance's EnvironmentFile (the DB holds only the token's hash). |
| +func runAdminLink(args []string) { |
| + fs := flag.NewFlagSet("admin-link", flag.ExitOnError) |
| + _ = fs.Parse(reorderFlagsFirst(fs, args)) |
| + host := strings.ToLower(strings.TrimSpace(fs.Arg(0))) |
| + if !validHost(host) { |
| + die("admin-link: invalid host %q", host) |
| + } |
| + envPath := filepath.Join(instanceDataDir, host+".env") |
| + env, err := readEnvFile(envPath) |
| + if err != nil { |
| + die("admin-link: %v", err) |
| + } |
| + tok := env["LCHAT_ADMIN_TOKEN"] |
| + if tok == "" { |
| + die("admin-link: no LCHAT_ADMIN_TOKEN in %s", envPath) |
| + } |
| + fmt.Printf("https://%s/admin#token=%s\n", host, tok) |
| +} |
| + |
| +// runDeploy: `lchat deploy <host>` (or --all) — restart the instance on the new |
| +// on-disk binary. SIGTERM drains it gracefully and the router retries dials across |
| +// the brief gap, so the swap drops no requests. Ship the new binary first (scp + |
| +// atomic mv over /opt/lchat/lchat); deploy then rolls it in. See |
| +// docs/instance-router.md. |
| +func runDeploy(args []string) { |
| + fs := flag.NewFlagSet("deploy", flag.ExitOnError) |
| + registryPath := fs.String("registry", envOr("LCHAT_REGISTRY", "/data/registry.db"), "router registry SQLite path") |
| + unitTmpl := fs.String("unit", envOr("LCHAT_UNIT", "lchat@"), "systemd instance unit template prefix") |
| + all := fs.Bool("all", false, "deploy every instance, one at a time") |
| + _ = fs.Parse(reorderFlagsFirst(fs, args)) |
| + |
| + reg, err := openRegistry(*registryPath) |
| + if err != nil { |
| + die("deploy: open registry: %v", err) |
| + } |
| + |
| + var hosts []string |
| + if *all { |
| + routes, err := reg.list() |
| + if err != nil { |
| + die("deploy: %v", err) |
| + } |
| + for _, r := range routes { |
| + if r.Kind == "instance" { |
| + hosts = append(hosts, r.Host) |
| + } |
| + } |
| + } else { |
| + h := strings.ToLower(strings.TrimSpace(fs.Arg(0))) |
| + if !validHost(h) { |
| + die("deploy: invalid host %q (or pass --all)", h) |
| + } |
| + hosts = []string{h} |
| + } |
| + if len(hosts) == 0 { |
| + die("deploy: no instances to deploy") |
| + } |
| + |
| + for _, host := range hosts { |
| + rt, err := reg.get(host) |
| + if err != nil { |
| + die("deploy %s: %v", host, err) |
| + } |
| + if rt == nil { |
| + die("deploy %s: unknown host (run add-instance first)", host) |
| + } |
| + if rt.Kind != "instance" { |
| + die("deploy %s: not an instance (kind %q) — only messenger instances are deployed", host, rt.Kind) |
| + } |
| + if err := systemctl("restart", *unitTmpl+host); err != nil { |
| + die("deploy %s: restart: %v", host, err) |
| + } |
| + // rt.Socket is what the router actually dials (stored at add-instance time), |
| + // so health-check exactly that rather than recomputing a path. |
| + if err := waitHealthy(rt.Socket, 20*time.Second); err != nil { |
| + die("deploy %s: %v", host, err) |
| + } |
| + fmt.Printf("deployed %s\n", host) |
| + } |
| +} |
| + |
| +// waitHealthy blocks until the instance answers an HTTP request on its socket, or |
| +// the timeout elapses. It retries, so it's robust to the restart window (socket |
| +// briefly absent, or a stale socket left by an abruptly-killed old process) — and |
| +// a broken new binary that never becomes healthy fails the deploy before `--all` |
| +// rolls it to the rest of the fleet. |
| +func waitHealthy(socket string, timeout time.Duration) error { |
| + deadline := time.Now().Add(timeout) |
| + client := &http.Client{ |
| + Timeout: 3 * time.Second, |
| + Transport: &http.Transport{ |
| + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { |
| + return (&net.Dialer{}).DialContext(ctx, "unix", socket) |
| + }, |
| + }, |
| + } |
| + var last error |
| + for time.Now().Before(deadline) { |
| + resp, err := client.Get("http://lchat/") |
| + if err == nil { |
| + resp.Body.Close() |
| + if resp.StatusCode < 500 { |
| + return nil |
| + } |
| + last = fmt.Errorf("unhealthy: HTTP %d", resp.StatusCode) |
| + } else { |
| + last = err |
| + } |
| + time.Sleep(150 * time.Millisecond) |
| + } |
| + return fmt.Errorf("instance not healthy within %s: %v", timeout, last) |
| +} |
| + |
| +// waitReachable blocks until the backend accepts a connection and returns any HTTP |
| +// response on its socket, or the timeout elapses. Unlike waitHealthy it accepts any |
| +// status and does not follow redirects — a sibling service's "/" may 3xx/4xx/5xx yet |
| +// still be up; only a failure to connect means "not up". |
| +func waitReachable(socket string, timeout time.Duration) error { |
| + deadline := time.Now().Add(timeout) |
| + client := &http.Client{ |
| + Timeout: 3 * time.Second, |
| + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, |
| + Transport: &http.Transport{ |
| + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { |
| + return (&net.Dialer{}).DialContext(ctx, "unix", socket) |
| + }, |
| + }, |
| + } |
| + var last error |
| + for time.Now().Before(deadline) { |
| + resp, err := client.Get("http://svc/") |
| + if err == nil { |
| + resp.Body.Close() |
| + return nil // any HTTP response means the backend is listening |
| + } |
| + last = err |
| + time.Sleep(150 * time.Millisecond) |
| + } |
| + return fmt.Errorf("backend not reachable within %s: %v", timeout, last) |
| +} |
| + |
| +// --- helpers --------------------------------------------------------------- |
| + |
| +func systemctl(args ...string) error { |
| + out, err := exec.Command("systemctl", args...).CombinedOutput() |
| + if err != nil { |
| + return fmt.Errorf("systemctl %s: %v: %s", strings.Join(args, " "), err, strings.TrimSpace(string(out))) |
| + } |
| + return nil |
| +} |
| + |
| +// writeEnvFile writes KEY=value lines (sorted for a stable file) at 0600 — it |
| +// holds the admin token, so it's operator-secret, like the platform env it |
| +// replaces. |
| +func writeEnvFile(path string, kv map[string]string) error { |
| + keys := make([]string, 0, len(kv)) |
| + for k := range kv { |
| + keys = append(keys, k) |
| + } |
| + sort.Strings(keys) |
| + var b strings.Builder |
| + for _, k := range keys { |
| + fmt.Fprintf(&b, "%s=%s\n", k, kv[k]) |
| + } |
| + return os.WriteFile(path, []byte(b.String()), 0o600) |
| +} |
| + |
| +// readEnvFile parses a KEY=value EnvironmentFile (blank lines and #comments |
| +// ignored). |
| +func readEnvFile(path string) (map[string]string, error) { |
| + f, err := os.Open(path) |
| + if err != nil { |
| + return nil, err |
| + } |
| + defer f.Close() |
| + m := map[string]string{} |
| + sc := bufio.NewScanner(f) |
| + for sc.Scan() { |
| + line := strings.TrimSpace(sc.Text()) |
| + if line == "" || strings.HasPrefix(line, "#") { |
| + continue |
| + } |
| + if k, v, ok := strings.Cut(line, "="); ok { |
| + m[strings.TrimSpace(k)] = strings.TrimSpace(v) |
| + } |
| + } |
| + return m, sc.Err() |
| +} |
| + |
| +// markInstanceFailed handles a partial add-instance: stop + disable the unit (so it |
| +// can't crash-loop) and record status=failed + last_error for `lchat instances`, |
| +// all best-effort. The env file and DB are KEPT — the env file is the only copy of |
| +// the admin token (the DB stores just its hash), so removing it could strand a |
| +// still-running instance with an unrecoverable token. add-instance overwrites a |
| +// non-ready route on retry, so the failed row doesn't block a re-run. |
| +func markInstanceFailed(reg *Registry, host, unit, reason string) { |
| + _ = systemctl("disable", "--now", unit) |
| + if r, err := reg.get(host); err == nil && r != nil { |
| + r.Status = "failed" |
| + r.LastError = reason |
| + _ = reg.upsert(*r) |
| + } |
| +} |
diff --git a/control_test.go b/control_test.go| new file mode 100644 |
| index 0000000..503ac0b |
| --- /dev/null |
| +++ b/control_test.go |
| @@ -0,0 +1,117 @@ |
| +package main |
| + |
| +import ( |
| + "flag" |
| + "os" |
| + "path/filepath" |
| + "strings" |
| + "testing" |
| +) |
| + |
| +func TestFlagsAfterPositional(t *testing.T) { |
| + fs := flag.NewFlagSet("t", flag.ContinueOnError) |
| + socket := fs.String("socket", "", "") |
| + all := fs.Bool("all", false, "") |
| + // host BEFORE the flags — the natural order that the stdlib parser drops. |
| + if err := fs.Parse(reorderFlagsFirst(fs, []string{"chat.example.com", "--socket", "/x.sock", "--all"})); err != nil { |
| + t.Fatalf("parse: %v", err) |
| + } |
| + if *socket != "/x.sock" { |
| + t.Errorf("socket = %q, want /x.sock", *socket) |
| + } |
| + if !*all { |
| + t.Error("all = false, want true") |
| + } |
| + if fs.Arg(0) != "chat.example.com" { |
| + t.Errorf("Arg(0) = %q, want chat.example.com", fs.Arg(0)) |
| + } |
| +} |
| + |
| +func TestReorderDoesNotPullFlagAsValue(t *testing.T) { |
| + fs := flag.NewFlagSet("t", flag.ContinueOnError) |
| + fs.String("socket", "", "") |
| + fs.String("router-unit", "", "") |
| + // --socket is given no value (next token is another flag); reorder must not |
| + // swallow --router-unit as --socket's value. |
| + out := reorderFlagsFirst(fs, []string{"host", "--socket", "--router-unit", "foo"}) |
| + if !strings.Contains(strings.Join(out, " "), "--router-unit foo") { |
| + t.Errorf("reorder paired a flag as a value: %v", out) |
| + } |
| +} |
| + |
| +func TestValidHost(t *testing.T) { |
| + ok := []string{"chat.example.com", "rtr-test.luan.chat", "a.b", "x1.y2.example.com"} |
| + bad := []string{"", "nodots", "-bad.example.com", "bad-.example.com", "has space.com", "Chat.Example.com", "a/b.com", "a..b"} |
| + for _, h := range ok { |
| + if !validHost(h) { |
| + t.Errorf("validHost(%q) = false, want true", h) |
| + } |
| + } |
| + for _, h := range bad { |
| + if validHost(h) { |
| + t.Errorf("validHost(%q) = true, want false", h) |
| + } |
| + } |
| +} |
| + |
| +func TestCheckSocketLen(t *testing.T) { |
| + if err := checkSocketLen("/run/lchat/chat.example.com.sock"); err != nil { |
| + t.Errorf("short path: unexpected error %v", err) |
| + } |
| + long := "/run/lchat/" + strings.Repeat("a", 120) + ".sock" |
| + if err := checkSocketLen(long); err == nil { |
| + t.Error("long path: expected error, got nil") |
| + } |
| +} |
| + |
| +func TestEnvFileRoundTrip(t *testing.T) { |
| + path := filepath.Join(t.TempDir(), "chat.example.com.env") |
| + in := map[string]string{"LCHAT_ADMIN_TOKEN": "tok-123", "LCHAT_HOME_ORIGIN": "https://home.example.com"} |
| + if err := writeEnvFile(path, in); err != nil { |
| + t.Fatalf("write: %v", err) |
| + } |
| + fi, err := os.Stat(path) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if fi.Mode().Perm() != 0o600 { |
| + t.Errorf("perms = %v, want -rw-------", fi.Mode().Perm()) |
| + } |
| + out, err := readEnvFile(path) |
| + if err != nil { |
| + t.Fatalf("read: %v", err) |
| + } |
| + for k, v := range in { |
| + if out[k] != v { |
| + t.Errorf("readEnvFile[%q] = %q, want %q", k, out[k], v) |
| + } |
| + } |
| +} |
| + |
| +func TestRegistryGet(t *testing.T) { |
| + reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db")) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if got, err := reg.get("missing.example.com"); err != nil || got != nil { |
| + t.Fatalf("get(missing) = %v, %v; want nil, nil", got, err) |
| + } |
| + if err := reg.upsert(Route{Host: "chat.example.com", Kind: "instance", Socket: "/run/lchat/chat.example.com.sock", Status: "ready"}); err != nil { |
| + t.Fatal(err) |
| + } |
| + got, err := reg.get("chat.example.com") |
| + if err != nil || got == nil { |
| + t.Fatalf("get(existing) = %v, %v", got, err) |
| + } |
| + if got.Socket != "/run/lchat/chat.example.com.sock" || got.Status != "ready" || got.Kind != "instance" { |
| + t.Errorf("get returned %+v", *got) |
| + } |
| + |
| + // delete stops routing the host. |
| + if err := reg.delete("chat.example.com"); err != nil { |
| + t.Fatalf("delete: %v", err) |
| + } |
| + if got, err := reg.get("chat.example.com"); err != nil || got != nil { |
| + t.Fatalf("after delete, get = %v, %v; want nil, nil", got, err) |
| + } |
| +} |
diff --git a/deploy/README.md b/deploy/README.md| new file mode 100644 |
| index 0000000..f3e19cd |
| --- /dev/null |
| +++ b/deploy/README.md |
| @@ -0,0 +1,53 @@ |
| +# Single-machine bootstrap |
| + |
| +Running the messenger behind the `lchat` router on one box (see |
| +`docs/instance-router.md`). The Go side is a single static binary — no runtime. |
| + |
| +```sh |
| +# 1. copy the binary, then let the binary provision the box |
| +scp lchat root@<box>:/opt/lchat/lchat |
| +ssh root@<box> '/opt/lchat/lchat bootstrap' # writes units + dirs, starts the router |
| + |
| +# 2. add an instance (mints token + env file, starts it, registers + reloads) |
| +ssh root@<box> '/opt/lchat/lchat add-instance chat.example.com' |
| +``` |
| + |
| +`lchat bootstrap` embeds the unit files (source `deploy/systemd/`) and writes |
| +them, creates `/data` + `/run/lchat`, and starts the router. To install by hand |
| +instead: `scp deploy/systemd/*.service` to `/etc/systemd/system/`, drop a |
| +`/etc/tmpfiles.d/lchat.conf` (`d /run/lchat 0710 root root -`), `daemon-reload`, |
| +`systemctl enable --now lchat-router`. |
| + |
| +- `lchat-router.service` — the front door (binds :80/:443, autocert HTTP-01). |
| +- `lchat@.service` — an instance template; runs one instance on |
| + `/run/lchat/<host>.sock`, reading its token + config from `/data/<host>.env`. |
| +- `lchat-bot@.service` — the optional per-instance Node sidecar (see Bots). |
| +- Runs as root here so it can bind :80/:443; for a non-root user add |
| + `AmbientCapabilities=CAP_NET_BIND_SERVICE` to the router unit and a dedicated |
| + `User=`/`Group=` with matching ownership on `/run/lchat` + `/data`. |
| + |
| +## Bots (sidecars) |
| + |
| +The sidecar is **Node**, not the Go binary (it reuses `web/static/crypto.js` for |
| +E2E — that's why it's Node), so a box that runs bots needs a Node runtime and the |
| +`agent/` code + its deps: |
| + |
| +```sh |
| +# one-time per box: Node 24 + the sidecar source + deps |
| +ssh root@<box> 'curl -fsSL https://deb.nodesource.com/setup_24.x | bash - && apt-get install -y nodejs' |
| +rsync -a agent web/static root@<box>:/opt/lchat/ # sidecar imports ../web/static/crypto.js |
| +ssh root@<box> 'cd /opt/lchat/agent && npm ci' |
| + |
| +# per instance: mint the three agent invites + start the sidecar |
| +# (API key via env, not a flag, so it doesn't leak into ps/shell history) |
| +ssh root@<box> 'ANTHROPIC_API_KEY=sk-ant-... /opt/lchat/lchat add-bot chat.example.com' |
| +``` |
| + |
| +`add-bot` mints an agent invite per identity (Claude / Eleven / Announcer) via the |
| +instance's admin API — running `agent/mint-invite.mjs`, which reuses `crypto.js` so |
| +the derived invite matches what the sidecar provisions with — writes |
| +`/data/<host>-bot.env` (invites + session paths + optional `ANTHROPIC_API_KEY`), |
| +and starts `lchat-bot@<host>`. Omit `ANTHROPIC_API_KEY` for a deterministic-only bot |
| +(Eleven still welcomes new members + answers slash commands). Update a bot's code |
| +with `systemctl restart lchat-bot@<host>` (it backfills on reconnect). Start the |
| +**instance before** the sidecar so Eleven baselines the roster correctly. |
diff --git a/deploy/systemd/lchat-bot@.service b/deploy/systemd/lchat-bot@.service| new file mode 100644 |
| index 0000000..4d826c7 |
| --- /dev/null |
| +++ b/deploy/systemd/lchat-bot@.service |
| @@ -0,0 +1,19 @@ |
| +[Unit] |
| +Description=lchat bot sidecar for %i |
| +After=network-online.target lchat@%i.service |
| +Wants=network-online.target |
| + |
| +[Service] |
| +# add-bot writes /data/%i-bot.env (0600) with LCHAT_BASE, the three agent invites, |
| +# session paths, and (optionally) ANTHROPIC_API_KEY. |
| +EnvironmentFile=-/data/%i-bot.env |
| +WorkingDirectory=/opt/lchat/agent |
| +# The sidecar is Node (not the static Go binary): the box needs a node runtime and |
| +# `npm ci` in /opt/lchat/agent (see deploy/README.md). It connects to its instance |
| +# over the public URL, backfills on reconnect, so a deploy is just a restart. |
| +ExecStart=/usr/bin/node /opt/lchat/agent/agent.mjs |
| +Restart=always |
| +RestartSec=2 |
| + |
| +[Install] |
| +WantedBy=multi-user.target |
diff --git a/deploy/systemd/lchat-router.service b/deploy/systemd/lchat-router.service| new file mode 100644 |
| index 0000000..c8c5c28 |
| --- /dev/null |
| +++ b/deploy/systemd/lchat-router.service |
| @@ -0,0 +1,14 @@ |
| +[Unit] |
| +Description=lchat router (TLS terminator + host router) |
| +After=network-online.target |
| +Wants=network-online.target |
| + |
| +[Service] |
| +ExecStart=/opt/lchat/lchat router --registry /data/registry.db --acme-cache /data/acme |
| +# add-instance triggers `systemctl reload lchat-router` to re-read the route table. |
| +ExecReload=/bin/kill -HUP $MAINPID |
| +Restart=always |
| +RestartSec=1 |
| + |
| +[Install] |
| +WantedBy=multi-user.target |
diff --git a/deploy/systemd/lchat@.service b/deploy/systemd/lchat@.service| new file mode 100644 |
| index 0000000..068d5a0 |
| --- /dev/null |
| +++ b/deploy/systemd/lchat@.service |
| @@ -0,0 +1,15 @@ |
| +[Unit] |
| +Description=lchat instance %i |
| +After=network-online.target |
| +Wants=network-online.target |
| + |
| +[Service] |
| +# add-instance writes /data/<host>.env (0600) with LCHAT_ADMIN_TOKEN + per-instance |
| +# config. The leading '-' makes it optional so a self-provisioning instance still boots. |
| +EnvironmentFile=-/data/%i.env |
| +ExecStart=/opt/lchat/lchat --socket /run/lchat/%i.sock --db /data/%i.db |
| +Restart=always |
| +RestartSec=1 |
| + |
| +[Install] |
| +WantedBy=multi-user.target |
diff --git a/docs/instance-migration.md b/docs/instance-migration.md| new file mode 100644 |
| index 0000000..77d98bf |
| --- /dev/null |
| +++ b/docs/instance-migration.md |
| @@ -0,0 +1,108 @@ |
| +# Migrating an instance from one server to another |
| + |
| +Status: **procedure** (validated once in practice). Moving a running messenger |
| +instance to a new box is **data movement + a DNS flip**, done so it's reversible: |
| +copy the DB, keep the old instance running as a fallback, and only cut DNS over |
| +once the new box is verified. The flip itself is reversible too. |
| + |
| +The key property: **keep the hostname the same**. Browser keys (and passkeys, for |
| +the home server) are per-origin, so an unchanged origin means members' keys, |
| +sessions, and history are all unaffected — only the DNS record moves. |
| + |
| +## Before you start |
| + |
| +- SSH to both the source and destination boxes. |
| +- The destination is already bootstrapped (`lchat bootstrap`; router running). |
| +- Know the instance's **admin token** and any per-instance config (see step 1). |
| + |
| +## Steps |
| + |
| +### 1. Recover the instance's config from the source |
| + |
| +The instance's DB stores only the **hash** of the admin token — the plaintext |
| +lives in the source's deploy env / secret store. Recover it, plus any per-instance |
| +config the instance reads (`LCHAT_PUSH_RELAY_URL`/`_TOKEN`, `LCHAT_HOME_ORIGIN`, |
| +`LCHAT_APP_STORE_URL`, APNs vars, …). You'll write these into the destination's |
| +EnvironmentFile in step 4. **Preserve the admin token** — minting a new one |
| +invalidates every saved admin link. |
| + |
| +### 2. Snapshot the DB consistently |
| + |
| +Use SQLite's `VACUUM INTO` — it produces a single, compacted, transaction-consistent |
| +file and is **safe on a live DB** (no need to stop the instance): |
| + |
| +```sh |
| +sqlite3 /path/to/lchat.db "VACUUM INTO '/tmp/snapshot.db'" |
| +sqlite3 /tmp/snapshot.db "PRAGMA integrity_check" # expect: ok |
| +``` |
| + |
| +If the source hides the DB behind an opaque store (many identically-named DB files), |
| +identify the right one by **content** — e.g. grep the files for the instance's |
| +`admin_token_hash` (`hex(sha256(admin token))`), which is unique per instance. |
| + |
| +### 3. Copy the snapshot to the destination |
| + |
| +Copy `/tmp/snapshot.db` to the destination as `/data/<host>.db` and verify it |
| +landed intact (`integrity_check` + a couple of row counts against the source). |
| + |
| +### 4. Import on the destination (preserve token + config) |
| + |
| +This is an **import**, not a fresh `add-instance` (which would mint a *new* token |
| +and a *new* DB). With the DB already in place: |
| + |
| +- write `/data/<host>.env` (mode `0600`) with the **existing** `LCHAT_ADMIN_TOKEN` |
| + + the recovered config vars; |
| +- register the route (`routes` row: host, kind `instance`, socket, db_path, |
| + status `ready`); |
| +- `systemctl enable --now lchat@<host>` and reload the router. |
| + |
| +Verify **before any DNS change**, over the instance's unix socket: `GET /` → 200, |
| +an admin-token request → 200, and the row counts match the source. |
| + |
| +> Tooling gap worth closing: `add-instance` mints fresh; there's no first-class |
| +> "import existing DB + token" command yet, so step 4 is done by hand. A `--admin-token` |
| +> flag (or an `import-instance` command) would make this one command. |
| + |
| +### 5. Gap check |
| + |
| +Confirm nothing was written to the source between the snapshot and the cutover |
| +(compare a row count, e.g. `messages`, source-now vs snapshot). For a busy |
| +instance, freeze writes or take the snapshot at cutover time — writes that land on |
| +the source after the snapshot are **not** on the destination. |
| + |
| +### 6. Cut over DNS |
| + |
| +Flip the host's A record to the destination's IP. **Lower the record's TTL ahead of |
| +time** (to ~60s, ideally a TTL-worth before the cutover) so the flip propagates |
| +fast; otherwise resolvers that cached the old IP keep hitting the old box until the |
| +old TTL expires (harmless while the old box is still up as a fallback). |
| + |
| +The TLS cert **can't be pre-issued**: autocert's HTTP-01 validates by resolving the |
| +host over public DNS, so the cert only issues once DNS points at the destination. |
| +Authoritative DNS updates in seconds (only resolver *caches* lag), so the cert |
| +issues on the first HTTPS hit right after the flip. |
| + |
| +### 7. Verify + keep the fallback |
| + |
| +`GET https://<host>/` → 200 with a valid (Let's Encrypt) cert, serving the migrated |
| +data. **Leave the source instance running** until you're confident — the migration |
| +is a copy, so flipping DNS back is an instant rollback. Retire the source only |
| +afterward. |
| + |
| +## Bots (sidecars) |
| + |
| +The instance's **bot agent-users migrate with the DB** (they're rows), but the |
| +**sidecar** — the Node process and its session files (the bots' identities) — is |
| +separate and does *not* come across in the DB copy. Do **not** re-run `add-bot` on |
| +the destination: that mints a *new* set of agent identities (duplicate bots in the |
| +Helpers catalog). Instead either copy the sidecar's session files to the |
| +destination and start `lchat-bot@<host>` there, or point the existing sidecar at |
| +the destination (it follows the host over DNS and reconnects with its persisted |
| +token). Plan the bot move as its own step. |
| + |
| +## What makes this safe |
| + |
| +- **Reversible**: the source is copied, never moved; DNS flip is reversible; the |
| + cert issues automatically post-flip. |
| +- **No key churn**: same origin ⇒ per-origin browser keys/passkeys untouched. |
| +- **Consistent snapshot**: `VACUUM INTO` captures a clean point-in-time DB. |
diff --git a/docs/instance-router.md b/docs/instance-router.md| new file mode 100644 |
| index 0000000..3683a81 |
| --- /dev/null |
| +++ b/docs/instance-router.md |
| @@ -0,0 +1,418 @@ |
| +# The instance router — a single-machine bootstrap |
| + |
| +Status: **proposal for discussion** (no code yet). This is the design note; when |
| +it ships the code becomes the source of truth. |
| + |
| +## The problem |
| + |
| +The messenger is a single static, no-cgo binary with SQLite compiled in, and it |
| +runs as **one fully-isolated instance per chat** — each with its own disk, admin |
| +token, and subdomain. The obvious way to run several of those on one machine is |
| +to reach for an orchestrator (Kamal, a PaaS, Kubernetes): it gives you TLS, |
| +subdomain routing, a per-app disk, env injection, and process supervision. |
| + |
| +But that's a lot of machinery around a single file you could `scp`. This doc |
| +describes a lighter alternative for the common case — many instances on one |
| +machine — that keeps the two properties process-per-instance gives you and that |
| +are worth keeping: |
| + |
| +- **Per-instance versioning / canary.** Deploy one instance on a new build while |
| + the others stay on the old one. A single shared process can't do that. |
| +- **Isolation.** A crash or a resource hog in one instance must not touch the |
| + others. |
| + |
| +…and adds **zero-downtime deploys**. Those reinforce each other: zero-downtime |
| +*means* "run the new version alongside the old and hand over," so per-instance |
| +versioning falls out of doing it right. |
| + |
| +No container platform required — just the binary, a small router, and systemd. |
| + |
| +## One binary, many ways to deploy |
| + |
| +The messenger makes no assumptions about how it's run. Put it behind nginx or |
| +Caddy, on a PaaS, in Kubernetes, under Kamal, or one-container-per-instance on |
| +any platform — all of those work, and nothing here precludes them. The instance |
| +binary stands alone. |
| + |
| +**What this doc describes is a batteries-included path for the common case:** |
| +bootstrapping a single machine from zero to a running, auto-TLS'd, multi-instance |
| +box with zero-downtime deploys and no external orchestrator. It's the *simplest* |
| +way, not the *only* way — the router below is optional convenience. |
| + |
| +## The shape |
| + |
| +``` |
| + :443 / :80 |
| + internet ──▶ lchat router ──(reverse proxy over unix sockets)──▶ lchat instance (chat-a) ── chat-a.db |
| + (autocert HTTP-01, lchat instance (chat-b) ── chat-b.db |
| + HostPolicy = route table, lchat instance (chat-c) ── chat-c.db |
| + atomic upstream map) … |
| + │ home server (home/) ── home.db ┐ service |
| + │ push relay (push/) ── push.db ┘ routes |
| + └── control API on a localhost-only socket (add / deploy / list) |
| + |
| + systemd supervises everything: router + lchat@<host> (instance) + lchat-bot@<host> (sidecar) + home/push units |
| +``` |
| + |
| +One binary, three roles (subcommands): |
| + |
| +- `lchat router` — the front door (TLS + routing + control plane). |
| +- `lchat instance` — one messenger instance (today's `main()` minus the TCP |
| + listener and minus the process-global env reads). |
| +- control subcommands — `add-instance`, `deploy`, `instances`, `admin-link`. |
| + |
| +## The router |
| + |
| +- **`:443`** — TLS via `golang.org/x/crypto/acme/autocert` (stdlib-adjacent, no |
| + DNS-API credentials). `HostPolicy` returns nil only for hosts in the route |
| + table, so the route list *is* the cert allowlist. `Cache: autocert.DirCache` on |
| + the persistent disk. HTTP-01 (well-known) is the challenge; a brand-new host is |
| + issued on first hit — no wildcard, no DNS-provider step. |
| +- **`:80`** — `m.HTTPHandler(nil)`: serves `/.well-known/acme-challenge/…`, |
| + 301-redirects everything else to https. |
| +- **Routing** — the route table maps **host → backend** (a unix socket at a |
| + version); `httputil.ReverseProxy` dials it. The upstream map is an |
| + `atomic.Pointer` so reloads and deploys swap it without dropping a connection. |
| + WebSocket upgrades pass through (Go ≥1.12). A backend is usually one instance — |
| + see *Backends*, below. |
| +- **Control API** — bound to a **localhost-only** unix socket, superadmin-gated; |
| + never exposed on a served hostname. This is the god surface (it can create |
| + instances and read admin links), so it stays off the public interface. |
| + |
| +Threat model is unchanged: TLS terminates at the router, but message bodies are |
| +already E2E ciphertext — the router sees exactly what the app sees, never |
| +plaintext. |
| + |
| +### Optional: wildcard TLS (DNS-01) |
| + |
| +HTTP-01 issues one cert per host on first hit — simple, needs no DNS credentials, |
| +but needs inbound `:80` and can't cover a host before it's requested. When that's |
| +not enough the cert source is **pluggable**: point the router at a DNS provider and |
| +it obtains a **wildcard** (`*.example.com`) over DNS-01 instead, covering every |
| +subdomain instantly with no port-80 dependency. The two can coexist — a wildcard |
| +for the base domain, HTTP-01 on demand for custom domains outside it. `HostPolicy` |
| +still gates which hosts are actually served; the wildcard only changes how the |
| +cert is obtained. Off by default (no DNS creds required); enable it by configuring |
| +a provider + credentials. |
| + |
| +## The instance process |
| + |
| +`lchat instance --db /data/<host>.db --socket /run/lchat/<host>.sock --config <file>` |
| + |
| +It's today's server with two changes: it listens on a **unix socket** instead of |
| +a TCP port, and it takes per-instance config from flags/a file the router writes |
| +rather than from process-global env (see below). Everything else is unchanged — |
| +its own SQLite (WAL), its own admin token (`ensureAdminToken`, in the DB), its own |
| +VAPID (`ensureVAPID`, in the DB). `handler()` is untouched, so `newTestServer` |
| +and the existing handler tests keep working as-is. |
| + |
| +## Backends: one per instance (default), one per version (option) |
| + |
| +The router maps **host → backend** (a socket at a version) and doesn't care how |
| +many hosts a backend serves. That leaves two ways to slice processes: |
| + |
| +**One backend per instance (the default).** Each instance is its own process, |
| +socket, and DB — full crash and resource isolation, and per-version deploys come |
| +*for free*: point one instance's unit at a new build and leave the rest on the old |
| +one (a `trunk` instance next to `stable` ones) with no coordination. Canary is |
| +just "which binary does this unit run." |
| + |
| +**One backend per version (an option, later).** Alternatively a single process |
| +serves *many* hosts at one version (an in-process host map, one DB per host), and |
| +the router sends each host to the process running its assigned version: |
| + |
| +``` |
| + stable-host ─┐ |
| + other-host ─┼─▶ backend @ stable (one process, many DBs) |
| + another-host ─┘ |
| + trunk-host ──▶ backend @ trunk (one process) |
| +``` |
| + |
| +This trades isolation for fewer processes — fewer Go runtimes (less RAM), fewer |
| +deploy targets — and the costs are real: a crash or OOM takes down *every* host on |
| +that version, there's no per-host resource cap, and deploying a version restarts |
| +all its hosts together (one big reconnect, not a staggered roll). Moving a host |
| +between versions also needs a clean SQLite hand-off (two processes must never write |
| +one DB file), so it suits hosts that live permanently on a channel (a dedicated |
| +trunk instance) more than ad-hoc per-instance canaries. |
| + |
| +**Recommendation:** default to one-backend-per-instance — isolation is cheap to |
| +keep, per-version is already free, and a dozen processes cost a few hundred MB |
| +(RAM isn't the binding resource; disk is). Keep the route table as host → backend |
| +so consolidating onto per-version backends stays a config change, not a rewrite, |
| +and reach for it only if process count ever actually bites. It needs the |
| +in-process host map, so it's the *second* thing to build, not the first. |
| + |
| +## Beyond messenger instances: sibling services |
| + |
| +A deployment usually runs sibling services next to the messenger that want the |
| +same TLS + host routing — for example the other two binaries in this repo: |
| + |
| +- **the home server (`home/`)** — the passkey-backup store, its own binary + DB. |
| + Its host *is* the WebAuthn RP origin, so it must keep the same hostname across |
| + any move (only the DNS record changes) or it orphans every passkey. |
| +- **the push relay (`push/`)** — a shared APNs relay that lets instances (yours |
| + *and* third parties') send iOS push without holding Apple's signing key (see |
| + `push-relay.md`). |
| + |
| +To the router these are just another **host → upstream** mapping: a `service`-kind |
| +route to a local unix socket, TLS via autocert, no lifecycle management (they |
| +deploy on their own cadence). That's the only difference from a messenger |
| +instance, which the router *also* create/deploy/version-manages. |
| + |
| +Because a route is nothing but host → upstream, **anyone running their own home |
| +server or push relay is first-class** — point a host at their backend the same |
| +way. This is deliberately the simple version (a flat mapping + a health check); it |
| +can be hardened — readiness gating, retries, per-service resource isolation — |
| +later, if it earns it. |
| + |
| +## Bots (sidecars) |
| + |
| +An instance can have a **sidecar** hosting the default agents (see `agent/`). It's |
| +a **Node** process, not part of the Go binary, and it connects to its instance as |
| +a **WebSocket client** over the instance's public host — authenticating with a |
| +one-time agent invite and persisting its session to disk. It serves no inbound |
| +traffic, so it never needs a socket or a cert, and the router never routes *to* it. |
| + |
| +**Why a separate process, and why Node?** Two different reasons. It's a *separate |
| +process* because it's a full E2E client — it holds its own keypair, and that |
| +private key must never live in the server process (the core rule), so a bot is a |
| +member that runs on a server, not a server feature (a separate *Go* process would |
| +satisfy this too). It's *Node* specifically because it imports `web/static/ |
| +crypto.js` directly — the same crypto the browser runs, on Node's WebCrypto — so |
| +its ciphertext can't drift from what clients expect. A Go rewrite would keep the |
| +separate-process property but give up that shared implementation: you'd |
| +re-implement crypto.js's exact wire format and keep it byte-compatible forever |
| +(correctness-critical, silent on failure). Cheaper ways to drop the Node install: |
| +bundle the sidecar into one executable (Node SEA), or run it off-box. |
| + |
| +That makes it the simplest thing to supervise: **one `lchat-bot@<host>.service` |
| +per instance**, restarted on deploy. Because the sidecar backfills on reconnect, a |
| +bot deploy is a plain restart — no socket-swap, no drain. Two notes: |
| + |
| +- The sidecar is Node, so its host needs a **Node runtime** (the Go server is a |
| + static binary; the bot is not); its `agent/` deps install with `npm`. |
| +- It reaches its instance at the public URL through the router — a normal client |
| + connection that happens to originate on the same box, so no special wiring. |
| + |
| +Its secrets (an API key, the persisted agent sessions) live in its own env + disk, |
| +the same way the messenger's admin token lives in the instance DB. |
| + |
| +## The registry (a small control DB) |
| + |
| +The router owns one SQLite table. Status lives here (not a flat file) so an |
| +add-in-flight survives a router restart, and it's where per-instance config that |
| +used to be env now lives: |
| + |
| +```sql |
| +CREATE TABLE routes ( |
| + host TEXT PRIMARY KEY, |
| + kind TEXT NOT NULL, -- 'instance' (messenger, lifecycle-managed) | 'service' (home, push, …) |
| + socket TEXT NOT NULL, -- unix socket the router proxies to |
| + status TEXT NOT NULL, -- pending|creating|starting|issuing|ready|failed |
| + last_error TEXT, |
| + -- instance-only fields (null for registered services): |
| + db_path TEXT, |
| + version TEXT, -- running binary version (per-instance skew is normal) |
| + app_store_url TEXT, -- was LCHAT_APP_STORE_URL |
| + home_origin TEXT, -- was LCHAT_HOME_ORIGIN |
| + push_relay_url TEXT, -- was LCHAT_PUSH_RELAY_URL |
| + push_relay_token TEXT, -- was LCHAT_PUSH_RELAY_TOKEN (per-instance bearer) |
| + created_at INTEGER |
| +); |
| +``` |
| + |
| +### Per-instance config: a per-instance EnvironmentFile |
| + |
| +Config stays env-shaped — the instance reads the same `LCHAT_*` vars it always |
| +has — but instead of process-global env it comes from a per-instance systemd |
| +**EnvironmentFile**, `/data/<host>.env` (mode `0600`), written by `add-instance`. |
| +No instance code changes: `EnvironmentFile=-/data/%i.env` in the unit, and the |
| +existing `os.Getenv` calls just work. |
| + |
| +| Var | Source | |
| +|---|---| |
| +| `LCHAT_ADMIN_TOKEN` | `/data/<host>.env` — the DB stores only its *hash*, so this file is the one recoverable copy (`admin-link` reads it) | |
| +| VAPID keypair | the instance DB (`ensureVAPID`) — self-generated, no config needed | |
| +| `LCHAT_APP_STORE_URL`, `LCHAT_HOME_ORIGIN`, `LCHAT_PUSH_RELAY_URL`/`_TOKEN` | `/data/<host>.env` (from `add-instance` flags) | |
| +| `LCHAT_DB` / socket path | derived from `<host>` by the unit | |
| +| APNs `.p8` | **not on instances at all** — the push relay holds it (see `push-relay.md`) | |
| + |
| +The `routes` table tracks routing + status; the EnvironmentFile is the source of |
| +per-instance config and the admin token. The push-relay work already did the |
| +hardest part — the one secret that can't sit safely in a per-host file (the Apple |
| +signer) is out of instances entirely, so each carries only a revocable bearer. |
| + |
| +## Provisioning the box |
| + |
| +Because the server is a static, no-cgo binary, a fresh box needs **no build |
| +toolchain and no language runtime** for the messenger, the router, or the |
| +home/push services — provisioning is just "copy binaries + unit files + make |
| +directories": |
| + |
| +1. `scp` the `lchat` binary (plus `home`/`push` and `litestream`) to the box. |
| +2. `lchat bootstrap` — writes the systemd units (router + `@` templates), creates |
| + `/data`, `/run/lchat`, and the autocert cache dir, sets `LimitNOFILE`, then |
| + enables and starts the router. |
| +3. Open inbound `:80` / `:443`. |
| + |
| +That's the whole dependency story for the Go side — nothing to `apt install`, no |
| +version to keep patched, no runtime to manage. A bare box with *no* Go on it is |
| +not a concern; it's the point. |
| + |
| +**The one runtime dependency is Node**, and only on a box that runs bot sidecars — |
| +the sidecar (`agent/`) is Node, not a static binary. Install a pinned Node and |
| +`npm ci` in `agent/`. If you'd rather keep the box to static binaries only, the |
| +sidecar can be bundled into a single executable (Node SEA / `pkg`) or run |
| +**off-box** entirely — it's just a WebSocket client to the public URL, so it need |
| +not live on the router box at all. For the common case, installing Node is the |
| +pragmatic choice. |
| + |
| +## add-instance |
| + |
| +`lchat add-instance <host>` runs the steps an operator would otherwise do by hand: |
| + |
| +``` |
| +mint token → write /data/<host>.env (0600, the unit's EnvironmentFile) |
| + → register route (status: creating) |
| + → systemctl enable --now lchat@<host> → wait for the socket |
| + → status: ready → systemctl reload lchat-router → print the admin link |
| +``` |
| + |
| +The route's `status` (`creating`/`ready`/`failed` + `last_error`) is recorded in |
| +the registry and shown by `lchat instances`. The cert issues lazily on the host's |
| +first HTTPS hit (proven fine in practice); pre-warming it |
| +(`m.GetCertificate(&tls.ClientHelloInfo{ServerName: host})`) and a fully async, |
| +retrying state machine are refinements the schema already allows. |
| +`lchat admin-link <host>` recovers the admin link from the EnvironmentFile (the DB |
| +holds only the token's hash). |
| + |
| +## Zero-downtime deploy (per instance) |
| + |
| +`lchat deploy <host>` (or `--all`) rolls an instance onto the new on-disk binary |
| +with no dropped requests — via **graceful drain + a retrying router**, rather than |
| +a dual-socket blue/green. (Simpler, and enough: clients already auto-reconnect, so |
| +the marginal gain of true zero-latency didn't justify alternating dual units and |
| +two processes sharing one SQLite DB.) |
| + |
| +1. Ship the new binary (`scp` + atomic `mv` over `/opt/lchat/lchat`). |
| +2. `systemctl restart lchat@<host>`. On **SIGTERM** the instance drains in-flight |
| + HTTP (`http.Server.Shutdown`) and closes WebSockets (so browsers reconnect), |
| + then exits; systemd starts the new process, which rebinds the socket. |
| +3. Across the brief gap (old exited, new not yet bound → dials fail |
| + `ENOENT`/`ECONNREFUSED`) the **router retries** the dial for a short window, so |
| + requests queue a beat instead of failing. |
| +4. `deploy` waits for the new process to answer an HTTP health check (retrying — |
| + catches a stale socket or a broken binary) before reporting success; `--all` |
| + stops on the first failure. |
| + |
| +Blast radius is **one instance**: its WebSocket clients reconnect once; HTTP |
| +requests see at most a little added latency. Canary = deploy one instance alone. |
| +Verified in practice: 120 requests against an instance during a live deploy all |
| +returned 200. |
| + |
| +The router itself changes rarely; when it does, its own front listener would get |
| +zero-downtime via `SO_REUSEPORT` / systemd socket activation (not yet wired — a |
| +router deploy is currently a ~1s blip for all hosts). |
| + |
| +The router itself changes rarely; when it does, its own front listener gets |
| +zero-downtime via `SO_REUSEPORT` / systemd socket activation. |
| + |
| +## Supervision (systemd) |
| + |
| +- `lchat-router.service` — `Restart=always`, socket-activated (its own |
| + zero-downtime). |
| +- `lchat@.service` — a template unit, one per instance. `MemoryMax=` + `CPUWeight=` |
| + give resource isolation; `Restart=always` gives crash isolation. Bump |
| + `LimitNOFILE` (instances × WS connections × SQLite files). |
| +- `lchat-bot@.service` — the optional per-instance sidecar (Node; see *Bots*). |
| + `Restart=always`; a deploy is just a restart (it backfills on reconnect). |
| + |
| +## Backups |
| + |
| +Collapsing onto one box means every instance's data is on one disk — a single |
| +failure loses everything, and there's no platform to restore from. |
| + |
| +- **Litestream per instance DB** → object storage, point-in-time restore. |
| +- **Back up the autocert cache dir** too — it holds issued certs *and their |
| + private keys*. Treat as key material (`0600`); losing it forces re-issue. |
| +- Encrypted attachments are stored as blobs **inside** SQLite (the `blobs` table), |
| + so the DB backup covers them — but they're also what drives disk growth (see |
| + sizing). |
| + |
| +## Sizing |
| + |
| +Server-side memory is light **by design**: crypto and image downscaling happen in |
| +the browser; the server only moves ciphertext. A modest box goes a long way. |
| + |
| +- **RAM:** each instance is ~20–40 MB idle (Go runtime + modernc SQLite + embedded |
| + assets); the router ~20 MB. A dozen instances is a few hundred MB. Set |
| + `MemoryMax` per instance so one can't OOM the box. |
| +- **CPU:** no server-side crypto or image work — just HTTP, SQLite, WebSocket |
| + fan-out, and TLS termination. Two vCPUs handle a small fleet comfortably. |
| +- **Disk is the binding resource, not RAM.** Attachments are blobs in SQLite, so |
| + DBs grow with media. Watch disk, and let Litestream offload history. If you |
| + outgrow anything first, it'll be disk. |
| + |
| +## Migrating an existing deployment onto the box |
| + |
| +Moving an instance from wherever it runs now onto this setup is data movement plus |
| +a DNS flip: |
| + |
| +1. **Snapshot the DB cleanly.** `VACUUM INTO` a fresh copy (folds the WAL into one |
| + consistent file); briefly freeze writes for a clean cut. |
| +2. **Copy it over** to the box as `/data/<host>.db`. |
| +3. **Register it** (`add-instance <host>` against the copied DB), which warms the |
| + cert. |
| +4. **Flip DNS** for the host to the box's IP; verify; retire the old deployment. |
| + |
| +Two things make this safe: |
| + |
| +- **Keep the hostname the same and there's no key work.** Browser keys (and |
| + passkeys, for the home server) are per-origin, so an unchanged origin leaves |
| + members' keys and history untouched — only the DNS record moves. |
| +- **A host on a domain not covered by a wildcard cert** simply exercises the |
| + on-demand HTTP-01 path — no special handling. |
| + |
| +**Rehearse the whole flow with a throwaway instance first** — under a temp host, |
| +run the router, `add-instance`, cert issuance, a socket-swap deploy, a |
| +crash-restart, and a Litestream restore, all at zero real-user risk before |
| +touching anything real. |
| + |
| +## What you're trading away (honest costs) |
| + |
| +1. **The router is a new SPOF** for all instances. But it's tiny, holds no |
| + per-request state, is `Restart=always` (sub-second recovery), and is smaller |
| + and more stable than a single-process design's whole-app SPOF. Its own deploys |
| + are socket-activated. |
| +2. **You own backups + supervision now.** Litestream + systemd cover what a |
| + platform did, but it's your operational surface — it has to actually be set up |
| + and monitored. |
| +3. **One box.** Multi-node failover is exactly where an orchestrator earns its |
| + keep; this is deliberately single-box. Revisit if that changes. |
| +4. **Migration is a real one-time project** — snapshot DBs, flip DNS, retire the |
| + old deployment, per instance. |
| + |
| +## Testing |
| + |
| +- Router: host→socket routing, unknown-host → 404, `HostPolicy` allowlist, atomic |
| + upstream swap under concurrent load, WebSocket pass-through. |
| +- `add-instance` state machine: creating / issuing / failed+retry transitions. |
| +- Instance: unchanged — `handler()` tests via `newTestServer` still exercise the |
| + real router. |
| +- Deploy: a socket swap drains without dropping an in-flight request (integration). |
| + |
| +## Deploy tooling (replaces deploy-instance.sh) |
| + |
| +- `lchat bootstrap` — one-time box setup: write systemd units, make dirs, start |
| + the router (no toolchain needed — just the copied binary). |
| +- `lchat add-instance <host>` — create + start + warm cert (async; `instances` |
| + shows status). |
| +- `lchat deploy <host> [--version …]` — zero-downtime socket swap. |
| +- `lchat deploy --all` — roll the fleet, one instance at a time. |
| +- `lchat admin-link <host>` — recover the admin link from the instance DB (no |
| + platform env lookup needed). |
| +- `lchat instances` — the status table. |
| + |
| +Operator specifics — box addresses, DNS/record ids, the concrete migration |
| +runbook — live in `CLAUDE.local.md`, not here. |
diff --git a/go.mod b/go.mod| index ac49e79..dfdda43 100644 |
| --- a/go.mod |
| +++ b/go.mod |
| @@ -6,6 +6,7 @@ require ( |
| github.com/SherClockHolmes/webpush-go v1.4.0 |
| github.com/coder/websocket v1.8.12 |
| github.com/go-webauthn/webauthn v0.11.2 |
| + golang.org/x/crypto v0.31.0 |
| modernc.org/sqlite v1.34.4 |
| ) |
| @@ -22,8 +23,9 @@ require ( |
| github.com/ncruces/go-strftime v0.1.9 // indirect |
| github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect |
| github.com/x448/float16 v0.8.4 // indirect |
| - golang.org/x/crypto v0.31.0 // indirect |
| + golang.org/x/net v0.25.0 // indirect |
| golang.org/x/sys v0.28.0 // indirect |
| + golang.org/x/text v0.21.0 // indirect |
| modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect |
| modernc.org/libc v1.55.3 // indirect |
| modernc.org/mathutil v1.6.0 // indirect |
diff --git a/go.sum b/go.sum| index 4fa1263..e5b4bb5 100644 |
| --- a/go.sum |
| +++ b/go.sum |
| @@ -58,6 +58,7 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= |
| golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= |
| golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= |
| golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= |
| +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= |
| golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= |
| golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= |
| golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= |
| @@ -97,6 +98,7 @@ golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= |
| golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= |
| golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= |
| golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= |
| +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= |
| golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= |
| golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= |
| golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= |
diff --git a/hub.go b/hub.go| index c88b37a..a3818a9 100644 |
| --- a/hub.go |
| +++ b/hub.go |
| @@ -37,6 +37,7 @@ type Hub struct { |
| register chan *client |
| unregister chan *client |
| broadcast chan outbound |
| + control chan func() // runs fn on the hub goroutine (safe map access) |
| clients map[*client]bool |
| } |
| @@ -45,6 +46,7 @@ func newHub() *Hub { |
| register: make(chan *client), |
| unregister: make(chan *client), |
| broadcast: make(chan outbound, 256), |
| + control: make(chan func()), |
| clients: make(map[*client]bool), |
| } |
| } |
| @@ -52,6 +54,8 @@ func newHub() *Hub { |
| func (h *Hub) run() { |
| for { |
| select { |
| + case fn := <-h.control: |
| + fn() |
| case c := <-h.register: |
| h.clients[c] = true |
| case c := <-h.unregister: |
| @@ -76,6 +80,23 @@ func (h *Hub) run() { |
| } |
| } |
| +// shutdown closes every connected client so their browsers reconnect — to the |
| +// new process, after a zero-downtime deploy (see docs/instance-router.md). It |
| +// runs the close on the hub goroutine so it can't race broadcast/register. The |
| +// closed send ends each client's writer, which closes the socket; the client's |
| +// read loop then unregisters (a no-op, already removed). |
| +func (h *Hub) shutdown() { |
| + done := make(chan struct{}) |
| + h.control <- func() { |
| + for c := range h.clients { |
| + delete(h.clients, c) |
| + close(c.send) |
| + } |
| + close(done) |
| + } |
| + <-done |
| +} |
| + |
| // broadcastToThread fans data out to the members of threadID only. |
| func (s *Server) broadcastToThread(threadID string, data []byte) { |
| members, err := s.db.memberUserIDs(threadID) |
diff --git a/hub_test.go b/hub_test.go| new file mode 100644 |
| index 0000000..9a68fb3 |
| --- /dev/null |
| +++ b/hub_test.go |
| @@ -0,0 +1,21 @@ |
| +package main |
| + |
| +import "testing" |
| + |
| +// TestHubShutdown checks that shutdown() closes every connected client's send |
| +// channel (which ends its writer and closes the socket, so browsers reconnect). |
| +func TestHubShutdown(t *testing.T) { |
| + h := newHub() |
| + go h.run() |
| + c := &client{send: make(chan []byte, 1)} |
| + h.register <- c |
| + h.shutdown() |
| + select { |
| + case _, ok := <-c.send: |
| + if ok { |
| + t.Fatal("send channel should be closed after shutdown") |
| + } |
| + default: |
| + t.Fatal("send channel not closed after shutdown") |
| + } |
| +} |
diff --git a/main.go b/main.go| index dc2cf13..06d6891 100644 |
| --- a/main.go |
| +++ b/main.go |
| @@ -1,6 +1,7 @@ |
| package main |
| import ( |
| + "context" |
| "crypto/rand" |
| "crypto/sha256" |
| "database/sql" |
| @@ -13,12 +14,15 @@ import ( |
| "io" |
| "io/fs" |
| "log" |
| + "net" |
| "net/http" |
| "net/url" |
| "os" |
| + "os/signal" |
| "strconv" |
| "strings" |
| "sync" |
| + "syscall" |
| "time" |
| "lchat/internal/apns" |
| @@ -73,7 +77,44 @@ type Server struct { |
| } |
| func main() { |
| + // Subcommands: `lchat router` runs the multi-instance front door and the |
| + // add-instance/instances/admin-link commands manage it; anything else (bare |
| + // `lchat` plus all existing flags) runs a single instance, so existing |
| + // invocations are unchanged. See docs/instance-router.md. |
| + if len(os.Args) > 1 { |
| + switch os.Args[1] { |
| + case "router": |
| + runRouter(os.Args[2:]) |
| + return |
| + case "bootstrap": |
| + runBootstrap(os.Args[2:]) |
| + return |
| + case "add-instance": |
| + runAddInstance(os.Args[2:]) |
| + return |
| + case "deploy": |
| + runDeploy(os.Args[2:]) |
| + return |
| + case "add-service": |
| + runAddService(os.Args[2:]) |
| + return |
| + case "add-bot": |
| + runAddBot(os.Args[2:]) |
| + return |
| + case "remove": |
| + runRemove(os.Args[2:]) |
| + return |
| + case "instances": |
| + runInstances(os.Args[2:]) |
| + return |
| + case "admin-link": |
| + runAdminLink(os.Args[2:]) |
| + return |
| + } |
| + } |
| + |
| addr := flag.String("addr", envOr("ADDR", ":1747"), "listen address (env: ADDR)") |
| + socket := flag.String("socket", envOr("LCHAT_SOCKET", ""), "unix socket to serve on instead of -addr (env: LCHAT_SOCKET); how the router fronts an instance") |
| dbPath := flag.String("db", envOr("LCHAT_DB", "lchat.db"), "path to SQLite database file (env: LCHAT_DB)") |
| resetAdmin := flag.Bool("reset-admin", false, "generate a new admin token and print its link") |
| resetDB := flag.Bool("reset-db", false, "delete and recreate the database") |
| @@ -133,10 +174,64 @@ func main() { |
| // call sweepDueScheduled directly, so only the real server runs the ticker. |
| go s.runScheduler() |
| - log.Printf("lchat listening on %s (db: %s)", *addr, *dbPath) |
| - if err := http.ListenAndServe(*addr, s.handler()); err != nil { |
| - log.Fatal(err) |
| + ln, where, err := listen(*addr, *socket) |
| + if err != nil { |
| + log.Fatalf("listen: %v", err) |
| + } |
| + log.Printf("lchat listening on %s (db: %s)", where, *dbPath) |
| + |
| + srv := &http.Server{Handler: s.handler()} |
| + go func() { |
| + if err := srv.Serve(ln); err != nil && err != http.ErrServerClosed { |
| + log.Fatalf("serve: %v", err) |
| + } |
| + }() |
| + |
| + // Graceful shutdown on SIGTERM/SIGINT (systemd stop/restart during a deploy): |
| + // stop accepting + drain in-flight HTTP, then close WebSockets so clients |
| + // reconnect to the new process. The router retries dials across the brief gap, |
| + // so no request is dropped. See docs/instance-router.md. |
| + sigCh := make(chan os.Signal, 1) |
| + signal.Notify(sigCh, syscall.SIGTERM, syscall.SIGINT) |
| + <-sigCh |
| + log.Printf("lchat: draining and shutting down") |
| + // Drain in-flight HTTP (e.g. a large upload) before exiting. Kept below the |
| + // router's dial-retry window so the socket-absent gap during a deploy stays |
| + // within what the router masks. |
| + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
| + defer cancel() |
| + _ = srv.Shutdown(ctx) |
| + hub.shutdown() |
| +} |
| + |
| +// listen returns the instance's listener. With -socket set it serves on a unix |
| +// domain socket (how the router fronts an instance — see docs/instance-router.md); |
| +// otherwise it's a TCP listener on addr (the standalone default, unchanged). A |
| +// stale socket left by a crashed process is removed first (but a socket another |
| +// process is still serving on refuses to start), and the socket is chmod 0600 so |
| +// only the router (same user) can dial it. |
| +func listen(addr, socketPath string) (net.Listener, string, error) { |
| + if socketPath == "" { |
| + ln, err := net.Listen("tcp", addr) |
| + return ln, addr, err |
| + } |
| + // Don't clobber a live socket: only a stale one (nothing listening) is removed. |
| + if c, err := net.DialTimeout("unix", socketPath, 200*time.Millisecond); err == nil { |
| + c.Close() |
| + return nil, "", fmt.Errorf("socket %s is already in use by another process", socketPath) |
| + } |
| + if err := os.Remove(socketPath); err != nil && !os.IsNotExist(err) { |
| + return nil, "", fmt.Errorf("remove stale socket %s: %w", socketPath, err) |
| + } |
| + ln, err := net.Listen("unix", socketPath) |
| + if err != nil { |
| + return nil, "", err |
| + } |
| + if err := os.Chmod(socketPath, 0o600); err != nil { |
| + ln.Close() |
| + return nil, "", fmt.Errorf("chmod socket %s: %w", socketPath, err) |
| } |
| + return ln, "unix:" + socketPath, nil |
| } |
| // handler wires up all routes. It's a method so tests can exercise the full |
diff --git a/registry.go b/registry.go| new file mode 100644 |
| index 0000000..9339d86 |
| --- /dev/null |
| +++ b/registry.go |
| @@ -0,0 +1,149 @@ |
| +package main |
| + |
| +import ( |
| + "database/sql" |
| + "time" |
| +) |
| + |
| +// Registry is the router's control DB — the set of hosts it serves and how to |
| +// reach each. It's deliberately separate from an instance's own SQLite: it holds |
| +// routing + operator config, never message data. See docs/instance-router.md. |
| +type Registry struct{ sql *sql.DB } |
| + |
| +// Route is one host the router fronts: a messenger instance it lifecycle-manages |
| +// (kind "instance") or a sibling service it only proxies (kind "service", e.g. |
| +// the home server or push relay). The instance-only fields hold per-instance |
| +// config that used to live in process env. |
| +type Route struct { |
| + Host string |
| + Kind string |
| + Socket string |
| + Status string |
| + LastError string |
| + DBPath string |
| + Version string |
| + AppStoreURL string |
| + HomeOrigin string |
| + PushRelayURL string |
| + PushRelayToken string |
| + CreatedAt int64 |
| +} |
| + |
| +func openRegistry(path string) (*Registry, error) { |
| + d, err := sql.Open("sqlite", path) |
| + if err != nil { |
| + return nil, err |
| + } |
| + // One writer at a time keeps WAL happy (mirrors openDB). |
| + d.SetMaxOpenConns(1) |
| + for _, p := range []string{ |
| + "PRAGMA journal_mode=WAL", |
| + "PRAGMA busy_timeout=5000", |
| + "PRAGMA synchronous=NORMAL", |
| + } { |
| + if _, err := d.Exec(p); err != nil { |
| + return nil, err |
| + } |
| + } |
| + r := &Registry{sql: d} |
| + if err := r.migrate(); err != nil { |
| + return nil, err |
| + } |
| + return r, nil |
| +} |
| + |
| +// migrate is additive (CREATE TABLE IF NOT EXISTS), like the instance schema, so |
| +// deploying new code over an existing registry is safe. |
| +func (r *Registry) migrate() error { |
| + _, err := r.sql.Exec(` |
| +CREATE TABLE IF NOT EXISTS routes ( |
| + host TEXT PRIMARY KEY, |
| + kind TEXT NOT NULL, |
| + socket TEXT NOT NULL, |
| + status TEXT NOT NULL DEFAULT 'ready', |
| + last_error TEXT NOT NULL DEFAULT '', |
| + db_path TEXT NOT NULL DEFAULT '', |
| + version TEXT NOT NULL DEFAULT '', |
| + app_store_url TEXT NOT NULL DEFAULT '', |
| + home_origin TEXT NOT NULL DEFAULT '', |
| + push_relay_url TEXT NOT NULL DEFAULT '', |
| + push_relay_token TEXT NOT NULL DEFAULT '', |
| + created_at INTEGER NOT NULL |
| +);`) |
| + return err |
| +} |
| + |
| +// list returns every route, ordered by host. |
| +func (r *Registry) list() ([]Route, error) { |
| + rows, err := r.sql.Query(` |
| +SELECT host, kind, socket, status, last_error, db_path, version, |
| + app_store_url, home_origin, push_relay_url, push_relay_token, created_at |
| +FROM routes ORDER BY host`) |
| + if err != nil { |
| + return nil, err |
| + } |
| + defer rows.Close() |
| + var out []Route |
| + for rows.Next() { |
| + var t Route |
| + if err := rows.Scan(&t.Host, &t.Kind, &t.Socket, &t.Status, &t.LastError, |
| + &t.DBPath, &t.Version, &t.AppStoreURL, &t.HomeOrigin, |
| + &t.PushRelayURL, &t.PushRelayToken, &t.CreatedAt); err != nil { |
| + return nil, err |
| + } |
| + out = append(out, t) |
| + } |
| + return out, rows.Err() |
| +} |
| + |
| +// get returns the route for host, or (nil, nil) if there is none. |
| +func (r *Registry) get(host string) (*Route, error) { |
| + var t Route |
| + err := r.sql.QueryRow(` |
| +SELECT host, kind, socket, status, last_error, db_path, version, |
| + app_store_url, home_origin, push_relay_url, push_relay_token, created_at |
| +FROM routes WHERE host = ?`, host).Scan(&t.Host, &t.Kind, &t.Socket, &t.Status, |
| + &t.LastError, &t.DBPath, &t.Version, &t.AppStoreURL, &t.HomeOrigin, |
| + &t.PushRelayURL, &t.PushRelayToken, &t.CreatedAt) |
| + if err == sql.ErrNoRows { |
| + return nil, nil |
| + } |
| + if err != nil { |
| + return nil, err |
| + } |
| + return &t, nil |
| +} |
| + |
| +// delete removes a route by host (data-safe: it stops routing the host but never |
| +// touches its DB or env file). |
| +func (r *Registry) delete(host string) error { |
| + _, err := r.sql.Exec(`DELETE FROM routes WHERE host = ?`, host) |
| + return err |
| +} |
| + |
| +// upsert inserts or updates a route by host. (add-instance / register will drive |
| +// this in a later step; it's here now so the router and its tests can seed |
| +// routes.) |
| +func (r *Registry) upsert(t Route) error { |
| + if t.CreatedAt == 0 { |
| + t.CreatedAt = time.Now().Unix() |
| + } |
| + if t.Status == "" { |
| + t.Status = "ready" |
| + } |
| + if t.Kind == "" { |
| + t.Kind = "instance" |
| + } |
| + _, err := r.sql.Exec(` |
| +INSERT INTO routes (host, kind, socket, status, last_error, db_path, version, |
| + app_store_url, home_origin, push_relay_url, push_relay_token, created_at) |
| +VALUES (?,?,?,?,?,?,?,?,?,?,?,?) |
| +ON CONFLICT(host) DO UPDATE SET |
| + kind=excluded.kind, socket=excluded.socket, status=excluded.status, |
| + last_error=excluded.last_error, db_path=excluded.db_path, version=excluded.version, |
| + app_store_url=excluded.app_store_url, home_origin=excluded.home_origin, |
| + push_relay_url=excluded.push_relay_url, push_relay_token=excluded.push_relay_token`, |
| + t.Host, t.Kind, t.Socket, t.Status, t.LastError, t.DBPath, t.Version, |
| + t.AppStoreURL, t.HomeOrigin, t.PushRelayURL, t.PushRelayToken, t.CreatedAt) |
| + return err |
| +} |
diff --git a/router.go b/router.go| new file mode 100644 |
| index 0000000..bae851b |
| --- /dev/null |
| +++ b/router.go |
| @@ -0,0 +1,215 @@ |
| +package main |
| + |
| +import ( |
| + "context" |
| + "errors" |
| + "flag" |
| + "fmt" |
| + "log" |
| + "net" |
| + "net/http" |
| + "net/http/httputil" |
| + "os" |
| + "os/signal" |
| + "sync/atomic" |
| + "syscall" |
| + "time" |
| + |
| + "golang.org/x/crypto/acme/autocert" |
| +) |
| + |
| +// router is the multi-instance front door: it terminates TLS (autocert) and |
| +// reverse-proxies each configured host to its backend's unix socket. The |
| +// host→backend map is swapped atomically, so reloads and zero-downtime deploys |
| +// never drop a live connection. See docs/instance-router.md. |
| +type router struct { |
| + reg *Registry |
| + upstreams atomic.Pointer[map[string]*httputil.ReverseProxy] |
| +} |
| + |
| +func newRouter(reg *Registry) (*router, error) { |
| + rt := &router{reg: reg} |
| + if err := rt.reload(); err != nil { |
| + return nil, err |
| + } |
| + return rt, nil |
| +} |
| + |
| +// reload rebuilds the host→backend map from the registry and swaps it in |
| +// atomically. Safe to call while serving. |
| +func (rt *router) reload() error { |
| + routes, err := rt.reg.list() |
| + if err != nil { |
| + return err |
| + } |
| + m := make(map[string]*httputil.ReverseProxy, len(routes)) |
| + for _, t := range routes { |
| + // Only serve ready routes: a 'creating'/'failed' entry has no working |
| + // backend, so proxying it (and minting a cert for it) would just stall → 502. |
| + if t.Socket == "" || t.Status != "ready" { |
| + continue |
| + } |
| + m[t.Host] = unixProxy(t.Socket) |
| + } |
| + rt.upstreams.Store(&m) |
| + return nil |
| +} |
| + |
| +func (rt *router) current() map[string]*httputil.ReverseProxy { |
| + if p := rt.upstreams.Load(); p != nil { |
| + return *p |
| + } |
| + return nil |
| +} |
| + |
| +// allowed reports whether host is configured — used as the autocert HostPolicy, |
| +// so a certificate is only ever requested for a host we actually serve. |
| +func (rt *router) allowed(host string) bool { |
| + _, ok := rt.current()[host] |
| + return ok |
| +} |
| + |
| +func (rt *router) ServeHTTP(w http.ResponseWriter, r *http.Request) { |
| + host := r.Host |
| + if h, _, err := net.SplitHostPort(host); err == nil { |
| + host = h |
| + } |
| + proxy, ok := rt.current()[host] |
| + if !ok { |
| + http.Error(w, "unknown host", http.StatusNotFound) |
| + return |
| + } |
| + proxy.ServeHTTP(w, r) |
| +} |
| + |
| +// unixProxy reverse-proxies to a backend listening on a unix socket, preserving |
| +// the client's Host header (so the backend sees the real hostname) and passing |
| +// WebSocket upgrades straight through. |
| +func unixProxy(socketPath string) *httputil.ReverseProxy { |
| + dialer := &net.Dialer{} |
| + return &httputil.ReverseProxy{ |
| + Rewrite: func(pr *httputil.ProxyRequest) { |
| + pr.SetXForwarded() |
| + pr.Out.URL.Scheme = "http" |
| + pr.Out.URL.Host = "lchat" // placeholder; the unix dialer ignores it |
| + pr.Out.Host = pr.In.Host // backend sees the real hostname |
| + }, |
| + Transport: &http.Transport{ |
| + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { |
| + return dialUnixWithRetry(ctx, dialer, socketPath) |
| + }, |
| + // Bound idle keep-alives so proxies replaced on reload don't leak their |
| + // pooled connections + reader goroutines on the long-running router. |
| + MaxIdleConns: 32, |
| + IdleConnTimeout: 90 * time.Second, |
| + }, |
| + } |
| +} |
| + |
| +// dialUnixWithRetry masks the brief gap during an instance restart: while the old |
| +// process has exited and the new one hasn't rebound the socket, dials fail with |
| +// ENOENT/ECONNREFUSED. Retry those for a short window so no in-flight request is |
| +// dropped; any other error fails fast. The window must exceed a deploy's |
| +// socket-absent gap — the instance drain (10s in main.go) plus rebind — so an |
| +// in-flight request is never 502'd mid-deploy; a sustained-down backend hangs this |
| +// long before 502 (rare: systemd Restart=always makes "down" transient). True |
| +// blue/green would remove the gap entirely (see docs/instance-router.md). |
| +func dialUnixWithRetry(ctx context.Context, d *net.Dialer, socketPath string) (net.Conn, error) { |
| + const window = 13 * time.Second |
| + deadline := time.Now().Add(window) |
| + for { |
| + conn, err := d.DialContext(ctx, "unix", socketPath) |
| + if err == nil { |
| + return conn, nil |
| + } |
| + if ctx.Err() != nil || !isRetryableDialErr(err) || time.Now().After(deadline) { |
| + return nil, err |
| + } |
| + select { |
| + case <-ctx.Done(): |
| + return nil, ctx.Err() |
| + case <-time.After(100 * time.Millisecond): |
| + } |
| + } |
| +} |
| + |
| +func isRetryableDialErr(err error) bool { |
| + return errors.Is(err, syscall.ECONNREFUSED) || errors.Is(err, syscall.ENOENT) |
| +} |
| + |
| +// runRouter is the `lchat router` subcommand: the shared front door for every |
| +// instance and sibling service on the box. |
| +func runRouter(args []string) { |
| + fs := flag.NewFlagSet("router", flag.ExitOnError) |
| + registryPath := fs.String("registry", envOr("LCHAT_REGISTRY", "/data/registry.db"), "router registry SQLite path (env: LCHAT_REGISTRY)") |
| + acmeCache := fs.String("acme-cache", envOr("LCHAT_ACME_CACHE", "/data/acme"), "autocert certificate cache dir (env: LCHAT_ACME_CACHE)") |
| + httpsAddr := fs.String("https", envOr("LCHAT_HTTPS_ADDR", ":443"), "HTTPS listen address") |
| + httpAddr := fs.String("http", envOr("LCHAT_HTTP_ADDR", ":80"), "HTTP listen address (ACME HTTP-01 challenge + redirect to https)") |
| + _ = fs.Parse(args) |
| + |
| + reg, err := openRegistry(*registryPath) |
| + if err != nil { |
| + log.Fatalf("router: open registry %s: %v", *registryPath, err) |
| + } |
| + rt, err := newRouter(reg) |
| + if err != nil { |
| + log.Fatalf("router: load routes: %v", err) |
| + } |
| + |
| + m := &autocert.Manager{ |
| + Prompt: autocert.AcceptTOS, |
| + Cache: autocert.DirCache(*acmeCache), |
| + HostPolicy: func(_ context.Context, host string) error { |
| + if rt.allowed(host) { |
| + return nil |
| + } |
| + return fmt.Errorf("acme: host %q not in registry", host) |
| + }, |
| + } |
| + |
| + // Reload routes on SIGHUP (add-instance/deploy will also trigger this via the |
| + // control socket in a later step). |
| + sighup := make(chan os.Signal, 1) |
| + signal.Notify(sighup, syscall.SIGHUP) |
| + go func() { |
| + for range sighup { |
| + if err := rt.reload(); err != nil { |
| + log.Printf("router: reload failed: %v", err) |
| + continue |
| + } |
| + log.Printf("router: reloaded (%d route(s))", len(rt.current())) |
| + } |
| + }() |
| + |
| + // ReadHeaderTimeout guards the public front door against Slowloris-style slow- |
| + // header connection exhaustion. No ReadTimeout/WriteTimeout — those would break |
| + // long-lived proxied WebSockets. |
| + httpSrv := &http.Server{Addr: *httpAddr, Handler: m.HTTPHandler(nil), ReadHeaderTimeout: 10 * time.Second} |
| + httpsSrv := &http.Server{Addr: *httpsAddr, Handler: rt, TLSConfig: m.TLSConfig(), ReadHeaderTimeout: 10 * time.Second} |
| + |
| + // :80 serves the ACME HTTP-01 challenge and redirects everything else to https. |
| + go func() { |
| + if err := httpSrv.ListenAndServe(); err != nil && err != http.ErrServerClosed { |
| + log.Fatalf("router: http listener %s: %v", *httpAddr, err) |
| + } |
| + }() |
| + go func() { |
| + log.Printf("router: serving %d route(s) on %s (registry %s, acme cache %s)", |
| + len(rt.current()), *httpsAddr, *registryPath, *acmeCache) |
| + if err := httpsSrv.ListenAndServeTLS("", ""); err != nil && err != http.ErrServerClosed { |
| + log.Fatalf("router: https listener %s: %v", *httpsAddr, err) |
| + } |
| + }() |
| + |
| + // Graceful shutdown on SIGTERM/SIGINT (e.g. a router binary upgrade): stop |
| + // accepting and drain in-flight HTTP before exiting. Proxied WebSockets are |
| + // hijacked (not drained); they reconnect through the new router. |
| + stop := make(chan os.Signal, 1) |
| + signal.Notify(stop, syscall.SIGTERM, syscall.SIGINT) |
| + <-stop |
| + log.Printf("router: draining and shutting down") |
| + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
| + defer cancel() |
| + _ = httpsSrv.Shutdown(ctx) |
| + _ = httpSrv.Shutdown(ctx) |
| +} |
diff --git a/router_test.go b/router_test.go| new file mode 100644 |
| index 0000000..7568b7a |
| --- /dev/null |
| +++ b/router_test.go |
| @@ -0,0 +1,275 @@ |
| +package main |
| + |
| +import ( |
| + "context" |
| + "fmt" |
| + "io" |
| + "net" |
| + "net/http" |
| + "net/http/httptest" |
| + "os" |
| + "path/filepath" |
| + "strings" |
| + "testing" |
| + "time" |
| + |
| + "github.com/coder/websocket" |
| +) |
| + |
| +func TestRouterProxiesByHost(t *testing.T) { |
| + // A backend on a unix socket that echoes the Host header it receives. |
| + sock := tmpSock(t) |
| + ln, err := net.Listen("unix", sock) |
| + if err != nil { |
| + t.Fatalf("listen unix: %v", err) |
| + } |
| + backend := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| + io.WriteString(w, "backend:"+r.Host) |
| + })} |
| + go backend.Serve(ln) |
| + t.Cleanup(func() { backend.Close() }) |
| + |
| + reg, err := openRegistry(filepath.Join(t.TempDir(), "registry.db")) |
| + if err != nil { |
| + t.Fatalf("open registry: %v", err) |
| + } |
| + if err := reg.upsert(Route{Host: "chat.example.com", Kind: "instance", Socket: sock}); err != nil { |
| + t.Fatalf("upsert: %v", err) |
| + } |
| + rt, err := newRouter(reg) |
| + if err != nil { |
| + t.Fatalf("new router: %v", err) |
| + } |
| + front := httptest.NewServer(rt) |
| + t.Cleanup(front.Close) |
| + |
| + // Known host is proxied, and the backend sees the real Host header. |
| + if body, code := proxyGet(t, front.URL, "chat.example.com"); code != 200 || body != "backend:chat.example.com" { |
| + t.Fatalf("known host: got %d %q, want 200 backend:chat.example.com", code, body) |
| + } |
| + // Unknown host → 404 (and, as HostPolicy, no cert would be issued for it). |
| + if _, code := proxyGet(t, front.URL, "nope.example.com"); code != http.StatusNotFound { |
| + t.Fatalf("unknown host: got %d, want 404", code) |
| + } |
| + if rt.allowed("nope.example.com") { |
| + t.Fatal("allowed() should be false for an unconfigured host") |
| + } |
| + |
| + // A new route isn't served until reload; after reload it is (atomic swap). |
| + if err := reg.upsert(Route{Host: "two.example.com", Kind: "instance", Socket: sock}); err != nil { |
| + t.Fatalf("upsert two: %v", err) |
| + } |
| + if rt.allowed("two.example.com") { |
| + t.Fatal("route should not be live before reload") |
| + } |
| + if err := rt.reload(); err != nil { |
| + t.Fatalf("reload: %v", err) |
| + } |
| + if body, code := proxyGet(t, front.URL, "two.example.com"); code != 200 || body != "backend:two.example.com" { |
| + t.Fatalf("after reload: got %d %q, want 200 backend:two.example.com", code, body) |
| + } |
| +} |
| + |
| +func TestRouterProxiesWebSocket(t *testing.T) { |
| + // A backend WebSocket echo server on a unix socket. |
| + sock := tmpSock(t) |
| + ln, err := net.Listen("unix", sock) |
| + if err != nil { |
| + t.Fatalf("listen unix: %v", err) |
| + } |
| + backend := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| + c, err := websocket.Accept(w, r, nil) |
| + if err != nil { |
| + return |
| + } |
| + defer c.Close(websocket.StatusNormalClosure, "") |
| + mt, data, err := c.Read(r.Context()) |
| + if err != nil { |
| + return |
| + } |
| + _ = c.Write(r.Context(), mt, append([]byte("echo:"), data...)) |
| + })} |
| + go backend.Serve(ln) |
| + t.Cleanup(func() { backend.Close() }) |
| + |
| + reg, err := openRegistry(filepath.Join(t.TempDir(), "registry.db")) |
| + if err != nil { |
| + t.Fatalf("open registry: %v", err) |
| + } |
| + // Route the loopback host so the client reaches it without a Host override |
| + // (ServeHTTP strips the port, matching "127.0.0.1"). |
| + if err := reg.upsert(Route{Host: "127.0.0.1", Kind: "instance", Socket: sock}); err != nil { |
| + t.Fatalf("upsert: %v", err) |
| + } |
| + rt, err := newRouter(reg) |
| + if err != nil { |
| + t.Fatalf("new router: %v", err) |
| + } |
| + front := httptest.NewServer(rt) |
| + t.Cleanup(front.Close) |
| + |
| + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) |
| + defer cancel() |
| + wsURL := "ws" + strings.TrimPrefix(front.URL, "http") + "/ws" |
| + c, _, err := websocket.Dial(ctx, wsURL, nil) |
| + if err != nil { |
| + t.Fatalf("ws dial through router: %v", err) |
| + } |
| + defer c.Close(websocket.StatusNormalClosure, "") |
| + if err := c.Write(ctx, websocket.MessageText, []byte("hi")); err != nil { |
| + t.Fatalf("ws write: %v", err) |
| + } |
| + _, data, err := c.Read(ctx) |
| + if err != nil { |
| + t.Fatalf("ws read: %v", err) |
| + } |
| + if string(data) != "echo:hi" { |
| + t.Fatalf("ws echo through router: got %q, want %q", data, "echo:hi") |
| + } |
| +} |
| + |
| +func TestIsRetryableDialErr(t *testing.T) { |
| + sock := tmpSock(t) // a path with no listener → ENOENT |
| + if _, err := net.Dial("unix", sock); err == nil { |
| + t.Fatal("expected a dial error for a missing socket") |
| + } else if !isRetryableDialErr(err) { |
| + t.Errorf("missing socket: isRetryableDialErr(%v) = false, want true", err) |
| + } |
| + if isRetryableDialErr(context.Canceled) { |
| + t.Error("context.Canceled should not be retryable") |
| + } |
| +} |
| + |
| +func TestRouterRetriesAcrossRestart(t *testing.T) { |
| + sock := tmpSock(t) |
| + reg, err := openRegistry(filepath.Join(t.TempDir(), "registry.db")) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := reg.upsert(Route{Host: "chat.example.com", Kind: "instance", Socket: sock}); err != nil { |
| + t.Fatal(err) |
| + } |
| + rt, err := newRouter(reg) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + front := httptest.NewServer(rt) |
| + t.Cleanup(front.Close) |
| + |
| + // Fire the request before any backend exists; the router must retry the dial. |
| + type res struct { |
| + code int |
| + body string |
| + } |
| + ch := make(chan res, 1) |
| + go func() { |
| + req, _ := http.NewRequest(http.MethodGet, front.URL+"/", nil) |
| + req.Host = "chat.example.com" |
| + resp, err := http.DefaultClient.Do(req) |
| + if err != nil { |
| + ch <- res{0, "err: " + err.Error()} |
| + return |
| + } |
| + b, _ := io.ReadAll(resp.Body) |
| + resp.Body.Close() |
| + ch <- res{resp.StatusCode, string(b)} |
| + }() |
| + |
| + // Bind the backend after a gap, simulating the instance restart window. |
| + time.Sleep(300 * time.Millisecond) |
| + ln, err := net.Listen("unix", sock) |
| + if err != nil { |
| + t.Fatalf("listen: %v", err) |
| + } |
| + backend := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| + io.WriteString(w, "late-backend") |
| + })} |
| + go backend.Serve(ln) |
| + t.Cleanup(func() { backend.Close() }) |
| + |
| + select { |
| + case r := <-ch: |
| + if r.code != 200 || r.body != "late-backend" { |
| + t.Fatalf("retry across gap: got %d %q, want 200 late-backend", r.code, r.body) |
| + } |
| + case <-time.After(10 * time.Second): |
| + t.Fatal("request never completed; router did not retry across the gap") |
| + } |
| +} |
| + |
| +func TestReloadOnlyServesReady(t *testing.T) { |
| + reg, err := openRegistry(filepath.Join(t.TempDir(), "registry.db")) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + for _, r := range []Route{ |
| + {Host: "ready.example.com", Kind: "instance", Socket: "/run/lchat/a.sock", Status: "ready"}, |
| + {Host: "failed.example.com", Kind: "instance", Socket: "/run/lchat/b.sock", Status: "failed"}, |
| + {Host: "creating.example.com", Kind: "instance", Socket: "/run/lchat/c.sock", Status: "creating"}, |
| + } { |
| + if err := reg.upsert(r); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + rt, err := newRouter(reg) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if !rt.allowed("ready.example.com") { |
| + t.Error("ready route should be served") |
| + } |
| + if rt.allowed("failed.example.com") { |
| + t.Error("failed route should not be served or cert-eligible") |
| + } |
| + if rt.allowed("creating.example.com") { |
| + t.Error("creating route should not be served") |
| + } |
| +} |
| + |
| +func TestWaitHealthy(t *testing.T) { |
| + serve := func(status int) string { |
| + sock := tmpSock(t) |
| + ln, err := net.Listen("unix", sock) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(status) })} |
| + go srv.Serve(ln) |
| + t.Cleanup(func() { srv.Close() }) |
| + return sock |
| + } |
| + if err := waitHealthy(serve(200), 3*time.Second); err != nil { |
| + t.Errorf("200 backend: waitHealthy = %v, want nil", err) |
| + } |
| + if err := waitHealthy(serve(500), 1*time.Second); err == nil { |
| + t.Error("500 backend: waitHealthy = nil, want error (a broken binary must fail the deploy gate)") |
| + } |
| +} |
| + |
| +// proxyGet sends GET base+"/" with an overridden Host header and returns the |
| +// response body + status code. |
| +func proxyGet(t *testing.T, base, host string) (string, int) { |
| + t.Helper() |
| + req, err := http.NewRequest(http.MethodGet, base+"/", nil) |
| + if err != nil { |
| + t.Fatalf("new request: %v", err) |
| + } |
| + req.Host = host |
| + resp, err := http.DefaultClient.Do(req) |
| + if err != nil { |
| + t.Fatalf("request: %v", err) |
| + } |
| + defer resp.Body.Close() |
| + b, _ := io.ReadAll(resp.Body) |
| + return string(b), resp.StatusCode |
| +} |
| + |
| +// tmpSock returns a short unix socket path (the sun_path limit is ~104 bytes, so |
| +// t.TempDir() can be too long on macOS) and schedules its removal. |
| +func tmpSock(t *testing.T) string { |
| + t.Helper() |
| + s := fmt.Sprintf("/tmp/lchat-rt-%d-%d.sock", os.Getpid(), time.Now().UnixNano()%1_000_000) |
| + os.Remove(s) |
| + t.Cleanup(func() { os.Remove(s) }) |
| + return s |
| +} |