| 1 | package main |
| 2 | |
| 3 | import ( |
| 4 | "bufio" |
| 5 | "context" |
| 6 | "crypto/tls" |
| 7 | "flag" |
| 8 | "fmt" |
| 9 | "log" |
| 10 | "net" |
| 11 | "net/http" |
| 12 | "os" |
| 13 | "os/exec" |
| 14 | "path/filepath" |
| 15 | "regexp" |
| 16 | "sort" |
| 17 | "strings" |
| 18 | "syscall" |
| 19 | "time" |
| 20 | |
| 21 | "github.com/benbjohnson/litestream" |
| 22 | "unicode" |
| 23 | "unicode/utf8" |
| 24 | ) |
| 25 | |
| 26 | // The control subcommands automate what an operator would otherwise do by hand to |
| 27 | // stand up an instance behind the router: mint the admin token, write the unit's |
| 28 | // EnvironmentFile, start the instance, register + reload the route. See |
| 29 | // docs/instance-router.md. |
| 30 | |
| 31 | // Instance paths are fixed by the systemd units (deploy/systemd/lchat@.service), |
| 32 | // so the control commands use these rather than per-invocation flags that would |
| 33 | // silently diverge from where the instance actually listens + stores. Vars with |
| 34 | // env overrides (cf. optDir) so tests can point them at temp dirs. |
| 35 | var ( |
| 36 | instanceDataDir = envOr("LCHAT_DATA_DIR", "/data") // <host>.db + <host>.env live here |
| 37 | instanceSocketDir = envOr("LCHAT_SOCKET_DIR", "/run/lchat") // <host>.sock lives here |
| 38 | ) |
| 39 | |
| 40 | // routerHTTPSAddr is where the local router serves TLS — prewarmCert dials it to |
| 41 | // trigger cert issuance for a new host. Loopback by default (the router binds |
| 42 | // :443 on the same box); overridable if the router listens elsewhere. |
| 43 | var routerHTTPSAddr = envOr("LCHAT_ROUTER_HTTPS_ADDR", "127.0.0.1:443") |
| 44 | |
| 45 | var hostnameRe = regexp.MustCompile(`^[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)+$`) |
| 46 | |
| 47 | // validHost reports whether h is a plausible DNS hostname. It's strict because |
| 48 | // <host> becomes a filename, a systemd instance name, and a unix socket path. |
| 49 | func validHost(h string) bool { |
| 50 | return len(h) > 0 && len(h) <= 253 && hostnameRe.MatchString(h) |
| 51 | } |
| 52 | |
| 53 | // checkSocketLen guards the unix sun_path limit (104 bytes on darwin/BSD, 108 on |
| 54 | // Linux — stay under the smaller so a path is portable). A too-long path fails at |
| 55 | // bind() with a cryptic error, so catch it early. |
| 56 | func checkSocketLen(path string) error { |
| 57 | if len(path) >= 104 { |
| 58 | return fmt.Errorf("socket path %q is too long (%d bytes; unix limit ~104)", path, len(path)) |
| 59 | } |
| 60 | return nil |
| 61 | } |
| 62 | |
| 63 | func die(format string, a ...any) { |
| 64 | fmt.Fprintf(os.Stderr, format+"\n", a...) |
| 65 | os.Exit(1) |
| 66 | } |
| 67 | |
| 68 | // reorderFlagsFirst moves positional args after all flags so the stdlib flag |
| 69 | // parser (which stops at the first positional) still sees every flag — letting |
| 70 | // `add-instance <host> --flag v` work as naturally as `--flag v <host>`. |
| 71 | func reorderFlagsFirst(fs *flag.FlagSet, args []string) []string { |
| 72 | var flags, positional []string |
| 73 | for i := 0; i < len(args); i++ { |
| 74 | a := args[i] |
| 75 | if a == "--" { |
| 76 | positional = append(positional, args[i+1:]...) |
| 77 | break |
| 78 | } |
| 79 | if len(a) > 1 && a[0] == '-' { |
| 80 | flags = append(flags, a) |
| 81 | name := strings.TrimLeft(a, "-") |
| 82 | if strings.IndexByte(name, '=') >= 0 { |
| 83 | continue // --flag=value carries its own value |
| 84 | } |
| 85 | f := fs.Lookup(name) |
| 86 | if f == nil { |
| 87 | continue // unknown flag; let Parse report it |
| 88 | } |
| 89 | if bf, ok := f.Value.(interface{ IsBoolFlag() bool }); ok && bf.IsBoolFlag() { |
| 90 | continue // bool flag takes no value |
| 91 | } |
| 92 | // Pull the value along, unless the next token is itself a flag (a |
| 93 | // missing value) — let Parse report that rather than silently eating it. |
| 94 | if i+1 < len(args) && !(len(args[i+1]) > 1 && args[i+1][0] == '-') { |
| 95 | i++ |
| 96 | flags = append(flags, args[i]) |
| 97 | } |
| 98 | continue |
| 99 | } |
| 100 | positional = append(positional, a) |
| 101 | } |
| 102 | return append(flags, positional...) |
| 103 | } |
| 104 | |
| 105 | // Sentinel errors so callers (the CLI, controld) can map failures to stable |
| 106 | // outcomes without parsing message strings. |
| 107 | var ( |
| 108 | errHostExists = fmt.Errorf("host already exists") |
| 109 | errInvalidHost = fmt.Errorf("invalid host") |
| 110 | ) |
| 111 | |
| 112 | // addInstanceOpts parameterizes one instance creation. Host is required; an |
| 113 | // empty AdminToken mints a fresh one (a non-empty one imports an existing token, |
| 114 | // the migration path). |
| 115 | type addInstanceOpts struct { |
| 116 | Host string |
| 117 | AdminToken string |
| 118 | AppStoreURL string |
| 119 | HomeOrigin string |
| 120 | PushRelayURL string |
| 121 | PushRelayToken string |
| 122 | // Entitlement mirrors the cloud ledger's plan flag ("free"/"paid") into the |
| 123 | // instance env (LCHAT_ENTITLEMENT) so its UI can gate billing affordances — |
| 124 | // the space menu's Upgrade. Empty for operator-created instances, which have |
| 125 | // no billing surface and show nothing. |
| 126 | Entitlement string |
| 127 | // GroupName seeds the space's display name (LCHAT_GROUP_NAME → the instance |
| 128 | // names itself at first boot; main.go applySeededName). The create flow asks |
| 129 | // for the name up front, so a provisioned space greets its admin by name |
| 130 | // instead of as "lchat". Seed-only: an already-named space ignores it. |
| 131 | GroupName string |
| 132 | HomeRegion string // geo seam (#550 Part 7): "" = the single implicit region |
| 133 | Provider string // co-op seam (#550 Part 10): "" = the single implicit provider |
| 134 | // Hibernate registers the instance as activator-managed (#550 Part 4 v2): |
| 135 | // no systemd unit is created — the router wakes it on first request. Needs |
| 136 | // the router running with LCHAT_HIBERNATE_BUCKET, or the host never answers. |
| 137 | Hibernate bool |
| 138 | // Canary flags the instance as a throwaway test host (registry.go: Route.Canary). |
| 139 | Canary bool |
| 140 | // Replication is the durability class on the activator's S3 replica |
| 141 | // (registry.go: stream/batched/daily; "" = stream) — set at birth so a |
| 142 | // provisioned space starts on its plan's rung instead of the most |
| 143 | // expensive one. Inert unless Hibernate. |
| 144 | Replication string |
| 145 | UnitTmpl string // default "lchat@" |
| 146 | RouterUnit string // default "lchat-router" |
| 147 | } |
| 148 | |
| 149 | // sanitizeGroupName makes an untrusted display name safe to carry in the |
| 150 | // KEY=value env file writeEnvFile emits: control characters (newlines would |
| 151 | // inject env lines) are dropped, whitespace trimmed, and length capped at 60 |
| 152 | // bytes on a rune boundary — the same cap the instance's own rename enforces. |
| 153 | func sanitizeGroupName(name string) string { |
| 154 | name = strings.Map(func(r rune) rune { |
| 155 | if unicode.IsControl(r) { |
| 156 | return -1 |
| 157 | } |
| 158 | return r |
| 159 | }, name) |
| 160 | name = strings.TrimSpace(name) |
| 161 | if len(name) > 60 { |
| 162 | cut := 60 |
| 163 | for cut > 0 && !utf8.RuneStart(name[cut]) { |
| 164 | cut-- |
| 165 | } |
| 166 | name = strings.TrimSpace(name[:cut]) |
| 167 | } |
| 168 | return name |
| 169 | } |
| 170 | |
| 171 | // addInstanceResult is what a successful creation yields. SetupLink may be empty |
| 172 | // with SetupErr set: the invite mint is best-effort (the admin link still |
| 173 | // bootstraps the instance), matching the CLI's long-standing behaviour. |
| 174 | type addInstanceResult struct { |
| 175 | Host string |
| 176 | AdminToken string |
| 177 | SetupLink string |
| 178 | SetupErr error |
| 179 | } |
| 180 | |
| 181 | // addInstance is the shared creation core behind `lchat add-instance` and the |
| 182 | // control daemon (controld.go). It is deliberately two named steps — register |
| 183 | // (the row) then materialize (the machinery) — because the planet-scale plan |
| 184 | // (#550 Part 4 v2 / Part 5) eventually makes "create a space" *just* register: |
| 185 | // a registry row plus a storage prefix, microseconds and uncapped, with |
| 186 | // materialization deferred to the first wake (hibernation kills the day-one |
| 187 | // blip). Today the two always run back to back and the caller's capacity cap |
| 188 | // (controld.atCapacity) guards the pair, exactly as it did before the split. |
| 189 | func addInstance(reg *Registry, opts addInstanceOpts) (*addInstanceResult, error) { |
| 190 | opts.GroupName = sanitizeGroupName(opts.GroupName) |
| 191 | route, token, err := registerInstance(reg, opts) |
| 192 | if err != nil { |
| 193 | return nil, err |
| 194 | } |
| 195 | return materializeInstance(reg, route, token, opts) |
| 196 | } |
| 197 | |
| 198 | // registerInstance records that the space exists: validate the host, mint (or |
| 199 | // import) the admin token, and write the registry row (status "creating"). |
| 200 | // Pure metadata — no env file, no unit, no process — so it never consumes box |
| 201 | // capacity. Placement (HomeRegion/Provider) is recorded here because it's a |
| 202 | // property of the space, not of any one materialization. |
| 203 | func registerInstance(reg *Registry, opts addInstanceOpts) (Route, string, error) { |
| 204 | host := strings.ToLower(strings.TrimSpace(opts.Host)) |
| 205 | if !validHost(host) { |
| 206 | return Route{}, "", fmt.Errorf("%w: %q (want a hostname like chat.example.com)", errInvalidHost, host) |
| 207 | } |
| 208 | socket := filepath.Join(instanceSocketDir, host+".sock") |
| 209 | if err := checkSocketLen(socket); err != nil { |
| 210 | return Route{}, "", fmt.Errorf("%w: %v", errInvalidHost, err) |
| 211 | } |
| 212 | |
| 213 | if existing, err := reg.get(host); err != nil { |
| 214 | return Route{}, "", err |
| 215 | } else if existing != nil && existing.Status == "ready" { |
| 216 | return Route{}, "", fmt.Errorf("%w: %q", errHostExists, host) |
| 217 | } |
| 218 | // A prior failed/creating attempt is overwritten on retry. |
| 219 | |
| 220 | // An imported AdminToken (migration): the instance's boot rewrites |
| 221 | // admin_token_hash from the env, so passing the old token over a pre-placed |
| 222 | // migrated DB leaves the hash — and every saved admin link — intact. |
| 223 | token := opts.AdminToken |
| 224 | if token == "" { |
| 225 | var err error |
| 226 | if token, err = randToken(); err != nil { |
| 227 | return Route{}, "", fmt.Errorf("token: %v", err) |
| 228 | } |
| 229 | } |
| 230 | |
| 231 | if !validReplication(opts.Replication) { |
| 232 | return Route{}, "", fmt.Errorf("unknown replication class %q (want %s, %s or %s)", |
| 233 | opts.Replication, replicationStream, replicationBatched, replicationDaily) |
| 234 | } |
| 235 | |
| 236 | route := Route{Host: host, Kind: "instance", Socket: socket, |
| 237 | DBPath: filepath.Join(instanceDataDir, host+".db"), Status: "creating", |
| 238 | AppStoreURL: opts.AppStoreURL, HomeOrigin: opts.HomeOrigin, PushRelayURL: opts.PushRelayURL, |
| 239 | HomeRegion: opts.HomeRegion, Provider: opts.Provider, Hibernate: opts.Hibernate, |
| 240 | Canary: opts.Canary, Replication: opts.Replication} |
| 241 | if err := reg.upsert(route); err != nil { |
| 242 | return Route{}, "", fmt.Errorf("register: %v", err) |
| 243 | } |
| 244 | return route, token, nil |
| 245 | } |
| 246 | |
| 247 | // materializeInstance turns a registered row into a running instance: env file, |
| 248 | // staged binary + live symlink, systemd unit, health check, router reload, |
| 249 | // setup invite. This is the half that consumes real capacity (a process, a |
| 250 | // socket, disk), which is why the capacity cap conceptually belongs to it — and |
| 251 | // the seam exists so it can later run at first *wake* rather than at create |
| 252 | // (#550 Part 4 v2). |
| 253 | func materializeInstance(reg *Registry, route Route, token string, opts addInstanceOpts) (*addInstanceResult, error) { |
| 254 | host := route.Host |
| 255 | socket := route.Socket |
| 256 | unitTmpl := opts.UnitTmpl |
| 257 | if unitTmpl == "" { |
| 258 | unitTmpl = "lchat@" |
| 259 | } |
| 260 | routerUnit := opts.RouterUnit |
| 261 | if routerUnit == "" { |
| 262 | routerUnit = "lchat-router" |
| 263 | } |
| 264 | |
| 265 | // The instance stores only the token's hash, so this is the one chance to |
| 266 | // capture the plaintext. Persist it (and per-instance config) to an |
| 267 | // EnvironmentFile the unit reads — that keeps it recoverable via `admin-link`. |
| 268 | env := instanceEnvMap(token, opts.AppStoreURL, opts.HomeOrigin, |
| 269 | opts.PushRelayURL, opts.PushRelayToken, opts.Entitlement, opts.GroupName) |
| 270 | envPath := filepath.Join(instanceDataDir, host+".env") |
| 271 | if err := writeEnvFile(envPath, env); err != nil { |
| 272 | return nil, fmt.Errorf("write env file: %v", err) |
| 273 | } |
| 274 | if route.Hibernate { |
| 275 | // Systemd-managed instances never care who owns the env file — systemd |
| 276 | // reads EnvironmentFile as root before dropping privileges. A hibernating |
| 277 | // instance's env is read by the ROUTER at wake time, and the router runs |
| 278 | // as the service user, so root-run add-instance must hand the file over |
| 279 | // (found live: the canary's first wake died on permission denied). |
| 280 | if err := matchOwner(envPath, instanceDataDir); err != nil { |
| 281 | return nil, fmt.Errorf("env file ownership: %v", err) |
| 282 | } |
| 283 | } |
| 284 | |
| 285 | // The instance's unit execs /opt/lchat/live/<host>; stage the running control |
| 286 | // binary and point that symlink at it, so ExecStart resolves on first start. |
| 287 | // A fresh instance therefore runs whatever build the operator's `lchat` is. |
| 288 | ver, err := ensureSelfStaged() |
| 289 | if err != nil { |
| 290 | return nil, fmt.Errorf("stage binary: %v", err) |
| 291 | } |
| 292 | if err := pointLive(host, ver); err != nil { |
| 293 | return nil, fmt.Errorf("point live symlink: %v", err) |
| 294 | } |
| 295 | route.Version = ver |
| 296 | |
| 297 | // The systemd half — the ONLY part a hibernating instance skips. The split |
| 298 | // is derived from what the activator's spawn (s3Backend.start) needs at |
| 299 | // first wake: the env file above (the spawn layers /data/<host>.env over a |
| 300 | // clean env, and the admin token's plaintext lives nowhere else — without |
| 301 | // it the woken instance would mint its own token and every saved admin |
| 302 | // link would be dead); the staged binary + live symlink above (the spawn |
| 303 | // execs /opt/lchat/live/<host>, and staging here keeps the instance under |
| 304 | // normal version management instead of self-healing onto the router's own |
| 305 | // build); and the ready row + router reload below (or no request can ever |
| 306 | // reach the host to wake it). What a hibernating instance must NOT have is |
| 307 | // a unit holding the socket — the instance refuses to clobber a live |
| 308 | // listener (listen() in main.go), so a running unit would turn every wake |
| 309 | // into "socket already in use". No DB is created either: the first wake |
| 310 | // finds no replica in S3 and boots fresh. And there's no health check here |
| 311 | // because nothing runs yet — the public-readiness probe below is the first |
| 312 | // request, and the wake it triggers is the health check. |
| 313 | if !route.Hibernate { |
| 314 | unit := unitTmpl + host |
| 315 | if err := systemctl("enable", "--now", unit); err != nil { |
| 316 | markInstanceFailed(reg, host, unit, "start: "+err.Error()) |
| 317 | return nil, fmt.Errorf("start %s: %v", unit, err) |
| 318 | } |
| 319 | if err := waitHealthy(socket, 15*time.Second); err != nil { |
| 320 | markInstanceFailed(reg, host, unit, "health: "+err.Error()) |
| 321 | return nil, err |
| 322 | } |
| 323 | } |
| 324 | |
| 325 | route.Status = "ready" |
| 326 | if err := reg.upsert(route); err != nil { |
| 327 | return nil, fmt.Errorf("mark ready: %v", err) |
| 328 | } |
| 329 | if route.Hibernate { |
| 330 | // Best-effort: give any live mesh peer this route before HRW could ever |
| 331 | // place it there (meshreplicate.go). A no-op when the mesh is off or |
| 332 | // this host is out of LCHAT_MESH_DOMAINS scope — self-hosters and |
| 333 | // non-mesh boxes are unaffected. Never fails materialization: the |
| 334 | // route and admin link stand regardless of replication's outcome. |
| 335 | if hc, hcErr := hibernateConfigFromEnv(); hcErr != nil { |
| 336 | log.Printf("mesh: skipping route replication for %s: %v", host, hcErr) |
| 337 | } else { |
| 338 | fanOutMeshRoute(hc, route, env) |
| 339 | } |
| 340 | } |
| 341 | if err := systemctl("reload", routerUnit); err != nil { |
| 342 | return nil, fmt.Errorf("reload %s: %v (route added; run: systemctl reload %s)", routerUnit, err, routerUnit) |
| 343 | } |
| 344 | |
| 345 | // Confirm the space answers through the PUBLIC path (router → this instance) |
| 346 | // before we hand back a setup link, so the moment a human clicks it they land |
| 347 | // on a ready admin sign-on — never a mid-provision error. This also warms the |
| 348 | // per-host TLS cert as a side effect (the handshake triggers autocert), so the |
| 349 | // first visitor doesn't race on-demand issuance. Best-effort: on failure we |
| 350 | // still return (the instance passed its own health check); the public path is |
| 351 | // almost certainly up a beat later. |
| 352 | // |
| 353 | // For a hibernating instance this probe is also its FIRST WAKE: the request |
| 354 | // makes the router's activator spawn the child, which binds the socket the |
| 355 | // setup-invite mint below dials. If the wake never lands (e.g. the router |
| 356 | // isn't running with LCHAT_HIBERNATE_BUCKET), the mint fails and we degrade |
| 357 | // to the usual "setup link unavailable" path — the route and admin link |
| 358 | // still stand. |
| 359 | confirmPublicReady(host, routerHTTPSAddr) |
| 360 | |
| 361 | // The first-user setup link: claiming it makes you the instance's admin and |
| 362 | // lands you in the chat view, where the Eleven setup conversation takes over. |
| 363 | // Minting can only fail against an unhealthy instance (we just health-checked), |
| 364 | // but if it does the admin link still bootstraps the old way. |
| 365 | res := &addInstanceResult{Host: host, AdminToken: token} |
| 366 | res.SetupLink, res.SetupErr = mintSetupInvite(socketClient(socket), "http://lchat", "https://"+host, token) |
| 367 | if res.SetupErr != nil { |
| 368 | // A hibernating instance may live behind the local socket in name only: |
| 369 | // the mesh can place it on a peer box, where the readiness probe above |
| 370 | // woke it — locally nothing listens, ever. Mint through the public path |
| 371 | // instead (the same router hop the member will use), which follows the |
| 372 | // mesh to wherever the instance actually runs; the replicated env means |
| 373 | // it holds the same admin token. Non-mesh boxes just get a second local |
| 374 | // try through the router, which is harmless. |
| 375 | link, perr := mintPublicSetupInvite(host, token) |
| 376 | if perr != nil { |
| 377 | res.SetupErr = fmt.Errorf("socket: %v; public: %v", res.SetupErr, perr) |
| 378 | res.SetupLink = "" |
| 379 | } else { |
| 380 | res.SetupLink, res.SetupErr = link, nil |
| 381 | } |
| 382 | } |
| 383 | return res, nil |
| 384 | } |
| 385 | |
| 386 | // mintPublicSetupInvite mints the setup link through the local router's public |
| 387 | // HTTPS front (SNI-pinned like confirmPublicReady, so the real route + cert |
| 388 | // are exercised) — the path that works no matter which mesh box the instance |
| 389 | // woke on. A var so the add-instance tests, which run no TLS router, can stub |
| 390 | // it alongside confirmPublicReady. |
| 391 | var mintPublicSetupInvite = func(host, adminToken string) (string, error) { |
| 392 | client := &http.Client{ |
| 393 | Timeout: 15 * time.Second, |
| 394 | Transport: &http.Transport{ |
| 395 | DialTLSContext: func(ctx context.Context, _, _ string) (net.Conn, error) { |
| 396 | return (&tls.Dialer{Config: &tls.Config{ServerName: host}}).DialContext(ctx, "tcp", routerHTTPSAddr) |
| 397 | }, |
| 398 | }, |
| 399 | } |
| 400 | return mintSetupInvite(client, "https://"+host, "https://"+host, adminToken) |
| 401 | } |
| 402 | |
| 403 | // runAddInstance: `lchat add-instance <host>` — mint token + env file, start the |
| 404 | // instance unit, register the route, reload the router. |
| 405 | func runAddInstance(args []string) { |
| 406 | fs := flag.NewFlagSet("add-instance", flag.ExitOnError) |
| 407 | registryPath := fs.String("registry", envOr("LCHAT_REGISTRY", "/data/registry.db"), "router registry SQLite path") |
| 408 | unitTmpl := fs.String("unit", envOr("LCHAT_UNIT", "lchat@"), "systemd instance unit template prefix") |
| 409 | routerUnit := fs.String("router-unit", envOr("LCHAT_ROUTER_UNIT", "lchat-router"), "router systemd unit to reload") |
| 410 | adminToken := fs.String("admin-token", "", "import: use this admin token instead of minting a new one (preserves saved admin links when migrating an instance whose DB is already in place)") |
| 411 | appStoreURL := fs.String("app-store-url", "", "per-instance LCHAT_APP_STORE_URL") |
| 412 | homeOrigin := fs.String("home-origin", "", "per-instance LCHAT_HOME_ORIGIN") |
| 413 | groupName := fs.String("group-name", "", "seed the space's display name (LCHAT_GROUP_NAME; the instance names itself at first boot — an already-named space ignores it)") |
| 414 | pushRelayURL := fs.String("push-relay-url", "", "per-instance LCHAT_PUSH_RELAY_URL") |
| 415 | pushRelayToken := fs.String("push-relay-token", "", "per-instance LCHAT_PUSH_RELAY_TOKEN") |
| 416 | homeRegion := fs.String("home-region", "", "placement region recorded as the instance's home_region (multi-edge, docs/multi-edge.md) — the mesh owns the space on a box whose LCHAT_MESH_REGION matches; empty = region-agnostic (today's behaviour)") |
| 417 | hibernate := fs.Bool("hibernate", false, "register as activator-managed (no systemd unit; the router wakes it on demand — requires the router to run with LCHAT_HIBERNATE_BUCKET)") |
| 418 | replication := fs.String("replication", "", "replication class on the activator's S3 replica: stream (default), batched, or daily — the per-plan durability ladder; inert without --hibernate") |
| 419 | canary := fs.Bool("canary", false, "mark as a throwaway test instance (a per-session canary) — a deliberate signal, never inferred from the hostname; lets fleet ops and a future scoped reset treat it as disposable") |
| 420 | _ = fs.Parse(reorderFlagsFirst(fs, args)) |
| 421 | |
| 422 | reg, err := openRegistry(*registryPath) |
| 423 | if err != nil { |
| 424 | die("add-instance: open registry: %v", err) |
| 425 | } |
| 426 | res, err := addInstance(reg, addInstanceOpts{ |
| 427 | Host: fs.Arg(0), |
| 428 | AdminToken: *adminToken, |
| 429 | AppStoreURL: *appStoreURL, |
| 430 | HomeOrigin: *homeOrigin, |
| 431 | GroupName: *groupName, |
| 432 | PushRelayURL: *pushRelayURL, |
| 433 | PushRelayToken: *pushRelayToken, |
| 434 | HomeRegion: strings.TrimSpace(*homeRegion), |
| 435 | Hibernate: *hibernate, |
| 436 | Canary: *canary, |
| 437 | Replication: strings.ToLower(strings.TrimSpace(*replication)), |
| 438 | UnitTmpl: *unitTmpl, |
| 439 | RouterUnit: *routerUnit, |
| 440 | }) |
| 441 | if err != nil { |
| 442 | die("add-instance: %v", err) |
| 443 | } |
| 444 | |
| 445 | fmt.Printf("added %s\n", res.Host) |
| 446 | if *hibernate { |
| 447 | fmt.Printf(" hibernating: no systemd unit — the router wakes it per request and sleeps it when idle\n") |
| 448 | } |
| 449 | if *canary { |
| 450 | fmt.Printf(" canary: flagged as a throwaway test instance\n") |
| 451 | } |
| 452 | if res.SetupErr != nil { |
| 453 | fmt.Printf(" (setup link unavailable: %v — mint later: lchat setup-invite %s)\n", res.SetupErr, res.Host) |
| 454 | } else { |
| 455 | fmt.Printf(" get started (one-time link, makes you the admin):\n %s\n", res.SetupLink) |
| 456 | } |
| 457 | fmt.Printf(" admin token link: https://%s/admin#token=%s\n (recover later: lchat admin-link %s / lchat setup-invite %s)\n", res.Host, res.AdminToken, res.Host, res.Host) |
| 458 | } |
| 459 | |
| 460 | // runAddService: `lchat add-service <host> --socket <path>` — front an |
| 461 | // externally-managed backend (the home server, push relay, or a third party's). |
| 462 | // Unlike an instance, the router only proxies it; it doesn't start or supervise |
| 463 | // the process. See docs/instance-router.md. |
| 464 | func runAddService(args []string) { |
| 465 | fs := flag.NewFlagSet("add-service", flag.ExitOnError) |
| 466 | registryPath := fs.String("registry", envOr("LCHAT_REGISTRY", "/data/registry.db"), "router registry SQLite path") |
| 467 | socket := fs.String("socket", "", "unix socket the service already listens on (required)") |
| 468 | routerUnit := fs.String("router-unit", envOr("LCHAT_ROUTER_UNIT", "lchat-router"), "router systemd unit to reload") |
| 469 | _ = fs.Parse(reorderFlagsFirst(fs, args)) |
| 470 | |
| 471 | host := strings.ToLower(strings.TrimSpace(fs.Arg(0))) |
| 472 | if !validHost(host) { |
| 473 | die("add-service: invalid host %q", host) |
| 474 | } |
| 475 | if *socket == "" { |
| 476 | die("add-service: --socket <path> is required (where the service listens)") |
| 477 | } |
| 478 | if err := checkSocketLen(*socket); err != nil { |
| 479 | die("add-service: %v", err) |
| 480 | } |
| 481 | reg, err := openRegistry(*registryPath) |
| 482 | if err != nil { |
| 483 | die("add-service: open registry: %v", err) |
| 484 | } |
| 485 | if existing, err := reg.get(host); err != nil { |
| 486 | die("add-service: %v", err) |
| 487 | } else if existing != nil { |
| 488 | die("add-service: host %q already exists (status %s)", host, existing.Status) |
| 489 | } |
| 490 | // The backend must already be up — the router only proxies it. Use a lax |
| 491 | // reachability check (any HTTP response, redirects not followed): a service's |
| 492 | // "/" may legitimately 3xx/4xx/5xx, unlike a messenger instance. |
| 493 | if err := waitReachable(*socket, 10*time.Second); err != nil { |
| 494 | die("add-service: %s not answering on %s: %v (start the service first)", host, *socket, err) |
| 495 | } |
| 496 | if err := reg.upsert(Route{Host: host, Kind: "service", Socket: *socket, Status: "ready"}); err != nil { |
| 497 | die("add-service: register: %v", err) |
| 498 | } |
| 499 | if err := systemctl("reload", *routerUnit); err != nil { |
| 500 | die("add-service: reload %s: %v (route added; run: systemctl reload %s)", *routerUnit, err, *routerUnit) |
| 501 | } |
| 502 | fmt.Printf("added service %s -> %s\n", host, *socket) |
| 503 | } |
| 504 | |
| 505 | // errUnknownHost lets callers (controld) map "no such instance" to a 404. |
| 506 | var errUnknownHost = fmt.Errorf("unknown host") |
| 507 | |
| 508 | // deleteInstance is the shared deprovisioning core behind the control daemon's |
| 509 | // DELETE (self-serve space deletion): stop the units, drop the route, reload the |
| 510 | // router, and ARCHIVE the instance's data — the DB/env move to |
| 511 | // /data/trash/<host>-<unix>/ rather than being destroyed, so an operator can |
| 512 | // still recover a space deleted in error until the trash is purged. The cert and |
| 513 | // live symlink are removed (both are re-creatable). |
| 514 | // |
| 515 | // A hibernating space's S3 replica (s3://<bucket>/<prefix>/<host>/) is |
| 516 | // deliberately NOT deleted here: controld holds no bucket credentials (only the |
| 517 | // router process does, and widening the root daemon's reach for this buys |
| 518 | // nothing), the prefix is opaque ciphertext of a DB whose authoritative copy |
| 519 | // just moved to trash, and the production bucket is versioned with a |
| 520 | // noncurrent-version expiry, so an orphaned prefix ages rather than grows. |
| 521 | // Reclaiming orphaned prefixes is deferred to the S3 lifecycle work — an |
| 522 | // offline sweep comparing bucket prefixes against the registry. |
| 523 | func deleteInstance(reg *Registry, host string, discard bool) error { |
| 524 | host = strings.ToLower(strings.TrimSpace(host)) |
| 525 | if !validHost(host) { |
| 526 | return fmt.Errorf("%w: %q", errInvalidHost, host) |
| 527 | } |
| 528 | rt, err := reg.get(host) |
| 529 | if err != nil { |
| 530 | return err |
| 531 | } |
| 532 | if rt == nil { |
| 533 | return fmt.Errorf("%w: %q", errUnknownHost, host) |
| 534 | } |
| 535 | if rt.Kind != "instance" { |
| 536 | return fmt.Errorf("%q is a %s route, not an instance", host, rt.Kind) |
| 537 | } |
| 538 | _ = systemctl("disable", "--now", "lchat-bot@"+host) // best-effort: a sidecar may not exist |
| 539 | if rt.Hibernate { |
| 540 | // Activator-managed: there is no lchat@ unit — stopping one is |
| 541 | // best-effort only, covering a half-flipped host. If the child is |
| 542 | // awake it's the ROUTER that puts it down: the route delete + reload |
| 543 | // below make the reload-time reconcile (retireExcept, activator.go) |
| 544 | // stop the child, take the final checkpoint, and release the S3 lease. |
| 545 | _ = systemctl("disable", "--now", "lchat@"+host) |
| 546 | } else if err := systemctl("disable", "--now", "lchat@"+host); err != nil { |
| 547 | return fmt.Errorf("stop unit: %v", err) |
| 548 | } |
| 549 | if err := reg.delete(host); err != nil { |
| 550 | return fmt.Errorf("deregister: %v", err) |
| 551 | } |
| 552 | if err := systemctl("reload", "lchat-router"); err != nil { |
| 553 | return fmt.Errorf("reload router: %v", err) |
| 554 | } |
| 555 | // The row is authoritatively gone here — tell the mesh peers so a |
| 556 | // destroyed space can't live on (or wake!) from a replicated copy |
| 557 | // (#1346). Best-effort like creation fan-out; a missed peer reconciles at |
| 558 | // the next mesh-seed --prune. No-op off-mesh. |
| 559 | if hc, hcErr := hibernateConfigFromEnv(); hcErr == nil { |
| 560 | fanOutMeshRouteDelete(hc, host) |
| 561 | } |
| 562 | if rt.Hibernate { |
| 563 | // The retire is asynchronous inside the router process; wait (bounded) |
| 564 | // for the child to release the socket so the archive renames below |
| 565 | // don't race its final checkpoint. Instant in the common case — the |
| 566 | // reaper's targets are asleep or never woken, so nothing listens. |
| 567 | // Best-effort on timeout: open fds follow the renamed inode, so a |
| 568 | // straggling checkpoint lands harmlessly inside the archive. |
| 569 | _ = waitSocketQuiet(rt.Socket, deleteReleaseTimeout) |
| 570 | } |
| 571 | // Archive the data (never destroy it here — the trash dir is the safety |
| 572 | // net) — except under discard, the demo tier's no-backups contract |
| 573 | // (docs/demo-spaces.md): a demo's files are removed outright, blob dir |
| 574 | // included, so nothing of a disclosed-24h space outlives it. |
| 575 | // The litestream meta dir rides along: it describes the archived DB, and a |
| 576 | // leftover would only confuse a future re-create of the same host (the wake |
| 577 | // path clears a stale one, but don't rely on it). |
| 578 | names := []string{host + ".db", host + ".db-wal", host + ".db-shm", host + ".db" + litestream.MetaDirSuffix, host + ".env", host + ".vapid"} |
| 579 | if discard { |
| 580 | for _, name := range append(names, host+"-blobs") { |
| 581 | if err := os.RemoveAll(filepath.Join(instanceDataDir, name)); err != nil { |
| 582 | return fmt.Errorf("discard %s: %v", name, err) |
| 583 | } |
| 584 | } |
| 585 | } else { |
| 586 | trash := filepath.Join(instanceDataDir, "trash", fmt.Sprintf("%s-%d", host, time.Now().Unix())) |
| 587 | if err := os.MkdirAll(trash, 0o700); err != nil { |
| 588 | return fmt.Errorf("trash dir: %v", err) |
| 589 | } |
| 590 | for _, name := range names { |
| 591 | src := filepath.Join(instanceDataDir, name) |
| 592 | if _, err := os.Stat(src); err == nil { |
| 593 | if err := os.Rename(src, filepath.Join(trash, name)); err != nil { |
| 594 | return fmt.Errorf("archive %s: %v", name, err) |
| 595 | } |
| 596 | } |
| 597 | } |
| 598 | } |
| 599 | _ = os.Remove(filepath.Join(instanceDataDir, "acme", host)) // cert re-issues if ever needed |
| 600 | _ = os.Remove(liveLink(host)) |
| 601 | return nil |
| 602 | } |
| 603 | |
| 604 | // deleteReleaseTimeout bounds how long deleteInstance waits for the activator |
| 605 | // to release a hibernating instance's socket before archiving. A var so tests |
| 606 | // can shrink the (rare) still-listening path. |
| 607 | var deleteReleaseTimeout = 30 * time.Second |
| 608 | |
| 609 | // runRemove: `lchat remove <host>` — stop routing a host. Data-safe by default: |
| 610 | // it deletes the route and reloads the router but leaves the instance running and |
| 611 | // its DB/env in place; --stop also stops+disables an instance's unit. |
| 612 | func runRemove(args []string) { |
| 613 | fs := flag.NewFlagSet("remove", flag.ExitOnError) |
| 614 | registryPath := fs.String("registry", envOr("LCHAT_REGISTRY", "/data/registry.db"), "router registry SQLite path") |
| 615 | unitTmpl := fs.String("unit", envOr("LCHAT_UNIT", "lchat@"), "systemd instance unit template prefix") |
| 616 | botUnit := fs.String("bot-unit", envOr("LCHAT_BOT_UNIT", "lchat-bot@"), "systemd bot unit template prefix") |
| 617 | routerUnit := fs.String("router-unit", envOr("LCHAT_ROUTER_UNIT", "lchat-router"), "router systemd unit to reload") |
| 618 | stop := fs.Bool("stop", false, "for an instance, also stop + disable its systemd unit (DB/env still kept)") |
| 619 | _ = fs.Parse(reorderFlagsFirst(fs, args)) |
| 620 | |
| 621 | host := strings.ToLower(strings.TrimSpace(fs.Arg(0))) |
| 622 | if !validHost(host) { |
| 623 | die("remove: invalid host %q", host) |
| 624 | } |
| 625 | reg, err := openRegistry(*registryPath) |
| 626 | if err != nil { |
| 627 | die("remove: open registry: %v", err) |
| 628 | } |
| 629 | rt, err := reg.get(host) |
| 630 | if err != nil { |
| 631 | die("remove: %v", err) |
| 632 | } |
| 633 | if rt == nil { |
| 634 | die("remove: unknown host %q", host) |
| 635 | } |
| 636 | if err := reg.delete(host); err != nil { |
| 637 | die("remove: %v", err) |
| 638 | } |
| 639 | if rt.Kind == "instance" { |
| 640 | // An instance's webhook-ingest route (in.<host>, add-hooks) is part of |
| 641 | // the instance: drop it too so the ingest URL dies with the space. |
| 642 | if err := reg.delete("in." + host); err != nil { |
| 643 | die("remove: %v", err) |
| 644 | } |
| 645 | } |
| 646 | if err := systemctl("reload", *routerUnit); err != nil { |
| 647 | die("remove: reload %s: %v", *routerUnit, err) |
| 648 | } |
| 649 | // Tombstone the mesh peers too (#1346) — same reasoning as deleteInstance. |
| 650 | if rt.Kind == "instance" { |
| 651 | if hc, hcErr := hibernateConfigFromEnv(); hcErr == nil { |
| 652 | fanOutMeshRouteDelete(hc, host) |
| 653 | } |
| 654 | } |
| 655 | if *stop && rt.Kind == "instance" { |
| 656 | _ = systemctl("disable", "--now", *botUnit+host) // best-effort: a sidecar may not exist |
| 657 | _ = systemctl("disable", "--now", "lchat-hooks@"+host) // best-effort: same for webhook ingest |
| 658 | if err := systemctl("disable", "--now", *unitTmpl+host); err != nil { |
| 659 | fmt.Printf("removed %s (route deleted), but stopping the unit failed: %v\n", host, err) |
| 660 | return |
| 661 | } |
| 662 | fmt.Printf("removed %s (route deleted, instance + sidecar stopped; DB/env kept in place)\n", host) |
| 663 | return |
| 664 | } |
| 665 | fmt.Printf("removed %s (route deleted, router reloaded)\n", host) |
| 666 | if rt.Kind == "instance" { |
| 667 | fmt.Printf(" note: instance %s%s is still running and its DB/env remain; pass --stop to stop it\n", *unitTmpl, host) |
| 668 | } |
| 669 | } |
| 670 | |
| 671 | // canReset reports why host may not be reset, or nil if it may. The canary gate |
| 672 | // IS the safety model: reset wipes a live DB, so it refuses anything not |
| 673 | // explicitly flagged throwaway (registry.go: Route.Canary) — a real member |
| 674 | // instance can never be wiped by this command, whatever its hostname. |
| 675 | func canReset(rt *Route) error { |
| 676 | if rt == nil { |
| 677 | return fmt.Errorf("unknown host") |
| 678 | } |
| 679 | if rt.Kind != "instance" { |
| 680 | return fmt.Errorf("%q is a %s route, not an instance", rt.Host, rt.Kind) |
| 681 | } |
| 682 | if !rt.Canary { |
| 683 | return fmt.Errorf("%q is not flagged as a canary — refusing to wipe a real instance's data. "+ |
| 684 | "If it is genuinely a throwaway: lchat canary %s --on", rt.Host, rt.Host) |
| 685 | } |
| 686 | return nil |
| 687 | } |
| 688 | |
| 689 | // archiveInstanceDB moves host's database (and blob dir) into /data/trash, |
| 690 | // keeping its env/vapid so the admin token + config survive — the wipe half of |
| 691 | // `lchat reset`. Recoverable, like remove/reaper (never a destroy). |
| 692 | func archiveInstanceDB(host string) error { |
| 693 | trash := filepath.Join(instanceDataDir, "trash", fmt.Sprintf("%s-reset-%d", host, time.Now().Unix())) |
| 694 | if err := os.MkdirAll(trash, 0o700); err != nil { |
| 695 | return fmt.Errorf("trash dir: %v", err) |
| 696 | } |
| 697 | names := []string{host + ".db", host + ".db-wal", host + ".db-shm", host + ".db" + litestream.MetaDirSuffix, host + "-blobs"} |
| 698 | for _, name := range names { |
| 699 | src := filepath.Join(instanceDataDir, name) |
| 700 | if _, err := os.Stat(src); err == nil { |
| 701 | if err := os.Rename(src, filepath.Join(trash, name)); err != nil { |
| 702 | return fmt.Errorf("archive %s: %v", name, err) |
| 703 | } |
| 704 | } |
| 705 | } |
| 706 | return nil |
| 707 | } |
| 708 | |
| 709 | // runReset: `lchat reset <host>` — wipe a CANARY's database in place for a fresh |
| 710 | // first-run, the operator-clean answer to "reset this throwaway" (the in-place |
| 711 | // `rm` an operator can't do under the hardening). Stop the unit, archive its DB |
| 712 | // to trash, restart — the instance process (User=lchat) recreates an empty DB |
| 713 | // with correct ownership on boot — then mint the fresh onboarding link. |
| 714 | func runReset(args []string) { |
| 715 | fs := flag.NewFlagSet("reset", flag.ExitOnError) |
| 716 | registryPath := fs.String("registry", envOr("LCHAT_REGISTRY", "/data/registry.db"), "router registry SQLite path") |
| 717 | unitTmpl := fs.String("unit", envOr("LCHAT_UNIT", "lchat@"), "systemd instance unit template prefix") |
| 718 | yes := fs.Bool("yes", false, "confirm the destructive wipe (required — reset deletes the DB)") |
| 719 | _ = fs.Parse(reorderFlagsFirst(fs, args)) |
| 720 | |
| 721 | host := strings.ToLower(strings.TrimSpace(fs.Arg(0))) |
| 722 | if !validHost(host) { |
| 723 | die("reset: invalid host %q", host) |
| 724 | } |
| 725 | reg, err := openRegistry(*registryPath) |
| 726 | if err != nil { |
| 727 | die("reset: open registry: %v", err) |
| 728 | } |
| 729 | rt, err := reg.get(host) |
| 730 | if err != nil { |
| 731 | die("reset: %v", err) |
| 732 | } |
| 733 | if err := canReset(rt); err != nil { |
| 734 | die("reset: %v", err) |
| 735 | } |
| 736 | if !*yes { |
| 737 | die("reset: this DELETES %s's database (archived to /data/trash; env/config kept). Re-run with --yes to confirm.", host) |
| 738 | } |
| 739 | |
| 740 | if err := systemctl("stop", *unitTmpl+host); err != nil { |
| 741 | die("reset: stop %s: %v", host, err) |
| 742 | } |
| 743 | if err := archiveInstanceDB(host); err != nil { |
| 744 | _ = systemctl("start", *unitTmpl+host) // never leave it stopped after a failed wipe |
| 745 | die("reset: %v", err) |
| 746 | } |
| 747 | if err := systemctl("start", *unitTmpl+host); err != nil { |
| 748 | die("reset: start %s: %v", host, err) |
| 749 | } |
| 750 | |
| 751 | // The instance is booting a fresh DB; mint the onboarding link once it answers. |
| 752 | env, err := readEnvFile(filepath.Join(instanceDataDir, host+".env")) |
| 753 | if err != nil { |
| 754 | die("reset: %v (instance restarted fresh; mint later: lchat setup-invite %s)", err, host) |
| 755 | } |
| 756 | socket := filepath.Join(instanceSocketDir, host+".sock") |
| 757 | var link string |
| 758 | for i := 0; i < 20; i++ { |
| 759 | if link, err = mintSetupInvite(socketClient(socket), "http://lchat", "https://"+host, env["LCHAT_ADMIN_TOKEN"]); err == nil { |
| 760 | break |
| 761 | } |
| 762 | time.Sleep(300 * time.Millisecond) |
| 763 | } |
| 764 | if err != nil { |
| 765 | fmt.Printf("reset %s — DB wiped (archived to /data/trash), instance restarted fresh.\n"+ |
| 766 | " (setup link not ready yet: %v — mint it: lchat setup-invite %s)\n", host, err, host) |
| 767 | return |
| 768 | } |
| 769 | fmt.Printf("reset %s — DB wiped (archived to /data/trash), instance restarted fresh.\n"+ |
| 770 | " onboarding link (one-time, makes you the admin):\n %s\n", host, link) |
| 771 | } |
| 772 | |
| 773 | // runInstances: `lchat instances` — the route/status table. |
| 774 | func runInstances(args []string) { |
| 775 | fs := flag.NewFlagSet("instances", flag.ExitOnError) |
| 776 | registryPath := fs.String("registry", envOr("LCHAT_REGISTRY", "/data/registry.db"), "router registry SQLite path") |
| 777 | _ = fs.Parse(reorderFlagsFirst(fs, args)) |
| 778 | reg, err := openRegistry(*registryPath) |
| 779 | if err != nil { |
| 780 | die("instances: %v", err) |
| 781 | } |
| 782 | routes, err := reg.list() |
| 783 | if err != nil { |
| 784 | die("instances: %v", err) |
| 785 | } |
| 786 | if len(routes) == 0 { |
| 787 | fmt.Println("(no routes)") |
| 788 | return |
| 789 | } |
| 790 | fmt.Printf("%-34s %-9s %-9s %-12s %-9s %-7s %-11s %s\n", "HOST", "KIND", "STATUS", "VERSION", "HIBERNATE", "CANARY", "REPLICATION", "SOCKET") |
| 791 | for _, r := range routes { |
| 792 | ver := r.Version |
| 793 | if ver == "" { |
| 794 | ver = "-" |
| 795 | } |
| 796 | // "-" for the common (systemd-managed) case keeps the table quiet; |
| 797 | // hibernating hosts stand out at a glance. |
| 798 | hib := "-" |
| 799 | if r.Hibernate { |
| 800 | hib = "yes" |
| 801 | } |
| 802 | // Replication class: hibernating hosts show their effective class |
| 803 | // (unset = stream); a class staged on a systemd-managed host shows too |
| 804 | // (it's inert but deliberate); "-" = nothing set, nothing replicating. |
| 805 | repl := r.Replication |
| 806 | if repl == "" { |
| 807 | if r.Hibernate { |
| 808 | repl = replicationStream |
| 809 | } else { |
| 810 | repl = "-" |
| 811 | } |
| 812 | } |
| 813 | can := "-" |
| 814 | if r.Canary { |
| 815 | can = "canary" |
| 816 | } |
| 817 | fmt.Printf("%-34s %-9s %-9s %-12s %-9s %-7s %-11s %s\n", r.Host, r.Kind, r.Status, ver, hib, can, repl, r.Socket) |
| 818 | if r.LastError != "" { |
| 819 | fmt.Printf(" last error: %s\n", r.LastError) |
| 820 | } |
| 821 | } |
| 822 | } |
| 823 | |
| 824 | // runReplication: `lchat replication <host> [class]` — read or set one |
| 825 | // instance's replication class on the activator's S3 replica (the per-plan |
| 826 | // durability ladder: stream / batched / daily; registry.go has the intervals). |
| 827 | // The class is read at wake time, so a set takes effect on the host's next |
| 828 | // wake — an awake instance keeps its current tick until it sleeps. Setting a |
| 829 | // class on a systemd-managed (non-hibernating) host is allowed but inert: it |
| 830 | // pre-stages the class for a later hibernate flip. |
| 831 | func runReplication(args []string) { |
| 832 | fs := flag.NewFlagSet("replication", flag.ExitOnError) |
| 833 | registryPath := fs.String("registry", envOr("LCHAT_REGISTRY", "/data/registry.db"), "router registry SQLite path") |
| 834 | routerUnit := fs.String("router-unit", envOr("LCHAT_ROUTER_UNIT", "lchat-router"), "router systemd unit to reload") |
| 835 | _ = fs.Parse(reorderFlagsFirst(fs, args)) |
| 836 | |
| 837 | host := strings.ToLower(strings.TrimSpace(fs.Arg(0))) |
| 838 | if !validHost(host) { |
| 839 | die("replication: invalid host %q (usage: lchat replication <host> [stream|batched|daily])", host) |
| 840 | } |
| 841 | reg, err := openRegistry(*registryPath) |
| 842 | if err != nil { |
| 843 | die("replication: open registry: %v", err) |
| 844 | } |
| 845 | rt, err := reg.get(host) |
| 846 | if err != nil { |
| 847 | die("replication: %v", err) |
| 848 | } |
| 849 | if rt == nil { |
| 850 | die("replication: unknown host %q", host) |
| 851 | } |
| 852 | if rt.Kind != "instance" { |
| 853 | die("replication: %q is a %s route, not an instance", host, rt.Kind) |
| 854 | } |
| 855 | |
| 856 | effective := func(class string) string { |
| 857 | if class == "" { |
| 858 | return replicationStream |
| 859 | } |
| 860 | return class |
| 861 | } |
| 862 | describe := map[string]string{ |
| 863 | replicationStream: "change-driven S3 sync (~1s) — a hard crash loses seconds", |
| 864 | replicationBatched: "S3 sync at most every 5m — a hard crash loses minutes", |
| 865 | replicationDaily: "S3 sync at wake, then at most every 24h — a hard crash loses up to a day", |
| 866 | } |
| 867 | |
| 868 | class := strings.ToLower(strings.TrimSpace(fs.Arg(1))) |
| 869 | if class == "" { |
| 870 | fmt.Printf("%s: %s (%s)\n", host, effective(rt.Replication), describe[effective(rt.Replication)]) |
| 871 | if !rt.Hibernate { |
| 872 | fmt.Println(" note: this host is systemd-managed (not hibernating) — the class is inert; the box-wide litestream replicates it") |
| 873 | } |
| 874 | return |
| 875 | } |
| 876 | if err := reg.setReplication(host, class); err != nil { |
| 877 | die("replication: %v", err) |
| 878 | } |
| 879 | fmt.Printf("%s replication → %s: %s\n", host, class, describe[class]) |
| 880 | fmt.Println(" every graceful sleep/deploy/shutdown still flushes in full — the class only bounds hard-crash loss while awake") |
| 881 | if !rt.Hibernate { |
| 882 | fmt.Println(" note: this host is systemd-managed (not hibernating) — the class is stored but inert until a hibernate flip") |
| 883 | return |
| 884 | } |
| 885 | // The router's route map only swaps on reload; without it the new class |
| 886 | // never reaches a wake. The registry write above is durable either way, so |
| 887 | // a failed reload is a warning (reload by hand), not a dead flip. |
| 888 | if err := systemctl("reload", *routerUnit); err != nil { |
| 889 | fmt.Printf(" WARNING: reload %s failed (%v) — the class is stored but inert until the router reloads\n", *routerUnit, err) |
| 890 | return |
| 891 | } |
| 892 | fmt.Println(" takes effect at the next wake (an awake instance keeps its current tick until it sleeps)") |
| 893 | } |
| 894 | |
| 895 | // runHibernate: `lchat hibernate <host> --on|--off` — flip one instance between |
| 896 | // systemd-managed (always-on) and activator-managed (hibernating). Per-host by |
| 897 | // design: a production canary can hibernate while the rest of the fleet stays |
| 898 | // on its units. --on requires the router to be running with |
| 899 | // LCHAT_HIBERNATE_BUCKET, or the host will sit unwakeable once its unit stops. |
| 900 | func runCanary(args []string) { |
| 901 | fs := flag.NewFlagSet("canary", flag.ExitOnError) |
| 902 | registryPath := fs.String("registry", envOr("LCHAT_REGISTRY", "/data/registry.db"), "router registry SQLite path") |
| 903 | on := fs.Bool("on", false, "mark the instance as a throwaway canary") |
| 904 | off := fs.Bool("off", false, "clear the canary flag (a real, keep-its-data instance)") |
| 905 | _ = fs.Parse(reorderFlagsFirst(fs, args)) |
| 906 | |
| 907 | if *on == *off { |
| 908 | die("canary: pass exactly one of --on / --off") |
| 909 | } |
| 910 | host := strings.ToLower(strings.TrimSpace(fs.Arg(0))) |
| 911 | if !validHost(host) { |
| 912 | die("canary: invalid host %q", host) |
| 913 | } |
| 914 | reg, err := openRegistry(*registryPath) |
| 915 | if err != nil { |
| 916 | die("canary: open registry: %v", err) |
| 917 | } |
| 918 | rt, err := reg.get(host) |
| 919 | if err != nil { |
| 920 | die("canary: %v", err) |
| 921 | } |
| 922 | if rt == nil { |
| 923 | die("canary: unknown host %q", host) |
| 924 | } |
| 925 | if rt.Kind != "instance" { |
| 926 | die("canary: %q is a %s route, not an instance", host, rt.Kind) |
| 927 | } |
| 928 | if err := reg.setCanary(host, *on); err != nil { |
| 929 | die("canary %s: %v", host, err) |
| 930 | } |
| 931 | if *on { |
| 932 | fmt.Printf("%s is flagged as a canary (throwaway test instance)\n", host) |
| 933 | } else { |
| 934 | fmt.Printf("%s is no longer flagged as a canary\n", host) |
| 935 | } |
| 936 | } |
| 937 | |
| 938 | func runHibernate(args []string) { |
| 939 | fs := flag.NewFlagSet("hibernate", flag.ExitOnError) |
| 940 | registryPath := fs.String("registry", envOr("LCHAT_REGISTRY", "/data/registry.db"), "router registry SQLite path") |
| 941 | unitTmpl := fs.String("unit", envOr("LCHAT_UNIT", "lchat@"), "systemd instance unit template prefix") |
| 942 | botUnit := fs.String("bot-unit", envOr("LCHAT_BOT_UNIT", "lchat-bot@"), "systemd bot unit template prefix") |
| 943 | routerUnit := fs.String("router-unit", envOr("LCHAT_ROUTER_UNIT", "lchat-router"), "router systemd unit to reload") |
| 944 | on := fs.Bool("on", false, "hand the instance to the activator: stop+disable its unit; the router wakes it per request") |
| 945 | off := fs.Bool("off", false, "hand the instance back to systemd: the activator releases it, then its unit is enabled+started") |
| 946 | _ = fs.Parse(reorderFlagsFirst(fs, args)) |
| 947 | |
| 948 | if *on == *off { |
| 949 | die("hibernate: pass exactly one of --on / --off") |
| 950 | } |
| 951 | host := strings.ToLower(strings.TrimSpace(fs.Arg(0))) |
| 952 | if !validHost(host) { |
| 953 | die("hibernate: invalid host %q", host) |
| 954 | } |
| 955 | reg, err := openRegistry(*registryPath) |
| 956 | if err != nil { |
| 957 | die("hibernate: open registry: %v", err) |
| 958 | } |
| 959 | rt, err := reg.get(host) |
| 960 | if err != nil { |
| 961 | die("hibernate: %v", err) |
| 962 | } |
| 963 | if rt == nil { |
| 964 | die("hibernate: unknown host %q", host) |
| 965 | } |
| 966 | if rt.Kind != "instance" { |
| 967 | die("hibernate: %q is a %s route, not an instance", host, rt.Kind) |
| 968 | } |
| 969 | if rt.Status != "ready" { |
| 970 | die("hibernate: %s is %s, not ready — fix the instance first", host, rt.Status) |
| 971 | } |
| 972 | if *on { |
| 973 | if rt.Hibernate && !unitActive(*unitTmpl+host) { |
| 974 | fmt.Printf("%s is already hibernating\n", host) |
| 975 | return |
| 976 | } |
| 977 | // Flag set but the unit still runs = a re-run after the litestream |
| 978 | // guard's two-step (the error message PROMISES this re-run works — |
| 979 | // hit live on the first bulk flip, where the early-exit above |
| 980 | // stranded hosts flagged-but-materialized). Fall through and finish |
| 981 | // the handover; hibernateOn is idempotent over an already-set flag. |
| 982 | if err := hibernateOn(reg, rt, *unitTmpl, *botUnit, *routerUnit); err != nil { |
| 983 | die("hibernate %s --on: %v", host, err) |
| 984 | } |
| 985 | fmt.Printf("%s is now activator-managed: unit stopped+disabled; the router wakes it per request and sleeps it when idle\n", host) |
| 986 | if _, err := os.Stat(botEnvPath(host)); err == nil { |
| 987 | fmt.Printf(" bot sidecar: co-managed by the activator — spawned on each wake, stopped at each sleep (%s%s disabled; --off hands it back to systemd)\n", *botUnit, host) |
| 988 | } |
| 989 | return |
| 990 | } |
| 991 | if !rt.Hibernate { |
| 992 | fmt.Printf("%s is not hibernating\n", host) |
| 993 | return |
| 994 | } |
| 995 | if err := hibernateOff(reg, rt, *unitTmpl, *botUnit, *routerUnit); err != nil { |
| 996 | die("hibernate %s --off: %v", host, err) |
| 997 | } |
| 998 | fmt.Printf("%s is back under systemd: unit enabled and healthy; the activator has released it\n", host) |
| 999 | } |
| 1000 | |
| 1001 | // hibernateOn flips a running, systemd-managed instance to activator-managed. |
| 1002 | // Ordering — chosen so the host is never left unserveable: |
| 1003 | // |
| 1004 | // 1. Flag the route. A registry write alone changes nothing live (the |
| 1005 | // router's map only swaps on reload), so a crash right after leaves the |
| 1006 | // unit serving exactly as before. |
| 1007 | // 2. Stop + disable the unit — BEFORE the reload, because once the route is |
| 1008 | // activator-managed the first wake's child must be able to bind the |
| 1009 | // socket, and the instance refuses to clobber a live listener (listen() |
| 1010 | // in main.go): a still-running unit would turn every wake into "socket |
| 1011 | // already in use" 503s. This opens the one unavoidable gap — the socket |
| 1012 | // is dead until the wake in step 4 — and requests in it ride the router's |
| 1013 | // 13s dial-retry window, the same shape as a deploy restart. Disable (not |
| 1014 | // just stop) so a reboot can't resurrect the unit to fight the activator. |
| 1015 | // The bot sidecar unit (if the host has one staged) goes down here too — |
| 1016 | // the activator co-spawns the sidecar with each wake from now on. |
| 1017 | // 3. Reload the router: the route becomes activator-managed. |
| 1018 | // 4. A warm request through the public path — best-effort, but it both |
| 1019 | // closes the gap immediately (the wake binds the socket, so any retrying |
| 1020 | // dials from step 2 connect) and proves the wake path works while the |
| 1021 | // operator is still watching. |
| 1022 | // |
| 1023 | // Nothing is checkpointed here and no S3 is touched: the DB is already local, |
| 1024 | // so the first wake's restore sees it as the warm cache and skips; replication |
| 1025 | // under the fresh lease is what backfills S3. |
| 1026 | // unitActive reports whether the systemd unit is running — the discriminator |
| 1027 | // between "already fully hibernating" and "flag set, handover unfinished" |
| 1028 | // (the litestream guard's two-step leaves exactly that state behind). |
| 1029 | func unitActive(unit string) bool { |
| 1030 | return systemctlQuery("is-active", "--quiet", unit) == nil |
| 1031 | } |
| 1032 | |
| 1033 | func hibernateOn(reg *Registry, rt *Route, unitTmpl, botUnit, routerUnit string) error { |
| 1034 | // Existing instances' env files were written for the systemd path (root |
| 1035 | // reads them); the router's activator reads them as the service user, so |
| 1036 | // hand the file over first — a flip that can't wake is worse than no flip. |
| 1037 | if err := matchOwner(filepath.Join(instanceDataDir, rt.Host+".env"), instanceDataDir); err != nil { |
| 1038 | return fmt.Errorf("env file ownership: %v", err) |
| 1039 | } |
| 1040 | if err := reg.setHibernate(rt.Host, true); err != nil { |
| 1041 | return fmt.Errorf("flag: %v", err) |
| 1042 | } |
| 1043 | // Two litestream processes on one DB share the same derived meta dir and |
| 1044 | // corrupt each other's LTX state (hit live on the first real flip: the |
| 1045 | // box-wide backup replicator still tracked the host when the activator's |
| 1046 | // embedded one started, and a config regen yanked files out from under |
| 1047 | // it — a stuck error loop and failed checkpoints). The flag is set above |
| 1048 | // ON PURPOSE before this guard: the operator's fix — re-running the backup |
| 1049 | // script, which excludes hibernate=1 hosts — only works because the flag |
| 1050 | // already reads true, and the flag is inert until the router reload that |
| 1051 | // happens only after this guard passes. |
| 1052 | if litestreamConfigTracks(rt.Host) { |
| 1053 | return fmt.Errorf("%s is still replicated by the box-wide litestream (%s): re-run deploy/fleet-backup.sh — it now excludes this host, the flag is already set — then run this flip again", rt.Host, litestreamConfigPath()) |
| 1054 | } |
| 1055 | if err := systemctl("disable", "--now", unitTmpl+rt.Host); err != nil { |
| 1056 | return fmt.Errorf("stop unit: %v (flag is set but the unit still runs — the host serves as before; re-run, or undo with `lchat hibernate %s --off`)", err, rt.Host) |
| 1057 | } |
| 1058 | if _, err := os.Stat(botEnvPath(rt.Host)); err == nil { |
| 1059 | // The bot sidecar hands over too: from here the activator co-spawns it |
| 1060 | // with each wake and stops it at each sleep, so the always-on unit |
| 1061 | // must go — two sidecars would double every bot reply. Before the |
| 1062 | // reload, like the instance unit, so a systemd sidecar and a |
| 1063 | // co-spawned one never overlap. Best-effort: a sidecar that won't |
| 1064 | // stop cleanly must not strand the flip (members > bots). |
| 1065 | _ = systemctl("disable", "--now", botUnit+rt.Host) |
| 1066 | // Same ownership handover as the instance env above, for the pieces |
| 1067 | // the co-spawned sidecar (a router child, running as the service |
| 1068 | // user) reads and writes. Non-recursive on purpose: the session files |
| 1069 | // inside were already written by the unit's service user, and add-bot |
| 1070 | // chowns fresh ones at creation. |
| 1071 | _ = matchOwner(botEnvPath(rt.Host), instanceDataDir) |
| 1072 | _ = matchOwner(botSessionDir(rt.Host), instanceDataDir) |
| 1073 | } |
| 1074 | if err := systemctl("reload", routerUnit); err != nil { |
| 1075 | return fmt.Errorf("reload %s: %v (unit is stopped — reload the router by hand to bring the host back)", routerUnit, err) |
| 1076 | } |
| 1077 | confirmPublicReady(rt.Host, routerHTTPSAddr) |
| 1078 | return nil |
| 1079 | } |
| 1080 | |
| 1081 | // hibernateOff flips an activator-managed instance back under systemd. The |
| 1082 | // mirror of hibernateOn, with one extra wait: |
| 1083 | // |
| 1084 | // 1. Clear the flag (inert until reload). |
| 1085 | // 2. Reload the router: the map swap stops routing requests through the |
| 1086 | // activator, and the reload-time reconcile (retireExcept, activator.go) |
| 1087 | // makes the activator release the host — stop the child if it's awake, |
| 1088 | // take the final checkpoint, give the S3 lease back. |
| 1089 | // 3. WAIT for the child to actually let go of the socket before starting the |
| 1090 | // unit. The release is asynchronous inside the router process and the |
| 1091 | // instance refuses to start over a live listener, so enabling the unit too |
| 1092 | // early would just crash-loop it against the child. Dial-probing the |
| 1093 | // socket is the narrowest thing observable from outside the router: quiet |
| 1094 | // means the child's listener is gone (an already-asleep host has none, so |
| 1095 | // this returns on the first probe). The final S3 checkpoint may still be |
| 1096 | // finishing when the unit starts — that's fine: the replica goes stale by |
| 1097 | // design once the flag is off, and a later flip back on wakes from the |
| 1098 | // local DB (the warm cache, which is the newer truth). |
| 1099 | // 4. Enable + start the unit and health-check it, exactly like add-instance. |
| 1100 | // A staged bot sidecar then goes back on its always-on unit (instance |
| 1101 | // first, then sidecar — the same bootstrap order add-bot documents). |
| 1102 | // |
| 1103 | // Requests between the child's stop and the unit binding ride the router's 13s |
| 1104 | // dial-retry window, same as a deploy restart. |
| 1105 | func hibernateOff(reg *Registry, rt *Route, unitTmpl, botUnit, routerUnit string) error { |
| 1106 | if err := reg.setHibernate(rt.Host, false); err != nil { |
| 1107 | return fmt.Errorf("flag: %v", err) |
| 1108 | } |
| 1109 | if err := systemctl("reload", routerUnit); err != nil { |
| 1110 | return fmt.Errorf("reload %s: %v (flag is cleared; reload by hand, then re-run)", routerUnit, err) |
| 1111 | } |
| 1112 | if err := waitSocketQuiet(rt.Socket, hibernateReleaseTimeout); err != nil { |
| 1113 | return fmt.Errorf("%v — the activator hasn't released %s yet (a long final checkpoint?); re-run once it has", err, rt.Host) |
| 1114 | } |
| 1115 | unit := unitTmpl + rt.Host |
| 1116 | if err := systemctl("enable", "--now", unit); err != nil { |
| 1117 | return fmt.Errorf("start %s: %v", unit, err) |
| 1118 | } |
| 1119 | if err := waitHealthy(rt.Socket, 20*time.Second); err != nil { |
| 1120 | return fmt.Errorf("%v (unit started but unhealthy — check journalctl -u %s)", err, unit) |
| 1121 | } |
| 1122 | // Hand the sidecar back too: the co-spawned one died with the activator's |
| 1123 | // child (the retire's stop takes the sidecar down first), so a host with |
| 1124 | // one staged returns to its always-on unit — after the instance is |
| 1125 | // healthy, matching add-bot's "instance first" bootstrap order. A failure |
| 1126 | // here doesn't fail the flip (the instance is fine); tell the operator. |
| 1127 | if _, err := os.Stat(botEnvPath(rt.Host)); err == nil { |
| 1128 | if err := systemctl("enable", "--now", botUnit+rt.Host); err != nil { |
| 1129 | fmt.Printf(" warning: sidecar unit %s%s failed to start: %v — start it by hand\n", botUnit, rt.Host, err) |
| 1130 | } |
| 1131 | } |
| 1132 | return nil |
| 1133 | } |
| 1134 | |
| 1135 | // hibernateReleaseTimeout bounds how long a flip-off waits for the activator to |
| 1136 | // release the child: the stop path's own budgets (next-due probe 5s + SIGTERM |
| 1137 | // drain 20s + final checkpoint 30s) total under a minute; leave headroom. A var |
| 1138 | // so tests can shrink the failure path. |
| 1139 | var hibernateReleaseTimeout = 90 * time.Second |
| 1140 | |
| 1141 | // waitSocketQuiet blocks until nothing accepts on the socket (or the timeout |
| 1142 | // elapses) — how the flip-off observes, from outside the router process, that |
| 1143 | // the activator's child has released the socket. A dial failure is the "quiet" |
| 1144 | // signal whether the socket file lingers (ECONNREFUSED) or not (ENOENT). |
| 1145 | func waitSocketQuiet(socket string, timeout time.Duration) error { |
| 1146 | deadline := time.Now().Add(timeout) |
| 1147 | for { |
| 1148 | c, err := net.DialTimeout("unix", socket, 500*time.Millisecond) |
| 1149 | if err != nil { |
| 1150 | return nil |
| 1151 | } |
| 1152 | c.Close() |
| 1153 | if time.Now().After(deadline) { |
| 1154 | return fmt.Errorf("socket %s still has a listener after %s", socket, timeout) |
| 1155 | } |
| 1156 | time.Sleep(250 * time.Millisecond) |
| 1157 | } |
| 1158 | } |
| 1159 | |
| 1160 | // runAdminLink: `lchat admin-link <host>` — recover the admin link from the |
| 1161 | // instance's EnvironmentFile (the DB holds only the token's hash). |
| 1162 | func runAdminLink(args []string) { |
| 1163 | fs := flag.NewFlagSet("admin-link", flag.ExitOnError) |
| 1164 | _ = fs.Parse(reorderFlagsFirst(fs, args)) |
| 1165 | host := strings.ToLower(strings.TrimSpace(fs.Arg(0))) |
| 1166 | if !validHost(host) { |
| 1167 | die("admin-link: invalid host %q", host) |
| 1168 | } |
| 1169 | envPath := filepath.Join(instanceDataDir, host+".env") |
| 1170 | env, err := readEnvFile(envPath) |
| 1171 | if err != nil { |
| 1172 | die("admin-link: %v", err) |
| 1173 | } |
| 1174 | tok := env["LCHAT_ADMIN_TOKEN"] |
| 1175 | if tok == "" { |
| 1176 | die("admin-link: no LCHAT_ADMIN_TOKEN in %s", envPath) |
| 1177 | } |
| 1178 | fmt.Printf("https://%s/admin#token=%s\n", host, tok) |
| 1179 | } |
| 1180 | |
| 1181 | // runDeploy: `lchat deploy <host>` (or --all) — restart the instance(s) so they |
| 1182 | // pick up a new binary. SIGTERM drains gracefully and the router retries dials |
| 1183 | // across the brief gap, so the swap drops no requests. |
| 1184 | // |
| 1185 | // With --version <ver> it first repoints the instance's /opt/lchat/live/<host> |
| 1186 | // symlink at that staged version (see `lchat stage`), then health-checks that the |
| 1187 | // instance actually reports that build — auto-rolling-back to the prior version if |
| 1188 | // it doesn't. That's the side-by-side path: `deploy test --version <ver>` moves one |
| 1189 | // instance while the rest stay put; `deploy --all --version <ver>` promotes. |
| 1190 | // |
| 1191 | // Without --version it just restarts on the instance's current version — the |
| 1192 | // classic "I've already swapped the binary" path. See docs/instance-router.md. |
| 1193 | func runDeploy(args []string) { |
| 1194 | fs := flag.NewFlagSet("deploy", flag.ExitOnError) |
| 1195 | registryPath := fs.String("registry", envOr("LCHAT_REGISTRY", "/data/registry.db"), "router registry SQLite path") |
| 1196 | unitTmpl := fs.String("unit", envOr("LCHAT_UNIT", "lchat@"), "systemd instance unit template prefix") |
| 1197 | all := fs.Bool("all", false, "deploy every instance, one at a time (skips pinned canaries — see --include-pinned)") |
| 1198 | verFlag := fs.String("version", "", "point the instance(s) at this staged version before restarting (see `lchat stage`)") |
| 1199 | pinFlag := fs.Bool("pin", false, "also pin the instance(s) to --version, so the auto-updater leaves them alone (the canary flow)") |
| 1200 | includePinned := fs.Bool("include-pinned", false, "let --all move pinned instances too (a pinned host is someone's live canary — override deliberately)") |
| 1201 | _ = fs.Parse(reorderFlagsFirst(fs, args)) |
| 1202 | |
| 1203 | ver := strings.TrimSpace(*verFlag) |
| 1204 | if ver != "" { |
| 1205 | if !validVersion(ver) { |
| 1206 | die("deploy: invalid version %q", ver) |
| 1207 | } |
| 1208 | if _, err := os.Stat(versionBinPath(ver)); err != nil { |
| 1209 | die("deploy: version %q is not staged — run `lchat stage --file <binary>` first", ver) |
| 1210 | } |
| 1211 | } |
| 1212 | if *pinFlag && ver == "" { |
| 1213 | die("deploy: --pin requires --version (a pin names one exact build)") |
| 1214 | } |
| 1215 | if *pinFlag && *all { |
| 1216 | die("deploy: --pin with --all would pin the whole fleet — pin one canary at a time") |
| 1217 | } |
| 1218 | |
| 1219 | unlock, err := acquireDeployLock() |
| 1220 | if err != nil { |
| 1221 | die("deploy: %v", err) |
| 1222 | } |
| 1223 | defer unlock() |
| 1224 | |
| 1225 | reg, err := openRegistry(*registryPath) |
| 1226 | if err != nil { |
| 1227 | die("deploy: open registry: %v", err) |
| 1228 | } |
| 1229 | hosts, err := deployTargets(reg, *all, fs.Arg(0), "deploy") |
| 1230 | if err != nil { |
| 1231 | die("%v", err) |
| 1232 | } |
| 1233 | |
| 1234 | for _, host := range hosts { |
| 1235 | rt, err := reg.get(host) |
| 1236 | if err != nil { |
| 1237 | die("deploy %s: %v", host, err) |
| 1238 | } |
| 1239 | if rt == nil { |
| 1240 | die("deploy %s: unknown host (run add-instance first)", host) |
| 1241 | } |
| 1242 | if rt.Kind != "instance" { |
| 1243 | die("deploy %s: not an instance (kind %q) — only messenger instances are deployed", host, rt.Kind) |
| 1244 | } |
| 1245 | // A fleet sweep leaves pinned instances alone: a pin is someone's live |
| 1246 | // canary (CLAUDE.md § per-session canaries), and until now only the |
| 1247 | // AUTO-updater honoured it — `deploy --all` was the loophole that |
| 1248 | // stomped a pinned canary onto the channel build while its pin sat |
| 1249 | // intact. Naming a host explicitly still deploys it (that's intent); |
| 1250 | // --include-pinned overrides the sweep on purpose. |
| 1251 | if *all && ver != "" && !*includePinned { |
| 1252 | if skip, why := sweepSkipsPinned(rt, ver); skip { |
| 1253 | fmt.Printf("skipped %s (%s — move it with `lchat deploy %s --version %s --pin`, or rerun with --include-pinned)\n", host, why, host, ver) |
| 1254 | continue |
| 1255 | } else if why != "" { |
| 1256 | fmt.Printf("note: %s: %s\n", host, why) |
| 1257 | } |
| 1258 | } |
| 1259 | if rt.Hibernate { |
| 1260 | // No unit to restart — and starting one would fight the activator's |
| 1261 | // child for the socket. A --version deploy is still meaningful as a |
| 1262 | // pure repoint (the next wake execs it — same dance as the |
| 1263 | // auto-updater's hibernating phase, and the rescue path for a bad |
| 1264 | // repointed build). A plain restart-deploy has nothing to restart. |
| 1265 | if ver == "" { |
| 1266 | if *all { |
| 1267 | fmt.Printf("skipped %s (hibernating — nothing to restart; use --version to repoint)\n", host) |
| 1268 | continue |
| 1269 | } |
| 1270 | die("deploy %s: instance is hibernating (no unit to restart) — use --version to repoint it for its next wake", host) |
| 1271 | } |
| 1272 | cur, _ := liveVersion(host) |
| 1273 | if cur == ver { |
| 1274 | fmt.Printf("deployed %s (already on %s)\n", host, ver) |
| 1275 | continue |
| 1276 | } |
| 1277 | if err := pointLive(host, ver); err != nil { |
| 1278 | die("deploy %s: %v", host, err) |
| 1279 | } |
| 1280 | if err := reg.setVersions(host, ver, cur); err != nil { |
| 1281 | die("deploy %s: repointed but recording the version failed: %v", host, err) |
| 1282 | } |
| 1283 | fmt.Printf("deployed %s @ %s (hibernating — no health check; runs it on next wake)\n", host, ver) |
| 1284 | if *pinFlag { |
| 1285 | // The pin lives in the instance's DB, written over its socket — |
| 1286 | // which wakes it, onto the build just repointed. Deliberate: a |
| 1287 | // pinned deploy is a canary, and its operator is about to poke |
| 1288 | // it anyway. |
| 1289 | if err := pinHost(rt, ver); err != nil { |
| 1290 | die("deploy %s: deployed @ %s but PINNING FAILED (%v) — the next update pass may move it; retry with `lchat update pin %s %s`", host, ver, err, host, ver) |
| 1291 | } |
| 1292 | fmt.Printf("pinned %s to %s (auto-updates paused; the pin write woke it)\n", host, ver) |
| 1293 | } |
| 1294 | continue |
| 1295 | } |
| 1296 | unit := *unitTmpl + host |
| 1297 | |
| 1298 | if ver == "" { |
| 1299 | // Restart on the current version (binary already in place). |
| 1300 | if err := systemctl("restart", unit); err != nil { |
| 1301 | die("deploy %s: restart: %v", host, err) |
| 1302 | } |
| 1303 | if err := waitHealthy(rt.Socket, 20*time.Second); err != nil { |
| 1304 | die("deploy %s: %v", host, err) |
| 1305 | } |
| 1306 | fmt.Printf("deployed %s\n", host) |
| 1307 | continue |
| 1308 | } |
| 1309 | |
| 1310 | // --version: repoint, restart, verify the build took, roll back if not. |
| 1311 | changed, err := deployHostVersion(reg, rt, unit, ver) |
| 1312 | if err != nil { |
| 1313 | die("deploy %s: %v", host, err) |
| 1314 | } |
| 1315 | if changed { |
| 1316 | fmt.Printf("deployed %s @ %s\n", host, ver) |
| 1317 | } else { |
| 1318 | fmt.Printf("deployed %s (already on %s)\n", host, ver) |
| 1319 | } |
| 1320 | if *pinFlag { |
| 1321 | // The deploy stands even if the pin write fails — say so precisely. |
| 1322 | if err := pinHost(rt, ver); err != nil { |
| 1323 | die("deploy %s: deployed @ %s but PINNING FAILED (%v) — the next update pass may move it; retry with `lchat update pin %s %s`", host, ver, err, host, ver) |
| 1324 | } |
| 1325 | fmt.Printf("pinned %s to %s (auto-updates paused)\n", host, ver) |
| 1326 | } else if prefs, err := fetchUpdatePrefs(rt.Socket); err == nil && prefs.Pin != "" && prefs.Pin != ver { |
| 1327 | // A stale pin silently freezes the instance on what was just |
| 1328 | // deployed — surface it so the operator decides. |
| 1329 | fmt.Printf("note: %s is still pinned to %s — auto-updates stay paused; `lchat update pin %s %s` to repin here, or `lchat update unpin %s` to resume its channel\n", |
| 1330 | host, prefs.Pin, host, ver, host) |
| 1331 | } |
| 1332 | } |
| 1333 | } |
| 1334 | |
| 1335 | // sweepSkipsPinned decides whether a fleet sweep (`deploy --all --version`) |
| 1336 | // leaves this host alone, returning the reason to print either way. Split |
| 1337 | // failure policies, both deliberate: |
| 1338 | // - a RUNNING host whose prefs can't be fetched is probably crashed — the |
| 1339 | // sweep is how it gets rescued, so deploy anyway (with a note); |
| 1340 | // - a HIBERNATING host whose parked DB can't be read fails CLOSED (skip), |
| 1341 | // matching the auto-updater: its pin may live only in the unreadable |
| 1342 | // copy, and "the one host that must not move, moves" is the failure |
| 1343 | // that matters. It converges once a wake leaves a warm local DB. |
| 1344 | func sweepSkipsPinned(rt *Route, ver string) (skip bool, why string) { |
| 1345 | var prefs updatePrefs |
| 1346 | var err error |
| 1347 | if rt.Hibernate { |
| 1348 | if prefs, err = hibernatingPrefs(rt.Host); err != nil { |
| 1349 | return true, fmt.Sprintf("hibernating and its parked prefs are unreadable (%v) — pin unknown, left alone", err) |
| 1350 | } |
| 1351 | } else if prefs, err = fetchUpdatePrefs(rt.Socket); err != nil { |
| 1352 | return false, fmt.Sprintf("couldn't read update prefs (%v) — deploying anyway", err) |
| 1353 | } |
| 1354 | if prefs.Pin != "" && prefs.Pin != ver { |
| 1355 | return true, "pinned to " + prefs.Pin + " — a canary" |
| 1356 | } |
| 1357 | return false, "" |
| 1358 | } |
| 1359 | |
| 1360 | // deployHostVersion is the one per-host apply dance, shared by `deploy |
| 1361 | // --version` and the auto-updater (updater.go): repoint the live symlink at a |
| 1362 | // staged version, restart, verify the instance actually reports that build, |
| 1363 | // roll back on failure, and record current+previous in the registry. Returns |
| 1364 | // changed=false when the host was already on ver. |
| 1365 | // |
| 1366 | // deployHealthTimeout is a var so tests can shrink the failure path's wait. |
| 1367 | var deployHealthTimeout = 20 * time.Second |
| 1368 | |
| 1369 | func deployHostVersion(reg *Registry, rt *Route, unit, ver string) (changed bool, err error) { |
| 1370 | host := rt.Host |
| 1371 | prev, _ := liveVersion(host) // "" if this instance isn't versioned yet |
| 1372 | if prev == ver { |
| 1373 | return false, nil |
| 1374 | } |
| 1375 | if err := pointLive(host, ver); err != nil { |
| 1376 | return false, err |
| 1377 | } |
| 1378 | if err := systemctl("restart", unit); err != nil { |
| 1379 | rollbackLive(host, prev, unit, rt.Socket) |
| 1380 | return false, fmt.Errorf("restart: %v (rolled back to %q)", err, prev) |
| 1381 | } |
| 1382 | if err := waitHealthyVersion(rt.Socket, ver, deployHealthTimeout); err != nil { |
| 1383 | rollbackLive(host, prev, unit, rt.Socket) |
| 1384 | return false, fmt.Errorf("failed health check (%v), rolled back to %q", err, prev) |
| 1385 | } |
| 1386 | rt.PrevVersion, rt.Version = prev, ver |
| 1387 | // Narrow UPDATE, not a full upsert: rt was read before a possibly-long |
| 1388 | // download, and writing every column back would clobber whatever the |
| 1389 | // activator or an operator changed meanwhile (next_due, hibernate). |
| 1390 | if err := reg.setVersions(host, ver, prev); err != nil { |
| 1391 | return true, fmt.Errorf("record version: %v", err) |
| 1392 | } |
| 1393 | restartBotSidecar(host) |
| 1394 | restartHooksSidecar(host) |
| 1395 | return true, nil |
| 1396 | } |
| 1397 | |
| 1398 | // restartBotSidecar best-effort re-execs a host's bot sidecar so it picks up the |
| 1399 | // same versioned binary the instance now runs — the bot execs /opt/lchat/live/<host> |
| 1400 | // too (deploy/systemd/lchat-bot@.service), so restarting it after the live symlink |
| 1401 | // moves keeps the bot's code in lock-step with the app. `try-restart` is a no-op when |
| 1402 | // no sidecar is active, so instances without a bot are untouched; a bot hiccup only |
| 1403 | // logs and never fails an otherwise-healthy instance deploy. |
| 1404 | func restartBotSidecar(host string) { |
| 1405 | if err := systemctl("try-restart", "lchat-bot@"+host); err != nil { |
| 1406 | log.Printf("deploy: bot sidecar restart for %s failed (instance is healthy on the new version): %v", host, err) |
| 1407 | } |
| 1408 | } |
| 1409 | |
| 1410 | // restartHooksSidecar: same lock-step contract for the webhook-ingest sidecar |
| 1411 | // (deploy/systemd/lchat-hooks@.service execs the live symlink too). |
| 1412 | func restartHooksSidecar(host string) { |
| 1413 | if err := systemctl("try-restart", "lchat-hooks@"+host); err != nil { |
| 1414 | log.Printf("deploy: hooks sidecar restart for %s failed (instance is healthy on the new version): %v", host, err) |
| 1415 | } |
| 1416 | } |
| 1417 | |
| 1418 | // acquireDeployLock serialises everything that runs the repoint/restart/ |
| 1419 | // health-check dance — deploy, rollback, and the timer-fired `update run` — |
| 1420 | // across processes, so an auto-update can never interleave with an operator's |
| 1421 | // deploy mid-swap. Non-blocking: the second caller gets a clear error rather |
| 1422 | // than silently queueing behind an unknown amount of work. |
| 1423 | func acquireDeployLock() (func(), error) { |
| 1424 | path := filepath.Join(optDir, ".deploy.lock") |
| 1425 | f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o644) |
| 1426 | if err != nil { |
| 1427 | return nil, err |
| 1428 | } |
| 1429 | if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { |
| 1430 | f.Close() |
| 1431 | return nil, fmt.Errorf("another deploy/rollback/update is running (%s is held)", path) |
| 1432 | } |
| 1433 | return func() { |
| 1434 | _ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN) |
| 1435 | f.Close() |
| 1436 | }, nil |
| 1437 | } |
| 1438 | |
| 1439 | // rollbackLive best-effort restores an instance to prev after a failed |
| 1440 | // --version deploy: repoint the symlink and restart. If there is no prev (the |
| 1441 | // instance had no version yet) it leaves the failed link in place — the operator |
| 1442 | // must intervene either way, and the die() that follows says so. |
| 1443 | func rollbackLive(host, prev, unit, socket string) { |
| 1444 | if prev == "" { |
| 1445 | return |
| 1446 | } |
| 1447 | if err := pointLive(host, prev); err != nil { |
| 1448 | return |
| 1449 | } |
| 1450 | _ = systemctl("restart", unit) |
| 1451 | _ = waitHealthy(socket, 20*time.Second) |
| 1452 | } |
| 1453 | |
| 1454 | // confirmPublicReady blocks until a freshly-created host answers an HTTPS request |
| 1455 | // through the local router (dialed at httpsAddr with host as the SNI, so the real |
| 1456 | // router route + cert are exercised), or a short deadline passes. It guarantees |
| 1457 | // the setup link works on first click — the visitor reaches a ready admin sign-on |
| 1458 | // rather than a mid-provision error — and warms the per-host TLS cert as a side |
| 1459 | // effect (the handshake triggers autocert issuance). Best-effort: it returns |
| 1460 | // quietly on timeout (the instance already passed its own socket health check, so |
| 1461 | // the public path is a beat behind at most). A var so tests can stub it. See |
| 1462 | // docs/cloud-provisioner.md. |
| 1463 | var confirmPublicReady = func(host, httpsAddr string) { |
| 1464 | client := &http.Client{ |
| 1465 | Timeout: 15 * time.Second, |
| 1466 | Transport: &http.Transport{ |
| 1467 | // Always dial the local router; the SNI/Host is the real host, so the |
| 1468 | // router serves that host's cert and routes to its instance. |
| 1469 | DialTLSContext: func(ctx context.Context, _, _ string) (net.Conn, error) { |
| 1470 | return (&tls.Dialer{Config: &tls.Config{ServerName: host}}).DialContext(ctx, "tcp", httpsAddr) |
| 1471 | }, |
| 1472 | }, |
| 1473 | } |
| 1474 | deadline := time.Now().Add(25 * time.Second) |
| 1475 | for time.Now().Before(deadline) { |
| 1476 | resp, err := client.Get("https://" + host + "/") |
| 1477 | if err == nil { |
| 1478 | resp.Body.Close() |
| 1479 | if publicReadyStatus(resp.StatusCode) { |
| 1480 | return |
| 1481 | } |
| 1482 | } |
| 1483 | time.Sleep(500 * time.Millisecond) |
| 1484 | } |
| 1485 | } |
| 1486 | |
| 1487 | // publicReadyStatus classifies one probe response: anything the instance |
| 1488 | // itself could have answered counts as ready, but 404 is the ROUTER's |
| 1489 | // "unknown host" (an instance's "/" always serves a page) — the route hasn't |
| 1490 | // landed on whichever box owns this host yet (mesh replication in flight). |
| 1491 | // Treating 404 as ready is exactly how a create once handed out a dead space, |
| 1492 | // so it waits like any 5xx. |
| 1493 | func publicReadyStatus(code int) bool { |
| 1494 | return code < 500 && code != http.StatusNotFound |
| 1495 | } |
| 1496 | |
| 1497 | // socketClient returns an HTTP client that dials an instance's unix socket |
| 1498 | // (request URLs use the placeholder host http://lchat). |
| 1499 | func socketClient(socket string) *http.Client { |
| 1500 | return &http.Client{ |
| 1501 | Timeout: 3 * time.Second, |
| 1502 | Transport: &http.Transport{ |
| 1503 | DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { |
| 1504 | return (&net.Dialer{}).DialContext(ctx, "unix", socket) |
| 1505 | }, |
| 1506 | }, |
| 1507 | } |
| 1508 | } |
| 1509 | |
| 1510 | // waitHealthy blocks until the instance answers an HTTP request on its socket, or |
| 1511 | // the timeout elapses. It retries, so it's robust to the restart window (socket |
| 1512 | // briefly absent, or a stale socket left by an abruptly-killed old process) — and |
| 1513 | // a broken new binary that never becomes healthy fails the deploy before `--all` |
| 1514 | // rolls it to the rest of the fleet. |
| 1515 | func waitHealthy(socket string, timeout time.Duration) error { |
| 1516 | deadline := time.Now().Add(timeout) |
| 1517 | client := socketClient(socket) |
| 1518 | var last error |
| 1519 | for time.Now().Before(deadline) { |
| 1520 | resp, err := client.Get("http://lchat/") |
| 1521 | if err == nil { |
| 1522 | resp.Body.Close() |
| 1523 | if resp.StatusCode < 500 { |
| 1524 | return nil |
| 1525 | } |
| 1526 | last = fmt.Errorf("unhealthy: HTTP %d", resp.StatusCode) |
| 1527 | } else { |
| 1528 | last = err |
| 1529 | } |
| 1530 | time.Sleep(150 * time.Millisecond) |
| 1531 | } |
| 1532 | return fmt.Errorf("instance not healthy within %s: %v", timeout, last) |
| 1533 | } |
| 1534 | |
| 1535 | // waitReachable blocks until the backend accepts a connection and returns any HTTP |
| 1536 | // response on its socket, or the timeout elapses. Unlike waitHealthy it accepts any |
| 1537 | // status and does not follow redirects — a sibling service's "/" may 3xx/4xx/5xx yet |
| 1538 | // still be up; only a failure to connect means "not up". |
| 1539 | func waitReachable(socket string, timeout time.Duration) error { |
| 1540 | deadline := time.Now().Add(timeout) |
| 1541 | client := &http.Client{ |
| 1542 | Timeout: 3 * time.Second, |
| 1543 | CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, |
| 1544 | Transport: &http.Transport{ |
| 1545 | DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { |
| 1546 | return (&net.Dialer{}).DialContext(ctx, "unix", socket) |
| 1547 | }, |
| 1548 | }, |
| 1549 | } |
| 1550 | var last error |
| 1551 | for time.Now().Before(deadline) { |
| 1552 | resp, err := client.Get("http://svc/") |
| 1553 | if err == nil { |
| 1554 | resp.Body.Close() |
| 1555 | return nil // any HTTP response means the backend is listening |
| 1556 | } |
| 1557 | last = err |
| 1558 | time.Sleep(150 * time.Millisecond) |
| 1559 | } |
| 1560 | return fmt.Errorf("backend not reachable within %s: %v", timeout, last) |
| 1561 | } |
| 1562 | |
| 1563 | // --- helpers --------------------------------------------------------------- |
| 1564 | |
| 1565 | // systemctl is a var so tests can stand in for systemd (e.g. bind the instance |
| 1566 | // socket themselves when they see "enable --now"); everything else in the |
| 1567 | // creation path — env files, registry, symlinks, health checks — runs for real. |
| 1568 | // systemctlQuery is the read-only twin of systemctl (is-active etc.) — its own |
| 1569 | // seam so tests asserting exact mutating-call sequences aren't polluted by |
| 1570 | // state queries. |
| 1571 | var systemctlQuery = func(args ...string) error { |
| 1572 | return exec.Command("systemctl", args...).Run() |
| 1573 | } |
| 1574 | |
| 1575 | var systemctl = func(args ...string) error { |
| 1576 | out, err := exec.Command("systemctl", args...).CombinedOutput() |
| 1577 | if err != nil { |
| 1578 | return fmt.Errorf("systemctl %s: %v: %s", strings.Join(args, " "), err, strings.TrimSpace(string(out))) |
| 1579 | } |
| 1580 | return nil |
| 1581 | } |
| 1582 | |
| 1583 | // matchOwner chowns path to whatever owns dir — the service user owns /data |
| 1584 | // (the box hardening), so this is "give the file to whoever runs the router" |
| 1585 | // without parsing systemd units for a user name. A no-op when both are |
| 1586 | // already the same (the single-user dev case). |
| 1587 | // litestreamConfigPath is where the box-wide backup replicator's config lives |
| 1588 | // (deploy/fleet-backup.sh writes it); overridable for tests and unusual boxes. |
| 1589 | func litestreamConfigPath() string { |
| 1590 | if p := os.Getenv("LCHAT_LITESTREAM_CONFIG"); p != "" { |
| 1591 | return p |
| 1592 | } |
| 1593 | return "/etc/litestream.yml" |
| 1594 | } |
| 1595 | |
| 1596 | // litestreamConfigTracks reports whether the box-wide litestream config still |
| 1597 | // lists the host's DB. A missing config means no box-wide replicator (dev |
| 1598 | // machines, self-hosters) — not tracked. The check is a plain line match |
| 1599 | // against the exact "path: /data/<host>.db" the generator emits; parsing YAML |
| 1600 | // for one substring would be ceremony. |
| 1601 | func litestreamConfigTracks(host string) bool { |
| 1602 | b, err := os.ReadFile(litestreamConfigPath()) |
| 1603 | if err != nil { |
| 1604 | return false |
| 1605 | } |
| 1606 | return strings.Contains(string(b), "path: "+filepath.Join(instanceDataDir, host+".db")+"\n") |
| 1607 | } |
| 1608 | |
| 1609 | func matchOwner(path, dir string) error { |
| 1610 | st, err := os.Stat(dir) |
| 1611 | if err != nil { |
| 1612 | return err |
| 1613 | } |
| 1614 | sys, ok := st.Sys().(*syscall.Stat_t) |
| 1615 | if !ok { |
| 1616 | return nil // non-unix stat: nothing to match, nothing to fix |
| 1617 | } |
| 1618 | return os.Chown(path, int(sys.Uid), int(sys.Gid)) |
| 1619 | } |
| 1620 | |
| 1621 | // writeEnvFile writes KEY=value lines (sorted for a stable file) at 0600 — it |
| 1622 | // holds the admin token, so it's operator-secret, like the platform env it |
| 1623 | // replaces. |
| 1624 | // instanceEnvMap builds a per-instance EnvironmentFile's contents — the ONE |
| 1625 | // definition, shared by materializeInstance and the mesh route receiver |
| 1626 | // (meshreplicate.go), so a replicated instance wakes with exactly the env its |
| 1627 | // creator wrote. Empty values are omitted, like the unit's "-" prefix. |
| 1628 | func instanceEnvMap(adminToken, appStoreURL, homeOrigin, pushRelayURL, pushRelayToken, entitlement, groupName string) map[string]string { |
| 1629 | env := map[string]string{"LCHAT_ADMIN_TOKEN": adminToken} |
| 1630 | for k, v := range map[string]string{ |
| 1631 | "LCHAT_APP_STORE_URL": appStoreURL, |
| 1632 | "LCHAT_HOME_ORIGIN": homeOrigin, |
| 1633 | "LCHAT_PUSH_RELAY_URL": pushRelayURL, |
| 1634 | "LCHAT_PUSH_RELAY_TOKEN": pushRelayToken, |
| 1635 | "LCHAT_ENTITLEMENT": entitlement, |
| 1636 | "LCHAT_GROUP_NAME": groupName, |
| 1637 | } { |
| 1638 | if v != "" { |
| 1639 | env[k] = v |
| 1640 | } |
| 1641 | } |
| 1642 | return env |
| 1643 | } |
| 1644 | |
| 1645 | func writeEnvFile(path string, kv map[string]string) error { |
| 1646 | keys := make([]string, 0, len(kv)) |
| 1647 | for k := range kv { |
| 1648 | keys = append(keys, k) |
| 1649 | } |
| 1650 | sort.Strings(keys) |
| 1651 | var b strings.Builder |
| 1652 | for _, k := range keys { |
| 1653 | fmt.Fprintf(&b, "%s=%s\n", k, kv[k]) |
| 1654 | } |
| 1655 | return os.WriteFile(path, []byte(b.String()), 0o600) |
| 1656 | } |
| 1657 | |
| 1658 | // readEnvFile parses a KEY=value EnvironmentFile (blank lines and #comments |
| 1659 | // ignored). |
| 1660 | func readEnvFile(path string) (map[string]string, error) { |
| 1661 | f, err := os.Open(path) |
| 1662 | if err != nil { |
| 1663 | return nil, err |
| 1664 | } |
| 1665 | defer f.Close() |
| 1666 | m := map[string]string{} |
| 1667 | sc := bufio.NewScanner(f) |
| 1668 | for sc.Scan() { |
| 1669 | line := strings.TrimSpace(sc.Text()) |
| 1670 | if line == "" || strings.HasPrefix(line, "#") { |
| 1671 | continue |
| 1672 | } |
| 1673 | if k, v, ok := strings.Cut(line, "="); ok { |
| 1674 | m[strings.TrimSpace(k)] = strings.TrimSpace(v) |
| 1675 | } |
| 1676 | } |
| 1677 | return m, sc.Err() |
| 1678 | } |
| 1679 | |
| 1680 | // markInstanceFailed handles a partial add-instance: stop + disable the unit (so it |
| 1681 | // can't crash-loop) and record status=failed + last_error for `lchat instances`, |
| 1682 | // all best-effort. The env file and DB are KEPT — the env file is the only copy of |
| 1683 | // the admin token (the DB stores just its hash), so removing it could strand a |
| 1684 | // still-running instance with an unrecoverable token. add-instance overwrites a |
| 1685 | // non-ready route on retry, so the failed row doesn't block a re-run. |
| 1686 | func markInstanceFailed(reg *Registry, host, unit, reason string) { |
| 1687 | _ = systemctl("disable", "--now", unit) |
| 1688 | if r, err := reg.get(host); err == nil && r != nil { |
| 1689 | r.Status = "failed" |
| 1690 | r.LastError = reason |
| 1691 | _ = reg.upsert(*r) |
| 1692 | } |
| 1693 | } |
| 1694 | |
amadan