| 1 | package main |
| 2 | |
| 3 | import ( |
| 4 | "database/sql" |
| 5 | "flag" |
| 6 | "fmt" |
| 7 | "net" |
| 8 | "net/http" |
| 9 | "os" |
| 10 | "path/filepath" |
| 11 | "reflect" |
| 12 | "strings" |
| 13 | "syscall" |
| 14 | "testing" |
| 15 | "time" |
| 16 | ) |
| 17 | |
| 18 | func TestFlagsAfterPositional(t *testing.T) { |
| 19 | fs := flag.NewFlagSet("t", flag.ContinueOnError) |
| 20 | socket := fs.String("socket", "", "") |
| 21 | all := fs.Bool("all", false, "") |
| 22 | // host BEFORE the flags — the natural order that the stdlib parser drops. |
| 23 | if err := fs.Parse(reorderFlagsFirst(fs, []string{"chat.example.com", "--socket", "/x.sock", "--all"})); err != nil { |
| 24 | t.Fatalf("parse: %v", err) |
| 25 | } |
| 26 | if *socket != "/x.sock" { |
| 27 | t.Errorf("socket = %q, want /x.sock", *socket) |
| 28 | } |
| 29 | if !*all { |
| 30 | t.Error("all = false, want true") |
| 31 | } |
| 32 | if fs.Arg(0) != "chat.example.com" { |
| 33 | t.Errorf("Arg(0) = %q, want chat.example.com", fs.Arg(0)) |
| 34 | } |
| 35 | } |
| 36 | |
| 37 | func TestReorderDoesNotPullFlagAsValue(t *testing.T) { |
| 38 | fs := flag.NewFlagSet("t", flag.ContinueOnError) |
| 39 | fs.String("socket", "", "") |
| 40 | fs.String("router-unit", "", "") |
| 41 | // --socket is given no value (next token is another flag); reorder must not |
| 42 | // swallow --router-unit as --socket's value. |
| 43 | out := reorderFlagsFirst(fs, []string{"host", "--socket", "--router-unit", "foo"}) |
| 44 | if !strings.Contains(strings.Join(out, " "), "--router-unit foo") { |
| 45 | t.Errorf("reorder paired a flag as a value: %v", out) |
| 46 | } |
| 47 | } |
| 48 | |
| 49 | func TestValidHost(t *testing.T) { |
| 50 | ok := []string{"chat.example.com", "rtr-test.example.com", "a.b", "x1.y2.example.com"} |
| 51 | bad := []string{"", "nodots", "-bad.example.com", "bad-.example.com", "has space.com", "Chat.Example.com", "a/b.com", "a..b"} |
| 52 | for _, h := range ok { |
| 53 | if !validHost(h) { |
| 54 | t.Errorf("validHost(%q) = false, want true", h) |
| 55 | } |
| 56 | } |
| 57 | for _, h := range bad { |
| 58 | if validHost(h) { |
| 59 | t.Errorf("validHost(%q) = true, want false", h) |
| 60 | } |
| 61 | } |
| 62 | } |
| 63 | |
| 64 | func TestCheckSocketLen(t *testing.T) { |
| 65 | if err := checkSocketLen("/run/lchat/chat.example.com.sock"); err != nil { |
| 66 | t.Errorf("short path: unexpected error %v", err) |
| 67 | } |
| 68 | long := "/run/lchat/" + strings.Repeat("a", 120) + ".sock" |
| 69 | if err := checkSocketLen(long); err == nil { |
| 70 | t.Error("long path: expected error, got nil") |
| 71 | } |
| 72 | } |
| 73 | |
| 74 | func TestEnvFileRoundTrip(t *testing.T) { |
| 75 | path := filepath.Join(t.TempDir(), "chat.example.com.env") |
| 76 | in := map[string]string{"LCHAT_ADMIN_TOKEN": "tok-123", "LCHAT_HOME_ORIGIN": "https://home.example.com"} |
| 77 | if err := writeEnvFile(path, in); err != nil { |
| 78 | t.Fatalf("write: %v", err) |
| 79 | } |
| 80 | fi, err := os.Stat(path) |
| 81 | if err != nil { |
| 82 | t.Fatal(err) |
| 83 | } |
| 84 | if fi.Mode().Perm() != 0o600 { |
| 85 | t.Errorf("perms = %v, want -rw-------", fi.Mode().Perm()) |
| 86 | } |
| 87 | out, err := readEnvFile(path) |
| 88 | if err != nil { |
| 89 | t.Fatalf("read: %v", err) |
| 90 | } |
| 91 | for k, v := range in { |
| 92 | if out[k] != v { |
| 93 | t.Errorf("readEnvFile[%q] = %q, want %q", k, out[k], v) |
| 94 | } |
| 95 | } |
| 96 | } |
| 97 | |
| 98 | func TestRegistryGet(t *testing.T) { |
| 99 | reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db")) |
| 100 | if err != nil { |
| 101 | t.Fatal(err) |
| 102 | } |
| 103 | if got, err := reg.get("missing.example.com"); err != nil || got != nil { |
| 104 | t.Fatalf("get(missing) = %v, %v; want nil, nil", got, err) |
| 105 | } |
| 106 | if err := reg.upsert(Route{Host: "chat.example.com", Kind: "instance", Socket: "/run/lchat/chat.example.com.sock", Status: "ready"}); err != nil { |
| 107 | t.Fatal(err) |
| 108 | } |
| 109 | got, err := reg.get("chat.example.com") |
| 110 | if err != nil || got == nil { |
| 111 | t.Fatalf("get(existing) = %v, %v", got, err) |
| 112 | } |
| 113 | if got.Socket != "/run/lchat/chat.example.com.sock" || got.Status != "ready" || got.Kind != "instance" { |
| 114 | t.Errorf("get returned %+v", *got) |
| 115 | } |
| 116 | |
| 117 | // delete stops routing the host. |
| 118 | if err := reg.delete("chat.example.com"); err != nil { |
| 119 | t.Fatalf("delete: %v", err) |
| 120 | } |
| 121 | if got, err := reg.get("chat.example.com"); err != nil || got != nil { |
| 122 | t.Fatalf("after delete, get = %v, %v; want nil, nil", got, err) |
| 123 | } |
| 124 | } |
| 125 | |
| 126 | // The register/materialize seam must be pure refactor: running the two named |
| 127 | // steps by hand ends in exactly the state `lchat add-instance` produces. |
| 128 | func TestRegisterThenMaterializeMatchesAddInstance(t *testing.T) { |
| 129 | reg := controlTestEnv(t) |
| 130 | |
| 131 | composed, err := addInstance(reg, addInstanceOpts{Host: "composed.example.com"}) |
| 132 | if err != nil { |
| 133 | t.Fatalf("addInstance: %v", err) |
| 134 | } |
| 135 | |
| 136 | route, token, err := registerInstance(reg, addInstanceOpts{Host: "split.example.com"}) |
| 137 | if err != nil { |
| 138 | t.Fatalf("register: %v", err) |
| 139 | } |
| 140 | // After register alone the space exists but nothing runs: a "creating" row, |
| 141 | // no env file (the token's only persistent home is written by materialize). |
| 142 | if rt, err := reg.get("split.example.com"); err != nil || rt == nil || rt.Status != "creating" { |
| 143 | t.Fatalf("after register, route = %+v, err %v; want status creating", rt, err) |
| 144 | } |
| 145 | if _, err := os.Stat(filepath.Join(instanceDataDir, "split.example.com.env")); !os.IsNotExist(err) { |
| 146 | t.Error("register wrote the env file — that's materialize's job") |
| 147 | } |
| 148 | split, err := materializeInstance(reg, route, token, addInstanceOpts{Host: "split.example.com"}) |
| 149 | if err != nil { |
| 150 | t.Fatalf("materialize: %v", err) |
| 151 | } |
| 152 | |
| 153 | // Both paths yield a ready route with the same host-independent shape… |
| 154 | for _, host := range []string{"composed.example.com", "split.example.com"} { |
| 155 | res := composed |
| 156 | if host == "split.example.com" { |
| 157 | res = split |
| 158 | } |
| 159 | rt, err := reg.get(host) |
| 160 | if err != nil || rt == nil { |
| 161 | t.Fatalf("get(%s): %+v, %v", host, rt, err) |
| 162 | } |
| 163 | if rt.Status != "ready" || rt.Kind != "instance" || rt.Version == "" { |
| 164 | t.Errorf("%s: route = %+v, want ready instance with a version", host, *rt) |
| 165 | } |
| 166 | // …an env file holding the admin token the result reports… |
| 167 | env, err := readEnvFile(filepath.Join(instanceDataDir, host+".env")) |
| 168 | if err != nil { |
| 169 | t.Fatalf("read env %s: %v", host, err) |
| 170 | } |
| 171 | if env["LCHAT_ADMIN_TOKEN"] != res.AdminToken { |
| 172 | t.Errorf("%s: env token != result token", host) |
| 173 | } |
| 174 | // …and a setup link minted against the started instance. |
| 175 | if !strings.HasPrefix(res.SetupLink, "https://"+host+"/join/setup#") { |
| 176 | t.Errorf("%s: setup link = %q", host, res.SetupLink) |
| 177 | } |
| 178 | } |
| 179 | |
| 180 | // Field-for-field, the two routes differ only in what derives from the host |
| 181 | // (paths, sockets) and the create timestamp. |
| 182 | a, _ := reg.get("composed.example.com") |
| 183 | b, _ := reg.get("split.example.com") |
| 184 | for _, rt := range []*Route{a, b} { |
| 185 | rt.Host, rt.Socket, rt.DBPath, rt.CreatedAt = "", "", "", 0 |
| 186 | } |
| 187 | if !reflect.DeepEqual(*a, *b) { |
| 188 | t.Errorf("routes diverge:\n composed %+v\n split %+v", *a, *b) |
| 189 | } |
| 190 | } |
| 191 | |
| 192 | // register() carries the geo/co-op placement seams onto the row, and |
| 193 | // materialize preserves them through its ready upsert. Nothing else may key on |
| 194 | // them yet — this test pins only the plumbing. |
| 195 | // `lchat add-instance --home-region <r>` records the region on the new route |
| 196 | // (multi-edge Phase 1): the flag flows through addInstanceOpts.HomeRegion into |
| 197 | // the registry row the mesh later places by, with surrounding whitespace |
| 198 | // trimmed. Exercises the real CLI entry point (runAddInstance) over the shared |
| 199 | // controlTestEnv fakes. |
| 200 | func TestAddInstanceHomeRegionFlag(t *testing.T) { |
| 201 | _ = controlTestEnv(t) // fakes: systemd (binds the socket), self-stage, dirs, public-ready |
| 202 | regPath := filepath.Join(t.TempDir(), "registry.db") |
| 203 | |
| 204 | runAddInstance([]string{"region.example.com", "--home-region", " ie ", "--registry", regPath}) |
| 205 | |
| 206 | reg, err := openRegistry(regPath) |
| 207 | if err != nil { |
| 208 | t.Fatal(err) |
| 209 | } |
| 210 | rt, err := reg.get("region.example.com") |
| 211 | if err != nil || rt == nil { |
| 212 | t.Fatalf("route = %+v, err %v", rt, err) |
| 213 | } |
| 214 | if rt.HomeRegion != "ie" { |
| 215 | t.Errorf("home_region = %q, want ie (flag value, trimmed)", rt.HomeRegion) |
| 216 | } |
| 217 | } |
| 218 | |
| 219 | func TestRegisterAcceptsRegionProvider(t *testing.T) { |
| 220 | reg := controlTestEnv(t) |
| 221 | |
| 222 | opts := addInstanceOpts{Host: "geo.example.com", HomeRegion: "eu-central-1", Provider: "hetzner"} |
| 223 | route, token, err := registerInstance(reg, opts) |
| 224 | if err != nil { |
| 225 | t.Fatalf("register: %v", err) |
| 226 | } |
| 227 | if rt, _ := reg.get("geo.example.com"); rt == nil || rt.HomeRegion != "eu-central-1" || rt.Provider != "hetzner" { |
| 228 | t.Fatalf("after register, route = %+v, want home_region/provider recorded", rt) |
| 229 | } |
| 230 | if _, err := materializeInstance(reg, route, token, opts); err != nil { |
| 231 | t.Fatalf("materialize: %v", err) |
| 232 | } |
| 233 | rt, _ := reg.get("geo.example.com") |
| 234 | if rt == nil || rt.Status != "ready" || rt.HomeRegion != "eu-central-1" || rt.Provider != "hetzner" { |
| 235 | t.Errorf("after materialize, route = %+v, want placement preserved on the ready row", rt) |
| 236 | } |
| 237 | } |
| 238 | |
| 239 | // A registry created before the geo/co-op columns existed must gain them on |
| 240 | // open, with existing rows reading as the implicit ""/"" placement — live boxes |
| 241 | // carry such DBs, so the migration has to be safe over real data. |
| 242 | func TestRegistryMigratesGeoProviderColumns(t *testing.T) { |
| 243 | path := filepath.Join(t.TempDir(), "reg.db") |
| 244 | |
| 245 | // Build the pre-seam schema by hand and seed a row, as a live box would have. |
| 246 | old, err := sql.Open("sqlite", path) |
| 247 | if err != nil { |
| 248 | t.Fatal(err) |
| 249 | } |
| 250 | if _, err := old.Exec(` |
| 251 | CREATE TABLE routes ( |
| 252 | host TEXT PRIMARY KEY, |
| 253 | kind TEXT NOT NULL, |
| 254 | socket TEXT NOT NULL, |
| 255 | status TEXT NOT NULL DEFAULT 'ready', |
| 256 | last_error TEXT NOT NULL DEFAULT '', |
| 257 | db_path TEXT NOT NULL DEFAULT '', |
| 258 | version TEXT NOT NULL DEFAULT '', |
| 259 | prev_version TEXT NOT NULL DEFAULT '', |
| 260 | app_store_url TEXT NOT NULL DEFAULT '', |
| 261 | home_origin TEXT NOT NULL DEFAULT '', |
| 262 | push_relay_url TEXT NOT NULL DEFAULT '', |
| 263 | push_relay_token TEXT NOT NULL DEFAULT '', |
| 264 | created_at INTEGER NOT NULL |
| 265 | ); |
| 266 | INSERT INTO routes (host, kind, socket, created_at) VALUES ('old.example.com', 'instance', '/run/lchat/old.sock', 1700000000); |
| 267 | `); err != nil { |
| 268 | t.Fatal(err) |
| 269 | } |
| 270 | if err := old.Close(); err != nil { |
| 271 | t.Fatal(err) |
| 272 | } |
| 273 | |
| 274 | reg, err := openRegistry(path) |
| 275 | if err != nil { |
| 276 | t.Fatalf("open over pre-seam registry: %v", err) |
| 277 | } |
| 278 | rt, err := reg.get("old.example.com") |
| 279 | if err != nil || rt == nil { |
| 280 | t.Fatalf("get pre-existing row: %+v, %v", rt, err) |
| 281 | } |
| 282 | if rt.HomeRegion != "" || rt.Provider != "" { |
| 283 | t.Errorf("pre-existing row: home_region=%q provider=%q, want the implicit \"\"/\"\"", rt.HomeRegion, rt.Provider) |
| 284 | } |
| 285 | // The pre-seam schema predates next_due too — the same open must add it. |
| 286 | if rt.NextDue != 0 { |
| 287 | t.Errorf("pre-existing row: next_due=%d, want 0", rt.NextDue) |
| 288 | } |
| 289 | |
| 290 | // New values round-trip through the migrated table. |
| 291 | rt.HomeRegion, rt.Provider = "us-east-1", "aws" |
| 292 | if err := reg.upsert(*rt); err != nil { |
| 293 | t.Fatalf("upsert with placement: %v", err) |
| 294 | } |
| 295 | got, err := reg.get("old.example.com") |
| 296 | if err != nil || got == nil || got.HomeRegion != "us-east-1" || got.Provider != "aws" { |
| 297 | t.Errorf("round-trip = %+v, %v", got, err) |
| 298 | } |
| 299 | |
| 300 | // migrate is idempotent — reopening must not trip on the added columns. |
| 301 | if _, err := openRegistry(path); err != nil { |
| 302 | t.Errorf("reopen after migration: %v", err) |
| 303 | } |
| 304 | } |
| 305 | |
| 306 | // The due-time index's registry half (activator.go's actDueStore): setNextDue |
| 307 | // is a narrow write, dueInstances returns exactly the ready instances whose |
| 308 | // time has arrived, and a route rewrite (deploy's read-modify-write upsert) |
| 309 | // must not clobber a sleeping host's due time. |
| 310 | func TestRegistryDueTimeIndex(t *testing.T) { |
| 311 | reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db")) |
| 312 | if err != nil { |
| 313 | t.Fatal(err) |
| 314 | } |
| 315 | seed := []Route{ |
| 316 | {Host: "sleepy.example.com", Kind: "instance", Socket: "/run/a.sock", Hibernate: true}, |
| 317 | {Host: "quiet.example.com", Kind: "instance", Socket: "/run/b.sock", Hibernate: true}, |
| 318 | {Host: "svc.example.com", Kind: "service", Socket: "/run/c.sock"}, |
| 319 | {Host: "broken.example.com", Kind: "instance", Socket: "/run/d.sock", Status: "failed", Hibernate: true}, |
| 320 | {Host: "manual.example.com", Kind: "instance", Socket: "/run/e.sock"}, // systemd-managed (hibernate off) |
| 321 | } |
| 322 | for _, rt := range seed { |
| 323 | if err := reg.upsert(rt); err != nil { |
| 324 | t.Fatalf("upsert %s: %v", rt.Host, err) |
| 325 | } |
| 326 | } |
| 327 | const now = int64(1_700_000_000) |
| 328 | for host, due := range map[string]int64{ |
| 329 | "sleepy.example.com": now - 5, // due |
| 330 | "svc.example.com": now - 5, // due but a service — never woken |
| 331 | "broken.example.com": now - 5, // due but not ready — never woken |
| 332 | "manual.example.com": now - 5, // due but NOT hibernating — always on, never due-woken |
| 333 | // quiet: next_due stays 0 — nothing scheduled, never due-woken |
| 334 | } { |
| 335 | if err := reg.setNextDue(host, due); err != nil { |
| 336 | t.Fatalf("setNextDue %s: %v", host, err) |
| 337 | } |
| 338 | } |
| 339 | |
| 340 | due, err := reg.dueInstances(now) |
| 341 | if err != nil { |
| 342 | t.Fatalf("dueInstances: %v", err) |
| 343 | } |
| 344 | if len(due) != 1 || due[0].Host != "sleepy.example.com" { |
| 345 | t.Fatalf("dueInstances = %+v, want just sleepy.example.com", due) |
| 346 | } |
| 347 | // Not yet due: a future timestamp stays out of the set. |
| 348 | if err := reg.setNextDue("sleepy.example.com", now+60); err != nil { |
| 349 | t.Fatal(err) |
| 350 | } |
| 351 | if due, _ := reg.dueInstances(now); len(due) != 0 { |
| 352 | t.Fatalf("future due time returned early: %+v", due) |
| 353 | } |
| 354 | |
| 355 | // Read-modify-write (how deploy/rollback update a route) carries next_due. |
| 356 | rt, err := reg.get("sleepy.example.com") |
| 357 | if err != nil || rt == nil { |
| 358 | t.Fatalf("get: %+v, %v", rt, err) |
| 359 | } |
| 360 | rt.Version = "v2" |
| 361 | if err := reg.upsert(*rt); err != nil { |
| 362 | t.Fatalf("upsert rewrite: %v", err) |
| 363 | } |
| 364 | if got, _ := reg.get("sleepy.example.com"); got.NextDue != now+60 { |
| 365 | t.Fatalf("route rewrite clobbered next_due: %d, want %d", got.NextDue, now+60) |
| 366 | } |
| 367 | |
| 368 | // The narrow setters stay orthogonal: the rewrite above carried the |
| 369 | // hibernate flag, and setHibernate must not touch next_due. |
| 370 | if got, _ := reg.get("sleepy.example.com"); !got.Hibernate { |
| 371 | t.Fatal("route rewrite dropped the hibernate flag") |
| 372 | } |
| 373 | if err := reg.setHibernate("sleepy.example.com", false); err != nil { |
| 374 | t.Fatalf("setHibernate: %v", err) |
| 375 | } |
| 376 | if got, _ := reg.get("sleepy.example.com"); got.Hibernate || got.NextDue != now+60 { |
| 377 | t.Fatalf("setHibernate clobbered the route: hibernate=%v next_due=%d", got.Hibernate, got.NextDue) |
| 378 | } |
| 379 | |
| 380 | // An unknown host is a quiet no-op — it can't be woken anyway. |
| 381 | if err := reg.setNextDue("ghost.example.com", now); err != nil { |
| 382 | t.Fatalf("setNextDue unknown host: %v", err) |
| 383 | } |
| 384 | } |
| 385 | |
| 386 | // A registry created before the hibernate column existed must gain it on open, |
| 387 | // with existing rows reading as hibernate=off — the live fleet's registry is |
| 388 | // exactly such a DB, so the migration must default every real instance to the |
| 389 | // classic systemd shape (the whole point of the per-host opt-in). |
| 390 | func TestRegistryMigratesHibernateColumn(t *testing.T) { |
| 391 | path := filepath.Join(t.TempDir(), "reg.db") |
| 392 | |
| 393 | // The schema as of the due-time index — everything current EXCEPT hibernate. |
| 394 | old, err := sql.Open("sqlite", path) |
| 395 | if err != nil { |
| 396 | t.Fatal(err) |
| 397 | } |
| 398 | if _, err := old.Exec(` |
| 399 | CREATE TABLE routes ( |
| 400 | host TEXT PRIMARY KEY, |
| 401 | kind TEXT NOT NULL, |
| 402 | socket TEXT NOT NULL, |
| 403 | status TEXT NOT NULL DEFAULT 'ready', |
| 404 | last_error TEXT NOT NULL DEFAULT '', |
| 405 | db_path TEXT NOT NULL DEFAULT '', |
| 406 | version TEXT NOT NULL DEFAULT '', |
| 407 | prev_version TEXT NOT NULL DEFAULT '', |
| 408 | app_store_url TEXT NOT NULL DEFAULT '', |
| 409 | home_origin TEXT NOT NULL DEFAULT '', |
| 410 | push_relay_url TEXT NOT NULL DEFAULT '', |
| 411 | push_relay_token TEXT NOT NULL DEFAULT '', |
| 412 | home_region TEXT NOT NULL DEFAULT '', |
| 413 | provider TEXT NOT NULL DEFAULT '', |
| 414 | next_due INTEGER NOT NULL DEFAULT 0, |
| 415 | created_at INTEGER NOT NULL |
| 416 | ); |
| 417 | INSERT INTO routes (host, kind, socket, created_at) VALUES ('old.example.com', 'instance', '/run/lchat/old.sock', 1700000000); |
| 418 | `); err != nil { |
| 419 | t.Fatal(err) |
| 420 | } |
| 421 | if err := old.Close(); err != nil { |
| 422 | t.Fatal(err) |
| 423 | } |
| 424 | |
| 425 | reg, err := openRegistry(path) |
| 426 | if err != nil { |
| 427 | t.Fatalf("open over pre-hibernate registry: %v", err) |
| 428 | } |
| 429 | rt, err := reg.get("old.example.com") |
| 430 | if err != nil || rt == nil { |
| 431 | t.Fatalf("get pre-existing row: %+v, %v", rt, err) |
| 432 | } |
| 433 | if rt.Hibernate { |
| 434 | t.Error("pre-existing row: hibernate=true, want the systemd-managed default (false)") |
| 435 | } |
| 436 | |
| 437 | // The flag round-trips through both the narrow setter and a full upsert. |
| 438 | if err := reg.setHibernate("old.example.com", true); err != nil { |
| 439 | t.Fatalf("setHibernate: %v", err) |
| 440 | } |
| 441 | if got, _ := reg.get("old.example.com"); got == nil || !got.Hibernate { |
| 442 | t.Fatalf("after setHibernate, route = %+v, want hibernate on", got) |
| 443 | } |
| 444 | rt, _ = reg.get("old.example.com") |
| 445 | rt.Hibernate = false |
| 446 | if err := reg.upsert(*rt); err != nil { |
| 447 | t.Fatalf("upsert with hibernate: %v", err) |
| 448 | } |
| 449 | if got, _ := reg.get("old.example.com"); got == nil || got.Hibernate { |
| 450 | t.Fatalf("after upsert, route = %+v, want hibernate off", got) |
| 451 | } |
| 452 | |
| 453 | // migrate is idempotent — reopening must not trip on the added column. |
| 454 | if _, err := openRegistry(path); err != nil { |
| 455 | t.Errorf("reopen after migration: %v", err) |
| 456 | } |
| 457 | } |
| 458 | |
| 459 | // A registry created before the replication column existed must gain it on |
| 460 | // open, with existing rows reading as "" — the stream class, i.e. exactly the |
| 461 | // pre-ladder behavior, so a deploy over the live fleet's registry changes no |
| 462 | // host's durability until an operator (or billing) says so. |
| 463 | func TestRegistryMigratesReplicationColumn(t *testing.T) { |
| 464 | path := filepath.Join(t.TempDir(), "reg.db") |
| 465 | |
| 466 | // The schema as of the hibernate flag — everything current EXCEPT replication. |
| 467 | old, err := sql.Open("sqlite", path) |
| 468 | if err != nil { |
| 469 | t.Fatal(err) |
| 470 | } |
| 471 | if _, err := old.Exec(` |
| 472 | CREATE TABLE routes ( |
| 473 | host TEXT PRIMARY KEY, |
| 474 | kind TEXT NOT NULL, |
| 475 | socket TEXT NOT NULL, |
| 476 | status TEXT NOT NULL DEFAULT 'ready', |
| 477 | last_error TEXT NOT NULL DEFAULT '', |
| 478 | db_path TEXT NOT NULL DEFAULT '', |
| 479 | version TEXT NOT NULL DEFAULT '', |
| 480 | prev_version TEXT NOT NULL DEFAULT '', |
| 481 | app_store_url TEXT NOT NULL DEFAULT '', |
| 482 | home_origin TEXT NOT NULL DEFAULT '', |
| 483 | push_relay_url TEXT NOT NULL DEFAULT '', |
| 484 | push_relay_token TEXT NOT NULL DEFAULT '', |
| 485 | home_region TEXT NOT NULL DEFAULT '', |
| 486 | provider TEXT NOT NULL DEFAULT '', |
| 487 | hibernate INTEGER NOT NULL DEFAULT 0, |
| 488 | next_due INTEGER NOT NULL DEFAULT 0, |
| 489 | created_at INTEGER NOT NULL |
| 490 | ); |
| 491 | INSERT INTO routes (host, kind, socket, hibernate, created_at) VALUES ('old.example.com', 'instance', '/run/lchat/old.sock', 1, 1700000000); |
| 492 | `); err != nil { |
| 493 | t.Fatal(err) |
| 494 | } |
| 495 | if err := old.Close(); err != nil { |
| 496 | t.Fatal(err) |
| 497 | } |
| 498 | |
| 499 | reg, err := openRegistry(path) |
| 500 | if err != nil { |
| 501 | t.Fatalf("open over pre-replication registry: %v", err) |
| 502 | } |
| 503 | rt, err := reg.get("old.example.com") |
| 504 | if err != nil || rt == nil { |
| 505 | t.Fatalf("get pre-existing row: %+v, %v", rt, err) |
| 506 | } |
| 507 | if rt.Replication != "" { |
| 508 | t.Errorf("pre-existing row: replication=%q, want \"\" (the stream default)", rt.Replication) |
| 509 | } |
| 510 | |
| 511 | // The class round-trips through the narrow setter, an upsert carries it, |
| 512 | // and setReplication stays orthogonal to the rest of the route. |
| 513 | if err := reg.setReplication("old.example.com", replicationDaily); err != nil { |
| 514 | t.Fatalf("setReplication: %v", err) |
| 515 | } |
| 516 | if got, _ := reg.get("old.example.com"); got == nil || got.Replication != replicationDaily || !got.Hibernate { |
| 517 | t.Fatalf("after setReplication, route = %+v, want daily + hibernate intact", got) |
| 518 | } |
| 519 | rt, _ = reg.get("old.example.com") |
| 520 | rt.Version = "v2" |
| 521 | if err := reg.upsert(*rt); err != nil { |
| 522 | t.Fatalf("upsert rewrite: %v", err) |
| 523 | } |
| 524 | if got, _ := reg.get("old.example.com"); got == nil || got.Replication != replicationDaily { |
| 525 | t.Fatalf("route rewrite dropped the class: %+v", got) |
| 526 | } |
| 527 | |
| 528 | // Garbage never lands: no write path may store a class the wake path would |
| 529 | // silently read as stream. |
| 530 | if err := reg.setReplication("old.example.com", "hourly"); err == nil { |
| 531 | t.Error("setReplication accepted an unknown class") |
| 532 | } |
| 533 | if got, _ := reg.get("old.example.com"); got == nil || got.Replication != replicationDaily { |
| 534 | t.Fatalf("rejected set still changed the row: %+v", got) |
| 535 | } |
| 536 | |
| 537 | // An unknown host is a quiet no-op, matching setHibernate/setNextDue. |
| 538 | if err := reg.setReplication("ghost.example.com", replicationBatched); err != nil { |
| 539 | t.Fatalf("setReplication unknown host: %v", err) |
| 540 | } |
| 541 | |
| 542 | // migrate is idempotent — reopening must not trip on the added column. |
| 543 | if _, err := openRegistry(path); err != nil { |
| 544 | t.Errorf("reopen after migration: %v", err) |
| 545 | } |
| 546 | } |
| 547 | |
| 548 | // The class→interval mapping is the ladder's contract: stream (and the unset |
| 549 | // "" every pre-ladder registry row reads as) keeps litestream's default, and |
| 550 | // the coarser classes stretch it — never the other way around. |
| 551 | func TestReplicationSyncInterval(t *testing.T) { |
| 552 | for class, want := range map[string]time.Duration{ |
| 553 | "": 0, |
| 554 | replicationStream: 0, |
| 555 | replicationBatched: 5 * time.Minute, |
| 556 | replicationDaily: 24 * time.Hour, |
| 557 | } { |
| 558 | if got := replicationSyncInterval(class); got != want { |
| 559 | t.Errorf("replicationSyncInterval(%q) = %v, want %v", class, got, want) |
| 560 | } |
| 561 | } |
| 562 | for _, ok := range []string{"", "stream", "batched", "daily"} { |
| 563 | if !validReplication(ok) { |
| 564 | t.Errorf("validReplication(%q) = false, want true", ok) |
| 565 | } |
| 566 | } |
| 567 | for _, bad := range []string{"hourly", "Stream", "off"} { |
| 568 | if validReplication(bad) { |
| 569 | t.Errorf("validReplication(%q) = true, want false", bad) |
| 570 | } |
| 571 | } |
| 572 | } |
| 573 | |
| 574 | // `add-instance --hibernate` produces a registered, unmaterialized-but- |
| 575 | // spawnable route: everything the activator's spawn needs at first wake (the |
| 576 | // env file with the admin token, a staged binary behind the live symlink, a |
| 577 | // ready+flagged route with the router reloaded) and nothing systemd (no unit |
| 578 | // enabled, no add-time health check, no DB — the first wake boots fresh). The |
| 579 | // public-readiness probe doubles as that first wake — simulated here by |
| 580 | // binding the socket when it fires — and the setup-invite mint rides the |
| 581 | // woken socket, so the space is claimable the moment add-instance returns. |
| 582 | func TestAddInstanceHibernate(t *testing.T) { |
| 583 | reg := controlTestEnv(t) |
| 584 | const host = "hib.example.com" |
| 585 | |
| 586 | // Replace the harness's systemd fake with a pure recorder: the hibernating |
| 587 | // path must never enable a unit, so nothing here binds the socket for it. |
| 588 | var calls []string |
| 589 | systemctl = func(args ...string) error { |
| 590 | calls = append(calls, strings.Join(args, " ")) |
| 591 | return nil |
| 592 | } |
| 593 | // Play the router+activator: the readiness probe is the first request, and |
| 594 | // the wake it triggers binds the instance socket — answering the health |
| 595 | // check and the invite mint like a freshly spawned child would. |
| 596 | confirmPublicReady = func(h, _ string) { |
| 597 | calls = append(calls, "confirm "+h) |
| 598 | ln, err := net.Listen("unix", filepath.Join(instanceSocketDir, h+".sock")) |
| 599 | if err != nil { |
| 600 | t.Errorf("bind woken socket: %v", err) |
| 601 | return |
| 602 | } |
| 603 | mux := http.NewServeMux() |
| 604 | mux.HandleFunc("GET /", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(200) }) |
| 605 | mux.HandleFunc("POST /api/admin/key-invites", func(w http.ResponseWriter, r *http.Request) { |
| 606 | writeJSON(w, 200, map[string]any{"ok": true}) |
| 607 | }) |
| 608 | srv := &http.Server{Handler: mux} |
| 609 | go srv.Serve(ln) |
| 610 | t.Cleanup(func() { srv.Close() }) |
| 611 | } |
| 612 | |
| 613 | res, err := addInstance(reg, addInstanceOpts{Host: host, Hibernate: true}) |
| 614 | if err != nil { |
| 615 | t.Fatalf("addInstance --hibernate: %v", err) |
| 616 | } |
| 617 | |
| 618 | // Registered: ready, flagged, under version management. |
| 619 | rt, err := reg.get(host) |
| 620 | if err != nil || rt == nil { |
| 621 | t.Fatalf("route = %+v, %v", rt, err) |
| 622 | } |
| 623 | if rt.Status != "ready" || !rt.Hibernate || rt.Version == "" { |
| 624 | t.Errorf("route = %+v, want a ready hibernating instance with a version", *rt) |
| 625 | } |
| 626 | |
| 627 | // Spawnable: the env file holds the admin token (the spawn's env layering |
| 628 | // and every saved admin link depend on it)… |
| 629 | env, err := readEnvFile(filepath.Join(instanceDataDir, host+".env")) |
| 630 | if err != nil { |
| 631 | t.Fatalf("read env: %v", err) |
| 632 | } |
| 633 | if env["LCHAT_ADMIN_TOKEN"] != res.AdminToken { |
| 634 | t.Error("env token != result token") |
| 635 | } |
| 636 | // …and the live symlink resolves to a staged binary (the spawn execs it). |
| 637 | if ver, err := liveVersion(host); err != nil || ver == "" { |
| 638 | t.Errorf("live symlink not staged: ver=%q err=%v", ver, err) |
| 639 | } |
| 640 | |
| 641 | // Unmaterialized: no unit, no DB; the router WAS reloaded (the route must |
| 642 | // serve before any request can wake it) and only then probed. |
| 643 | reloadIdx, confirmIdx := -1, -1 |
| 644 | for i, c := range calls { |
| 645 | if strings.HasPrefix(c, "enable ") { |
| 646 | t.Errorf("a systemd unit was touched: %q", c) |
| 647 | } |
| 648 | if c == "reload lchat-router" { |
| 649 | reloadIdx = i |
| 650 | } |
| 651 | if c == "confirm "+host { |
| 652 | confirmIdx = i |
| 653 | } |
| 654 | } |
| 655 | if reloadIdx == -1 || confirmIdx == -1 || confirmIdx < reloadIdx { |
| 656 | t.Errorf("want router reload before the wake probe, got calls %v", calls) |
| 657 | } |
| 658 | if _, err := os.Stat(filepath.Join(instanceDataDir, host+".db")); !os.IsNotExist(err) { |
| 659 | t.Error("a DB was created — the first wake should boot fresh") |
| 660 | } |
| 661 | |
| 662 | // Claimable: the setup link was minted through the woken socket. |
| 663 | if !strings.HasPrefix(res.SetupLink, "https://"+host+"/join/setup#") { |
| 664 | t.Errorf("setup link = %q (err %v)", res.SetupLink, res.SetupErr) |
| 665 | } |
| 666 | } |
| 667 | |
| 668 | // The flip-on ordering (hibernateOn): flag first — it's inert until a reload — |
| 669 | // then the unit stops BEFORE the router reloads (a wake must be able to bind |
| 670 | // the socket), then the warm request closes the gap. A crash at any seam |
| 671 | // leaves the host serveable. |
| 672 | func TestHibernateOnOrdering(t *testing.T) { |
| 673 | reg := controlTestEnv(t) |
| 674 | const host = "flip-on.example.com" |
| 675 | if _, err := addInstance(reg, addInstanceOpts{Host: host}); err != nil { |
| 676 | t.Fatalf("addInstance: %v", err) |
| 677 | } |
| 678 | |
| 679 | var calls []string |
| 680 | flagAtDisable, flagAtReload := false, false |
| 681 | systemctl = func(args ...string) error { |
| 682 | call := strings.Join(args, " ") |
| 683 | calls = append(calls, call) |
| 684 | rt, _ := reg.get(host) |
| 685 | switch call { |
| 686 | case "disable --now lchat@" + host: |
| 687 | flagAtDisable = rt.Hibernate |
| 688 | case "reload lchat-router": |
| 689 | flagAtReload = rt.Hibernate |
| 690 | } |
| 691 | return nil |
| 692 | } |
| 693 | confirmPublicReady = func(h, _ string) { calls = append(calls, "confirm "+h) } |
| 694 | |
| 695 | rt, _ := reg.get(host) |
| 696 | if err := hibernateOn(reg, rt, "lchat@", "lchat-bot@", "lchat-router"); err != nil { |
| 697 | t.Fatalf("hibernateOn: %v", err) |
| 698 | } |
| 699 | |
| 700 | want := []string{"disable --now lchat@" + host, "reload lchat-router", "confirm " + host} |
| 701 | if !reflect.DeepEqual(calls, want) { |
| 702 | t.Errorf("flip-on sequence = %v, want %v", calls, want) |
| 703 | } |
| 704 | if !flagAtDisable || !flagAtReload { |
| 705 | t.Errorf("flag not set before unit stop/reload (disable=%v reload=%v) — a crash mid-flip would strand the ordering", flagAtDisable, flagAtReload) |
| 706 | } |
| 707 | if got, _ := reg.get(host); got == nil || !got.Hibernate { |
| 708 | t.Error("flag not set after flip-on") |
| 709 | } |
| 710 | } |
| 711 | |
| 712 | // The flip-off ordering (hibernateOff): flag cleared, router reloaded (which |
| 713 | // makes the activator release the host), and the unit started ONLY after the |
| 714 | // activator's child has let the socket go — verified by a fake child that |
| 715 | // holds the socket until the reload lands, releasing it a beat later like the |
| 716 | // real (asynchronous) reconcile. |
| 717 | func TestHibernateOffOrdering(t *testing.T) { |
| 718 | reg := controlTestEnv(t) |
| 719 | const host = "flip-off.example.com" |
| 720 | sock := filepath.Join(instanceSocketDir, host+".sock") |
| 721 | if err := reg.upsert(Route{Host: host, Kind: "instance", Socket: sock, Status: "ready", Hibernate: true}); err != nil { |
| 722 | t.Fatal(err) |
| 723 | } |
| 724 | |
| 725 | // The fake activator child, awake and holding the socket. |
| 726 | child, err := net.Listen("unix", sock) |
| 727 | if err != nil { |
| 728 | t.Fatal(err) |
| 729 | } |
| 730 | childSrv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(200) })} |
| 731 | go childSrv.Serve(child) |
| 732 | t.Cleanup(func() { childSrv.Close() }) |
| 733 | |
| 734 | var calls []string |
| 735 | systemctl = func(args ...string) error { |
| 736 | call := strings.Join(args, " ") |
| 737 | calls = append(calls, call) |
| 738 | switch { |
| 739 | case call == "reload lchat-router": |
| 740 | // The reconcile's release is asynchronous inside the router |
| 741 | // process — model that: the child lets go a beat after the reload. |
| 742 | go func() { |
| 743 | time.Sleep(300 * time.Millisecond) |
| 744 | childSrv.Close() |
| 745 | }() |
| 746 | case call == "enable --now lchat@"+host: |
| 747 | // The ordering crux: by unit-start time the child must be gone… |
| 748 | if c, err := net.DialTimeout("unix", sock, 100*time.Millisecond); err == nil { |
| 749 | c.Close() |
| 750 | t.Error("unit started while the activator's child still held the socket") |
| 751 | } |
| 752 | // …and the flag off, so systemd unambiguously owns the host. |
| 753 | if rt, _ := reg.get(host); rt.Hibernate { |
| 754 | t.Error("unit started with the hibernate flag still set") |
| 755 | } |
| 756 | // Bind like a started unit would (clearing the stale socket file |
| 757 | // first, as the instance's listen() does), so waitHealthy passes. |
| 758 | _ = os.Remove(sock) |
| 759 | ln, err := net.Listen("unix", sock) |
| 760 | if err != nil { |
| 761 | return err |
| 762 | } |
| 763 | srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(200) })} |
| 764 | go srv.Serve(ln) |
| 765 | t.Cleanup(func() { srv.Close() }) |
| 766 | } |
| 767 | return nil |
| 768 | } |
| 769 | |
| 770 | rt, _ := reg.get(host) |
| 771 | if err := hibernateOff(reg, rt, "lchat@", "lchat-bot@", "lchat-router"); err != nil { |
| 772 | t.Fatalf("hibernateOff: %v", err) |
| 773 | } |
| 774 | want := []string{"reload lchat-router", "enable --now lchat@" + host} |
| 775 | if !reflect.DeepEqual(calls, want) { |
| 776 | t.Errorf("flip-off sequence = %v, want %v", calls, want) |
| 777 | } |
| 778 | if got, _ := reg.get(host); got == nil || got.Hibernate { |
| 779 | t.Error("flag still set after flip-off") |
| 780 | } |
| 781 | } |
| 782 | |
| 783 | // A host with a bot sidecar staged (its -bot.env exists) hands the sidecar |
| 784 | // over at the hibernate flips: --on disables the always-on unit BEFORE the |
| 785 | // router reload (the activator co-spawns from then on; a systemd sidecar and |
| 786 | // a co-spawned one would double every bot reply), and --off re-enables it |
| 787 | // LAST — only after the instance unit is healthy, add-bot's "instance first" |
| 788 | // bootstrap order. A host without one (the two ordering tests above) never |
| 789 | // touches the bot unit. |
| 790 | func TestHibernateFlipHandsSidecarUnitOver(t *testing.T) { |
| 791 | reg := controlTestEnv(t) |
| 792 | const host = "flip-bots.example.com" |
| 793 | sock := filepath.Join(instanceSocketDir, host+".sock") |
| 794 | if err := reg.upsert(Route{Host: host, Kind: "instance", Socket: sock, Status: "ready"}); err != nil { |
| 795 | t.Fatal(err) |
| 796 | } |
| 797 | // The instance env (hibernateOn chowns it) and the sidecar staging — |
| 798 | // exactly what add-instance + add-bot leave on disk. |
| 799 | if err := os.WriteFile(filepath.Join(instanceDataDir, host+".env"), []byte("LCHAT_ADMIN_TOKEN=t\n"), 0o600); err != nil { |
| 800 | t.Fatal(err) |
| 801 | } |
| 802 | if err := os.WriteFile(botEnvPath(host), []byte("LCHAT_BASE=https://"+host+"\n"), 0o600); err != nil { |
| 803 | t.Fatal(err) |
| 804 | } |
| 805 | if err := os.MkdirAll(botSessionDir(host), 0o750); err != nil { |
| 806 | t.Fatal(err) |
| 807 | } |
| 808 | |
| 809 | var calls []string |
| 810 | systemctl = func(args ...string) error { |
| 811 | call := strings.Join(args, " ") |
| 812 | calls = append(calls, call) |
| 813 | if call == "enable --now lchat@"+host { |
| 814 | // Bind like a started unit so hibernateOff's health check passes. |
| 815 | ln, err := net.Listen("unix", sock) |
| 816 | if err != nil { |
| 817 | return err |
| 818 | } |
| 819 | srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(200) })} |
| 820 | go srv.Serve(ln) |
| 821 | t.Cleanup(func() { srv.Close() }) |
| 822 | } |
| 823 | return nil |
| 824 | } |
| 825 | |
| 826 | rt, _ := reg.get(host) |
| 827 | if err := hibernateOn(reg, rt, "lchat@", "lchat-bot@", "lchat-router"); err != nil { |
| 828 | t.Fatalf("hibernateOn: %v", err) |
| 829 | } |
| 830 | wantOn := []string{"disable --now lchat@" + host, "disable --now lchat-bot@" + host, "reload lchat-router"} |
| 831 | if !reflect.DeepEqual(calls, wantOn) { |
| 832 | t.Errorf("flip-on sequence = %v, want %v", calls, wantOn) |
| 833 | } |
| 834 | |
| 835 | calls = nil |
| 836 | rt, _ = reg.get(host) |
| 837 | if err := hibernateOff(reg, rt, "lchat@", "lchat-bot@", "lchat-router"); err != nil { |
| 838 | t.Fatalf("hibernateOff: %v", err) |
| 839 | } |
| 840 | wantOff := []string{"reload lchat-router", "enable --now lchat@" + host, "enable --now lchat-bot@" + host} |
| 841 | if !reflect.DeepEqual(calls, wantOff) { |
| 842 | t.Errorf("flip-off sequence = %v, want %v", calls, wantOff) |
| 843 | } |
| 844 | } |
| 845 | |
| 846 | // waitSocketQuiet is the flip-off's release observation: quiet means no |
| 847 | // listener (dial failure), whether or not the socket file lingers. |
| 848 | func TestWaitSocketQuiet(t *testing.T) { |
| 849 | sock := tmpSock(t) |
| 850 | // No listener: quiet immediately. |
| 851 | if err := waitSocketQuiet(sock, time.Second); err != nil { |
| 852 | t.Errorf("no listener: %v, want quiet", err) |
| 853 | } |
| 854 | // A held listener: not quiet — the bounded wait must report it. |
| 855 | ln, err := net.Listen("unix", sock) |
| 856 | if err != nil { |
| 857 | t.Fatal(err) |
| 858 | } |
| 859 | defer ln.Close() |
| 860 | if err := waitSocketQuiet(sock, 400*time.Millisecond); err == nil { |
| 861 | t.Error("live listener reported quiet — the flip-off would start a unit against a live child") |
| 862 | } |
| 863 | } |
| 864 | |
| 865 | // matchOwner hands a root-written env file to the service user (the owner of |
| 866 | // /data) so the router's activator can read it at wake time — the bug the |
| 867 | // production canary's first wake hit. Unprivileged test: same-owner is a |
| 868 | // clean no-op and the owners really do match afterwards. |
| 869 | func TestMatchOwner(t *testing.T) { |
| 870 | dir := t.TempDir() |
| 871 | path := filepath.Join(dir, "x.env") |
| 872 | if err := os.WriteFile(path, []byte("A=b\n"), 0o600); err != nil { |
| 873 | t.Fatal(err) |
| 874 | } |
| 875 | if err := matchOwner(path, dir); err != nil { |
| 876 | t.Fatalf("matchOwner: %v", err) |
| 877 | } |
| 878 | ds, _ := os.Stat(dir) |
| 879 | fs, _ := os.Stat(path) |
| 880 | if ds.Sys().(*syscall.Stat_t).Uid != fs.Sys().(*syscall.Stat_t).Uid { |
| 881 | t.Fatalf("owner mismatch after matchOwner") |
| 882 | } |
| 883 | } |
| 884 | |
| 885 | // The flip-window guard: while the box-wide litestream config still tracks a |
| 886 | // host's DB, flipping it on must stop before touching the unit — two |
| 887 | // replicators share one meta dir and corrupt it (hit live on the first real |
| 888 | // flip). The flag must already be SET when the guard fires, because the |
| 889 | // operator's fix (re-running the backup script, which excludes hibernate=1 |
| 890 | // hosts) depends on it — and the flag is inert until the reload the guard |
| 891 | // prevents. |
| 892 | func TestHibernateOnRefusesWhileBoxLitestreamTracks(t *testing.T) { |
| 893 | reg := controlTestEnv(t) |
| 894 | const host = "flip-guard.example.com" |
| 895 | if _, err := addInstance(reg, addInstanceOpts{Host: host}); err != nil { |
| 896 | t.Fatalf("addInstance: %v", err) |
| 897 | } |
| 898 | |
| 899 | conf := filepath.Join(t.TempDir(), "litestream.yml") |
| 900 | body := "dbs:\n - path: " + filepath.Join(instanceDataDir, host+".db") + "\n" |
| 901 | if err := os.WriteFile(conf, []byte(body), 0o644); err != nil { |
| 902 | t.Fatal(err) |
| 903 | } |
| 904 | t.Setenv("LCHAT_LITESTREAM_CONFIG", conf) |
| 905 | |
| 906 | var calls []string |
| 907 | systemctl = func(args ...string) error { |
| 908 | calls = append(calls, strings.Join(args, " ")) |
| 909 | return nil |
| 910 | } |
| 911 | confirmPublicReady = func(string, string) { t.Fatal("must not warm while guarded") } |
| 912 | |
| 913 | rt, _ := reg.get(host) |
| 914 | err := hibernateOn(reg, rt, "lchat@", "lchat-bot@", "lchat-router") |
| 915 | if err == nil || !strings.Contains(err.Error(), "fleet-backup.sh") { |
| 916 | t.Fatalf("want the re-run-fleet-backup refusal, got %v", err) |
| 917 | } |
| 918 | if len(calls) != 0 { |
| 919 | t.Fatalf("guarded flip touched systemd: %v", calls) |
| 920 | } |
| 921 | if got, _ := reg.get(host); !got.Hibernate { |
| 922 | t.Fatalf("flag must be set when the guard fires (the operator's fix depends on it)") |
| 923 | } |
| 924 | |
| 925 | // The exact two-step flow the error prescribes: the config regen drops the |
| 926 | // host (the flag now reads true), and the re-run flip proceeds. |
| 927 | if err := os.WriteFile(conf, []byte("dbs: []\n"), 0o644); err != nil { |
| 928 | t.Fatal(err) |
| 929 | } |
| 930 | confirmPublicReady = func(string, string) {} |
| 931 | rt, _ = reg.get(host) |
| 932 | if err := hibernateOn(reg, rt, "lchat@", "lchat-bot@", "lchat-router"); err != nil { |
| 933 | t.Fatalf("re-run flip after exclusion: %v", err) |
| 934 | } |
| 935 | if len(calls) == 0 || calls[0] != "disable --now lchat@"+host { |
| 936 | t.Fatalf("re-run flip sequence = %v", calls) |
| 937 | } |
| 938 | } |
| 939 | |
| 940 | // The litestream guard's two-step leaves flag-set-but-unit-running behind; the |
| 941 | // promised re-run must FINISH the handover, not early-exit on the flag (hit |
| 942 | // live on the first bulk flip: "already hibernating" with the unit still up). |
| 943 | func TestHibernateOnRerunFinishesHandover(t *testing.T) { |
| 944 | reg := controlTestEnv(t) |
| 945 | const host = "flip-rerun.example.com" |
| 946 | if _, err := addInstance(reg, addInstanceOpts{Host: host}); err != nil { |
| 947 | t.Fatal(err) |
| 948 | } |
| 949 | if err := reg.setHibernate(host, true); err != nil { // the guard's leftover state |
| 950 | t.Fatal(err) |
| 951 | } |
| 952 | oldQ := systemctlQuery |
| 953 | systemctlQuery = func(args ...string) error { return nil } // unit reads active |
| 954 | defer func() { systemctlQuery = oldQ }() |
| 955 | var calls []string |
| 956 | systemctl = func(args ...string) error { calls = append(calls, strings.Join(args, " ")); return nil } |
| 957 | confirmPublicReady = func(string, string) { calls = append(calls, "confirm") } |
| 958 | |
| 959 | rt, _ := reg.get(host) |
| 960 | if err := hibernateOn(reg, rt, "lchat@", "lchat-bot@", "lchat-router"); err != nil { |
| 961 | t.Fatalf("re-run: %v", err) |
| 962 | } |
| 963 | if len(calls) == 0 || calls[0] != "disable --now lchat@"+host { |
| 964 | t.Fatalf("re-run must stop the unit first, got %v", calls) |
| 965 | } |
| 966 | } |
| 967 | |
| 968 | func TestRegistryMigratesCanaryColumn(t *testing.T) { |
| 969 | path := filepath.Join(t.TempDir(), "reg.db") |
| 970 | |
| 971 | // The schema as of the replication column — everything current EXCEPT canary. |
| 972 | old, err := sql.Open("sqlite", path) |
| 973 | if err != nil { |
| 974 | t.Fatal(err) |
| 975 | } |
| 976 | if _, err := old.Exec(` |
| 977 | CREATE TABLE routes ( |
| 978 | host TEXT PRIMARY KEY, |
| 979 | kind TEXT NOT NULL, |
| 980 | socket TEXT NOT NULL, |
| 981 | status TEXT NOT NULL DEFAULT 'ready', |
| 982 | last_error TEXT NOT NULL DEFAULT '', |
| 983 | db_path TEXT NOT NULL DEFAULT '', |
| 984 | version TEXT NOT NULL DEFAULT '', |
| 985 | prev_version TEXT NOT NULL DEFAULT '', |
| 986 | app_store_url TEXT NOT NULL DEFAULT '', |
| 987 | home_origin TEXT NOT NULL DEFAULT '', |
| 988 | push_relay_url TEXT NOT NULL DEFAULT '', |
| 989 | push_relay_token TEXT NOT NULL DEFAULT '', |
| 990 | home_region TEXT NOT NULL DEFAULT '', |
| 991 | provider TEXT NOT NULL DEFAULT '', |
| 992 | hibernate INTEGER NOT NULL DEFAULT 0, |
| 993 | replication TEXT NOT NULL DEFAULT '', |
| 994 | next_due INTEGER NOT NULL DEFAULT 0, |
| 995 | created_at INTEGER NOT NULL |
| 996 | ); |
| 997 | INSERT INTO routes (host, kind, socket, hibernate, created_at) VALUES ('old.example.com', 'instance', '/run/lchat/old.sock', 1, 1700000000); |
| 998 | `); err != nil { |
| 999 | t.Fatal(err) |
| 1000 | } |
| 1001 | if err := old.Close(); err != nil { |
| 1002 | t.Fatal(err) |
| 1003 | } |
| 1004 | |
| 1005 | reg, err := openRegistry(path) |
| 1006 | if err != nil { |
| 1007 | t.Fatalf("open over pre-canary registry: %v", err) |
| 1008 | } |
| 1009 | // Existing rows read as canary=false — a live instance is never a throwaway |
| 1010 | // by default; the flag is only ever set deliberately. |
| 1011 | rt, err := reg.get("old.example.com") |
| 1012 | if err != nil || rt == nil { |
| 1013 | t.Fatalf("get pre-existing row: %+v, %v", rt, err) |
| 1014 | } |
| 1015 | if rt.Canary { |
| 1016 | t.Error("pre-existing row: canary=true, want the never-inferred default (false)") |
| 1017 | } |
| 1018 | |
| 1019 | // The flag round-trips through the narrow setter, an upsert carries it, and |
| 1020 | // setCanary stays orthogonal to the rest of the route (hibernate intact). |
| 1021 | if err := reg.setCanary("old.example.com", true); err != nil { |
| 1022 | t.Fatalf("setCanary: %v", err) |
| 1023 | } |
| 1024 | if got, _ := reg.get("old.example.com"); got == nil || !got.Canary || !got.Hibernate { |
| 1025 | t.Fatalf("after setCanary, route = %+v, want canary + hibernate intact", got) |
| 1026 | } |
| 1027 | rt, _ = reg.get("old.example.com") |
| 1028 | rt.Version = "v2" |
| 1029 | if err := reg.upsert(*rt); err != nil { |
| 1030 | t.Fatalf("upsert rewrite: %v", err) |
| 1031 | } |
| 1032 | if got, _ := reg.get("old.example.com"); got == nil || !got.Canary { |
| 1033 | t.Fatalf("route rewrite dropped the canary flag: %+v", got) |
| 1034 | } |
| 1035 | // Clearing it round-trips too, and an unknown host is a quiet no-op. |
| 1036 | if err := reg.setCanary("old.example.com", false); err != nil { |
| 1037 | t.Fatalf("setCanary off: %v", err) |
| 1038 | } |
| 1039 | if got, _ := reg.get("old.example.com"); got == nil || got.Canary { |
| 1040 | t.Fatalf("setCanary off left it flagged: %+v", got) |
| 1041 | } |
| 1042 | if err := reg.setCanary("ghost.example.com", true); err != nil { |
| 1043 | t.Fatalf("setCanary unknown host should no-op: %v", err) |
| 1044 | } |
| 1045 | } |
| 1046 | |
| 1047 | func TestCanResetGatesOnTheCanaryFlag(t *testing.T) { |
| 1048 | // The whole safety model: reset refuses anything not flagged a throwaway. |
| 1049 | if err := canReset(nil); err == nil { |
| 1050 | t.Error("canReset(nil) should refuse an unknown host") |
| 1051 | } |
| 1052 | if err := canReset(&Route{Host: "push.example.com", Kind: "service", Canary: true}); err == nil { |
| 1053 | t.Error("canReset should refuse a non-instance route even if flagged") |
| 1054 | } |
| 1055 | real := &Route{Host: "family.example.com", Kind: "instance", Canary: false} |
| 1056 | if err := canReset(real); err == nil { |
| 1057 | t.Fatal("canReset MUST refuse a real (unflagged) instance — this is the data-safety gate") |
| 1058 | } |
| 1059 | canary := &Route{Host: "test-x.11ch.at", Kind: "instance", Canary: true} |
| 1060 | if err := canReset(canary); err != nil { |
| 1061 | t.Fatalf("canReset should allow a flagged canary instance, got: %v", err) |
| 1062 | } |
| 1063 | } |
| 1064 | |
| 1065 | func TestArchiveInstanceDBMovesDBKeepsEnv(t *testing.T) { |
| 1066 | dir := t.TempDir() |
| 1067 | oldData := instanceDataDir |
| 1068 | instanceDataDir = dir |
| 1069 | defer func() { instanceDataDir = oldData }() |
| 1070 | |
| 1071 | host := "test-x.11ch.at" |
| 1072 | // Seed the DB set + a blob dir + the env/vapid that must SURVIVE. |
| 1073 | for _, f := range []string{host + ".db", host + ".db-wal", host + ".db-shm", host + ".env", host + ".vapid"} { |
| 1074 | if err := os.WriteFile(filepath.Join(dir, f), []byte("x"), 0o600); err != nil { |
| 1075 | t.Fatal(err) |
| 1076 | } |
| 1077 | } |
| 1078 | if err := os.MkdirAll(filepath.Join(dir, host+"-blobs"), 0o700); err != nil { |
| 1079 | t.Fatal(err) |
| 1080 | } |
| 1081 | |
| 1082 | if err := archiveInstanceDB(host); err != nil { |
| 1083 | t.Fatalf("archiveInstanceDB: %v", err) |
| 1084 | } |
| 1085 | |
| 1086 | // DB + blobs gone from /data; env + vapid kept in place. |
| 1087 | for _, gone := range []string{host + ".db", host + ".db-wal", host + ".db-shm", host + "-blobs"} { |
| 1088 | if _, err := os.Stat(filepath.Join(dir, gone)); !os.IsNotExist(err) { |
| 1089 | t.Errorf("%s should have been archived out of the data dir", gone) |
| 1090 | } |
| 1091 | } |
| 1092 | for _, kept := range []string{host + ".env", host + ".vapid"} { |
| 1093 | if _, err := os.Stat(filepath.Join(dir, kept)); err != nil { |
| 1094 | t.Errorf("%s must survive a reset (admin token + config): %v", kept, err) |
| 1095 | } |
| 1096 | } |
| 1097 | // And the DB is recoverable under /data/trash, not destroyed. |
| 1098 | trashes, _ := filepath.Glob(filepath.Join(dir, "trash", host+"-reset-*", host+".db")) |
| 1099 | if len(trashes) != 1 { |
| 1100 | t.Errorf("archived DB should be recoverable in /data/trash, found %d", len(trashes)) |
| 1101 | } |
| 1102 | } |
| 1103 | |
| 1104 | func TestSanitizeGroupName(t *testing.T) { |
| 1105 | long := strings.Repeat("é", 40) // 80 bytes — the cap must cut on a rune boundary |
| 1106 | cases := []struct{ in, want string }{ |
| 1107 | {" The Campbells ", "The Campbells"}, |
| 1108 | {"Line\nBREAK=1\ninjection", "LineBREAK=1injection"}, // env-line injection defused |
| 1109 | {"tabs\tand\rreturns", "tabsandreturns"}, |
| 1110 | {long, strings.Repeat("é", 30)}, |
| 1111 | {"\n\t ", ""}, |
| 1112 | } |
| 1113 | for _, c := range cases { |
| 1114 | if got := sanitizeGroupName(c.in); got != c.want { |
| 1115 | t.Errorf("sanitizeGroupName(%q) = %q, want %q", c.in, got, c.want) |
| 1116 | } |
| 1117 | } |
| 1118 | } |
| 1119 | |
| 1120 | // A seeded display name lands in the instance env (LCHAT_GROUP_NAME) already |
| 1121 | // sanitized — the instance names itself from it at first boot. |
| 1122 | func TestAddInstanceSeedsGroupName(t *testing.T) { |
| 1123 | reg := controlTestEnv(t) |
| 1124 | host := "named.example.com" |
| 1125 | if _, err := addInstance(reg, addInstanceOpts{Host: host, GroupName: " The Campbells\n "}); err != nil { |
| 1126 | t.Fatalf("addInstance: %v", err) |
| 1127 | } |
| 1128 | env, err := readEnvFile(filepath.Join(instanceDataDir, host+".env")) |
| 1129 | if err != nil { |
| 1130 | t.Fatalf("read env: %v", err) |
| 1131 | } |
| 1132 | if env["LCHAT_GROUP_NAME"] != "The Campbells" { |
| 1133 | t.Errorf("LCHAT_GROUP_NAME = %q, want %q", env["LCHAT_GROUP_NAME"], "The Campbells") |
| 1134 | } |
| 1135 | |
| 1136 | // No name → no env line at all (the instance keeps its default behaviour). |
| 1137 | host2 := "unnamed.example.com" |
| 1138 | if _, err := addInstance(reg, addInstanceOpts{Host: host2}); err != nil { |
| 1139 | t.Fatalf("addInstance: %v", err) |
| 1140 | } |
| 1141 | env2, err := readEnvFile(filepath.Join(instanceDataDir, host2+".env")) |
| 1142 | if err != nil { |
| 1143 | t.Fatalf("read env: %v", err) |
| 1144 | } |
| 1145 | if _, present := env2["LCHAT_GROUP_NAME"]; present { |
| 1146 | t.Error("empty group name still wrote LCHAT_GROUP_NAME") |
| 1147 | } |
| 1148 | } |
| 1149 | |
| 1150 | // applySeededName consumes LCHAT_GROUP_NAME exactly once: it names a fresh |
| 1151 | // space and never overrides a name an admin has since chosen. |
| 1152 | func TestApplySeededName(t *testing.T) { |
| 1153 | db := newTestDB(t) |
| 1154 | t.Setenv("LCHAT_GROUP_NAME", "Book Club") |
| 1155 | if err := applySeededName(db); err != nil { |
| 1156 | t.Fatal(err) |
| 1157 | } |
| 1158 | if got := db.getGroupName(); got != "Book Club" { |
| 1159 | t.Errorf("seeded name = %q, want Book Club", got) |
| 1160 | } |
| 1161 | // A later boot with a different env value must not clobber the set name. |
| 1162 | t.Setenv("LCHAT_GROUP_NAME", "Usurper") |
| 1163 | if err := applySeededName(db); err != nil { |
| 1164 | t.Fatal(err) |
| 1165 | } |
| 1166 | if got := db.getGroupName(); got != "Book Club" { |
| 1167 | t.Errorf("after re-seed, name = %q, want Book Club (seed-only)", got) |
| 1168 | } |
| 1169 | } |
| 1170 | |
| 1171 | // The readiness probe's status vocabulary: an instance's own answers (2xx-4xx |
| 1172 | // bar 404) mean ready; 404 is the router's "unknown host" (a route still |
| 1173 | // replicating to its owner box) and must keep the probe waiting, like any 5xx. |
| 1174 | func TestPublicReadyStatus(t *testing.T) { |
| 1175 | for code, want := range map[int]bool{200: true, 302: true, 401: true, 404: false, 500: false, 502: false} { |
| 1176 | if got := publicReadyStatus(code); got != want { |
| 1177 | t.Errorf("publicReadyStatus(%d) = %v, want %v", code, got, want) |
| 1178 | } |
| 1179 | } |
| 1180 | } |
| 1181 | |
| 1182 | // A hibernating instance the mesh placed on a peer box never listens on the |
| 1183 | // local socket — the setup link then mints through the public path (the same |
| 1184 | // router hop members use, which follows the mesh to wherever the instance |
| 1185 | // woke). Both paths failing surfaces both errors and no link. |
| 1186 | func TestAddInstanceSetupLinkPublicFallback(t *testing.T) { |
| 1187 | reg := controlTestEnv(t) |
| 1188 | systemctl = func(args ...string) error { return nil } |
| 1189 | oldMint := mintPublicSetupInvite |
| 1190 | defer func() { mintPublicSetupInvite = oldMint }() |
| 1191 | confirmPublicReady = func(string, string) { /* woke on the peer — nothing local */ } |
| 1192 | var mintedHost, mintedToken string |
| 1193 | mintPublicSetupInvite = func(host, token string) (string, error) { |
| 1194 | mintedHost, mintedToken = host, token |
| 1195 | return "https://" + host + "/join/setup#PUB", nil |
| 1196 | } |
| 1197 | |
| 1198 | res, err := addInstance(reg, addInstanceOpts{Host: "peerowned.example.com", Hibernate: true}) |
| 1199 | if err != nil { |
| 1200 | t.Fatalf("addInstance: %v", err) |
| 1201 | } |
| 1202 | if res.SetupErr != nil || res.SetupLink != "https://peerowned.example.com/join/setup#PUB" { |
| 1203 | t.Fatalf("setup link = %q (err %v), want the public-path mint", res.SetupLink, res.SetupErr) |
| 1204 | } |
| 1205 | if mintedHost != "peerowned.example.com" || mintedToken != res.AdminToken { |
| 1206 | t.Errorf("public mint saw host %q, token match %v", mintedHost, mintedToken == res.AdminToken) |
| 1207 | } |
| 1208 | |
| 1209 | mintPublicSetupInvite = func(string, string) (string, error) { return "", fmt.Errorf("no route") } |
| 1210 | res2, err := addInstance(reg, addInstanceOpts{Host: "dark.example.com", Hibernate: true}) |
| 1211 | if err != nil { |
| 1212 | t.Fatalf("addInstance: %v", err) |
| 1213 | } |
| 1214 | if res2.SetupLink != "" || res2.SetupErr == nil { |
| 1215 | t.Fatalf("double failure: link %q, err %v — want no link + a surfaced error", res2.SetupLink, res2.SetupErr) |
| 1216 | } |
| 1217 | if !strings.Contains(res2.SetupErr.Error(), "public") { |
| 1218 | t.Errorf("error should name both paths: %v", res2.SetupErr) |
| 1219 | } |
| 1220 | } |
| 1221 | |
amadan