| 1 | package main |
| 2 | |
| 3 | import ( |
| 4 | "bytes" |
| 5 | "encoding/json" |
| 6 | "net" |
| 7 | "net/http" |
| 8 | "net/http/httptest" |
| 9 | "os" |
| 10 | "path/filepath" |
| 11 | "strings" |
| 12 | "sync/atomic" |
| 13 | "testing" |
| 14 | ) |
| 15 | |
| 16 | // controlTestEnv redirects every path the creation core touches into temp dirs |
| 17 | // and replaces systemd with a fake that binds the instance's unix socket itself |
| 18 | // (serving the health check and the setup-invite mint), so addInstance runs its |
| 19 | // real state machine — env files, registry, symlinks, health checks — with no |
| 20 | // systemd and no /data. |
| 21 | func controlTestEnv(t *testing.T) *Registry { |
| 22 | t.Helper() |
| 23 | |
| 24 | oldData, oldSock, oldOpt, oldSelf, oldSysctl, oldDisk, oldReady := instanceDataDir, instanceSocketDir, optDir, selfPath, systemctl, freeDiskBytes, confirmPublicReady |
| 25 | t.Cleanup(func() { |
| 26 | instanceDataDir, instanceSocketDir, optDir, selfPath, systemctl, freeDiskBytes, confirmPublicReady = oldData, oldSock, oldOpt, oldSelf, oldSysctl, oldDisk, oldReady |
| 27 | }) |
| 28 | confirmPublicReady = func(string, string) {} // no router to dial in tests |
| 29 | |
| 30 | instanceDataDir = t.TempDir() |
| 31 | optDir = t.TempDir() |
| 32 | // The socket dir must stay under the ~104-byte sun_path limit, and t.TempDir |
| 33 | // embeds the test name — use a plain short temp dir instead. |
| 34 | sockDir, err := os.MkdirTemp("", "lcs") |
| 35 | if err != nil { |
| 36 | t.Fatal(err) |
| 37 | } |
| 38 | t.Cleanup(func() { os.RemoveAll(sockDir) }) |
| 39 | instanceSocketDir = sockDir |
| 40 | |
| 41 | // Stage a stub file, not the multi-MB test binary. |
| 42 | stub := filepath.Join(t.TempDir(), "stub") |
| 43 | if err := os.WriteFile(stub, []byte("#!/bin/sh\n"), 0o755); err != nil { |
| 44 | t.Fatal(err) |
| 45 | } |
| 46 | selfPath = func() (string, error) { return stub, nil } |
| 47 | |
| 48 | freeDiskBytes = func(string) (uint64, error) { return 100 << 30, nil } |
| 49 | |
| 50 | // The systemd stand-in: "enable --now lchat@<host>" binds the instance socket |
| 51 | // and answers like a healthy instance; everything else is a no-op. |
| 52 | systemctl = func(args ...string) error { |
| 53 | if len(args) == 3 && args[0] == "enable" && args[1] == "--now" && strings.HasPrefix(args[2], "lchat@") { |
| 54 | host := strings.TrimPrefix(args[2], "lchat@") |
| 55 | ln, err := net.Listen("unix", filepath.Join(instanceSocketDir, host+".sock")) |
| 56 | if err != nil { |
| 57 | return err |
| 58 | } |
| 59 | mux := http.NewServeMux() |
| 60 | mux.HandleFunc("GET /", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(200) }) |
| 61 | mux.HandleFunc("POST /api/admin/key-invites", func(w http.ResponseWriter, r *http.Request) { |
| 62 | writeJSON(w, 200, map[string]any{"ok": true}) |
| 63 | }) |
| 64 | srv := &http.Server{Handler: mux} |
| 65 | go srv.Serve(ln) |
| 66 | t.Cleanup(func() { srv.Close() }) |
| 67 | } |
| 68 | return nil |
| 69 | } |
| 70 | |
| 71 | reg, err := openRegistry(filepath.Join(t.TempDir(), "registry.db")) |
| 72 | if err != nil { |
| 73 | t.Fatal(err) |
| 74 | } |
| 75 | return reg |
| 76 | } |
| 77 | |
| 78 | func newTestControld(t *testing.T, reg *Registry, maxInstances int) (*httptest.Server, string) { |
| 79 | t.Helper() |
| 80 | token := "test-control-token" |
| 81 | c := &controld{reg: reg, tokenHash: hashToken(token), maxInstances: maxInstances, minFreeGB: 5} |
| 82 | srv := httptest.NewServer(c.handler()) |
| 83 | t.Cleanup(srv.Close) |
| 84 | return srv, token |
| 85 | } |
| 86 | |
| 87 | func controlCreate(t *testing.T, srv *httptest.Server, token, host string) *http.Response { |
| 88 | t.Helper() |
| 89 | body, _ := json.Marshal(map[string]string{"host": host}) |
| 90 | req, _ := http.NewRequest("POST", srv.URL+"/instances", bytes.NewReader(body)) |
| 91 | if token != "" { |
| 92 | req.Header.Set("Authorization", "Bearer "+token) |
| 93 | } |
| 94 | resp, err := http.DefaultClient.Do(req) |
| 95 | if err != nil { |
| 96 | t.Fatal(err) |
| 97 | } |
| 98 | t.Cleanup(func() { resp.Body.Close() }) |
| 99 | return resp |
| 100 | } |
| 101 | |
| 102 | func decodeBody(t *testing.T, resp *http.Response) map[string]any { |
| 103 | t.Helper() |
| 104 | var m map[string]any |
| 105 | if err := json.NewDecoder(resp.Body).Decode(&m); err != nil { |
| 106 | t.Fatalf("decode body: %v", err) |
| 107 | } |
| 108 | return m |
| 109 | } |
| 110 | |
| 111 | // wakeOnProbe replaces confirmPublicReady with a fake first wake: it binds the |
| 112 | // host's socket and answers like a freshly spawned child (health check + the |
| 113 | // setup-invite mint) — which is exactly what the router's activator does when |
| 114 | // the create-path readiness probe hits a hibernating host. controlTestEnv's |
| 115 | // cleanup restores the real confirmPublicReady. |
| 116 | func wakeOnProbe(t *testing.T) { |
| 117 | t.Helper() |
| 118 | confirmPublicReady = func(h, _ string) { |
| 119 | ln, err := net.Listen("unix", filepath.Join(instanceSocketDir, h+".sock")) |
| 120 | if err != nil { |
| 121 | t.Errorf("bind woken socket: %v", err) |
| 122 | return |
| 123 | } |
| 124 | mux := http.NewServeMux() |
| 125 | mux.HandleFunc("GET /", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(200) }) |
| 126 | mux.HandleFunc("POST /api/admin/key-invites", func(w http.ResponseWriter, r *http.Request) { |
| 127 | writeJSON(w, 200, map[string]any{"ok": true}) |
| 128 | }) |
| 129 | srv := &http.Server{Handler: mux} |
| 130 | go srv.Serve(ln) |
| 131 | t.Cleanup(func() { srv.Close() }) |
| 132 | } |
| 133 | } |
| 134 | |
| 135 | // socketTripwire binds a host's instance socket and counts connections — proof |
| 136 | // that a code path stayed a pure metadata read (dialed no socket, woke nothing). |
| 137 | func socketTripwire(t *testing.T, host string) *atomic.Int32 { |
| 138 | t.Helper() |
| 139 | var dials atomic.Int32 |
| 140 | ln, err := net.Listen("unix", filepath.Join(instanceSocketDir, host+".sock")) |
| 141 | if err != nil { |
| 142 | t.Fatal(err) |
| 143 | } |
| 144 | t.Cleanup(func() { ln.Close() }) |
| 145 | go func() { |
| 146 | for { |
| 147 | c, err := ln.Accept() |
| 148 | if err != nil { |
| 149 | return |
| 150 | } |
| 151 | dials.Add(1) |
| 152 | c.Close() |
| 153 | } |
| 154 | }() |
| 155 | return &dials |
| 156 | } |
| 157 | |
| 158 | // The replication endpoint: records the class on the registry route and |
| 159 | // reloads the router exactly once per actual change — a matching class is a |
| 160 | // no-op with no reload (the cloud sweep re-pushes every pass), and garbage |
| 161 | // classes / unknown hosts never touch anything. |
| 162 | func TestControldReplicationEndpoint(t *testing.T) { |
| 163 | reg := controlTestEnv(t) |
| 164 | if err := reg.upsert(Route{Host: "space.11ch.at", Kind: "instance", Socket: "/run/s.sock", Hibernate: true}); err != nil { |
| 165 | t.Fatal(err) |
| 166 | } |
| 167 | if err := reg.upsert(Route{Host: "svc.11ch.at", Kind: "service", Socket: "/run/v.sock"}); err != nil { |
| 168 | t.Fatal(err) |
| 169 | } |
| 170 | srv, token := newTestControld(t, reg, 40) |
| 171 | |
| 172 | var reloads int |
| 173 | systemctl = func(args ...string) error { |
| 174 | if len(args) == 2 && args[0] == "reload" { |
| 175 | reloads++ |
| 176 | } |
| 177 | return nil |
| 178 | } |
| 179 | |
| 180 | post := func(host, class, tok string) *http.Response { |
| 181 | t.Helper() |
| 182 | body, _ := json.Marshal(map[string]string{"class": class}) |
| 183 | req, _ := http.NewRequest("POST", srv.URL+"/instances/"+host+"/replication", bytes.NewReader(body)) |
| 184 | if tok != "" { |
| 185 | req.Header.Set("Authorization", "Bearer "+tok) |
| 186 | } |
| 187 | resp, err := http.DefaultClient.Do(req) |
| 188 | if err != nil { |
| 189 | t.Fatal(err) |
| 190 | } |
| 191 | t.Cleanup(func() { resp.Body.Close() }) |
| 192 | return resp |
| 193 | } |
| 194 | |
| 195 | if resp := post("space.11ch.at", "daily", ""); resp.StatusCode != http.StatusUnauthorized { |
| 196 | t.Errorf("no token: status = %d, want 401", resp.StatusCode) |
| 197 | } |
| 198 | if resp := post("space.11ch.at", "hourly", token); resp.StatusCode != http.StatusBadRequest { |
| 199 | t.Errorf("bad class: status = %d, want 400", resp.StatusCode) |
| 200 | } |
| 201 | if resp := post("ghost.11ch.at", "daily", token); resp.StatusCode != http.StatusNotFound { |
| 202 | t.Errorf("unknown host: status = %d, want 404", resp.StatusCode) |
| 203 | } |
| 204 | if resp := post("svc.11ch.at", "daily", token); resp.StatusCode != http.StatusNotFound { |
| 205 | t.Errorf("service route: status = %d, want 404", resp.StatusCode) |
| 206 | } |
| 207 | if rt, _ := reg.get("space.11ch.at"); rt.Replication != "" || reloads != 0 { |
| 208 | t.Fatalf("rejected requests left marks: class=%q reloads=%d", rt.Replication, reloads) |
| 209 | } |
| 210 | |
| 211 | resp := post("space.11ch.at", "daily", token) |
| 212 | if resp.StatusCode != http.StatusOK { |
| 213 | t.Fatalf("set: status = %d, want 200", resp.StatusCode) |
| 214 | } |
| 215 | if body := decodeBody(t, resp); body["changed"] != true { |
| 216 | t.Errorf("set: changed = %v, want true", body["changed"]) |
| 217 | } |
| 218 | if rt, _ := reg.get("space.11ch.at"); rt.Replication != "daily" { |
| 219 | t.Fatalf("class not recorded: %q", rt.Replication) |
| 220 | } |
| 221 | if reloads != 1 { |
| 222 | t.Fatalf("reloads after set = %d, want 1", reloads) |
| 223 | } |
| 224 | |
| 225 | // Idempotent re-push (the sweep's steady state): no change, no reload. |
| 226 | resp = post("space.11ch.at", "daily", token) |
| 227 | if resp.StatusCode != http.StatusOK { |
| 228 | t.Fatalf("re-push: status = %d, want 200", resp.StatusCode) |
| 229 | } |
| 230 | if body := decodeBody(t, resp); body["changed"] != false { |
| 231 | t.Errorf("re-push: changed = %v, want false", body["changed"]) |
| 232 | } |
| 233 | if reloads != 1 { |
| 234 | t.Fatalf("re-push reloaded the router: %d reloads", reloads) |
| 235 | } |
| 236 | } |
| 237 | |
| 238 | // A create request carrying a replication class produces a route born on that |
| 239 | // rung (what the provisioner sends for a fresh free space). |
| 240 | func TestControldCreateBornClassed(t *testing.T) { |
| 241 | reg := controlTestEnv(t) |
| 242 | srv, token := newTestControld(t, reg, 40) |
| 243 | |
| 244 | body, _ := json.Marshal(map[string]string{"host": "born.11ch.at", "replication": "daily"}) |
| 245 | req, _ := http.NewRequest("POST", srv.URL+"/instances", bytes.NewReader(body)) |
| 246 | req.Header.Set("Authorization", "Bearer "+token) |
| 247 | resp, err := http.DefaultClient.Do(req) |
| 248 | if err != nil { |
| 249 | t.Fatal(err) |
| 250 | } |
| 251 | defer resp.Body.Close() |
| 252 | if resp.StatusCode != http.StatusCreated { |
| 253 | t.Fatalf("create: status = %d, want 201", resp.StatusCode) |
| 254 | } |
| 255 | rt, err := reg.get("born.11ch.at") |
| 256 | if err != nil || rt == nil { |
| 257 | t.Fatalf("route: %+v, %v", rt, err) |
| 258 | } |
| 259 | if rt.Replication != "daily" { |
| 260 | t.Errorf("born class = %q, want daily", rt.Replication) |
| 261 | } |
| 262 | } |
| 263 | |
| 264 | // A create carrying home_region records it on the route (multi-edge Phase 1): |
| 265 | // the field crosses the socket in controlCreateReq → addInstanceOpts.HomeRegion |
| 266 | // → the registry row the mesh later places by (mesh.candidatesFor). The value is |
| 267 | // trimmed but not otherwise touched; omitted leaves the space region-agnostic. |
| 268 | func TestControldCreateStampsHomeRegion(t *testing.T) { |
| 269 | reg := controlTestEnv(t) |
| 270 | srv, token := newTestControld(t, reg, 40) |
| 271 | |
| 272 | create := func(host, region string) { |
| 273 | t.Helper() |
| 274 | payload := map[string]string{"host": host} |
| 275 | if region != "" { |
| 276 | payload["home_region"] = region |
| 277 | } |
| 278 | body, _ := json.Marshal(payload) |
| 279 | req, _ := http.NewRequest("POST", srv.URL+"/instances", bytes.NewReader(body)) |
| 280 | req.Header.Set("Authorization", "Bearer "+token) |
| 281 | resp, err := http.DefaultClient.Do(req) |
| 282 | if err != nil { |
| 283 | t.Fatal(err) |
| 284 | } |
| 285 | defer resp.Body.Close() |
| 286 | if resp.StatusCode != http.StatusCreated { |
| 287 | t.Fatalf("create %s: status = %d, want 201", host, resp.StatusCode) |
| 288 | } |
| 289 | } |
| 290 | |
| 291 | create("ie-space.11ch.at", " ie ") // trimmed crossing the boundary |
| 292 | if rt, _ := reg.get("ie-space.11ch.at"); rt == nil || rt.HomeRegion != "ie" { |
| 293 | t.Fatalf("route = %+v, want home_region=ie", rt) |
| 294 | } |
| 295 | // Omitted ⇒ region-agnostic, exactly today's behaviour (home_region ""). |
| 296 | create("agnostic.11ch.at", "") |
| 297 | if rt, _ := reg.get("agnostic.11ch.at"); rt == nil || rt.HomeRegion != "" { |
| 298 | t.Fatalf("route = %+v, want empty home_region", rt) |
| 299 | } |
| 300 | } |
| 301 | |
| 302 | func TestControldRequiresToken(t *testing.T) { |
| 303 | reg := controlTestEnv(t) |
| 304 | srv, _ := newTestControld(t, reg, 40) |
| 305 | |
| 306 | if resp := controlCreate(t, srv, "", "a.example.com"); resp.StatusCode != http.StatusUnauthorized { |
| 307 | t.Errorf("no token: status = %d, want 401", resp.StatusCode) |
| 308 | } |
| 309 | if resp := controlCreate(t, srv, "wrong", "a.example.com"); resp.StatusCode != http.StatusUnauthorized { |
| 310 | t.Errorf("bad token: status = %d, want 401", resp.StatusCode) |
| 311 | } |
| 312 | } |
| 313 | |
| 314 | func TestControldCreatesInstance(t *testing.T) { |
| 315 | reg := controlTestEnv(t) |
| 316 | srv, token := newTestControld(t, reg, 40) |
| 317 | |
| 318 | resp := controlCreate(t, srv, token, "new.example.com") |
| 319 | if resp.StatusCode != http.StatusCreated { |
| 320 | t.Fatalf("status = %d, want 201", resp.StatusCode) |
| 321 | } |
| 322 | m := decodeBody(t, resp) |
| 323 | if m["host"] != "new.example.com" || m["url"] != "https://new.example.com" { |
| 324 | t.Errorf("host/url = %v/%v", m["host"], m["url"]) |
| 325 | } |
| 326 | setup, _ := m["setup_link"].(string) |
| 327 | if !strings.HasPrefix(setup, "https://new.example.com/join/setup#") { |
| 328 | t.Errorf("setup_link = %q", setup) |
| 329 | } |
| 330 | admin, _ := m["admin_link"].(string) |
| 331 | if !strings.HasPrefix(admin, "https://new.example.com/admin#token=") { |
| 332 | t.Errorf("admin_link = %q", admin) |
| 333 | } |
| 334 | |
| 335 | // The route is registered ready and the env file holds the same token. |
| 336 | rt, err := reg.get("new.example.com") |
| 337 | if err != nil || rt == nil || rt.Status != "ready" || rt.Kind != "instance" { |
| 338 | t.Fatalf("route = %+v, err %v", rt, err) |
| 339 | } |
| 340 | env, err := readEnvFile(filepath.Join(instanceDataDir, "new.example.com.env")) |
| 341 | if err != nil { |
| 342 | t.Fatal(err) |
| 343 | } |
| 344 | if !strings.HasSuffix(admin, env["LCHAT_ADMIN_TOKEN"]) { |
| 345 | t.Error("admin_link token does not match the env file") |
| 346 | } |
| 347 | } |
| 348 | |
| 349 | func TestControldConfirmsPublicReady(t *testing.T) { |
| 350 | reg := controlTestEnv(t) |
| 351 | var confirmed string |
| 352 | confirmPublicReady = func(host, addr string) { confirmed = host } |
| 353 | srv, token := newTestControld(t, reg, 40) |
| 354 | |
| 355 | if resp := controlCreate(t, srv, token, "ready.example.com"); resp.StatusCode != http.StatusCreated { |
| 356 | t.Fatalf("status = %d, want 201", resp.StatusCode) |
| 357 | } |
| 358 | // Readiness through the public path is confirmed for the new host before the |
| 359 | // setup link is returned, so a click lands on a ready admin sign-on. |
| 360 | if confirmed != "ready.example.com" { |
| 361 | t.Errorf("confirmPublicReady called with %q, want ready.example.com", confirmed) |
| 362 | } |
| 363 | } |
| 364 | |
| 365 | func TestControldAtCapacity(t *testing.T) { |
| 366 | reg := controlTestEnv(t) |
| 367 | srv, token := newTestControld(t, reg, 1) |
| 368 | |
| 369 | if err := reg.upsert(Route{Host: "full.example.com", Kind: "instance", Socket: "/x.sock", Status: "ready"}); err != nil { |
| 370 | t.Fatal(err) |
| 371 | } |
| 372 | resp := controlCreate(t, srv, token, "new.example.com") |
| 373 | if resp.StatusCode != http.StatusServiceUnavailable { |
| 374 | t.Fatalf("status = %d, want 503", resp.StatusCode) |
| 375 | } |
| 376 | if m := decodeBody(t, resp); m["error"] != "at_capacity" { |
| 377 | t.Errorf("error = %v, want at_capacity", m["error"]) |
| 378 | } |
| 379 | } |
| 380 | |
| 381 | func TestControldServiceRoutesDontCountTowardCapacity(t *testing.T) { |
| 382 | reg := controlTestEnv(t) |
| 383 | srv, token := newTestControld(t, reg, 1) |
| 384 | |
| 385 | // A 'service' route (home, push, cloud) must not eat instance capacity. |
| 386 | if err := reg.upsert(Route{Host: "home.example.com", Kind: "service", Socket: "/h.sock", Status: "ready"}); err != nil { |
| 387 | t.Fatal(err) |
| 388 | } |
| 389 | if resp := controlCreate(t, srv, token, "new.example.com"); resp.StatusCode != http.StatusCreated { |
| 390 | t.Errorf("status = %d, want 201", resp.StatusCode) |
| 391 | } |
| 392 | } |
| 393 | |
| 394 | func TestControldDiskFloor(t *testing.T) { |
| 395 | reg := controlTestEnv(t) |
| 396 | srv, token := newTestControld(t, reg, 40) |
| 397 | freeDiskBytes = func(string) (uint64, error) { return 1 << 30, nil } // 1 GB free, floor is 5 |
| 398 | |
| 399 | resp := controlCreate(t, srv, token, "new.example.com") |
| 400 | if resp.StatusCode != http.StatusServiceUnavailable { |
| 401 | t.Fatalf("status = %d, want 503", resp.StatusCode) |
| 402 | } |
| 403 | if m := decodeBody(t, resp); m["error"] != "at_capacity" { |
| 404 | t.Errorf("error = %v, want at_capacity", m["error"]) |
| 405 | } |
| 406 | } |
| 407 | |
| 408 | func TestControldHostExists(t *testing.T) { |
| 409 | reg := controlTestEnv(t) |
| 410 | srv, token := newTestControld(t, reg, 40) |
| 411 | |
| 412 | if err := reg.upsert(Route{Host: "taken.example.com", Kind: "instance", Socket: "/x.sock", Status: "ready"}); err != nil { |
| 413 | t.Fatal(err) |
| 414 | } |
| 415 | resp := controlCreate(t, srv, token, "taken.example.com") |
| 416 | if resp.StatusCode != http.StatusConflict { |
| 417 | t.Fatalf("status = %d, want 409", resp.StatusCode) |
| 418 | } |
| 419 | if m := decodeBody(t, resp); m["error"] != "host_exists" { |
| 420 | t.Errorf("error = %v, want host_exists", m["error"]) |
| 421 | } |
| 422 | } |
| 423 | |
| 424 | func TestControldInvalidHost(t *testing.T) { |
| 425 | reg := controlTestEnv(t) |
| 426 | srv, token := newTestControld(t, reg, 40) |
| 427 | |
| 428 | resp := controlCreate(t, srv, token, "Bad Host!") |
| 429 | if resp.StatusCode != http.StatusBadRequest { |
| 430 | t.Fatalf("status = %d, want 400", resp.StatusCode) |
| 431 | } |
| 432 | if m := decodeBody(t, resp); m["error"] != "invalid_host" { |
| 433 | t.Errorf("error = %v, want invalid_host", m["error"]) |
| 434 | } |
| 435 | } |
| 436 | |
| 437 | func TestControldDeleteInstance(t *testing.T) { |
| 438 | reg := controlTestEnv(t) |
| 439 | srv, token := newTestControld(t, reg, 40) |
| 440 | |
| 441 | // Create a real (fake-systemd) instance first. |
| 442 | if resp := controlCreate(t, srv, token, "gone.example.com"); resp.StatusCode != http.StatusCreated { |
| 443 | t.Fatalf("create: %d", resp.StatusCode) |
| 444 | } |
| 445 | |
| 446 | del := func(tok, host string) *http.Response { |
| 447 | req, _ := http.NewRequest("DELETE", srv.URL+"/instances/"+host, nil) |
| 448 | if tok != "" { |
| 449 | req.Header.Set("Authorization", "Bearer "+tok) |
| 450 | } |
| 451 | resp, err := http.DefaultClient.Do(req) |
| 452 | if err != nil { |
| 453 | t.Fatal(err) |
| 454 | } |
| 455 | t.Cleanup(func() { resp.Body.Close() }) |
| 456 | return resp |
| 457 | } |
| 458 | |
| 459 | if r := del("", "gone.example.com"); r.StatusCode != http.StatusUnauthorized { |
| 460 | t.Fatalf("no token: %d, want 401", r.StatusCode) |
| 461 | } |
| 462 | if r := del(token, "never-was.example.com"); r.StatusCode != http.StatusNotFound { |
| 463 | t.Fatalf("unknown host: %d, want 404", r.StatusCode) |
| 464 | } |
| 465 | if r := del(token, "gone.example.com"); r.StatusCode != http.StatusOK { |
| 466 | t.Fatalf("delete: %d, want 200", r.StatusCode) |
| 467 | } |
| 468 | |
| 469 | // The route is gone… |
| 470 | if rt, _ := reg.get("gone.example.com"); rt != nil { |
| 471 | t.Errorf("route still present after delete: %+v", rt) |
| 472 | } |
| 473 | // …the live data files are gone from /data… |
| 474 | if _, err := os.Stat(filepath.Join(instanceDataDir, "gone.example.com.env")); !os.IsNotExist(err) { |
| 475 | t.Error("env file still in /data — should be archived") |
| 476 | } |
| 477 | // …but ARCHIVED into the trash, not destroyed (the operator safety net). |
| 478 | matches, err := filepath.Glob(filepath.Join(instanceDataDir, "trash", "gone.example.com-*", "gone.example.com.env")) |
| 479 | if err != nil || len(matches) != 1 { |
| 480 | t.Errorf("archived env not found in trash (matches=%v err=%v)", matches, err) |
| 481 | } |
| 482 | } |
| 483 | |
| 484 | // ---- hibernate-by-default creates (#550) --------------------------------------- |
| 485 | |
| 486 | // With LCHAT_CONTROL_HIBERNATE_CREATES on, a provisioner create is born |
| 487 | // hibernating: routed + env + staged binary, setup link minted through the |
| 488 | // probe-triggered first wake — but no systemd unit and no DB (the wake boots |
| 489 | // fresh). |
| 490 | func TestControldHibernateCreates(t *testing.T) { |
| 491 | reg := controlTestEnv(t) |
| 492 | // Pure recorder: the hibernating path must never enable a unit, so nothing |
| 493 | // here binds the socket for it — only the fake wake below does. |
| 494 | var calls []string |
| 495 | systemctl = func(args ...string) error { |
| 496 | calls = append(calls, strings.Join(args, " ")) |
| 497 | return nil |
| 498 | } |
| 499 | wakeOnProbe(t) |
| 500 | |
| 501 | token := "test-control-token" |
| 502 | c := &controld{reg: reg, tokenHash: hashToken(token), maxInstances: 40, minFreeGB: 5, hibernateCreates: true} |
| 503 | srv := httptest.NewServer(c.handler()) |
| 504 | t.Cleanup(srv.Close) |
| 505 | |
| 506 | resp := controlCreate(t, srv, token, "hib.11ch.at") |
| 507 | if resp.StatusCode != http.StatusCreated { |
| 508 | t.Fatalf("status = %d, want 201", resp.StatusCode) |
| 509 | } |
| 510 | m := decodeBody(t, resp) |
| 511 | if setup, _ := m["setup_link"].(string); !strings.HasPrefix(setup, "https://hib.11ch.at/join/setup#") { |
| 512 | t.Errorf("setup_link = %q — the probe-woken instance should have minted it", setup) |
| 513 | } |
| 514 | |
| 515 | rt, err := reg.get("hib.11ch.at") |
| 516 | if err != nil || rt == nil || rt.Status != "ready" || !rt.Hibernate { |
| 517 | t.Fatalf("route = %+v, err %v; want a ready hibernating instance", rt, err) |
| 518 | } |
| 519 | for _, call := range calls { |
| 520 | if strings.HasPrefix(call, "enable ") { |
| 521 | t.Errorf("a systemd unit was touched: %q", call) |
| 522 | } |
| 523 | } |
| 524 | if _, err := os.Stat(filepath.Join(instanceDataDir, "hib.11ch.at.db")); !os.IsNotExist(err) { |
| 525 | t.Error("a DB was created — the first wake should boot fresh") |
| 526 | } |
| 527 | // The env file IS written — the future wake's spawn layers it (admin token). |
| 528 | if _, err := os.Stat(filepath.Join(instanceDataDir, "hib.11ch.at.env")); err != nil { |
| 529 | t.Errorf("env file: %v", err) |
| 530 | } |
| 531 | } |
| 532 | |
| 533 | // The flag defaults OFF: without it, provisioner creates stay systemd-managed — |
| 534 | // the flip is a config decision, never a side effect of deploying this code. |
| 535 | func TestControldHibernateCreatesDefaultOff(t *testing.T) { |
| 536 | reg := controlTestEnv(t) |
| 537 | srv, token := newTestControld(t, reg, 40) |
| 538 | |
| 539 | if resp := controlCreate(t, srv, token, "plain.11ch.at"); resp.StatusCode != http.StatusCreated { |
| 540 | t.Fatalf("status = %d, want 201", resp.StatusCode) |
| 541 | } |
| 542 | rt, _ := reg.get("plain.11ch.at") |
| 543 | if rt == nil || rt.Hibernate { |
| 544 | t.Fatalf("route = %+v; the default create must stay systemd-managed", rt) |
| 545 | } |
| 546 | } |
| 547 | |
| 548 | // A hibernating space that has never woken has no local DB — its stats are |
| 549 | // definitional (nothing to count, nothing set up, no content), answered from |
| 550 | // the registry row alone. The tripwire proves no socket was dialed and the |
| 551 | // recorder proves no unit was touched: a reaper sweep over dormant spaces must |
| 552 | // never wake them. |
| 553 | func TestControldStatsNeverWokenHibernating(t *testing.T) { |
| 554 | reg := controlTestEnv(t) |
| 555 | var calls []string |
| 556 | systemctl = func(args ...string) error { |
| 557 | calls = append(calls, strings.Join(args, " ")) |
| 558 | return nil |
| 559 | } |
| 560 | |
| 561 | const host = "dormant.11ch.at" |
| 562 | sock := filepath.Join(instanceSocketDir, host+".sock") |
| 563 | if err := reg.upsert(Route{Host: host, Kind: "instance", Socket: sock, Status: "ready", Hibernate: true}); err != nil { |
| 564 | t.Fatal(err) |
| 565 | } |
| 566 | dials := socketTripwire(t, host) |
| 567 | srv, token := newTestControld(t, reg, 40) |
| 568 | |
| 569 | req, _ := http.NewRequest("GET", srv.URL+"/instances/"+host+"/stats", nil) |
| 570 | req.Header.Set("Authorization", "Bearer "+token) |
| 571 | resp, err := http.DefaultClient.Do(req) |
| 572 | if err != nil { |
| 573 | t.Fatal(err) |
| 574 | } |
| 575 | defer resp.Body.Close() |
| 576 | if resp.StatusCode != http.StatusOK { |
| 577 | t.Fatalf("stats = HTTP %d, want 200 (definitional, not unknown_host)", resp.StatusCode) |
| 578 | } |
| 579 | var body struct { |
| 580 | Members int `json:"members"` |
| 581 | SetupComplete bool `json:"setup_complete"` |
| 582 | HasContent bool `json:"has_content"` |
| 583 | } |
| 584 | if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { |
| 585 | t.Fatal(err) |
| 586 | } |
| 587 | if body.Members != 0 || body.SetupComplete || body.HasContent { |
| 588 | t.Errorf("stats = %+v, want the definitional 0/false/false", body) |
| 589 | } |
| 590 | if n := dials.Load(); n != 0 { |
| 591 | t.Errorf("stats dialed the instance socket %d time(s) — it must stay a pure metadata read", n) |
| 592 | } |
| 593 | if len(calls) != 0 { |
| 594 | t.Errorf("stats touched systemd: %v", calls) |
| 595 | } |
| 596 | } |
| 597 | |
| 598 | // An asleep hibernating space (woken before, now stopped) answers from its |
| 599 | // local DB — the warm cache the activator left at the last sleep — read-only, |
| 600 | // again with zero socket touches. |
| 601 | func TestControldStatsAsleepHibernatingAnswersFromFile(t *testing.T) { |
| 602 | reg := controlTestEnv(t) |
| 603 | const host = "napping.11ch.at" |
| 604 | |
| 605 | // The warm-cache DB: the pre-seeded admin, never set up, no content. |
| 606 | db, err := openDB(filepath.Join(instanceDataDir, host+".db")) |
| 607 | if err != nil { |
| 608 | t.Fatal(err) |
| 609 | } |
| 610 | if _, err := db.sql.Exec(`INSERT INTO users(id, created_at) VALUES('admin',1)`); err != nil { |
| 611 | t.Fatal(err) |
| 612 | } |
| 613 | db.sql.Close() |
| 614 | |
| 615 | sock := filepath.Join(instanceSocketDir, host+".sock") |
| 616 | if err := reg.upsert(Route{Host: host, Kind: "instance", Socket: sock, Status: "ready", Hibernate: true}); err != nil { |
| 617 | t.Fatal(err) |
| 618 | } |
| 619 | dials := socketTripwire(t, host) |
| 620 | srv, token := newTestControld(t, reg, 40) |
| 621 | |
| 622 | req, _ := http.NewRequest("GET", srv.URL+"/instances/"+host+"/stats", nil) |
| 623 | req.Header.Set("Authorization", "Bearer "+token) |
| 624 | resp, err := http.DefaultClient.Do(req) |
| 625 | if err != nil { |
| 626 | t.Fatal(err) |
| 627 | } |
| 628 | defer resp.Body.Close() |
| 629 | if resp.StatusCode != http.StatusOK { |
| 630 | t.Fatalf("stats = HTTP %d, want 200", resp.StatusCode) |
| 631 | } |
| 632 | var body struct { |
| 633 | Members int `json:"members"` |
| 634 | SetupComplete bool `json:"setup_complete"` |
| 635 | HasContent bool `json:"has_content"` |
| 636 | } |
| 637 | if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { |
| 638 | t.Fatal(err) |
| 639 | } |
| 640 | if body.Members != 1 || body.SetupComplete || body.HasContent { |
| 641 | t.Errorf("stats = %+v, want 1/false/false (from the warm-cache DB)", body) |
| 642 | } |
| 643 | if n := dials.Load(); n != 0 { |
| 644 | t.Errorf("stats dialed the instance socket %d time(s) — the DB file is the source", n) |
| 645 | } |
| 646 | } |
| 647 | |
| 648 | // The definitional answer is ONLY for hibernating rows: a host with no route at |
| 649 | // all, and a registered systemd-managed instance whose DB is missing (a failed |
| 650 | // create), both stay 404 — the reaper keeps skipping those, as before. |
| 651 | func TestControldStatsMissingStays404(t *testing.T) { |
| 652 | reg := controlTestEnv(t) |
| 653 | srv, token := newTestControld(t, reg, 40) |
| 654 | |
| 655 | stat := func(host string) int { |
| 656 | req, _ := http.NewRequest("GET", srv.URL+"/instances/"+host+"/stats", nil) |
| 657 | req.Header.Set("Authorization", "Bearer "+token) |
| 658 | resp, err := http.DefaultClient.Do(req) |
| 659 | if err != nil { |
| 660 | t.Fatal(err) |
| 661 | } |
| 662 | resp.Body.Close() |
| 663 | return resp.StatusCode |
| 664 | } |
| 665 | |
| 666 | if code := stat("never-was.11ch.at"); code != http.StatusNotFound { |
| 667 | t.Errorf("unregistered host: %d, want 404", code) |
| 668 | } |
| 669 | if err := reg.upsert(Route{Host: "broken.11ch.at", Kind: "instance", Socket: "/x.sock", Status: "failed"}); err != nil { |
| 670 | t.Fatal(err) |
| 671 | } |
| 672 | if code := stat("broken.11ch.at"); code != http.StatusNotFound { |
| 673 | t.Errorf("non-hibernating row without a DB: %d, want 404", code) |
| 674 | } |
| 675 | } |
| 676 | |
| 677 | // Under hibernate-creates the capacity cap counts MATERIALIZED (systemd-managed) |
| 678 | // instances only — hibernating rows cost no process, so they don't consume the |
| 679 | // box's RAM budget; a separate, much higher ceiling bounds total registrations. |
| 680 | func TestControldCapCountsMaterializedOnlyWhenHibernateCreates(t *testing.T) { |
| 681 | reg := controlTestEnv(t) |
| 682 | |
| 683 | // Three hibernating rows against maxInstances=1: with the flag on they |
| 684 | // don't count, so a (hibernating) create still succeeds… |
| 685 | for _, h := range []string{"h1.11ch.at", "h2.11ch.at", "h3.11ch.at"} { |
| 686 | if err := reg.upsert(Route{Host: h, Kind: "instance", Socket: "/run/" + h + ".sock", Status: "ready", Hibernate: true}); err != nil { |
| 687 | t.Fatal(err) |
| 688 | } |
| 689 | } |
| 690 | token := "test-control-token" |
| 691 | c := &controld{reg: reg, tokenHash: hashToken(token), maxInstances: 1, maxRegistered: 10, minFreeGB: 5, hibernateCreates: true} |
| 692 | srv := httptest.NewServer(c.handler()) |
| 693 | t.Cleanup(srv.Close) |
| 694 | if resp := controlCreate(t, srv, token, "new1.11ch.at"); resp.StatusCode != http.StatusCreated { |
| 695 | t.Fatalf("create with 3 hibernating rows: %d, want 201 (they must not count)", resp.StatusCode) |
| 696 | } |
| 697 | |
| 698 | // …a materialized row DOES hold the maxInstances cap… |
| 699 | if err := reg.upsert(Route{Host: "always-on.11ch.at", Kind: "instance", Socket: "/run/a.sock", Status: "ready"}); err != nil { |
| 700 | t.Fatal(err) |
| 701 | } |
| 702 | resp := controlCreate(t, srv, token, "new2.11ch.at") |
| 703 | if resp.StatusCode != http.StatusServiceUnavailable { |
| 704 | t.Fatalf("create at the materialized cap: %d, want 503", resp.StatusCode) |
| 705 | } |
| 706 | if m := decodeBody(t, resp); m["error"] != "at_capacity" { |
| 707 | t.Errorf("error = %v, want at_capacity", m["error"]) |
| 708 | } |
| 709 | |
| 710 | // …and the registration ceiling bounds total rows, hibernating included |
| 711 | // (5 instance rows now: h1-h3 + new1 hibernating, always-on materialized). |
| 712 | c2 := &controld{reg: reg, tokenHash: hashToken(token), maxInstances: 40, maxRegistered: 5, minFreeGB: 5, hibernateCreates: true} |
| 713 | srv2 := httptest.NewServer(c2.handler()) |
| 714 | t.Cleanup(srv2.Close) |
| 715 | resp2 := controlCreate(t, srv2, token, "new3.11ch.at") |
| 716 | if resp2.StatusCode != http.StatusServiceUnavailable { |
| 717 | t.Fatalf("create at the registration ceiling: %d, want 503", resp2.StatusCode) |
| 718 | } |
| 719 | if m := decodeBody(t, resp2); m["error"] != "at_capacity" { |
| 720 | t.Errorf("error = %v, want at_capacity", m["error"]) |
| 721 | } |
| 722 | } |
| 723 | |
| 724 | // --- materialize (the controld-mediated fix for mesh-replicated routes' first wake) --- |
| 725 | |
| 726 | func controlMaterialize(t *testing.T, srv *httptest.Server, token, host string) *http.Response { |
| 727 | t.Helper() |
| 728 | req, _ := http.NewRequest("POST", srv.URL+"/instances/"+host+"/materialize", nil) |
| 729 | if token != "" { |
| 730 | req.Header.Set("Authorization", "Bearer "+token) |
| 731 | } |
| 732 | resp, err := http.DefaultClient.Do(req) |
| 733 | if err != nil { |
| 734 | t.Fatal(err) |
| 735 | } |
| 736 | t.Cleanup(func() { resp.Body.Close() }) |
| 737 | return resp |
| 738 | } |
| 739 | |
| 740 | func TestControldMaterializeRequiresToken(t *testing.T) { |
| 741 | reg := controlTestEnv(t) |
| 742 | srv, _ := newTestControld(t, reg, 40) |
| 743 | if resp := controlMaterialize(t, srv, "", "a.example.com"); resp.StatusCode != http.StatusUnauthorized { |
| 744 | t.Errorf("no token: status = %d, want 401", resp.StatusCode) |
| 745 | } |
| 746 | } |
| 747 | |
| 748 | func TestControldMaterializeUnknownHost(t *testing.T) { |
| 749 | reg := controlTestEnv(t) |
| 750 | srv, token := newTestControld(t, reg, 40) |
| 751 | resp := controlMaterialize(t, srv, token, "never-registered.example.com") |
| 752 | if resp.StatusCode != http.StatusNotFound { |
| 753 | t.Fatalf("status = %d, want 404", resp.StatusCode) |
| 754 | } |
| 755 | if m := decodeBody(t, resp); m["error"] != "unknown_host" { |
| 756 | t.Errorf("error = %v, want unknown_host", m["error"]) |
| 757 | } |
| 758 | } |
| 759 | |
| 760 | // Mirrors the exact scenario found live 2026-07-13: a route with a registry |
| 761 | // row but no live symlink (meshreplicate.go's meshRouteHandler only ever |
| 762 | // upserts the row) must come out materialized — staged binary + live symlink |
| 763 | // — after one call, ready for s3Backend.start to exec. |
| 764 | func TestControldMaterializeStagesAndPointsLive(t *testing.T) { |
| 765 | reg := controlTestEnv(t) |
| 766 | if err := reg.upsert(Route{Host: "seeded.11ch.at", Kind: "instance", Socket: "/run/seeded.sock", Hibernate: true}); err != nil { |
| 767 | t.Fatal(err) |
| 768 | } |
| 769 | if _, err := os.Stat(liveLink("seeded.11ch.at")); err == nil { |
| 770 | t.Fatal("test setup: live symlink should not exist yet") |
| 771 | } |
| 772 | |
| 773 | srv, token := newTestControld(t, reg, 40) |
| 774 | resp := controlMaterialize(t, srv, token, "seeded.11ch.at") |
| 775 | if resp.StatusCode != http.StatusOK { |
| 776 | t.Fatalf("status = %d, want 200", resp.StatusCode) |
| 777 | } |
| 778 | m := decodeBody(t, resp) |
| 779 | if m["ok"] != true || m["version"] == "" { |
| 780 | t.Fatalf("body = %v, want ok:true + a version", m) |
| 781 | } |
| 782 | |
| 783 | ver, err := liveVersion("seeded.11ch.at") |
| 784 | if err != nil || ver == "" { |
| 785 | t.Fatalf("liveVersion after materialize: %v, %v — symlink was not created", ver, err) |
| 786 | } |
| 787 | if _, err := os.Stat(versionBinPath(ver)); err != nil { |
| 788 | t.Fatalf("staged binary missing at the version path: %v", err) |
| 789 | } |
| 790 | } |
| 791 | |
| 792 | // A second call (the retry a wake would make) must not fail just because the |
| 793 | // symlink already exists — pointLive happily repoints at the same version. |
| 794 | func TestControldMaterializeIdempotent(t *testing.T) { |
| 795 | reg := controlTestEnv(t) |
| 796 | if err := reg.upsert(Route{Host: "seeded2.11ch.at", Kind: "instance", Socket: "/run/seeded2.sock", Hibernate: true}); err != nil { |
| 797 | t.Fatal(err) |
| 798 | } |
| 799 | srv, token := newTestControld(t, reg, 40) |
| 800 | for i := 0; i < 2; i++ { |
| 801 | if resp := controlMaterialize(t, srv, token, "seeded2.11ch.at"); resp.StatusCode != http.StatusOK { |
| 802 | t.Fatalf("call %d: status = %d, want 200", i, resp.StatusCode) |
| 803 | } |
| 804 | } |
| 805 | } |
| 806 | |
| 807 | // A box-level default home origin fills provisioner creates that name none |
| 808 | // (LCHAT_CONTROL_HOME_ORIGIN — non-production fleets front a different home); |
| 809 | // an explicit request value still wins. |
| 810 | func TestControldHomeOriginDefault(t *testing.T) { |
| 811 | reg := controlTestEnv(t) |
| 812 | c := &controld{reg: reg, tokenHash: hashToken("test-control-token"), maxInstances: 10, minFreeGB: 5, |
| 813 | homeOrigin: "https://home.ie.dev.example"} |
| 814 | srv := httptest.NewServer(c.handler()) |
| 815 | defer srv.Close() |
| 816 | |
| 817 | create := func(payload map[string]string) { |
| 818 | t.Helper() |
| 819 | body, _ := json.Marshal(payload) |
| 820 | req, _ := http.NewRequest("POST", srv.URL+"/instances", bytes.NewReader(body)) |
| 821 | req.Header.Set("Authorization", "Bearer test-control-token") |
| 822 | resp, err := http.DefaultClient.Do(req) |
| 823 | if err != nil { |
| 824 | t.Fatal(err) |
| 825 | } |
| 826 | defer resp.Body.Close() |
| 827 | if resp.StatusCode != http.StatusCreated { |
| 828 | t.Fatalf("create %v: HTTP %d", payload, resp.StatusCode) |
| 829 | } |
| 830 | } |
| 831 | envOrigin := func(host string) string { |
| 832 | t.Helper() |
| 833 | env, err := readEnvFile(filepath.Join(instanceDataDir, host+".env")) |
| 834 | if err != nil { |
| 835 | t.Fatal(err) |
| 836 | } |
| 837 | return env["LCHAT_HOME_ORIGIN"] |
| 838 | } |
| 839 | |
| 840 | create(map[string]string{"host": "defaulted.example.com"}) |
| 841 | if got := envOrigin("defaulted.example.com"); got != "https://home.ie.dev.example" { |
| 842 | t.Fatalf("defaulted create's LCHAT_HOME_ORIGIN = %q", got) |
| 843 | } |
| 844 | create(map[string]string{"host": "explicit.example.com", "home_origin": "https://home.other.example"}) |
| 845 | if got := envOrigin("explicit.example.com"); got != "https://home.other.example" { |
| 846 | t.Fatalf("explicit create's LCHAT_HOME_ORIGIN = %q, want the request's own", got) |
| 847 | } |
| 848 | } |
| 849 | |
amadan