| 1 | package activator |
| 2 | |
| 3 | // Tests for the zero-downtime-deploys rev 19 amendment |
| 4 | // (docs/superpowers/specs/2026-10-13-zero-downtime-deploys-rev19-amendment.md): |
| 5 | // a generation's socket keeps the flag-off file name in a directory of its |
| 6 | // own, a generation launch matches a flag-off launch, the directory goes |
| 7 | // once the generation is proven gone, and a flat host too long for a |
| 8 | // generation path keeps its canonical socket as a foreign row does. |
| 9 | |
| 10 | import ( |
| 11 | "context" |
| 12 | "encoding/json" |
| 13 | "errors" |
| 14 | "maps" |
| 15 | "os" |
| 16 | "path/filepath" |
| 17 | "slices" |
| 18 | "strings" |
| 19 | "testing" |
| 20 | "time" |
| 21 | |
| 22 | "github.com/carlosframework/platform/internal/instancectl" |
| 23 | "github.com/carlosframework/platform/internal/provision" |
| 24 | "github.com/carlosframework/platform/internal/registry" |
| 25 | ) |
| 26 | |
| 27 | // goneProc is a process whose leader has exited. |
| 28 | type goneProc struct{} |
| 29 | |
| 30 | func (goneProc) Stop(context.Context) error { return nil } |
| 31 | func (goneProc) Exited() bool { return true } |
| 32 | |
| 33 | // envMap is an environment list ("K=V") as a map. |
| 34 | func envMap(t *testing.T, env []string) map[string]string { |
| 35 | t.Helper() |
| 36 | m := make(map[string]string, len(env)) |
| 37 | for _, kv := range env { |
| 38 | k, v, ok := strings.Cut(kv, "=") |
| 39 | if !ok { |
| 40 | continue // a multi-line value's continuation in `env` output |
| 41 | } |
| 42 | m[k] = v |
| 43 | } |
| 44 | return m |
| 45 | } |
| 46 | |
| 47 | // sameLaunch checks rule 2 for one layout: the two argvs differ only in |
| 48 | // --socket's directory, and the two environments only in |
| 49 | // CARLOS_CONTROL_SOCKET, which names each launch's own socket + ".control". |
| 50 | func sameLaunch(t *testing.T, offArgv, genArgv []string, offEnv, genEnv map[string]string, offSock, genSock string) { |
| 51 | t.Helper() |
| 52 | i := slices.Index(offArgv, "--socket") |
| 53 | j := slices.Index(genArgv, "--socket") |
| 54 | if i < 0 || j != i || i+1 >= len(offArgv) || len(genArgv) != len(offArgv) { |
| 55 | t.Fatalf("argv shapes differ: flag off %v, generation %v", offArgv, genArgv) |
| 56 | } |
| 57 | if offArgv[i+1] != offSock || genArgv[i+1] != genSock { |
| 58 | t.Fatalf("--socket: flag off %q (want %q), generation %q (want %q)", offArgv[i+1], offSock, genArgv[i+1], genSock) |
| 59 | } |
| 60 | if offSock == genSock { |
| 61 | t.Fatalf("the generation launch bound the canonical socket %s", offSock) |
| 62 | } |
| 63 | if a, b := filepath.Base(offArgv[i+1]), filepath.Base(genArgv[i+1]); a != b { |
| 64 | t.Fatalf("--socket file name: flag off %q, generation %q; rule 2 wants the same", a, b) |
| 65 | } |
| 66 | rest := func(argv []string) []string { return slices.Delete(slices.Clone(argv), i+1, i+2) } |
| 67 | if !slices.Equal(rest(offArgv), rest(genArgv)) { |
| 68 | t.Fatalf("argv differs beyond --socket: flag off %v, generation %v", offArgv, genArgv) |
| 69 | } |
| 70 | if offEnv["CARLOS_CONTROL_SOCKET"] != offSock+".control" || genEnv["CARLOS_CONTROL_SOCKET"] != genSock+".control" { |
| 71 | t.Fatalf("CARLOS_CONTROL_SOCKET: flag off %q, generation %q", offEnv["CARLOS_CONTROL_SOCKET"], genEnv["CARLOS_CONTROL_SOCKET"]) |
| 72 | } |
| 73 | off, gen := maps.Clone(offEnv), maps.Clone(genEnv) |
| 74 | delete(off, "CARLOS_CONTROL_SOCKET") |
| 75 | delete(gen, "CARLOS_CONTROL_SOCKET") |
| 76 | if !maps.Equal(off, gen) { |
| 77 | for k := range off { |
| 78 | if gen[k] != off[k] { |
| 79 | t.Errorf("env %s: flag off %q, generation %q", k, off[k], gen[k]) |
| 80 | } |
| 81 | } |
| 82 | for k := range gen { |
| 83 | if _, ok := off[k]; !ok { |
| 84 | t.Errorf("env %s: only the generation launch has it (%q)", k, gen[k]) |
| 85 | } |
| 86 | } |
| 87 | t.FailNow() |
| 88 | } |
| 89 | if off["CARLOS_HOST"] == "" { |
| 90 | t.Fatal("neither launch carries CARLOS_HOST") |
| 91 | } |
| 92 | } |
| 93 | |
| 94 | // Rule 2, flat: a generation's exec has the flag-off argv and agent-owned |
| 95 | // environment apart from the socket's directory and the control path. |
| 96 | func TestFlatGenerationLaunchMatchesTheFlagOffLaunch(t *testing.T) { |
| 97 | t.Parallel() |
| 98 | requireUnixSockets(t) |
| 99 | b := testExecBackend(t) |
| 100 | b.RunDirOverride = shortRunDir(t) |
| 101 | host := "shop.example.com" |
| 102 | argvPath, envPath := stageRecordingLiveFullEnv(t, b, host, "v1") |
| 103 | rt := flatGenRoute(b, host) |
| 104 | launch := func(n uint64, perGen bool) ([]string, map[string]string, string) { |
| 105 | t.Helper() |
| 106 | _ = os.Remove(argvPath) |
| 107 | _ = os.Remove(envPath) |
| 108 | ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) |
| 109 | defer cancel() |
| 110 | var ( |
| 111 | proc Proc |
| 112 | g Gen |
| 113 | ) |
| 114 | done := make(chan struct{}) |
| 115 | go func() { |
| 116 | proc, g, _ = b.StartGen(ctx, rt, n, perGen) // the recorder never binds; a health error is expected |
| 117 | close(done) |
| 118 | }() |
| 119 | argv := strings.Split(strings.TrimSpace(waitForFile(t, argvPath, 3*time.Second)), "\n") |
| 120 | env := envMap(t, strings.Split(strings.TrimSpace(waitForFile(t, envPath, 3*time.Second)), "\n")) |
| 121 | cancel() // recorded: end the health wait the recorder never answers |
| 122 | <-done |
| 123 | if proc != nil { |
| 124 | _ = proc.Stop(context.Background()) |
| 125 | } |
| 126 | return argv, env, g.Socket |
| 127 | } |
| 128 | offArgv, offEnv, offSock := launch(5, false) |
| 129 | genArgv, genEnv, genSock := launch(6, true) |
| 130 | if want := filepath.Join(b.RunDirOverride, "g", "6", host+".sock"); genSock != want { |
| 131 | t.Fatalf("generation socket %s, want %s", genSock, want) |
| 132 | } |
| 133 | sameLaunch(t, offArgv, genArgv, offEnv, genEnv, rt.Socket, genSock) |
| 134 | if offSock != rt.Socket { |
| 135 | t.Fatalf("flag-off handle names %s, want the canonical %s", offSock, rt.Socket) |
| 136 | } |
| 137 | fi, err := os.Stat(filepath.Dir(genSock)) |
| 138 | if err != nil || !fi.IsDir() || fi.Mode().Perm() != 0o750 { |
| 139 | t.Fatalf("generation directory %s: %v, %v; want a 0750 directory", filepath.Dir(genSock), fi, err) |
| 140 | } |
| 141 | } |
| 142 | |
| 143 | // Rule 2, nested: the generation's launch file has the flag-off argv and |
| 144 | // agent-owned environment apart from the socket's directory and the control |
| 145 | // path, and prepare made the generation's own directory before the start. |
| 146 | func TestNestedGenerationLaunchMatchesTheFlagOffLaunch(t *testing.T) { |
| 147 | t.Parallel() |
| 148 | ctl := &fakeControl{} |
| 149 | unit := &fakeUnit{} |
| 150 | b, rt := nestedBackend(t, ctl, unit) |
| 151 | launchFile := filepath.Join(provision.InstanceDir(b.DataDir, rt.Host), "launch.run.json") |
| 152 | var ( |
| 153 | serveAt string |
| 154 | dirAtStart bool |
| 155 | stop func() |
| 156 | ) |
| 157 | ctl.onStart = func() { |
| 158 | fi, err := os.Stat(filepath.Dir(serveAt)) |
| 159 | dirAtStart = err == nil && fi.IsDir() |
| 160 | stop = serveOK(t, serveAt) |
| 161 | } |
| 162 | launch := func(n uint64, perGen bool, sock, inv string) instancectl.Launch { |
| 163 | t.Helper() |
| 164 | serveAt, dirAtStart = sock, false |
| 165 | ctl.mu.Lock() |
| 166 | ctl.inv = inv |
| 167 | ctl.mu.Unlock() |
| 168 | unit.set("active", inv, true, true) |
| 169 | p, g, err := b.StartGen(context.Background(), rt, n, perGen) |
| 170 | if err != nil { |
| 171 | t.Fatalf("StartGen(%d, %v): %v", n, perGen, err) |
| 172 | } |
| 173 | if g.Socket != sock { |
| 174 | t.Fatalf("handle socket %s, want %s", g.Socket, sock) |
| 175 | } |
| 176 | if !dirAtStart { |
| 177 | t.Fatalf("%s did not exist when root was asked to start the unit", filepath.Dir(sock)) |
| 178 | } |
| 179 | raw, err := os.ReadFile(launchFile) |
| 180 | if err != nil { |
| 181 | t.Fatal(err) |
| 182 | } |
| 183 | var l instancectl.Launch |
| 184 | if err := json.Unmarshal(raw, &l); err != nil { |
| 185 | t.Fatal(err) |
| 186 | } |
| 187 | unit.set("inactive", inv, false, false) |
| 188 | waitFor(t, "the unit to exit", func() bool { return p.Exited() }) |
| 189 | ctx, cancel := context.WithTimeout(context.Background(), time.Second) |
| 190 | defer cancel() |
| 191 | _ = p.Stop(ctx) |
| 192 | if stop != nil { |
| 193 | stop() |
| 194 | stop = nil |
| 195 | } |
| 196 | return l |
| 197 | } |
| 198 | genSock := filepath.Join(filepath.Dir(rt.Socket), "g9", "app.sock") |
| 199 | off := launch(8, false, rt.Socket, "inv1") |
| 200 | gen := launch(9, true, genSock, "inv2") |
| 201 | sameLaunch(t, off.Argv, gen.Argv, envMap(t, off.Env), envMap(t, gen.Env), rt.Socket, genSock) |
| 202 | } |
| 203 | |
| 204 | // Rule 4: a flat host whose generation path would not fit a unix socket |
| 205 | // address is handled as a foreign row is: no generation path, a generation |
| 206 | // on its canonical socket, and an overlap that runs as a hold. |
| 207 | func TestOverLongFlatHostKeepsItsCanonicalSocket(t *testing.T) { |
| 208 | requireUnixSockets(t) |
| 209 | b := testExecBackend(t) |
| 210 | b.RunDirOverride = shortRunDir(t) |
| 211 | host := strings.Repeat("a", 60-len(".example.com")) + ".example.com" // 60 bytes |
| 212 | stageServingLive(t, b, host) |
| 213 | rt := flatGenRoute(b, host) |
| 214 | if l, _ := b.Layout(rt); l != LayoutFlat { |
| 215 | t.Fatalf("fixture classifies %q, want flat", l) |
| 216 | } |
| 217 | if l := b.GenLayout(rt); l != LayoutForeign { |
| 218 | t.Fatalf("GenLayout = %s, want foreign", l) |
| 219 | } |
| 220 | split := &SplitBackend{Exec: b} |
| 221 | if l := split.GenLayout(rt); l != LayoutForeign { |
| 222 | t.Fatalf("SplitBackend.GenLayout = %s, want foreign", l) |
| 223 | } |
| 224 | if g, ok := b.GenPaths(rt, 9); ok { |
| 225 | t.Fatalf("GenPaths = %+v, true; want no generation path", g) |
| 226 | } |
| 227 | a := &Service{backend: b, entries: map[string]*actEntry{host: {route: rt}}} |
| 228 | if m := a.EffectiveMode(host, ModeOverlap); m != ModeHold { |
| 229 | t.Fatalf("EffectiveMode(overlap) = %s, want hold, as for a foreign row", m) |
| 230 | } |
| 231 | ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
| 232 | defer cancel() |
| 233 | proc, g, err := b.StartGen(ctx, rt, 9, true) |
| 234 | if err != nil { |
| 235 | t.Fatalf("StartGen: %v", err) |
| 236 | } |
| 237 | defer proc.Stop(context.Background()) |
| 238 | if g.Socket != rt.Socket || g.Control != rt.Socket+".control" { |
| 239 | t.Fatalf("handle = %+v, want the canonical socket", g) |
| 240 | } |
| 241 | if !isSocket(rt.Socket) { |
| 242 | t.Fatalf("%s is not a bound socket", rt.Socket) |
| 243 | } |
| 244 | if _, err := os.Stat(filepath.Join(b.RunDirOverride, "g")); !os.IsNotExist(err) { |
| 245 | t.Fatalf("a generation directory was made for a host with no generation path (stat: %v)", err) |
| 246 | } |
| 247 | |
| 248 | // The same host one byte shorter has generation paths. |
| 249 | short := host[1:] |
| 250 | if l := b.GenLayout(flatGenRoute(b, short)); l != LayoutFlat { |
| 251 | t.Fatalf("GenLayout(%d-byte host) = %s, want flat", len(short), l) |
| 252 | } |
| 253 | if _, ok := b.GenPaths(flatGenRoute(b, short), 9); !ok { |
| 254 | t.Fatalf("GenPaths(%d-byte host) has no generation path", len(short)) |
| 255 | } |
| 256 | } |
| 257 | |
| 258 | // Rule 3: once a generation's termination is proven, its own empty |
| 259 | // directory goes. A directory its app left a socket in stays, a sibling's |
| 260 | // directory is never touched (empty or not), and a generation on the |
| 261 | // canonical socket touches nothing. |
| 262 | func TestRetiredGenerationDirectoryIsRemovedAndASiblingsIsNot(t *testing.T) { |
| 263 | root := t.TempDir() |
| 264 | flat := registry.Route{Host: "shop.example.com", Socket: filepath.Join(root, "shop.example.com.sock")} |
| 265 | nestedRun := filepath.Join(root, "instances", "shop.example.com") |
| 266 | nested := registry.Route{Host: "shop.example.com", Socket: filepath.Join(nestedRun, "app.sock")} |
| 267 | mk := func(dir string, files ...string) { |
| 268 | t.Helper() |
| 269 | if err := os.MkdirAll(dir, 0o750); err != nil { |
| 270 | t.Fatal(err) |
| 271 | } |
| 272 | for _, f := range files { |
| 273 | if err := os.WriteFile(filepath.Join(dir, f), nil, 0o600); err != nil { |
| 274 | t.Fatal(err) |
| 275 | } |
| 276 | } |
| 277 | } |
| 278 | exists := func(p string) bool { _, err := os.Lstat(p); return err == nil } |
| 279 | a := &Service{} |
| 280 | |
| 281 | for _, tc := range []struct { |
| 282 | name string |
| 283 | rt registry.Route |
| 284 | own string // generation 9's directory |
| 285 | siblings []string |
| 286 | }{ |
| 287 | {"flat", flat, filepath.Join(root, "g", "9"), []string{filepath.Join(root, "g", "8"), filepath.Join(root, "g", "10")}}, |
| 288 | {"nested", nested, filepath.Join(nestedRun, "g9"), []string{filepath.Join(nestedRun, "g8"), filepath.Join(nestedRun, "g10")}}, |
| 289 | } { |
| 290 | t.Run(tc.name, func(t *testing.T) { |
| 291 | sock := filepath.Join(tc.own, filepath.Base(tc.rt.Socket)) |
| 292 | g := Gen{N: 9, Socket: sock, Control: sock + ".control"} |
| 293 | mk(tc.own) |
| 294 | mk(tc.siblings[0], filepath.Base(tc.rt.Socket)) // a running sibling's socket |
| 295 | mk(tc.siblings[1]) // an empty sibling |
| 296 | mk(filepath.Dir(tc.rt.Socket), filepath.Base(tc.rt.Socket)) |
| 297 | |
| 298 | if a.genTerminated(tc.rt, g, &maintenanceProc{}) { |
| 299 | t.Fatal("a running generation read as terminated") |
| 300 | } |
| 301 | if !exists(tc.own) { |
| 302 | t.Fatal("the directory went before the termination proof") |
| 303 | } |
| 304 | |
| 305 | mk(tc.own, filepath.Base(tc.rt.Socket)) // the app left its socket behind |
| 306 | if !a.genTerminated(tc.rt, g, goneProc{}) { |
| 307 | t.Fatal("an exited generation read as running") |
| 308 | } |
| 309 | if !exists(sock) { |
| 310 | t.Fatal("the generation's own leftover socket was removed; only an empty directory goes") |
| 311 | } |
| 312 | |
| 313 | if err := os.Remove(sock); err != nil { |
| 314 | t.Fatal(err) |
| 315 | } |
| 316 | if !a.genTerminated(tc.rt, g, goneProc{}) { |
| 317 | t.Fatal("an exited generation read as running") |
| 318 | } |
| 319 | if exists(tc.own) { |
| 320 | t.Fatalf("the retired generation's empty directory %s is still there", tc.own) |
| 321 | } |
| 322 | for _, s := range tc.siblings { |
| 323 | if !exists(s) { |
| 324 | t.Fatalf("a sibling's directory %s was removed", s) |
| 325 | } |
| 326 | } |
| 327 | if !exists(tc.rt.Socket) { |
| 328 | t.Fatal("the canonical socket was touched") |
| 329 | } |
| 330 | |
| 331 | // A handleless generation proven gone is cleaned the same way. |
| 332 | mk(tc.own) |
| 333 | if !a.handlelessGenEmpty(tc.rt, g) { |
| 334 | t.Fatal("a handleless generation with no child read as running") |
| 335 | } |
| 336 | if exists(tc.own) { |
| 337 | t.Fatalf("the handleless generation's empty directory %s is still there", tc.own) |
| 338 | } |
| 339 | |
| 340 | // A generation on the canonical socket has no directory of its |
| 341 | // own: the run dir it sits in is never a candidate. |
| 342 | canon := Gen{N: 9, Socket: tc.rt.Socket, Control: tc.rt.Socket + ".control"} |
| 343 | if err := os.Remove(tc.rt.Socket); err != nil { |
| 344 | t.Fatal(err) |
| 345 | } |
| 346 | for _, s := range tc.siblings { |
| 347 | if err := os.RemoveAll(s); err != nil { |
| 348 | t.Fatal(err) |
| 349 | } |
| 350 | } |
| 351 | if tc.name == "flat" { |
| 352 | if err := os.Remove(filepath.Join(root, "g")); err != nil { |
| 353 | t.Fatal(err) |
| 354 | } |
| 355 | } |
| 356 | if !a.genTerminated(tc.rt, canon, goneProc{}) { |
| 357 | t.Fatal("an exited canonical generation read as running") |
| 358 | } |
| 359 | if !exists(filepath.Dir(tc.rt.Socket)) { |
| 360 | t.Fatal("a canonical generation's run dir was removed") |
| 361 | } |
| 362 | }) |
| 363 | } |
| 364 | } |
| 365 | |
| 366 | // Astra, review of 19c07b2b (P2 1): a flat host too long for a generation |
| 367 | // path binds its canonical socket with perGen true, and a canonical link a |
| 368 | // box left from the old naming must not fail that bind, as it does not on |
| 369 | // a flag-off spawn or StartStaged's foreign arm. |
| 370 | func TestOverLongFlatHostClearsALeftoverCanonicalLink(t *testing.T) { |
| 371 | requireUnixSockets(t) |
| 372 | b := testExecBackend(t) |
| 373 | b.RunDirOverride = shortRunDir(t) |
| 374 | host := strings.Repeat("a", 60-len(".example.com")) + ".example.com" // 60 bytes |
| 375 | stageServingLive(t, b, host) |
| 376 | rt := flatGenRoute(b, host) |
| 377 | if err := os.Symlink("/run/carlos/g/0123456789ab.3.sock", rt.Socket); err != nil { |
| 378 | t.Fatal(err) |
| 379 | } |
| 380 | ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
| 381 | defer cancel() |
| 382 | proc, g, err := b.StartGen(ctx, rt, 9, true) |
| 383 | if err != nil { |
| 384 | t.Fatalf("StartGen over a leftover canonical link: %v", err) |
| 385 | } |
| 386 | defer proc.Stop(context.Background()) |
| 387 | if g.Socket != rt.Socket { |
| 388 | t.Fatalf("handle socket %s, want the canonical %s", g.Socket, rt.Socket) |
| 389 | } |
| 390 | if !isSocket(rt.Socket) { |
| 391 | t.Fatalf("%s is not a bound socket: the leftover link was not cleared", rt.Socket) |
| 392 | } |
| 393 | } |
| 394 | |
| 395 | // Astra, review of 19c07b2b (P2 2): a failed wake's generation directory |
| 396 | // goes once failedWakeStopped's whole proof holds, the host cgroup's |
| 397 | // included where the generation had no child of its own, and never before. |
| 398 | func TestFailedWakeRemovesItsDirectoryOnlyOnceProven(t *testing.T) { |
| 399 | t.Parallel() |
| 400 | running := &sigtermIgnorer{} |
| 401 | exited := &sigtermIgnorer{} |
| 402 | exited.exited.Store(true) |
| 403 | for _, row := range []struct { |
| 404 | name string |
| 405 | proc Proc |
| 406 | killGen error // nil: the generation's own child was proven empty |
| 407 | populated bool // the host cgroup still holds something |
| 408 | want bool |
| 409 | }{ |
| 410 | {"leader still running", running, nil, false, false}, |
| 411 | {"child not proven empty", exited, errors.New("kill gen: timeout"), false, false}, |
| 412 | {"uncontained, host cgroup populated", exited, errNoGenContainment, true, false}, |
| 413 | {"uncontained, host cgroup empty", exited, errNoGenContainment, false, true}, |
| 414 | {"contained, child proven empty", exited, nil, true, true}, |
| 415 | } { |
| 416 | t.Run(row.name, func(t *testing.T) { |
| 417 | a, b, _, _ := testActivator(t, time.Hour) |
| 418 | kb := &killGenBackend{fakeBackend: b, err: row.killGen} |
| 419 | a.backend = &uncontainedStartBackend{unprovenStartBackend: &unprovenStartBackend{killGenBackend: kb}, populated: row.populated} |
| 420 | root := t.TempDir() |
| 421 | rt := testRoute("shop.example.com") |
| 422 | rt.Socket = filepath.Join(root, "shop.example.com.sock") |
| 423 | own := filepath.Join(root, "g", "4") |
| 424 | sibling := filepath.Join(root, "g", "3") |
| 425 | for _, d := range []string{own, sibling} { |
| 426 | if err := os.MkdirAll(d, 0o750); err != nil { |
| 427 | t.Fatal(err) |
| 428 | } |
| 429 | } |
| 430 | g := Gen{N: 4, Socket: filepath.Join(own, "shop.example.com.sock")} |
| 431 | g.Control = g.Socket + ".control" |
| 432 | if got := a.failedWakeStopped(rt, g, row.proc, LayoutFlat); got != row.want { |
| 433 | t.Fatalf("failedWakeStopped = %v, want %v", got, row.want) |
| 434 | } |
| 435 | if _, err := os.Stat(own); os.IsNotExist(err) != row.want { |
| 436 | t.Fatalf("own directory removed = %v, want %v (stat: %v)", os.IsNotExist(err), row.want, err) |
| 437 | } |
| 438 | if _, err := os.Stat(sibling); err != nil { |
| 439 | t.Fatalf("a sibling's empty directory was touched: %v", err) |
| 440 | } |
| 441 | }) |
| 442 | } |
| 443 | } |
| 444 | |
amadan