| 1 | //go:build linux |
| 2 | |
| 3 | package cgroup |
| 4 | |
| 5 | import ( |
| 6 | "bufio" |
| 7 | "context" |
| 8 | "errors" |
| 9 | "fmt" |
| 10 | "io" |
| 11 | "os" |
| 12 | "os/exec" |
| 13 | "path/filepath" |
| 14 | "strconv" |
| 15 | "strings" |
| 16 | "sync" |
| 17 | "syscall" |
| 18 | "testing" |
| 19 | "time" |
| 20 | ) |
| 21 | |
| 22 | // The generation layout against a real kernel (zero-downtime plan 2, Task |
| 23 | // 4). Opt-in: CARLOS_GEN_KERNEL=1 inside a disposable unit with |
| 24 | // Delegate=yes (the plan's recipe). Without the variable, or without a |
| 25 | // delegated cgroup v2 subtree to work in, every test here skips. |
| 26 | |
| 27 | func genKernelManager(t *testing.T) *Manager { |
| 28 | t.Helper() |
| 29 | if os.Getenv("CARLOS_GEN_KERNEL") != "1" { |
| 30 | t.Skip("set CARLOS_GEN_KERNEL=1 inside a unit with Delegate=yes (zero-downtime plan 2, Task 4)") |
| 31 | } |
| 32 | raw, err := os.ReadFile("/proc/self/cgroup") |
| 33 | if err != nil { |
| 34 | t.Skipf("cgroup v2 delegation unavailable: %v", err) |
| 35 | } |
| 36 | rel, ok := ParseSelfCgroup(string(raw)) |
| 37 | if !ok { |
| 38 | t.Skip("cgroup v2 delegation unavailable: no unified cgroup path") |
| 39 | } |
| 40 | root := filepath.Join("/sys/fs/cgroup", rel) |
| 41 | if filepath.Base(root) == "supervise" { |
| 42 | root = filepath.Dir(root) // an earlier test in this process moved us |
| 43 | } |
| 44 | sup := filepath.Join(root, "supervise") |
| 45 | if err := os.MkdirAll(sup, 0o755); err != nil { |
| 46 | t.Skipf("cgroup v2 delegation unavailable: %v", err) |
| 47 | } |
| 48 | if err := os.WriteFile(filepath.Join(sup, "cgroup.procs"), []byte(strconv.Itoa(os.Getpid())), 0o644); err != nil { |
| 49 | t.Skipf("cgroup v2 delegation unavailable: %v", err) |
| 50 | } |
| 51 | if err := enableChildren(root); err != nil { |
| 52 | t.Skipf("cgroup v2 delegation unavailable: %v", err) |
| 53 | } |
| 54 | return &Manager{root: root, logf: t.Logf} |
| 55 | } |
| 56 | |
| 57 | func genKernelCmd(mode string) *exec.Cmd { |
| 58 | cmd := exec.Command(os.Args[0], "-test.run=^TestGenKernelChild$", "-test.count=1") |
| 59 | cmd.Env = []string{"CARLOS_GEN_CHILD=" + mode} |
| 60 | cmd.Stderr = os.Stderr // a child that fails says why |
| 61 | return cmd |
| 62 | } |
| 63 | |
| 64 | // TestGenKernelChild is the synthetic process: "echo" answers each line it |
| 65 | // reads, standing in for an incumbent serving requests; "hog" touches |
| 66 | // memory until something kills it. |
| 67 | func TestGenKernelChild(t *testing.T) { |
| 68 | switch os.Getenv("CARLOS_GEN_CHILD") { |
| 69 | case "echo": |
| 70 | fmt.Println("ready") |
| 71 | sc := bufio.NewScanner(os.Stdin) |
| 72 | for sc.Scan() { |
| 73 | fmt.Println(sc.Text()) |
| 74 | } |
| 75 | os.Exit(0) |
| 76 | case "hog": |
| 77 | // Wait for the parent's go-ahead, which it gives only after it has |
| 78 | // checked that this process is inside its capped cgroup. EOF means |
| 79 | // the parent is gone: exit rather than allocate unwatched. |
| 80 | if _, err := bufio.NewReader(os.Stdin).ReadString('\n'); err != nil { |
| 81 | fmt.Fprintln(os.Stderr, "hog: no go-ahead:", err) |
| 82 | os.Exit(4) |
| 83 | } |
| 84 | // One populating mmap: the kernel faults every page in without |
| 85 | // returning to user space, so memory.high's throttle (a sleep on |
| 86 | // each return) cannot slow the climb to memory.max. A hog that |
| 87 | // touches pages one at a time crawls at the throttle, swaps, and |
| 88 | // can sit under the limit for minutes instead of dying. |
| 89 | mem, err := syscall.Mmap(-1, 0, 256<<20, syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_ANON|syscall.MAP_PRIVATE|syscall.MAP_POPULATE) |
| 90 | if err != nil { |
| 91 | fmt.Fprintln(os.Stderr, "hog: mmap:", err) |
| 92 | os.Exit(2) |
| 93 | } |
| 94 | _ = mem |
| 95 | // Bounded: a hog that survives 1 GiB in all was not confined. |
| 96 | hold := make([][]byte, 0, 192) |
| 97 | for len(hold) < cap(hold) { |
| 98 | b := make([]byte, 4<<20) |
| 99 | for i := 0; i < len(b); i += 4096 { |
| 100 | b[i] = 1 |
| 101 | } |
| 102 | hold = append(hold, b) |
| 103 | } |
| 104 | fmt.Fprintln(os.Stderr, "hog: survived 1 GiB: not confined") |
| 105 | os.Exit(3) |
| 106 | default: |
| 107 | // Synthetic child only. It returns rather than skips, so a kernel |
| 108 | // run's -test.run 'GenKernel' shows no SKIP line (as TestShimKernelChild). |
| 109 | return |
| 110 | } |
| 111 | } |
| 112 | |
| 113 | type echoChild struct { |
| 114 | in io.WriteCloser |
| 115 | out *bufio.Reader |
| 116 | } |
| 117 | |
| 118 | func (e *echoChild) answer(t *testing.T, send, want string) { |
| 119 | t.Helper() |
| 120 | got := make(chan string, 1) |
| 121 | go func() { |
| 122 | if send != "" { |
| 123 | fmt.Fprintln(e.in, send) |
| 124 | } |
| 125 | line, _ := e.out.ReadString('\n') |
| 126 | got <- line |
| 127 | }() |
| 128 | select { |
| 129 | case line := <-got: |
| 130 | if line != want+"\n" { |
| 131 | t.Fatalf("incumbent answered %q, want %q", line, want) |
| 132 | } |
| 133 | case <-time.After(5 * time.Second): |
| 134 | t.Fatal("incumbent stopped answering") |
| 135 | } |
| 136 | } |
| 137 | |
| 138 | func (e *echoChild) ping(t *testing.T) { t.Helper(); e.answer(t, "ping", "ping") } |
| 139 | |
| 140 | func echoPipes(t *testing.T, cmd *exec.Cmd) *echoChild { |
| 141 | t.Helper() |
| 142 | in, err := cmd.StdinPipe() |
| 143 | if err != nil { |
| 144 | t.Fatal(err) |
| 145 | } |
| 146 | out, err := cmd.StdoutPipe() |
| 147 | if err != nil { |
| 148 | t.Fatal(err) |
| 149 | } |
| 150 | return &echoChild{in: in, out: bufio.NewReader(out)} |
| 151 | } |
| 152 | |
| 153 | func expectFile(t *testing.T, path, want string) { |
| 154 | t.Helper() |
| 155 | b, err := os.ReadFile(path) |
| 156 | if err != nil { |
| 157 | t.Fatal(err) |
| 158 | } |
| 159 | if got := strings.TrimSpace(string(b)); got != want { |
| 160 | t.Fatalf("%s = %q, want %q", path, got, want) |
| 161 | } |
| 162 | } |
| 163 | |
| 164 | func oomKills(t *testing.T, dir string) int { |
| 165 | t.Helper() |
| 166 | raw, err := os.ReadFile(filepath.Join(dir, "memory.events")) |
| 167 | if err != nil { |
| 168 | t.Fatal(err) |
| 169 | } |
| 170 | for _, line := range strings.Split(string(raw), "\n") { |
| 171 | if f := strings.Fields(line); len(f) == 2 && f[0] == "oom_kill" { |
| 172 | n, _ := strconv.Atoi(f[1]) |
| 173 | return n |
| 174 | } |
| 175 | } |
| 176 | t.Fatalf("%s/memory.events has no oom_kill line", dir) |
| 177 | return 0 |
| 178 | } |
| 179 | |
| 180 | // cgroupOf returns pid's unified cgroup. It reports rather than fails, so |
| 181 | // a caller holding a process that must not run on can kill it first. |
| 182 | func cgroupOf(pid int) (string, error) { |
| 183 | raw, err := os.ReadFile(fmt.Sprintf("/proc/%d/cgroup", pid)) |
| 184 | if err != nil { |
| 185 | return "", err |
| 186 | } |
| 187 | rel, ok := ParseSelfCgroup(string(raw)) |
| 188 | if !ok { |
| 189 | return "", fmt.Errorf("pid %d has no unified cgroup", pid) |
| 190 | } |
| 191 | return rel, nil |
| 192 | } |
| 193 | |
| 194 | // kernelKid is a synthetic child the test started. Its one waiter runs at |
| 195 | // most once, so the test body and the cleanup can both wait on it. |
| 196 | type kernelKid struct { |
| 197 | cmd *exec.Cmd |
| 198 | once sync.Once |
| 199 | done chan struct{} |
| 200 | err error |
| 201 | } |
| 202 | |
| 203 | func (k *kernelKid) reaped() <-chan struct{} { |
| 204 | k.once.Do(func() { |
| 205 | go func() { k.err = k.cmd.Wait(); close(k.done) }() |
| 206 | }) |
| 207 | return k.done |
| 208 | } |
| 209 | |
| 210 | func (k *kernelKid) wait() error { <-k.reaped(); return k.err } |
| 211 | |
| 212 | // reapOnCleanup is registered right after a spawn, before anything that |
| 213 | // can fail: whatever the test's outcome, the child ends. It tries the |
| 214 | // cgroup's own kill first, on a fresh budget, then kills the process |
| 215 | // directly, closes the pipes the test holds, and reaps on a bounded wait. |
| 216 | // A child the spawn never started is left alone. |
| 217 | func reapOnCleanup(t *testing.T, m *Manager, host, child string, cmd *exec.Cmd, pipes ...io.Closer) *kernelKid { |
| 218 | t.Helper() |
| 219 | k := &kernelKid{cmd: cmd, done: make(chan struct{})} |
| 220 | t.Cleanup(func() { |
| 221 | closePipes := func() { |
| 222 | for _, p := range pipes { |
| 223 | if p != nil { |
| 224 | _ = p.Close() |
| 225 | } |
| 226 | } |
| 227 | } |
| 228 | if cmd == nil || cmd.Process == nil { |
| 229 | closePipes() |
| 230 | return |
| 231 | } |
| 232 | if c, err := m.OpenCleanupChild(host, child); err == nil { |
| 233 | ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) |
| 234 | if err := c.KillAndEmpty(ctx); err != nil { |
| 235 | t.Logf("cleanup: %s/%s: cgroup kill: %v (killing pid %d directly)", host, child, err, cmd.Process.Pid) |
| 236 | } |
| 237 | cancel() |
| 238 | _ = c.Close() |
| 239 | } |
| 240 | _ = cmd.Process.Kill() |
| 241 | closePipes() |
| 242 | select { |
| 243 | case <-k.reaped(): |
| 244 | case <-time.After(5 * time.Second): |
| 245 | t.Errorf("cleanup: pid %d was not reaped within 5s", cmd.Process.Pid) |
| 246 | } |
| 247 | }) |
| 248 | return k |
| 249 | } |
| 250 | |
| 251 | // Spec §12, Containment: "a memory-hog successor that overruns its child |
| 252 | // limit and is OOM-killed while the incumbent keeps serving." A small |
| 253 | // limit, not a tier: the accounting path takes any limit, and 64M makes |
| 254 | // the hog die in seconds rather than after a gigabyte of swap. |
| 255 | func TestGenOOMKernel(t *testing.T) { |
| 256 | m := genKernelManager(t) |
| 257 | const host = "oom.example" |
| 258 | small := Limit{MemoryMax: 64 << 20} |
| 259 | parent := m.hostDir(host) |
| 260 | // Registered first, so it runs last: after every child is reaped. |
| 261 | t.Cleanup(func() { |
| 262 | m.RemoveChild(host, GenChild(1)) |
| 263 | m.RemoveChild(host, GenChild(2)) |
| 264 | m.CleanupHost(host) |
| 265 | }) |
| 266 | |
| 267 | var echo *echoChild |
| 268 | incumbentCmd, err := m.StartChild(host, GenChild(1), small, small, func() *exec.Cmd { |
| 269 | cmd := genKernelCmd("echo") |
| 270 | echo = echoPipes(t, cmd) |
| 271 | return cmd |
| 272 | }) |
| 273 | var echoIn io.Closer |
| 274 | if echo != nil { |
| 275 | echoIn = echo.in |
| 276 | } |
| 277 | incumbent := reapOnCleanup(t, m, host, GenChild(1), incumbentCmd, echoIn) |
| 278 | if err != nil { |
| 279 | t.Fatal(err) |
| 280 | } |
| 281 | echo.answer(t, "", "ready") |
| 282 | expectFile(t, filepath.Join(parent, "memory.max"), "67108864") |
| 283 | expectFile(t, filepath.Join(parent, "memory.oom.group"), "0") |
| 284 | expectFile(t, filepath.Join(parent, GenChild(1), "memory.oom.group"), "1") |
| 285 | got1, err := cgroupOf(incumbent.cmd.Process.Pid) |
| 286 | if want := filepath.Join(strings.TrimPrefix(m.Root(), "/sys/fs/cgroup/"), ExecName(host), GenChild(1)); err != nil || got1 != want { |
| 287 | t.Fatalf("incumbent runs in %s (%v), want %s", got1, err, want) |
| 288 | } |
| 289 | |
| 290 | // The raise happens before the successor spawns. |
| 291 | if err := m.SetParentLimit(host, OverlapLimit(small, small)); err != nil { |
| 292 | t.Fatal(err) |
| 293 | } |
| 294 | expectFile(t, filepath.Join(parent, "memory.max"), "134217728") |
| 295 | |
| 296 | var hogIn io.WriteCloser |
| 297 | var hogPipeErr error |
| 298 | hogCmd, err := m.StartChild(host, GenChild(2), small, small, func() *exec.Cmd { |
| 299 | cmd := genKernelCmd("hog") |
| 300 | hogIn, hogPipeErr = cmd.StdinPipe() |
| 301 | return cmd |
| 302 | }) |
| 303 | hog := reapOnCleanup(t, m, host, GenChild(2), hogCmd, hogIn) |
| 304 | if err != nil { |
| 305 | t.Fatal(err) |
| 306 | } |
| 307 | if hogPipeErr != nil { |
| 308 | t.Fatalf("hog stdin: %v", hogPipeErr) |
| 309 | } |
| 310 | // StartChild refuses a generation whose limits were not written (Task 7), |
| 311 | // but prove the confinement here anyway, on the real kernel, before the |
| 312 | // hog runs: it waits on stdin. Any failure from here kills it: the |
| 313 | // cleanup registered above runs. |
| 314 | g2 := filepath.Join(parent, GenChild(2)) |
| 315 | wantG2 := filepath.Join(strings.TrimPrefix(m.Root(), "/sys/fs/cgroup/"), ExecName(host), GenChild(2)) |
| 316 | if got, err := os.ReadFile(filepath.Join(g2, "memory.max")); err != nil || strings.TrimSpace(string(got)) != "67108864" { |
| 317 | t.Fatalf("g2 memory.max = %q, %v; want 67108864 before the hog runs", got, err) |
| 318 | } |
| 319 | if gotCg, err := cgroupOf(hog.cmd.Process.Pid); err != nil || gotCg != wantG2 { |
| 320 | t.Fatalf("hog runs in %s (%v), want %s", gotCg, err, wantG2) |
| 321 | } |
| 322 | if _, err := fmt.Fprintln(hogIn, "go"); err != nil { |
| 323 | t.Fatalf("hog go-ahead: %v", err) |
| 324 | } |
| 325 | select { |
| 326 | case <-hog.reaped(): |
| 327 | err = hog.err |
| 328 | case <-time.After(60 * time.Second): |
| 329 | t.Fatal("the hog was not OOM-killed within 60s") |
| 330 | } |
| 331 | var exitErr *exec.ExitError |
| 332 | if !errors.As(err, &exitErr) { |
| 333 | t.Fatalf("hog exit: %v", err) |
| 334 | } |
| 335 | if ws, ok := exitErr.Sys().(syscall.WaitStatus); !ok || !ws.Signaled() || ws.Signal() != syscall.SIGKILL { |
| 336 | t.Fatalf("hog ended with %v, want SIGKILL from the OOM killer", err) |
| 337 | } |
| 338 | if n := oomKills(t, filepath.Join(parent, GenChild(2))); n < 1 { |
| 339 | t.Fatalf("g2 memory.events oom_kill = %d, want at least 1", n) |
| 340 | } |
| 341 | if n := oomKills(t, filepath.Join(parent, GenChild(1))); n != 0 { |
| 342 | t.Fatalf("the incumbent's child saw %d OOM kills", n) |
| 343 | } |
| 344 | echo.ping(t) // the incumbent keeps serving |
| 345 | |
| 346 | // The restore happens after the retiring child is empty. |
| 347 | m.RemoveChild(host, GenChild(2)) |
| 348 | if _, err := os.Stat(filepath.Join(parent, GenChild(2))); !os.IsNotExist(err) { |
| 349 | t.Fatalf("empty g2 not removed: %v", err) |
| 350 | } |
| 351 | if err := m.SetParentLimit(host, small); err != nil { |
| 352 | t.Fatal(err) |
| 353 | } |
| 354 | expectFile(t, filepath.Join(parent, "memory.max"), "67108864") |
| 355 | echo.ping(t) |
| 356 | } |
| 357 | |
| 358 | // Strict containment against the kernel: per-generation reservation, the |
| 359 | // parent limit through one API, retirement of one generation, and the |
| 360 | // parent's kill taking every generation. |
| 361 | func TestGenStrictKernel(t *testing.T) { |
| 362 | m := genKernelManager(t) |
| 363 | const host = "strictgen.example" |
| 364 | dir := m.hostDir(host) |
| 365 | parent, err := m.PrepareStrictParent(host, TierDefault) |
| 366 | if err != nil { |
| 367 | t.Fatal(err) |
| 368 | } |
| 369 | // Registered first, so it runs last: after every child is reaped. The |
| 370 | // handles close before the parent's, then the directories go. |
| 371 | var handles []*Prepared |
| 372 | t.Cleanup(func() { |
| 373 | for i := len(handles) - 1; i >= 0; i-- { |
| 374 | _ = handles[i].Close() |
| 375 | } |
| 376 | _ = parent.Close() |
| 377 | for n := uint64(1); n <= 3; n++ { |
| 378 | m.RemoveChild(host, GenChild(n)) |
| 379 | } |
| 380 | m.CleanupHost(host) |
| 381 | }) |
| 382 | expectFile(t, filepath.Join(dir, "memory.oom.group"), "0") |
| 383 | expectFile(t, filepath.Join(dir, "memory.max"), "536870912") |
| 384 | if b, err := os.ReadFile(filepath.Join(dir, "cgroup.subtree_control")); err != nil || !strings.Contains(string(b), "memory") { |
| 385 | t.Fatalf("parent subtree_control = %q, %v", b, err) |
| 386 | } |
| 387 | refused := genKernelCmd("echo") |
| 388 | err = parent.Start(refused) |
| 389 | reapOnCleanup(t, m, host, "", refused) |
| 390 | if !errors.Is(err, ErrStrictState) { |
| 391 | t.Fatalf("a generation parent accepted a process: %v", err) |
| 392 | } |
| 393 | |
| 394 | prepareGen := func(n uint64) *Prepared { |
| 395 | t.Helper() |
| 396 | g, err := parent.PrepareGen(n, TierDefault) |
| 397 | if err != nil { |
| 398 | t.Fatal(err) |
| 399 | } |
| 400 | handles = append(handles, g) |
| 401 | return g |
| 402 | } |
| 403 | start := func(g *Prepared, n uint64) (*kernelKid, *echoChild) { |
| 404 | t.Helper() |
| 405 | cmd := genKernelCmd("echo") |
| 406 | e := echoPipes(t, cmd) |
| 407 | err := g.Start(cmd) |
| 408 | k := reapOnCleanup(t, m, host, GenChild(n), cmd, e.in) |
| 409 | if err != nil { |
| 410 | t.Fatal(err) |
| 411 | } |
| 412 | e.answer(t, "", "ready") |
| 413 | return k, e |
| 414 | } |
| 415 | g1 := prepareGen(1) |
| 416 | kid1, echo1 := start(g1, 1) |
| 417 | expectFile(t, filepath.Join(dir, GenChild(1), "memory.oom.group"), "1") |
| 418 | if err := parent.SetParentLimit(OverlapLimit(TierDefault, TierDefault)); err != nil { |
| 419 | t.Fatal(err) |
| 420 | } |
| 421 | expectFile(t, filepath.Join(dir, "memory.max"), "1073741824") |
| 422 | g2 := prepareGen(2) |
| 423 | kid2, echo2 := start(g2, 2) |
| 424 | base := filepath.Join(strings.TrimPrefix(m.Root(), "/sys/fs/cgroup/"), ExecName(host)) |
| 425 | if got, err := cgroupOf(kid1.cmd.Process.Pid); err != nil || got != filepath.Join(base, GenChild(1)) { |
| 426 | t.Fatalf("g1 process in %s (%v)", got, err) |
| 427 | } |
| 428 | if got, err := cgroupOf(kid2.cmd.Process.Pid); err != nil || got != filepath.Join(base, GenChild(2)) { |
| 429 | t.Fatalf("g2 process in %s (%v)", got, err) |
| 430 | } |
| 431 | echo1.ping(t) |
| 432 | |
| 433 | // Retire g1 alone. |
| 434 | ctx1, cancel1 := context.WithTimeout(t.Context(), 3*time.Second) |
| 435 | defer cancel1() |
| 436 | if err := g1.KillAndEmpty(ctx1); err != nil { |
| 437 | t.Fatal(err) |
| 438 | } |
| 439 | _ = kid1.wait() |
| 440 | echo2.ping(t) |
| 441 | if err := g1.CloseRemove(); err != nil { |
| 442 | t.Fatal(err) |
| 443 | } |
| 444 | if _, err := os.Stat(filepath.Join(dir, GenChild(1))); !os.IsNotExist(err) { |
| 445 | t.Fatalf("g1 directory survived CloseRemove: %v", err) |
| 446 | } |
| 447 | if err := parent.SetParentLimit(TierDefault); err != nil { |
| 448 | t.Fatal(err) |
| 449 | } |
| 450 | expectFile(t, filepath.Join(dir, "memory.max"), "536870912") |
| 451 | |
| 452 | // A limit this API did not set stops the next spawn. |
| 453 | if err := os.WriteFile(filepath.Join(dir, "memory.max"), []byte("268435456"), 0o644); err != nil { |
| 454 | t.Fatal(err) |
| 455 | } |
| 456 | g3 := prepareGen(3) |
| 457 | refused3 := genKernelCmd("echo") |
| 458 | err = g3.Start(refused3) |
| 459 | reapOnCleanup(t, m, host, GenChild(3), refused3) |
| 460 | if !errors.Is(err, ErrStrictRefused) { |
| 461 | t.Fatalf("spawn under a foreign parent limit: %v", err) |
| 462 | } |
| 463 | if err := g3.CloseRemove(); err != nil { |
| 464 | t.Fatal(err) |
| 465 | } |
| 466 | |
| 467 | // A terminal stop: the parent's kill takes every generation. Its own |
| 468 | // budget, not the one g1's retirement spent. |
| 469 | ctx2, cancel2 := context.WithTimeout(t.Context(), 3*time.Second) |
| 470 | defer cancel2() |
| 471 | if err := parent.KillAndEmpty(ctx2); err != nil { |
| 472 | t.Fatal(err) |
| 473 | } |
| 474 | _ = kid2.wait() |
| 475 | if err := g2.CloseRemove(); err != nil { |
| 476 | t.Fatal(err) |
| 477 | } |
| 478 | if err := parent.Close(); err != nil { |
| 479 | t.Fatal(err) |
| 480 | } |
| 481 | m.CleanupHost(host) |
| 482 | } |
| 483 | |
| 484 | // sleepKid starts nothing itself: it is the build for a tenant that sleeps |
| 485 | // until killed. |
| 486 | func sleepKid() *exec.Cmd { return exec.Command("/bin/sleep", "60") } |
| 487 | |
| 488 | // staysUp fails the test if k's process exits within a moment of its start |
| 489 | // (amadan #272's symptom: a child SIGKILLed as it lands), and checks that |
| 490 | // the process is the tenant, exec'd, in want (a path below /sys/fs/cgroup). |
| 491 | func staysUp(t *testing.T, label string, k *kernelKid, want string) { |
| 492 | t.Helper() |
| 493 | select { |
| 494 | case <-k.reaped(): |
| 495 | t.Fatalf("%s: the child died as it landed: %v", label, k.err) |
| 496 | case <-time.After(500 * time.Millisecond): |
| 497 | } |
| 498 | pid := k.cmd.Process.Pid |
| 499 | rel, err := cgroupOf(pid) |
| 500 | if err != nil || "/sys/fs/cgroup/"+rel != want { |
| 501 | t.Fatalf("%s: pid %d runs in %q (%v), want %s", label, pid, rel, err, want) |
| 502 | } |
| 503 | if comm, err := os.ReadFile(fmt.Sprintf("/proc/%d/comm", pid)); err != nil || strings.TrimSpace(string(comm)) != "sleep" { |
| 504 | t.Fatalf("%s: pid %d is %q (%v), want the exec'd tenant, sleep", label, pid, comm, err) |
| 505 | } |
| 506 | } |
| 507 | |
| 508 | func inodeOf(t *testing.T, path string) uint64 { |
| 509 | t.Helper() |
| 510 | var st syscall.Stat_t |
| 511 | if err := syscall.Lstat(path, &st); err != nil { |
| 512 | t.Fatalf("stat %s: %v", path, err) |
| 513 | } |
| 514 | return st.Ino |
| 515 | } |
| 516 | |
| 517 | // The respawn scenario amadan #272 found with clone3 (a clone3 into a |
| 518 | // cgroup that took cgroup.kill is SIGKILLed at once), now through the |
| 519 | // shim: a flat start (a worker's) lands in carlos-exec@<host> placed before |
| 520 | // exec, a cleanup kills and proves it empty, and each respawn gets a fresh |
| 521 | // cgroup (new inode, the tier written again) and stays up as the tenant. |
| 522 | // The activator's TestClaimContainmentCleanupRespawn runs the same sequence |
| 523 | // through ExecBackend.KillHost, but needs a delegation with io to arm |
| 524 | // cgroup.Setup; this one runs under a user manager's (cpu memory pids). |
| 525 | func TestGenKernelShimRespawnAfterKill(t *testing.T) { |
| 526 | m := genKernelManager(t) |
| 527 | const host = "respawn.example" |
| 528 | dir := m.hostDir(host) |
| 529 | t.Cleanup(func() { m.CleanupHost(host) }) |
| 530 | tier := strconv.FormatInt(TierDefault.MemoryMax, 10) |
| 531 | var last uint64 |
| 532 | for attempt := range 3 { |
| 533 | label := fmt.Sprintf("spawn %d", attempt) |
| 534 | cmd, err := m.Start(host, TierDefault, sleepKid) |
| 535 | if err != nil { |
| 536 | t.Fatalf("%s: %v", label, err) |
| 537 | } |
| 538 | k := &kernelKid{cmd: cmd, done: make(chan struct{})} |
| 539 | t.Cleanup(func() { _ = cmd.Process.Kill(); <-k.reaped() }) |
| 540 | staysUp(t, label, k, dir) |
| 541 | ino := inodeOf(t, dir) |
| 542 | if attempt > 0 && ino == last { |
| 543 | t.Fatalf("%s reused the killed host cgroup", label) |
| 544 | } |
| 545 | last = ino |
| 546 | if got := strings.TrimSpace(readFixture(t, filepath.Join(dir, "memory.max"))); got != tier { |
| 547 | t.Fatalf("%s: memory.max = %q, want the tier", label, got) |
| 548 | } |
| 549 | c, err := m.OpenCleanup(host) |
| 550 | if err != nil { |
| 551 | t.Fatal(err) |
| 552 | } |
| 553 | ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) |
| 554 | err = c.KillAndEmpty(ctx) |
| 555 | cancel() |
| 556 | _ = c.Close() |
| 557 | if err != nil { |
| 558 | t.Fatalf("%s: KillAndEmpty: %v", label, err) |
| 559 | } |
| 560 | select { |
| 561 | case <-k.reaped(): |
| 562 | case <-time.After(5 * time.Second): |
| 563 | t.Fatalf("%s: the killed child was not reaped", label) |
| 564 | } |
| 565 | } |
| 566 | } |
| 567 | |
| 568 | // amadan #272, real-unit check (d), through the shim: a flat start puts its |
| 569 | // process directly in carlos-exec@<host>, and a generation start beside it |
| 570 | // is refused (ErrLegacyParentPopulated) before it builds anything or makes |
| 571 | // a g* child. The recovery control: once the direct member has exited, the |
| 572 | // same host starts a capped g1 under a parent that holds no process. |
| 573 | func TestGenKernelGenerationRefusedBesideAFlatMember(t *testing.T) { |
| 574 | m := genKernelManager(t) |
| 575 | const host = "legacy.example" |
| 576 | parent := m.hostDir(host) |
| 577 | t.Cleanup(func() { |
| 578 | m.RemoveChild(host, GenChild(1)) |
| 579 | m.CleanupHost(host) |
| 580 | }) |
| 581 | legacy, err := m.Start(host, TierDefault, sleepKid) |
| 582 | if err != nil { |
| 583 | t.Fatal(err) |
| 584 | } |
| 585 | lk := &kernelKid{cmd: legacy, done: make(chan struct{})} |
| 586 | t.Cleanup(func() { _ = legacy.Process.Kill(); <-lk.reaped() }) |
| 587 | staysUp(t, "flat", lk, parent) |
| 588 | |
| 589 | built := false |
| 590 | cmd, err := m.StartChild(host, GenChild(1), TierDefault, TierDefault, func() *exec.Cmd { built = true; return sleepKid() }) |
| 591 | if !errors.Is(err, ErrLegacyParentPopulated) || cmd != nil || built { |
| 592 | t.Fatalf("StartChild beside a flat member = %v, %v (built %v): want ErrLegacyParentPopulated, nothing built", cmd, err, built) |
| 593 | } |
| 594 | if _, err := os.Stat(filepath.Join(parent, GenChild(1))); !os.IsNotExist(err) { |
| 595 | t.Fatalf("the refused start made g1: %v", err) |
| 596 | } |
| 597 | |
| 598 | _ = legacy.Process.Kill() |
| 599 | <-lk.reaped() |
| 600 | for deadline := time.Now().Add(5 * time.Second); strings.TrimSpace(readFixture(t, filepath.Join(parent, "cgroup.procs"))) != ""; time.Sleep(20 * time.Millisecond) { |
| 601 | if time.Now().After(deadline) { |
| 602 | t.Fatal("the parent still holds the killed flat process") |
| 603 | } |
| 604 | } |
| 605 | g1, err := m.StartChild(host, GenChild(1), TierDefault, TierDefault, sleepKid) |
| 606 | if err != nil { |
| 607 | t.Fatalf("g1 once the flat member exited: %v", err) |
| 608 | } |
| 609 | gk := reapOnCleanup(t, m, host, GenChild(1), g1) |
| 610 | staysUp(t, "g1", gk, filepath.Join(parent, GenChild(1))) |
| 611 | if got := strings.TrimSpace(readFixture(t, filepath.Join(parent, GenChild(1), "memory.max"))); got != strconv.FormatInt(TierDefault.MemoryMax, 10) { |
| 612 | t.Errorf("g1 memory.max = %q, want the tier", got) |
| 613 | } |
| 614 | if got := strings.TrimSpace(readFixture(t, filepath.Join(parent, "cgroup.procs"))); got != "" { |
| 615 | t.Errorf("the parent holds processes directly: %q", got) |
| 616 | } |
| 617 | } |
| 618 | |
| 619 | // Review focus 4 on a real kernel: a generation's shim held before its |
| 620 | // placement is registered in flight and still in the agent's own cgroup, so |
| 621 | // the kernel says the host is empty. The host's cleanup must not: Populated |
| 622 | // is true, Members is ErrPlacementInFlight, and KillAndEmpty waits, killing |
| 623 | // nothing, until the placement resolves (here: the shim is SIGKILLed, so |
| 624 | // the start is refused as a placement failure) and only then proves the |
| 625 | // host empty. |
| 626 | func TestGenKernelCleanupWaitsForAPlacementInFlight(t *testing.T) { |
| 627 | m := genKernelManager(t) |
| 628 | const host = "inflight.example" |
| 629 | parent := m.hostDir(host) |
| 630 | t.Cleanup(func() { |
| 631 | m.RemoveChild(host, GenChild(1)) |
| 632 | m.CleanupHost(host) |
| 633 | }) |
| 634 | setShimVar(t, &shimTestHold, 1) |
| 635 | held := make(chan int, 1) |
| 636 | orig := startCmd |
| 637 | t.Cleanup(func() { startCmd = orig }) |
| 638 | startCmd = func(cmd *exec.Cmd) error { |
| 639 | if err := orig(cmd); err != nil { |
| 640 | return err |
| 641 | } |
| 642 | pid, specFD := cmd.Process.Pid, len(cmd.ExtraFiles) // the shim's spec fd: see TestShimKernelSignalBeforePlacement |
| 643 | go func() { |
| 644 | if waitShimHeld(pid, specFD) { |
| 645 | held <- pid |
| 646 | } else { |
| 647 | held <- 0 |
| 648 | } |
| 649 | }() |
| 650 | return nil |
| 651 | } |
| 652 | started := make(chan error, 1) |
| 653 | go func() { |
| 654 | _, err := m.StartChild(host, GenChild(1), TierDefault, TierDefault, sleepKid) |
| 655 | started <- err |
| 656 | }() |
| 657 | pid := <-held |
| 658 | if pid == 0 { |
| 659 | t.Fatal("the shim never reached its hold before placement") |
| 660 | } |
| 661 | t.Cleanup(func() { _ = syscall.Kill(pid, syscall.SIGKILL) }) |
| 662 | |
| 663 | c, err := m.OpenCleanup(host) |
| 664 | if err != nil { |
| 665 | t.Fatal(err) |
| 666 | } |
| 667 | t.Cleanup(func() { _ = c.Close() }) |
| 668 | events := readFixture(t, filepath.Join(parent, "cgroup.events")) |
| 669 | if !strings.Contains(events, "populated 0") { |
| 670 | t.Fatalf("the parent's cgroup.events = %q: want the kernel to see it empty, or this proves nothing", events) |
| 671 | } |
| 672 | if populated, err := c.Populated(); err != nil || !populated { |
| 673 | t.Fatalf("Populated = %v, %v with a shim held before placement: want true", populated, err) |
| 674 | } |
| 675 | if _, err := c.Members(); !errors.Is(err, ErrPlacementInFlight) { |
| 676 | t.Fatalf("Members = %v, want ErrPlacementInFlight", err) |
| 677 | } |
| 678 | ctx, cancel := context.WithTimeout(t.Context(), 200*time.Millisecond) |
| 679 | err = c.KillAndEmpty(ctx) |
| 680 | cancel() |
| 681 | if !errors.Is(err, ErrPlacementInFlight) || !errors.Is(err, context.DeadlineExceeded) { |
| 682 | t.Fatalf("KillAndEmpty with a placement in flight = %v: want ErrPlacementInFlight, unproven", err) |
| 683 | } |
| 684 | if procGone(pid) { |
| 685 | t.Fatal("the held shim died: the unproven proof killed it, or the hold did not hold") |
| 686 | } |
| 687 | |
| 688 | proof := make(chan error, 1) |
| 689 | go func() { |
| 690 | ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) |
| 691 | defer cancel() |
| 692 | proof <- c.KillAndEmpty(ctx) |
| 693 | }() |
| 694 | select { |
| 695 | case err := <-proof: |
| 696 | t.Fatalf("KillAndEmpty returned %v while the placement was still in flight", err) |
| 697 | case <-time.After(200 * time.Millisecond): |
| 698 | } |
| 699 | if err := syscall.Kill(pid, syscall.SIGKILL); err != nil { |
| 700 | t.Fatal(err) |
| 701 | } |
| 702 | if err := <-started; !errors.Is(err, ErrLimitNotApplied) || !isPlacementError(err) { |
| 703 | t.Fatalf("the generation start = %v, want ErrLimitNotApplied from the placement failure", err) |
| 704 | } |
| 705 | if err := <-proof; err != nil { |
| 706 | t.Fatalf("KillAndEmpty once the placement resolved = %v, want the proof", err) |
| 707 | } |
| 708 | assertNothingInFlight(t, m) |
| 709 | } |
| 710 | |
| 711 | // A generation start whose exec.CommandContext context ends while its shim |
| 712 | // is held before placement: exec.Cmd kills the shim, nothing runs, the |
| 713 | // start is not the box's condition (not ErrLimitNotApplied), the empty |
| 714 | // child is removed, and nothing is left in flight. |
| 715 | func TestGenKernelCancelledGenerationStartIsNotABoxCondition(t *testing.T) { |
| 716 | m := genKernelManager(t) |
| 717 | const host = "cancel.example" |
| 718 | parent := m.hostDir(host) |
| 719 | t.Cleanup(func() { |
| 720 | m.RemoveChild(host, GenChild(1)) |
| 721 | m.CleanupHost(host) |
| 722 | }) |
| 723 | setShimVar(t, &shimTestHold, 1) |
| 724 | ctx, cancel := context.WithCancel(t.Context()) |
| 725 | defer cancel() |
| 726 | held := make(chan bool, 1) |
| 727 | orig := startCmd |
| 728 | t.Cleanup(func() { startCmd = orig }) |
| 729 | startCmd = func(cmd *exec.Cmd) error { |
| 730 | if err := orig(cmd); err != nil { |
| 731 | return err |
| 732 | } |
| 733 | pid, specFD := cmd.Process.Pid, len(cmd.ExtraFiles) |
| 734 | go func() { |
| 735 | ok := waitShimHeld(pid, specFD) |
| 736 | held <- ok |
| 737 | cancel() |
| 738 | }() |
| 739 | return nil |
| 740 | } |
| 741 | marker := filepath.Join(t.TempDir(), "ran") |
| 742 | var built *exec.Cmd |
| 743 | cmd, err := m.StartChild(host, GenChild(1), TierDefault, TierDefault, func() *exec.Cmd { |
| 744 | built = exec.CommandContext(ctx, "/bin/sh", "-c", `touch "$0"; sleep 60`, marker) |
| 745 | return built |
| 746 | }) |
| 747 | if !<-held { |
| 748 | t.Fatal("the shim never reached its hold before placement") |
| 749 | } |
| 750 | if err == nil || cmd != nil || errors.Is(err, ErrLimitNotApplied) { |
| 751 | t.Fatalf("StartChild = %v, %v: want a failure that is not ErrLimitNotApplied", cmd, err) |
| 752 | } |
| 753 | // The context's own error, bare, as exec.Cmd.Start answers an ended |
| 754 | // context (whole-branch review, finding 3). |
| 755 | if err != context.Canceled { |
| 756 | t.Fatalf("StartChild = %v: want context.Canceled itself", err) |
| 757 | } |
| 758 | t.Logf("cancelled generation start: %v", err) |
| 759 | assertReaped(t, built) |
| 760 | assertNoMarker(t, marker) |
| 761 | assertNothingInFlight(t, m) |
| 762 | if _, err := os.Stat(filepath.Join(parent, GenChild(1))); !os.IsNotExist(err) { |
| 763 | t.Fatalf("the empty g1 survived: %v", err) |
| 764 | } |
| 765 | } |
| 766 | |
amadan