amadan

carlos / platform Public

zdd plan 2 task 4: kernel suite: an OOM-killed successor leaves the incumbent serving

TestGenOOMKernel and TestGenStrictKernel run the generation layout against
a real kernel. Opt-in: CARLOS_GEN_KERNEL=1 inside a unit with Delegate=yes;
otherwise they skip with a message naming why.

Passed on DOCS-T1 under `systemd-run --user -p Delegate=yes`: both PASS
(OOM test 14s, strict test 0.07s). Plain `go test` run: all three SKIP.

The synthetic hog allocates with one MAP_POPULATE mmap, not a page-touch
loop. The loop was throttled by memory.high on each return to user space,
thrashed against swap and its own text pages, and had not died after 60s
(memory.events: high 9497, oom_kill 0). One populating mmap climbs to
memory.max inside the kernel and is OOM-killed in seconds.

The dev-box run (plan Step 4) was not done: it touches production
infrastructure and needs Paul's approval.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude f54813c27d959f929fe8b030455f6ab594837c62 parent e85733a
1 file changed, +359 −0
  • internal/cgroup/gen_kernel_linux_test.go +359 −0
diff --git a/internal/cgroup/gen_kernel_linux_test.go b/internal/cgroup/gen_kernel_linux_test.go
new file mode 100644
index 000000000..099674e19
--- /dev/null
+++ b/internal/cgroup/gen_kernel_linux_test.go
@@ -0,0 +1,359 @@
+//go:build linux
+
+package cgroup
+
+import (
+ "bufio"
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strconv"
+ "strings"
+ "syscall"
+ "testing"
+ "time"
+)
+
+// The generation layout against a real kernel (zero-downtime plan 2, Task
+// 4). Opt-in: CARLOS_GEN_KERNEL=1 inside a disposable unit with
+// Delegate=yes (the plan's recipe). Without the variable, or without a
+// delegated cgroup v2 subtree to work in, every test here skips.
+
+func genKernelManager(t *testing.T) *Manager {
+ t.Helper()
+ if os.Getenv("CARLOS_GEN_KERNEL") != "1" {
+ t.Skip("set CARLOS_GEN_KERNEL=1 inside a unit with Delegate=yes (zero-downtime plan 2, Task 4)")
+ }
+ raw, err := os.ReadFile("/proc/self/cgroup")
+ if err != nil {
+ t.Skipf("cgroup v2 delegation unavailable: %v", err)
+ }
+ rel, ok := ParseSelfCgroup(string(raw))
+ if !ok {
+ t.Skip("cgroup v2 delegation unavailable: no unified cgroup path")
+ }
+ root := filepath.Join("/sys/fs/cgroup", rel)
+ if filepath.Base(root) == "supervise" {
+ root = filepath.Dir(root) // an earlier test in this process moved us
+ }
+ sup := filepath.Join(root, "supervise")
+ if err := os.MkdirAll(sup, 0o755); err != nil {
+ t.Skipf("cgroup v2 delegation unavailable: %v", err)
+ }
+ if err := os.WriteFile(filepath.Join(sup, "cgroup.procs"), []byte(strconv.Itoa(os.Getpid())), 0o644); err != nil {
+ t.Skipf("cgroup v2 delegation unavailable: %v", err)
+ }
+ if err := enableChildren(root); err != nil {
+ t.Skipf("cgroup v2 delegation unavailable: %v", err)
+ }
+ return &Manager{root: root, logf: t.Logf}
+}
+
+func genKernelCmd(mode string) *exec.Cmd {
+ cmd := exec.Command(os.Args[0], "-test.run=^TestGenKernelChild$", "-test.count=1")
+ cmd.Env = []string{"CARLOS_GEN_CHILD=" + mode}
+ return cmd
+}
+
+// TestGenKernelChild is the synthetic process: "echo" answers each line it
+// reads, standing in for an incumbent serving requests; "hog" touches
+// memory until something kills it.
+func TestGenKernelChild(t *testing.T) {
+ switch os.Getenv("CARLOS_GEN_CHILD") {
+ case "echo":
+ fmt.Println("ready")
+ sc := bufio.NewScanner(os.Stdin)
+ for sc.Scan() {
+ fmt.Println(sc.Text())
+ }
+ os.Exit(0)
+ case "hog":
+ // One populating mmap: the kernel faults every page in without
+ // returning to user space, so memory.high's throttle (a sleep on
+ // each return) cannot slow the climb to memory.max. A hog that
+ // touches pages one at a time crawls at the throttle, swaps, and
+ // can sit under the limit for minutes instead of dying.
+ if mem, err := syscall.Mmap(-1, 0, 512<<20, syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_ANON|syscall.MAP_PRIVATE|syscall.MAP_POPULATE); err == nil {
+ _ = mem
+ }
+ var hold [][]byte
+ for {
+ b := make([]byte, 4<<20)
+ for i := 0; i < len(b); i += 4096 {
+ b[i] = 1
+ }
+ hold = append(hold, b)
+ }
+ default:
+ t.Skip("synthetic child only")
+ }
+}
+
+type echoChild struct {
+ in io.WriteCloser
+ out *bufio.Reader
+}
+
+func (e *echoChild) answer(t *testing.T, send, want string) {
+ t.Helper()
+ got := make(chan string, 1)
+ go func() {
+ if send != "" {
+ fmt.Fprintln(e.in, send)
+ }
+ line, _ := e.out.ReadString('\n')
+ got <- line
+ }()
+ select {
+ case line := <-got:
+ if line != want+"\n" {
+ t.Fatalf("incumbent answered %q, want %q", line, want)
+ }
+ case <-time.After(5 * time.Second):
+ t.Fatal("incumbent stopped answering")
+ }
+}
+
+func (e *echoChild) ping(t *testing.T) { t.Helper(); e.answer(t, "ping", "ping") }
+
+func echoPipes(t *testing.T, cmd *exec.Cmd) *echoChild {
+ t.Helper()
+ in, err := cmd.StdinPipe()
+ if err != nil {
+ t.Fatal(err)
+ }
+ out, err := cmd.StdoutPipe()
+ if err != nil {
+ t.Fatal(err)
+ }
+ return &echoChild{in: in, out: bufio.NewReader(out)}
+}
+
+func expectFile(t *testing.T, path, want string) {
+ t.Helper()
+ b, err := os.ReadFile(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got := strings.TrimSpace(string(b)); got != want {
+ t.Fatalf("%s = %q, want %q", path, got, want)
+ }
+}
+
+func oomKills(t *testing.T, dir string) int {
+ t.Helper()
+ raw, err := os.ReadFile(filepath.Join(dir, "memory.events"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, line := range strings.Split(string(raw), "\n") {
+ if f := strings.Fields(line); len(f) == 2 && f[0] == "oom_kill" {
+ n, _ := strconv.Atoi(f[1])
+ return n
+ }
+ }
+ t.Fatalf("%s/memory.events has no oom_kill line", dir)
+ return 0
+}
+
+func cgroupOf(t *testing.T, pid int) string {
+ t.Helper()
+ raw, err := os.ReadFile(fmt.Sprintf("/proc/%d/cgroup", pid))
+ if err != nil {
+ t.Fatal(err)
+ }
+ rel, ok := ParseSelfCgroup(string(raw))
+ if !ok {
+ t.Fatalf("pid %d has no unified cgroup", pid)
+ }
+ return rel
+}
+
+// Spec §12, Containment: "a memory-hog successor that overruns its child
+// limit and is OOM-killed while the incumbent keeps serving." A small
+// limit, not a tier: the accounting path takes any limit, and 64M makes
+// the hog die in seconds rather than after a gigabyte of swap.
+func TestGenOOMKernel(t *testing.T) {
+ m := genKernelManager(t)
+ const host = "oom.example"
+ small := Limit{MemoryMax: 64 << 20}
+ parent := m.hostDir(host)
+
+ var echo *echoChild
+ incumbent, err := m.StartChild(host, GenChild(1), small, small, func() *exec.Cmd {
+ cmd := genKernelCmd("echo")
+ echo = echoPipes(t, cmd)
+ return cmd
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() {
+ if c, err := m.OpenCleanupChild(host, GenChild(1)); err == nil {
+ ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+ _ = c.KillAndEmpty(ctx)
+ cancel()
+ _ = c.Close()
+ }
+ _ = incumbent.Wait()
+ m.RemoveChild(host, GenChild(1))
+ m.CleanupHost(host)
+ })
+ echo.answer(t, "", "ready")
+ expectFile(t, filepath.Join(parent, "memory.max"), "67108864")
+ expectFile(t, filepath.Join(parent, "memory.oom.group"), "0")
+ expectFile(t, filepath.Join(parent, GenChild(1), "memory.oom.group"), "1")
+ if got, want := cgroupOf(t, incumbent.Process.Pid), filepath.Join(strings.TrimPrefix(m.Root(), "/sys/fs/cgroup/"), ExecName(host), GenChild(1)); got != want {
+ t.Fatalf("incumbent runs in %s, want %s", got, want)
+ }
+
+ // The raise happens before the successor spawns.
+ if err := m.SetParentLimit(host, OverlapLimit(small, small)); err != nil {
+ t.Fatal(err)
+ }
+ expectFile(t, filepath.Join(parent, "memory.max"), "134217728")
+
+ hog, err := m.StartChild(host, GenChild(2), small, small, func() *exec.Cmd { return genKernelCmd("hog") })
+ if err != nil {
+ t.Fatal(err)
+ }
+ waited := make(chan error, 1)
+ go func() { waited <- hog.Wait() }()
+ select {
+ case err = <-waited:
+ case <-time.After(60 * time.Second):
+ _ = hog.Process.Kill()
+ t.Fatal("the hog was not OOM-killed within 60s")
+ }
+ var exitErr *exec.ExitError
+ if !errors.As(err, &exitErr) {
+ t.Fatalf("hog exit: %v", err)
+ }
+ if ws, ok := exitErr.Sys().(syscall.WaitStatus); !ok || !ws.Signaled() || ws.Signal() != syscall.SIGKILL {
+ t.Fatalf("hog ended with %v, want SIGKILL from the OOM killer", err)
+ }
+ if n := oomKills(t, filepath.Join(parent, GenChild(2))); n < 1 {
+ t.Fatalf("g2 memory.events oom_kill = %d, want at least 1", n)
+ }
+ if n := oomKills(t, filepath.Join(parent, GenChild(1))); n != 0 {
+ t.Fatalf("the incumbent's child saw %d OOM kills", n)
+ }
+ echo.ping(t) // the incumbent keeps serving
+
+ // The restore happens after the retiring child is empty.
+ m.RemoveChild(host, GenChild(2))
+ if _, err := os.Stat(filepath.Join(parent, GenChild(2))); !os.IsNotExist(err) {
+ t.Fatalf("empty g2 not removed: %v", err)
+ }
+ if err := m.SetParentLimit(host, small); err != nil {
+ t.Fatal(err)
+ }
+ expectFile(t, filepath.Join(parent, "memory.max"), "67108864")
+ echo.ping(t)
+}
+
+// Strict containment against the kernel: per-generation reservation, the
+// parent limit through one API, retirement of one generation, and the
+// parent's kill taking every generation.
+func TestGenStrictKernel(t *testing.T) {
+ m := genKernelManager(t)
+ const host = "strictgen.example"
+ dir := m.hostDir(host)
+ parent, err := m.PrepareStrictParent(host, TierDefault)
+ if err != nil {
+ t.Fatal(err)
+ }
+ expectFile(t, filepath.Join(dir, "memory.oom.group"), "0")
+ expectFile(t, filepath.Join(dir, "memory.max"), "536870912")
+ if b, err := os.ReadFile(filepath.Join(dir, "cgroup.subtree_control")); err != nil || !strings.Contains(string(b), "memory") {
+ t.Fatalf("parent subtree_control = %q, %v", b, err)
+ }
+ if err := parent.Start(genKernelCmd("echo")); !errors.Is(err, ErrStrictState) {
+ t.Fatalf("a generation parent accepted a process: %v", err)
+ }
+
+ start := func(g *Prepared) (*exec.Cmd, *echoChild) {
+ t.Helper()
+ cmd := genKernelCmd("echo")
+ e := echoPipes(t, cmd)
+ if err := g.Start(cmd); err != nil {
+ t.Fatal(err)
+ }
+ e.answer(t, "", "ready")
+ return cmd, e
+ }
+ g1, err := parent.PrepareGen(1, TierDefault)
+ if err != nil {
+ t.Fatal(err)
+ }
+ cmd1, echo1 := start(g1)
+ expectFile(t, filepath.Join(dir, GenChild(1), "memory.oom.group"), "1")
+ if err := parent.SetParentLimit(OverlapLimit(TierDefault, TierDefault)); err != nil {
+ t.Fatal(err)
+ }
+ expectFile(t, filepath.Join(dir, "memory.max"), "1073741824")
+ g2, err := parent.PrepareGen(2, TierDefault)
+ if err != nil {
+ t.Fatal(err)
+ }
+ cmd2, echo2 := start(g2)
+ base := filepath.Join(strings.TrimPrefix(m.Root(), "/sys/fs/cgroup/"), ExecName(host))
+ if got := cgroupOf(t, cmd1.Process.Pid); got != filepath.Join(base, GenChild(1)) {
+ t.Fatalf("g1 process in %s", got)
+ }
+ if got := cgroupOf(t, cmd2.Process.Pid); got != filepath.Join(base, GenChild(2)) {
+ t.Fatalf("g2 process in %s", got)
+ }
+ echo1.ping(t)
+
+ // Retire g1 alone.
+ ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second)
+ defer cancel()
+ if err := g1.KillAndEmpty(ctx); err != nil {
+ t.Fatal(err)
+ }
+ _ = cmd1.Wait()
+ echo2.ping(t)
+ if err := g1.CloseRemove(); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := os.Stat(filepath.Join(dir, GenChild(1))); !os.IsNotExist(err) {
+ t.Fatalf("g1 directory survived CloseRemove: %v", err)
+ }
+ if err := parent.SetParentLimit(TierDefault); err != nil {
+ t.Fatal(err)
+ }
+ expectFile(t, filepath.Join(dir, "memory.max"), "536870912")
+
+ // A limit this API did not set stops the next spawn.
+ if err := os.WriteFile(filepath.Join(dir, "memory.max"), []byte("268435456"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ g3, err := parent.PrepareGen(3, TierDefault)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := g3.Start(genKernelCmd("echo")); !errors.Is(err, ErrStrictRefused) {
+ t.Fatalf("spawn under a foreign parent limit: %v", err)
+ }
+ if err := g3.CloseRemove(); err != nil {
+ t.Fatal(err)
+ }
+
+ // A terminal stop: the parent's kill takes every generation.
+ if err := parent.KillAndEmpty(ctx); err != nil {
+ t.Fatal(err)
+ }
+ _ = cmd2.Wait()
+ if err := g2.CloseRemove(); err != nil {
+ t.Fatal(err)
+ }
+ if err := parent.Close(); err != nil {
+ t.Fatal(err)
+ }
+ m.CleanupHost(host)
+}