| 1 | //go:build linux |
| 2 | |
| 3 | package activator |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "errors" |
| 8 | "fmt" |
| 9 | "os" |
| 10 | "os/exec" |
| 11 | "path/filepath" |
| 12 | "slices" |
| 13 | "strconv" |
| 14 | "strings" |
| 15 | "testing" |
| 16 | "time" |
| 17 | |
| 18 | "github.com/carlosframework/platform/internal/cgroup" |
| 19 | "github.com/carlosframework/platform/internal/provision" |
| 20 | "github.com/carlosframework/platform/internal/registry" |
| 21 | ) |
| 22 | |
| 23 | // Amadan #272, real-unit check (d): on the dev box (arm64, kernel 6.12, |
| 24 | // systemd 257) a generation started under a carlos-exec@<host> that still |
| 25 | // held a process directly ran with no memory limit, silently, while the |
| 26 | // strict path refused. This runs the same shape against the real cgroup |
| 27 | // filesystem: a flat start (the older agent's layout) puts a process |
| 28 | // directly in the parent, and a generation start beside it must be refused |
| 29 | // before it creates a g* child or execs anything. The unit-level cases |
| 30 | // (internal/cgroup gen_test.go, containment_test.go) use a fake cgroupfs; |
| 31 | // only this one proves the kernel's side. Same gate as the claim-spawn |
| 32 | // kernel test: a disposable delegated user service, and the synthetic |
| 33 | // child TestClaimSpawnKernelChild. |
| 34 | func TestGenerationStartRefusedBesideADirectParentMember(t *testing.T) { |
| 35 | if os.Getenv("CARLOS_CLAIM_SPAWN_KERNEL") != "1" { |
| 36 | t.Skip("requires a disposable delegated user service") |
| 37 | } |
| 38 | manager := cgroup.Setup(t.Logf) |
| 39 | if manager == nil { |
| 40 | t.Fatal("missing delegated cgroup") |
| 41 | } |
| 42 | const host = "genparent.example" |
| 43 | parent := filepath.Join(manager.Root(), cgroup.ExecName(host)) |
| 44 | t.Cleanup(func() { manager.CleanupHost(host) }) |
| 45 | |
| 46 | legacy, err := manager.Start(host, cgroup.TierDefault, func() *exec.Cmd { |
| 47 | cmd := exec.Command(os.Args[0], "-test.run=^TestClaimSpawnKernelChild$", "-test.count=1") |
| 48 | cmd.Env = []string{"CARLOS_CLAIM_SPAWN_CHILD=1"} |
| 49 | cmd.Dir = t.TempDir() |
| 50 | return cmd |
| 51 | }) |
| 52 | if err != nil || legacy == nil || legacy.Process == nil { |
| 53 | t.Fatalf("flat start: %v, %v", legacy, err) |
| 54 | } |
| 55 | t.Cleanup(func() { |
| 56 | _ = legacy.Process.Kill() |
| 57 | _ = legacy.Wait() |
| 58 | }) |
| 59 | procs, err := os.ReadFile(filepath.Join(parent, "cgroup.procs")) |
| 60 | if err != nil || !strings.Contains(string(procs), "\n") { |
| 61 | t.Fatalf("the flat start's process is not a direct member of %s: %q, %v", parent, procs, err) |
| 62 | } |
| 63 | |
| 64 | b := testExecBackend(t) |
| 65 | b.Cgroups = manager |
| 66 | b.GenContainment = true |
| 67 | ran := false |
| 68 | cmd, _, err := b.spawnInCgroupAt(registry.Route{Host: host}, cgroup.GenChild(1), true, func() *exec.Cmd { |
| 69 | ran = true |
| 70 | return exec.Command("/bin/true") |
| 71 | }) |
| 72 | if !errors.Is(err, cgroup.ErrLegacyParentPopulated) || cmd != nil { |
| 73 | t.Fatalf("generation start beside a direct member = %v, %v: want ErrLegacyParentPopulated and no command", cmd, err) |
| 74 | } |
| 75 | if ran { |
| 76 | t.Error("the generation's command was built for exec") |
| 77 | } |
| 78 | ents, err := os.ReadDir(parent) |
| 79 | if err != nil { |
| 80 | t.Fatal(err) |
| 81 | } |
| 82 | for _, e := range ents { |
| 83 | if e.IsDir() { |
| 84 | t.Errorf("the refused start left %s under the parent", e.Name()) |
| 85 | } |
| 86 | } |
| 87 | |
| 88 | // Positive controls (zdd readiness round 2, Task 7): the refusal above |
| 89 | // says nothing unless the same start succeeds, capped, where the parent |
| 90 | // holds no process directly. The sibling control runs on another host; |
| 91 | // the recovery control is this host once its direct member is gone. |
| 92 | t.Run("sibling control", func(t *testing.T) { genParentSiblingControl(t, manager) }) |
| 93 | t.Run("recovery control", func(t *testing.T) { genParentRecoveryControl(t, b, host, parent, legacy) }) |
| 94 | } |
| 95 | |
| 96 | // genParentKernelChild is the sleeper TestClaimSpawnKernelChild runs as. |
| 97 | func genParentKernelChild(t *testing.T) func() *exec.Cmd { |
| 98 | dir := t.TempDir() |
| 99 | return func() *exec.Cmd { |
| 100 | cmd := exec.Command(os.Args[0], "-test.run=^TestClaimSpawnKernelChild$", "-test.count=1") |
| 101 | cmd.Env = []string{"CARLOS_CLAIM_SPAWN_CHILD=1"} |
| 102 | cmd.Dir = dir |
| 103 | return cmd |
| 104 | } |
| 105 | } |
| 106 | |
| 107 | // readCgroupFile reads one cgroupfs file, trimmed. |
| 108 | func readCgroupFile(t *testing.T, path string) string { |
| 109 | t.Helper() |
| 110 | b, err := os.ReadFile(path) |
| 111 | if err != nil { |
| 112 | t.Fatalf("read %s: %v", path, err) |
| 113 | } |
| 114 | return strings.TrimSpace(string(b)) |
| 115 | } |
| 116 | |
| 117 | // expectGenParent checks a generation parent the layout made: no process |
| 118 | // of its own, memory delegated to its children, memory.oom.group=0. |
| 119 | func expectGenParent(t *testing.T, parent string) { |
| 120 | t.Helper() |
| 121 | if got := readCgroupFile(t, filepath.Join(parent, "cgroup.procs")); got != "" { |
| 122 | t.Errorf("the parent holds processes directly: %q", got) |
| 123 | } |
| 124 | if got := readCgroupFile(t, filepath.Join(parent, "cgroup.subtree_control")); !slices.Contains(strings.Fields(got), "memory") { |
| 125 | t.Errorf("the parent's subtree_control is %q, want memory delegated", got) |
| 126 | } |
| 127 | if got := readCgroupFile(t, filepath.Join(parent, "memory.oom.group")); got != "0" { |
| 128 | t.Errorf("the parent's memory.oom.group = %q, want 0", got) |
| 129 | } |
| 130 | } |
| 131 | |
| 132 | // expectCappedGen checks that cmd runs in host's parent/<child> and that the |
| 133 | // child carries the default tier. |
| 134 | func expectCappedGen(t *testing.T, host, parent, child string, cmd *exec.Cmd) { |
| 135 | t.Helper() |
| 136 | tier := strconv.FormatInt(cgroup.TierDefault.MemoryMax, 10) |
| 137 | if got := readCgroupFile(t, filepath.Join(parent, child, "memory.max")); got != tier { |
| 138 | t.Errorf("%s memory.max = %q, want the tier %s", child, got, tier) |
| 139 | } |
| 140 | own, ok := cgroup.ParseSelfCgroup(readCgroupFile(t, filepath.Join("/proc", strconv.Itoa(cmd.Process.Pid), "cgroup"))) |
| 141 | if want := "/" + cgroup.ExecName(host) + "/" + child; !ok || !strings.HasSuffix(own, want) { |
| 142 | t.Errorf("pid %d runs in %q, want .../%s", cmd.Process.Pid, own, want[1:]) |
| 143 | } |
| 144 | } |
| 145 | |
| 146 | // genParentSiblingControl: on a host whose parent holds no process, g1 |
| 147 | // starts capped, and g2 starts capped beside the live g1 once the parent is |
| 148 | // raised, as a swap's overlap does. |
| 149 | func genParentSiblingControl(t *testing.T, manager *cgroup.Manager) { |
| 150 | const host = "genparent-ok.example" |
| 151 | b := &ExecBackend{Cgroups: manager, GenContainment: true, DataDir: t.TempDir(), OptDir: t.TempDir()} |
| 152 | rt := registry.Route{Host: host, Kind: "instance", DBPath: provision.DefaultDBPath(b.DataDir, host), |
| 153 | Socket: filepath.Join(provision.SocketDir, host+".sock")} |
| 154 | parent := filepath.Join(manager.Root(), cgroup.ExecName(host)) |
| 155 | t.Cleanup(func() { manager.CleanupHost(host) }) // registered first, so it runs last |
| 156 | build := genParentKernelChild(t) |
| 157 | |
| 158 | g1, _, err := b.spawnInCgroupAt(rt, cgroup.GenChild(1), true, build) |
| 159 | if err != nil || g1 == nil { |
| 160 | t.Fatalf("g1 under an empty parent: %v, %v", g1, err) |
| 161 | } |
| 162 | t.Cleanup(func() { _ = g1.Process.Kill() }) |
| 163 | g1Done := spawnStaysUp(t, "g1", g1) |
| 164 | expectGenParent(t, parent) |
| 165 | expectCappedGen(t, host, parent, cgroup.GenChild(1), g1) |
| 166 | |
| 167 | if err := manager.SetParentLimit(host, cgroup.OverlapLimit(cgroup.TierDefault, cgroup.TierDefault)); err != nil { |
| 168 | t.Fatal(err) |
| 169 | } |
| 170 | g2, _, err := b.spawnInCgroupAt(rt, cgroup.GenChild(2), true, build) |
| 171 | if err != nil || g2 == nil { |
| 172 | t.Fatalf("g2 beside a live g1: %v, %v", g2, err) |
| 173 | } |
| 174 | t.Cleanup(func() { _ = g2.Process.Kill() }) |
| 175 | g2Done := spawnStaysUp(t, "g2", g2) |
| 176 | expectGenParent(t, parent) |
| 177 | expectCappedGen(t, host, parent, cgroup.GenChild(2), g2) |
| 178 | select { |
| 179 | case err := <-g1Done: |
| 180 | t.Fatalf("g1 died while g2 started beside it: %v", err) |
| 181 | default: |
| 182 | } |
| 183 | |
| 184 | for _, g := range []struct { |
| 185 | n uint64 |
| 186 | done <-chan error |
| 187 | }{{2, g2Done}, {1, g1Done}} { |
| 188 | ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) |
| 189 | err := b.KillGen(ctx, rt, g.n) |
| 190 | cancel() |
| 191 | if err != nil { |
| 192 | t.Fatalf("KillGen %d: %v", g.n, err) |
| 193 | } |
| 194 | reaped(t, fmt.Sprintf("g%d", g.n), g.done) |
| 195 | } |
| 196 | } |
| 197 | |
| 198 | // genParentRecoveryControl: once the direct member the refusal named has |
| 199 | // exited, the same host starts a capped g1 under a parent the layout made. |
| 200 | // This is the manual recovery amadan #272's ruling relies on. |
| 201 | func genParentRecoveryControl(t *testing.T, b *ExecBackend, host, parent string, legacy *exec.Cmd) { |
| 202 | _ = legacy.Process.Kill() |
| 203 | _ = legacy.Wait() |
| 204 | deadline := time.Now().Add(5 * time.Second) |
| 205 | for readCgroupFile(t, filepath.Join(parent, "cgroup.procs")) != "" { |
| 206 | if time.Now().After(deadline) { |
| 207 | t.Fatal("the parent still holds the killed legacy process") |
| 208 | } |
| 209 | time.Sleep(20 * time.Millisecond) |
| 210 | } |
| 211 | g1, _, err := b.spawnInCgroupAt(registry.Route{Host: host}, cgroup.GenChild(1), true, genParentKernelChild(t)) |
| 212 | if err != nil || g1 == nil { |
| 213 | t.Fatalf("g1 after the legacy process exited: %v, %v", g1, err) |
| 214 | } |
| 215 | t.Cleanup(func() { _ = g1.Process.Kill() }) // a failed check below must not leave g1 running |
| 216 | done := spawnStaysUp(t, "recovered g1", g1) |
| 217 | expectGenParent(t, parent) |
| 218 | expectCappedGen(t, host, parent, cgroup.GenChild(1), g1) |
| 219 | _ = g1.Process.Kill() |
| 220 | reaped(t, "recovered g1", done) |
| 221 | b.Cgroups.RemoveChild(host, cgroup.GenChild(1)) |
| 222 | } |
| 223 | |
amadan