| 1 | package cgroup |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "fmt" |
| 7 | "io" |
| 8 | "os" |
| 9 | "os/exec" |
| 10 | "path/filepath" |
| 11 | "slices" |
| 12 | "strconv" |
| 13 | "strings" |
| 14 | "sync" |
| 15 | "syscall" |
| 16 | "testing" |
| 17 | "time" |
| 18 | ) |
| 19 | |
| 20 | // quietOOM points the oom_score_adj write at a temp dir, so a fake start |
| 21 | // that reports our own PID never re-scores the test process. |
| 22 | func quietOOM(t *testing.T) { |
| 23 | t.Helper() |
| 24 | dir := t.TempDir() |
| 25 | orig := oomScoreAdjPath |
| 26 | oomScoreAdjPath = func(pid int) string { return filepath.Join(dir, strconv.Itoa(pid)) } |
| 27 | t.Cleanup(func() { oomScoreAdjPath = orig }) |
| 28 | } |
| 29 | |
| 30 | func writeFixture(t *testing.T, path, body string) { |
| 31 | t.Helper() |
| 32 | if err := os.WriteFile(path, []byte(body), 0o644); err != nil { |
| 33 | t.Fatal(err) |
| 34 | } |
| 35 | } |
| 36 | |
| 37 | func readFixture(t *testing.T, path string) string { |
| 38 | t.Helper() |
| 39 | b, err := os.ReadFile(path) |
| 40 | if err != nil { |
| 41 | t.Fatalf("read %s: %v", path, err) |
| 42 | } |
| 43 | return string(b) |
| 44 | } |
| 45 | |
| 46 | // genManager is a delegated fixture root whose host dir already lists the |
| 47 | // four controllers, as a real child of the delegated root would. A temp |
| 48 | // directory is no cgroupfs, so no shim can place a process there: the |
| 49 | // placement is faked (FakePlacementForTest), as if the kernel had placed |
| 50 | // it. genFixture is the same root with the real placement. |
| 51 | func genManager(t *testing.T, host string) (*Manager, string) { |
| 52 | t.Helper() |
| 53 | m, parent := genFixture(t, host) |
| 54 | FakePlacementForTest(t) |
| 55 | return m, parent |
| 56 | } |
| 57 | |
| 58 | func genFixture(t *testing.T, host string) (*Manager, string) { |
| 59 | t.Helper() |
| 60 | quietOOM(t) |
| 61 | m, err := SetupAt(newDelegatedRoot(t), t.Logf) |
| 62 | if err != nil { |
| 63 | t.Fatal(err) |
| 64 | } |
| 65 | parent := m.hostDir(host) |
| 66 | if err := os.MkdirAll(parent, 0o755); err != nil { |
| 67 | t.Fatal(err) |
| 68 | } |
| 69 | writeFixture(t, filepath.Join(parent, "cgroup.controllers"), "cpu io memory pids\n") |
| 70 | return m, parent |
| 71 | } |
| 72 | |
| 73 | func TestGenAndRoleChildNames(t *testing.T) { |
| 74 | if GenChild(7) != "g7" || RoleChild("sidecar") != "role-sidecar" { |
| 75 | t.Fatalf("GenChild(7) = %q, RoleChild(sidecar) = %q", GenChild(7), RoleChild("sidecar")) |
| 76 | } |
| 77 | for name, want := range map[string]bool{ |
| 78 | "g7": true, "g18446744073709551615": true, "role-seed": true, "role-sidecar": true, |
| 79 | "g0": false, "g07": false, "g": false, "gx": false, "g-1": false, |
| 80 | "role-": false, "role-a/b": false, "role-..": false, "child": false, |
| 81 | "carlos-exec@h.example.com": false, "../g7": false, |
| 82 | } { |
| 83 | if got := isChildName(name); got != want { |
| 84 | t.Errorf("isChildName(%q) = %v, want %v", name, got, want) |
| 85 | } |
| 86 | } |
| 87 | } |
| 88 | |
| 89 | func TestOverlapAndValidParentLimits(t *testing.T) { |
| 90 | if got := OverlapLimit(TierDefault, TierChildren); got.MemoryMax != 1536<<20 { |
| 91 | t.Errorf("OverlapLimit(512M, 1G) = %d", got.MemoryMax) |
| 92 | } |
| 93 | for max, want := range map[int64]bool{ |
| 94 | 512 << 20: true, 1 << 30: true, 1536 << 20: true, 2 << 30: true, |
| 95 | 0: false, 64 << 20: false, 3 << 30: false, |
| 96 | } { |
| 97 | if got := ValidParentLimit(Limit{MemoryMax: max}); got != want { |
| 98 | t.Errorf("ValidParentLimit(%d) = %v, want %v", max, got, want) |
| 99 | } |
| 100 | } |
| 101 | } |
| 102 | |
| 103 | func TestStartChildPutsTheTierOnTheGenerationAndTheParent(t *testing.T) { |
| 104 | const host = "h.example.com" |
| 105 | m, parent := genManager(t, host) |
| 106 | fakeStart(t, func(*exec.Cmd) error { return nil }) |
| 107 | if _, err := m.StartChild(host, GenChild(1), TierDefault, TierDefault, buildTrue); err != nil { |
| 108 | t.Fatal(err) |
| 109 | } |
| 110 | high := strconv.FormatInt(HighOf(TierDefault.MemoryMax), 10) |
| 111 | child := filepath.Join(parent, "g1") |
| 112 | for file, want := range map[string]string{ |
| 113 | filepath.Join(child, "memory.max"): "536870912", |
| 114 | filepath.Join(child, "memory.high"): high, |
| 115 | filepath.Join(child, "memory.swap.max"): "536870912", |
| 116 | filepath.Join(child, "memory.oom.group"): "1", |
| 117 | filepath.Join(parent, "memory.max"): "536870912", |
| 118 | filepath.Join(parent, "memory.high"): high, |
| 119 | filepath.Join(parent, "memory.swap.max"): "536870912", |
| 120 | filepath.Join(parent, "memory.oom.group"): "0", |
| 121 | filepath.Join(parent, "cgroup.subtree_control"): "+cpu +memory +io +pids", |
| 122 | } { |
| 123 | if got := readFixture(t, file); got != want { |
| 124 | t.Errorf("%s = %q, want %q", file, got, want) |
| 125 | } |
| 126 | } |
| 127 | if _, err := os.Stat(filepath.Join(parent, "cgroup.procs")); !os.IsNotExist(err) { |
| 128 | t.Errorf("the parent must hold no process, but its cgroup.procs was written (%v)", err) |
| 129 | } |
| 130 | } |
| 131 | |
| 132 | func TestStartChildRoleChildCarriesNoLimitOfItsOwn(t *testing.T) { |
| 133 | const host = "h.example.com" |
| 134 | m, parent := genManager(t, host) |
| 135 | fakeStart(t, func(*exec.Cmd) error { return nil }) |
| 136 | if _, err := m.StartChild(host, RoleChild("sidecar"), TierDefault, Limit{}, buildTrue); err != nil { |
| 137 | t.Fatal(err) |
| 138 | } |
| 139 | if _, err := os.Stat(filepath.Join(parent, "role-sidecar", "memory.max")); !os.IsNotExist(err) { |
| 140 | t.Errorf("a role child got a limit of its own: %v", err) |
| 141 | } |
| 142 | if got := readFixture(t, filepath.Join(parent, "memory.max")); got != "536870912" { |
| 143 | t.Errorf("parent memory.max = %q, want the tier", got) |
| 144 | } |
| 145 | } |
| 146 | |
| 147 | // A successor spawns while the incumbent lives: the swap has raised the |
| 148 | // parent on purpose, and the spawn must not lower it. |
| 149 | func TestStartChildLeavesALiveParentAlone(t *testing.T) { |
| 150 | const host = "h.example.com" |
| 151 | m, parent := genManager(t, host) |
| 152 | fakeStart(t, func(*exec.Cmd) error { return nil }) |
| 153 | writeFixture(t, filepath.Join(parent, "cgroup.events"), "populated 1\nfrozen 0\n") |
| 154 | writeFixture(t, filepath.Join(parent, "cgroup.subtree_control"), "cpu memory io pids\n") |
| 155 | writeFixture(t, filepath.Join(parent, "memory.max"), "1073741824") |
| 156 | if _, err := m.StartChild(host, GenChild(2), TierDefault, TierDefault, buildTrue); err != nil { |
| 157 | t.Fatal(err) |
| 158 | } |
| 159 | if got := readFixture(t, filepath.Join(parent, "memory.max")); got != "1073741824" { |
| 160 | t.Errorf("a successor's spawn changed the raised parent to %q", got) |
| 161 | } |
| 162 | } |
| 163 | |
| 164 | // Review focus 1: an agent killed between the raise and the restore leaves |
| 165 | // the parent at two tiers. The next cold wake puts it back. |
| 166 | func TestStartChildResetsAStaleRaisedParentOnAColdWake(t *testing.T) { |
| 167 | const host = "h.example.com" |
| 168 | m, parent := genManager(t, host) |
| 169 | fakeStart(t, func(*exec.Cmd) error { return nil }) |
| 170 | writeFixture(t, filepath.Join(parent, "cgroup.events"), "populated 0\nfrozen 0\n") |
| 171 | writeFixture(t, filepath.Join(parent, "memory.max"), "1073741824") |
| 172 | if _, err := m.StartChild(host, GenChild(3), TierDefault, TierDefault, buildTrue); err != nil { |
| 173 | t.Fatal(err) |
| 174 | } |
| 175 | if got := readFixture(t, filepath.Join(parent, "memory.max")); got != "536870912" { |
| 176 | t.Errorf("cold wake left the parent at %q, want the tier", got) |
| 177 | } |
| 178 | } |
| 179 | |
| 180 | func TestStartChildRefusesAForeignChildName(t *testing.T) { |
| 181 | m, _ := genManager(t, "h.example.com") |
| 182 | calls := fakeStart(t, func(*exec.Cmd) error { return nil }) |
| 183 | if _, err := m.StartChild("h.example.com", "../carlos-exec@other.example.com", TierDefault, TierDefault, buildTrue); err == nil { |
| 184 | t.Fatal("a child name outside the layout was accepted") |
| 185 | } |
| 186 | if len(*calls) != 0 { |
| 187 | t.Fatalf("spawned %d times for a refused child name", len(*calls)) |
| 188 | } |
| 189 | } |
| 190 | |
| 191 | func TestSetParentLimitWritesTheBackstopAndReportsFailure(t *testing.T) { |
| 192 | const host = "h.example.com" |
| 193 | m, parent := genManager(t, host) |
| 194 | if err := m.SetParentLimit(host, OverlapLimit(TierDefault, TierDefault)); err != nil { |
| 195 | t.Fatal(err) |
| 196 | } |
| 197 | for file, want := range map[string]string{ |
| 198 | "memory.max": "1073741824", |
| 199 | "memory.high": strconv.FormatInt(HighOf(1<<30), 10), |
| 200 | "memory.swap.max": "1073741824", |
| 201 | "memory.oom.group": "0", |
| 202 | } { |
| 203 | if got := readFixture(t, filepath.Join(parent, file)); got != want { |
| 204 | t.Errorf("%s = %q, want %q", file, got, want) |
| 205 | } |
| 206 | } |
| 207 | // A raise that did not land must say so: the swap stops on it. |
| 208 | if err := os.Remove(filepath.Join(parent, "memory.max")); err != nil { |
| 209 | t.Fatal(err) |
| 210 | } |
| 211 | if err := os.Mkdir(filepath.Join(parent, "memory.max"), 0o755); err != nil { |
| 212 | t.Fatal(err) |
| 213 | } |
| 214 | if err := m.SetParentLimit(host, TierDefault); err == nil { |
| 215 | t.Fatal("a limit write that failed reported success") |
| 216 | } |
| 217 | var nilM *Manager |
| 218 | if err := nilM.SetParentLimit(host, TierDefault); err != nil { |
| 219 | t.Fatalf("nil manager: %v, want nil (no delegation, nothing to set)", err) |
| 220 | } |
| 221 | } |
| 222 | |
| 223 | // Review focus 2: rmdir refuses a cgroup with child cgroups, empty or not. |
| 224 | func TestSweepRemovesEmptyGenerationChildrenThenTheParent(t *testing.T) { |
| 225 | root := newDelegatedRoot(t) |
| 226 | stale := filepath.Join(root, ExecName("old.example.com")) |
| 227 | for _, d := range []string{"g3", "role-seed"} { |
| 228 | if err := os.MkdirAll(filepath.Join(stale, d), 0o755); err != nil { |
| 229 | t.Fatal(err) |
| 230 | } |
| 231 | } |
| 232 | live := filepath.Join(root, ExecName("live.example.com")) |
| 233 | if err := os.MkdirAll(filepath.Join(live, "g4"), 0o755); err != nil { |
| 234 | t.Fatal(err) |
| 235 | } |
| 236 | // A file stands in for a member: rmdir refuses the directory. |
| 237 | writeFixture(t, filepath.Join(live, "g4", "cgroup.procs"), "123\n") |
| 238 | if _, err := SetupAt(root, t.Logf); err != nil { |
| 239 | t.Fatal(err) |
| 240 | } |
| 241 | if _, err := os.Stat(stale); !os.IsNotExist(err) { |
| 242 | t.Errorf("stale generation parent survived the sweep: %v", err) |
| 243 | } |
| 244 | if _, err := os.Stat(filepath.Join(live, "g4")); err != nil { |
| 245 | t.Errorf("a populated generation child must be left alone: %v", err) |
| 246 | } |
| 247 | } |
| 248 | |
| 249 | func TestRemoveChildRemovesOnlyItsOwnNames(t *testing.T) { |
| 250 | const host = "h.example.com" |
| 251 | m, parent := genManager(t, host) |
| 252 | for _, d := range []string{"g5", "keep"} { |
| 253 | if err := os.Mkdir(filepath.Join(parent, d), 0o755); err != nil { |
| 254 | t.Fatal(err) |
| 255 | } |
| 256 | } |
| 257 | m.RemoveChild(host, "g5") |
| 258 | m.RemoveChild(host, "keep") |
| 259 | m.RemoveChild(host, "../"+ExecName("other.example.com")) |
| 260 | if _, err := os.Stat(filepath.Join(parent, "g5")); !os.IsNotExist(err) { |
| 261 | t.Errorf("empty g5 survived: %v", err) |
| 262 | } |
| 263 | if _, err := os.Stat(filepath.Join(parent, "keep")); err != nil { |
| 264 | t.Errorf("a foreign child was removed: %v", err) |
| 265 | } |
| 266 | var nilM *Manager |
| 267 | nilM.RemoveChild(host, "g5") // must not panic |
| 268 | } |
| 269 | |
| 270 | // Flag off: Start keeps today's single-level layout. |
| 271 | func TestStartKeepsTheSingleLevelLayout(t *testing.T) { |
| 272 | quietOOM(t) |
| 273 | m, err := SetupAt(newDelegatedRoot(t), t.Logf) |
| 274 | if err != nil { |
| 275 | t.Fatal(err) |
| 276 | } |
| 277 | fakeStart(t, func(*exec.Cmd) error { return nil }) |
| 278 | if _, err := m.Start("h.example.com", TierDefault, buildTrue); err != nil { |
| 279 | t.Fatal(err) |
| 280 | } |
| 281 | dir := m.hostDir("h.example.com") |
| 282 | if got := readFixture(t, filepath.Join(dir, "memory.oom.group")); got != "1" { |
| 283 | t.Errorf("host cgroup memory.oom.group = %q, want 1", got) |
| 284 | } |
| 285 | if _, err := os.Stat(filepath.Join(dir, "cgroup.subtree_control")); !os.IsNotExist(err) { |
| 286 | t.Errorf("flag-off Start delegated controllers: %v", err) |
| 287 | } |
| 288 | entries, err := os.ReadDir(dir) |
| 289 | if err != nil { |
| 290 | t.Fatal(err) |
| 291 | } |
| 292 | for _, e := range entries { |
| 293 | if e.IsDir() { |
| 294 | t.Errorf("flag-off Start made a child cgroup %s", e.Name()) |
| 295 | } |
| 296 | } |
| 297 | } |
| 298 | |
| 299 | // Zero-downtime final review wave 2 (F1 completion): StartChildGated's |
| 300 | // before-start hook is asked after the whole preparation, the parent's tier |
| 301 | // and the generation child's limits both written, and a refusal starts |
| 302 | // nothing and comes back as the hook's own error. A hook that agrees |
| 303 | // starts exactly once, as StartChild does. |
| 304 | func TestStartChildGatedAsksAfterThePreparation(t *testing.T) { |
| 305 | const host = "h.example.com" |
| 306 | m, parent := genManager(t, host) |
| 307 | calls := fakeStart(t, func(*exec.Cmd) error { return nil }) |
| 308 | refused := errors.New("the attempt was superseded") |
| 309 | var prepared bool |
| 310 | cmd, err := m.StartChildGated(host, GenChild(1), TierDefault, TierDefault, buildTrue, func() error { |
| 311 | _, perr := os.Stat(filepath.Join(parent, "memory.max")) |
| 312 | _, cerr := os.Stat(filepath.Join(parent, "g1", "memory.max")) |
| 313 | prepared = perr == nil && cerr == nil |
| 314 | return refused |
| 315 | }) |
| 316 | if err != refused || cmd != nil { |
| 317 | t.Fatalf("StartChildGated = %v, %v: want the hook's own error and no command", cmd, err) |
| 318 | } |
| 319 | if !prepared { |
| 320 | t.Fatal("the hook was asked before the parent's tier and the child's limits were written") |
| 321 | } |
| 322 | if len(*calls) != 0 { |
| 323 | t.Fatalf("started %d times after the hook refused, want none", len(*calls)) |
| 324 | } |
| 325 | if _, err := m.StartChildGated(host, GenChild(2), TierDefault, TierDefault, buildTrue, func() error { return nil }); err != nil { |
| 326 | t.Fatal(err) |
| 327 | } |
| 328 | if len(*calls) != 1 { |
| 329 | t.Fatalf("started %d times with an agreeing hook, want 1", len(*calls)) |
| 330 | } |
| 331 | } |
| 332 | |
| 333 | // With no manager (no delegation) the hook still guards the one plain |
| 334 | // start. |
| 335 | func TestStartGatedWithNoManagerAsksBeforeThePlainStart(t *testing.T) { |
| 336 | calls := fakeStart(t, func(*exec.Cmd) error { return nil }) |
| 337 | refused := errors.New("lease lost") |
| 338 | var m *Manager |
| 339 | if cmd, err := m.StartGated("h.example.com", TierDefault, buildTrue, func() error { return refused }); err != refused || cmd != nil { |
| 340 | t.Fatalf("StartGated = %v, %v: want the hook's own error and no command", cmd, err) |
| 341 | } |
| 342 | if cmd, err := m.StartChildGated("h.example.com", GenChild(1), TierDefault, TierDefault, buildTrue, func() error { return refused }); err != refused || cmd != nil { |
| 343 | t.Fatalf("StartChildGated = %v, %v: want the hook's own error and no command", cmd, err) |
| 344 | } |
| 345 | if len(*calls) != 0 { |
| 346 | t.Fatalf("started %d times after the hook refused, want none", len(*calls)) |
| 347 | } |
| 348 | } |
| 349 | |
| 350 | // legacyParent is genManager over a manager whose log lines are kept, with |
| 351 | // the parent's own cgroup.procs listing pids: the shape the dev box showed |
| 352 | // (amadan #272, check (d)), a carlos-exec@<host> an older agent started a |
| 353 | // process in directly. |
| 354 | func legacyParent(t *testing.T, host, procs string) (*Manager, string, *[]string) { |
| 355 | t.Helper() |
| 356 | m, parent := genManager(t, host) |
| 357 | var lines []string |
| 358 | m.logf = func(format string, args ...any) { lines = append(lines, fmt.Sprintf(format, args...)) } |
| 359 | writeFixture(t, filepath.Join(parent, "cgroup.procs"), procs) |
| 360 | return m, parent, &lines |
| 361 | } |
| 362 | |
| 363 | func noGenerationChild(t *testing.T, parent string) { |
| 364 | t.Helper() |
| 365 | ents, err := os.ReadDir(parent) |
| 366 | if err != nil { |
| 367 | t.Fatal(err) |
| 368 | } |
| 369 | for _, e := range ents { |
| 370 | if e.IsDir() { |
| 371 | t.Errorf("the refused start created %s under the parent", e.Name()) |
| 372 | } |
| 373 | } |
| 374 | } |
| 375 | |
| 376 | // Flag blocker (amadan #272): a generation started under a parent that |
| 377 | // still holds a process directly runs with no memory limit, because a |
| 378 | // cgroup with processes of its own cannot delegate the memory controller. |
| 379 | // The start is refused, names the host and the count, and creates nothing. |
| 380 | func TestAPopulatedLegacyParentIsRefusedBeforeAnyChildExists(t *testing.T) { |
| 381 | const host = "h.example.com" |
| 382 | m, parent, lines := legacyParent(t, host, "4242\n4243\n") |
| 383 | calls := fakeStart(t, func(*exec.Cmd) error { return nil }) |
| 384 | cmd, err := m.StartChild(host, GenChild(1), TierDefault, TierDefault, buildTrue) |
| 385 | if !errors.Is(err, ErrLegacyParentPopulated) || cmd != nil { |
| 386 | t.Fatalf("StartChild = %v, %v: want ErrLegacyParentPopulated and no command", cmd, err) |
| 387 | } |
| 388 | if len(*calls) != 0 { |
| 389 | t.Fatalf("started %d times under a populated parent", len(*calls)) |
| 390 | } |
| 391 | noGenerationChild(t, parent) |
| 392 | if _, serr := os.Stat(filepath.Join(parent, "cgroup.subtree_control")); !os.IsNotExist(serr) { |
| 393 | t.Errorf("the refused start delegated controllers anyway (%v)", serr) |
| 394 | } |
| 395 | if len(*lines) != 1 || !strings.Contains((*lines)[0], host) || !strings.Contains((*lines)[0], "2 process") { |
| 396 | t.Errorf("log lines = %q, want one naming %s and 2 processes", *lines, host) |
| 397 | } |
| 398 | } |
| 399 | |
| 400 | // A parent that holds a process directly is refused whatever children it |
| 401 | // has (plan review #12): an empty g1 left from an earlier generation does |
| 402 | // not make the parent a generation parent. |
| 403 | func TestAPopulatedParentWithAStaleEmptyGenerationIsRefused(t *testing.T) { |
| 404 | const host = "h.example.com" |
| 405 | m, parent, _ := legacyParent(t, host, "4242\n") |
| 406 | if err := os.MkdirAll(filepath.Join(parent, "g1"), 0o755); err != nil { |
| 407 | t.Fatal(err) |
| 408 | } |
| 409 | writeFixture(t, filepath.Join(parent, "g1", "cgroup.procs"), "") |
| 410 | writeFixture(t, filepath.Join(parent, "cgroup.events"), "populated 1\nfrozen 0\n") |
| 411 | calls := fakeStart(t, func(*exec.Cmd) error { return nil }) |
| 412 | if _, err := m.StartChild(host, GenChild(2), TierDefault, TierDefault, buildTrue); !errors.Is(err, ErrLegacyParentPopulated) { |
| 413 | t.Fatalf("StartChild = %v, want ErrLegacyParentPopulated", err) |
| 414 | } |
| 415 | if len(*calls) != 0 { |
| 416 | t.Fatalf("started %d times", len(*calls)) |
| 417 | } |
| 418 | if _, serr := os.Stat(filepath.Join(parent, "g2")); !os.IsNotExist(serr) { |
| 419 | t.Errorf("the refused start created g2 (%v)", serr) |
| 420 | } |
| 421 | } |
| 422 | |
| 423 | // The refusal is about direct members only. A parent whose processes all |
| 424 | // live in generation children is the layout working: cgroup.procs is empty |
| 425 | // and cgroup.events reports the descendants. |
| 426 | func TestAnEmptyParentWithLiveGenerationsStillStartsASuccessor(t *testing.T) { |
| 427 | const host = "h.example.com" |
| 428 | m, parent, _ := legacyParent(t, host, "") |
| 429 | if err := os.MkdirAll(filepath.Join(parent, "g1"), 0o755); err != nil { |
| 430 | t.Fatal(err) |
| 431 | } |
| 432 | writeFixture(t, filepath.Join(parent, "g1", "cgroup.procs"), "5151\n") |
| 433 | writeFixture(t, filepath.Join(parent, "cgroup.events"), "populated 1\nfrozen 0\n") |
| 434 | writeFixture(t, filepath.Join(parent, "cgroup.subtree_control"), "cpu memory io pids\n") |
| 435 | writeFixture(t, filepath.Join(parent, "memory.max"), "1073741824") |
| 436 | calls := fakeStart(t, func(*exec.Cmd) error { return nil }) |
| 437 | if _, err := m.StartChild(host, GenChild(2), TierDefault, TierDefault, buildTrue); err != nil { |
| 438 | t.Fatalf("StartChild beside a live generation: %v", err) |
| 439 | } |
| 440 | if len(*calls) != 1 { |
| 441 | t.Fatalf("started %d times, want 1", len(*calls)) |
| 442 | } |
| 443 | if got := readFixture(t, filepath.Join(parent, "memory.max")); got != "1073741824" { |
| 444 | t.Errorf("the successor's start lowered the raised parent to %q", got) |
| 445 | } |
| 446 | } |
| 447 | |
| 448 | // A parent cgroup.procs that cannot be read is not known to be empty, so it |
| 449 | // is refused as populated is (parentPopulated's rule: unreadable counts as |
| 450 | // populated). A parent that has no cgroup.procs yet is a fresh directory. |
| 451 | func TestAnUnreadableParentProcsFileIsRefusedAndAnAbsentOneIsNot(t *testing.T) { |
| 452 | const host = "h.example.com" |
| 453 | m, parent, _ := legacyParent(t, host, "") |
| 454 | if err := os.Remove(filepath.Join(parent, "cgroup.procs")); err != nil { |
| 455 | t.Fatal(err) |
| 456 | } |
| 457 | fakeStart(t, func(*exec.Cmd) error { return nil }) |
| 458 | if _, err := m.StartChild(host, GenChild(1), TierDefault, TierDefault, buildTrue); err != nil { |
| 459 | t.Fatalf("a parent with no cgroup.procs file: %v", err) |
| 460 | } |
| 461 | if err := os.MkdirAll(filepath.Join(parent, "cgroup.procs"), 0o755); err != nil { // reading a directory fails |
| 462 | t.Fatal(err) |
| 463 | } |
| 464 | if _, err := m.StartChild(host, GenChild(2), TierDefault, TierDefault, buildTrue); !errors.Is(err, ErrLegacyParentPopulated) { |
| 465 | t.Fatalf("StartChild over an unreadable cgroup.procs = %v, want ErrLegacyParentPopulated", err) |
| 466 | } |
| 467 | } |
| 468 | |
| 469 | // Task 7 (zdd flag readiness round 2): a generation whose limits cannot be |
| 470 | // applied does not start. Each case below is one of the branches that used |
| 471 | // to log a line and run the generation uncapped (recon C, A to G). Each |
| 472 | // refuses with ErrLimitNotApplied before anything is exec'd (the build |
| 473 | // function is never even called) and leaves no child cgroup behind. |
| 474 | |
| 475 | // refusedChild runs StartChild for g1 at the default tier and checks the |
| 476 | // refusal: ErrLimitNotApplied (or want), no command, nothing started, and |
| 477 | // the command never built. |
| 478 | func refusedChild(t *testing.T, m *Manager, host, child string, own Limit, want error) error { |
| 479 | t.Helper() |
| 480 | calls := fakeStart(t, func(*exec.Cmd) error { return nil }) |
| 481 | built := false |
| 482 | cmd, err := m.StartChild(host, child, TierDefault, own, func() *exec.Cmd { built = true; return buildTrue() }) |
| 483 | if !errors.Is(err, want) || cmd != nil { |
| 484 | t.Fatalf("StartChild(%s) = %v, %v: want %v and no command", child, cmd, err, want) |
| 485 | } |
| 486 | if len(*calls) != 0 || built { |
| 487 | t.Fatalf("StartChild(%s): built %v, started %d times; want neither", child, built, len(*calls)) |
| 488 | } |
| 489 | return err |
| 490 | } |
| 491 | |
| 492 | // noChildBut is noGenerationChild for a fixture that plants a directory |
| 493 | // under the parent (a control file made a directory so its write fails): |
| 494 | // the planted ones are the only directories allowed. |
| 495 | func noChildBut(t *testing.T, parent string, planted ...string) { |
| 496 | t.Helper() |
| 497 | ents, err := os.ReadDir(parent) |
| 498 | if err != nil { |
| 499 | t.Fatal(err) |
| 500 | } |
| 501 | for _, e := range ents { |
| 502 | if e.IsDir() && !slices.Contains(planted, e.Name()) { |
| 503 | t.Errorf("the refused start created %s under the parent", e.Name()) |
| 504 | } |
| 505 | } |
| 506 | } |
| 507 | |
| 508 | // A: the parent cgroup cannot be made. It used to start the generation in |
| 509 | // the agent's own cgroup, unaccounted and uncapped. The flat start (flag |
| 510 | // off) keeps its best-effort plain start. |
| 511 | func TestAGenerationParentThatCannotBeMadeIsRefused(t *testing.T) { |
| 512 | const host = "h.example.com" |
| 513 | m, parent := genManager(t, host) |
| 514 | if err := os.RemoveAll(parent); err != nil { |
| 515 | t.Fatal(err) |
| 516 | } |
| 517 | writeFixture(t, parent, "a file where the parent goes") // MkdirAll: ENOTDIR |
| 518 | refusedChild(t, m, host, GenChild(1), TierDefault, ErrLimitNotApplied) |
| 519 | if fi, err := os.Stat(parent); err != nil || fi.IsDir() { |
| 520 | t.Fatalf("the refused start replaced the parent: %v", err) |
| 521 | } |
| 522 | calls := fakeStart(t, func(*exec.Cmd) error { return nil }) |
| 523 | if cmd, err := m.Start(host, TierDefault, buildTrue); err != nil || cmd == nil || len(*calls) != 1 { |
| 524 | t.Fatalf("flag-off Start = %v, %v after %d starts: want today's plain start", cmd, err, len(*calls)) |
| 525 | } |
| 526 | } |
| 527 | |
| 528 | // B: a parent with live generations is left as it is, but only when it |
| 529 | // delegates memory to its children. Without memory in its subtree_control |
| 530 | // (absent, unreadable, or listing other controllers) a new g<n> under it |
| 531 | // would get no memory.max at all. |
| 532 | func TestALiveParentThatDoesNotDelegateMemoryIsRefused(t *testing.T) { |
| 533 | const host = "h.example.com" |
| 534 | for name, subtree := range map[string]*string{ |
| 535 | "absent": nil, |
| 536 | "without memory": ptr("cpu pids\n"), |
| 537 | "empty": ptr(""), |
| 538 | } { |
| 539 | t.Run(name, func(t *testing.T) { |
| 540 | m, parent := genManager(t, host) |
| 541 | writeFixture(t, filepath.Join(parent, "cgroup.events"), "populated 1\nfrozen 0\n") |
| 542 | if subtree != nil { |
| 543 | writeFixture(t, filepath.Join(parent, "cgroup.subtree_control"), *subtree) |
| 544 | } |
| 545 | refusedChild(t, m, host, GenChild(2), TierDefault, ErrLimitNotApplied) |
| 546 | noGenerationChild(t, parent) |
| 547 | }) |
| 548 | } |
| 549 | t.Run("unreadable", func(t *testing.T) { |
| 550 | m, parent := genManager(t, host) |
| 551 | writeFixture(t, filepath.Join(parent, "cgroup.events"), "populated 1\nfrozen 0\n") |
| 552 | if err := os.Mkdir(filepath.Join(parent, "cgroup.subtree_control"), 0o755); err != nil { |
| 553 | t.Fatal(err) |
| 554 | } |
| 555 | refusedChild(t, m, host, GenChild(2), TierDefault, ErrLimitNotApplied) |
| 556 | noChildBut(t, parent, "cgroup.subtree_control") |
| 557 | }) |
| 558 | // Both spellings delegate: the kernel reads back "memory", and this |
| 559 | // package's own write (and its fixtures) store "+memory". |
| 560 | for _, subtree := range []string{"cpu memory io pids\n", "+cpu +memory +io +pids"} { |
| 561 | m, parent := genManager(t, host) |
| 562 | writeFixture(t, filepath.Join(parent, "cgroup.events"), "populated 1\nfrozen 0\n") |
| 563 | writeFixture(t, filepath.Join(parent, "cgroup.subtree_control"), subtree) |
| 564 | calls := fakeStart(t, func(*exec.Cmd) error { return nil }) |
| 565 | if _, err := m.StartChild(host, GenChild(2), TierDefault, TierDefault, buildTrue); err != nil || len(*calls) != 1 { |
| 566 | t.Fatalf("subtree_control %q: StartChild = %v after %d starts, want one start", subtree, err, len(*calls)) |
| 567 | } |
| 568 | } |
| 569 | } |
| 570 | |
| 571 | func ptr(s string) *string { return &s } |
| 572 | |
| 573 | // C: the controllers cannot be delegated on a cold start. A parent with no |
| 574 | // memory controller to hand on, or a subtree_control write that fails, is |
| 575 | // ErrLimitNotApplied; a write the kernel answers EBUSY is the populated |
| 576 | // parent race (a process landed in the parent after the cgroup.procs |
| 577 | // check), so it is ErrLegacyParentPopulated. Nothing else is written: no |
| 578 | // parent limit, no child. |
| 579 | func TestAParentWhoseControllersCannotBeDelegatedIsRefused(t *testing.T) { |
| 580 | const host = "h.example.com" |
| 581 | t.Run("no memory controller", func(t *testing.T) { |
| 582 | m, parent := genManager(t, host) |
| 583 | writeFixture(t, filepath.Join(parent, "cgroup.controllers"), "cpu pids\n") |
| 584 | refusedChild(t, m, host, GenChild(1), TierDefault, ErrLimitNotApplied) |
| 585 | noGenerationChild(t, parent) |
| 586 | if _, err := os.Stat(filepath.Join(parent, "memory.max")); !os.IsNotExist(err) { |
| 587 | t.Errorf("the refused start wrote the parent's limit: %v", err) |
| 588 | } |
| 589 | }) |
| 590 | for name, tc := range map[string]struct { |
| 591 | write error |
| 592 | want error |
| 593 | }{ |
| 594 | "the write is refused": {syscall.EACCES, ErrLimitNotApplied}, |
| 595 | "a process landed in the parent": {syscall.EBUSY, ErrLegacyParentPopulated}, |
| 596 | } { |
| 597 | t.Run(name, func(t *testing.T) { |
| 598 | m, parent := genManager(t, host) |
| 599 | orig := writeSubtreeControl |
| 600 | writeSubtreeControl = func(path string, _ []byte) error { |
| 601 | return &os.PathError{Op: "write", Path: path, Err: tc.write} |
| 602 | } |
| 603 | t.Cleanup(func() { writeSubtreeControl = orig }) |
| 604 | err := refusedChild(t, m, host, GenChild(1), TierDefault, tc.want) |
| 605 | noGenerationChild(t, parent) |
| 606 | if tc.want == ErrLegacyParentPopulated && errors.Is(err, ErrLimitNotApplied) { |
| 607 | t.Error("the EBUSY race reads as a limit failure") |
| 608 | } |
| 609 | if _, err := os.Stat(filepath.Join(parent, "memory.max")); !os.IsNotExist(err) { |
| 610 | t.Errorf("the refused start wrote the parent's limit: %v", err) |
| 611 | } |
| 612 | }) |
| 613 | } |
| 614 | } |
| 615 | |
| 616 | // D: the parent's backstop cannot be written. A role child (the legacy |
| 617 | // sidecar, the seed) has no limit of its own, so without the parent's it |
| 618 | // would have none at all. |
| 619 | func TestAParentWhoseLimitCannotBeWrittenIsRefused(t *testing.T) { |
| 620 | const host = "h.example.com" |
| 621 | for _, file := range []string{"memory.max", "memory.oom.group"} { |
| 622 | for _, child := range []string{GenChild(1), RoleChild("sidecar")} { |
| 623 | m, parent := genManager(t, host) |
| 624 | if err := os.Mkdir(filepath.Join(parent, file), 0o755); err != nil { // WriteFile: EISDIR |
| 625 | t.Fatal(err) |
| 626 | } |
| 627 | own := Limit{} |
| 628 | if child == GenChild(1) { |
| 629 | own = TierDefault |
| 630 | } |
| 631 | refusedChild(t, m, host, child, own, ErrLimitNotApplied) |
| 632 | noChildBut(t, parent, file) |
| 633 | } |
| 634 | } |
| 635 | } |
| 636 | |
| 637 | // E: the generation's own child cgroup cannot be made. It used to start in |
| 638 | // the agent's cgroup, unaccounted and uncapped. |
| 639 | func TestAGenerationChildThatCannotBeMadeIsRefused(t *testing.T) { |
| 640 | const host = "h.example.com" |
| 641 | m, parent := genManager(t, host) |
| 642 | writeFixture(t, filepath.Join(parent, "g1"), "a file where g1 goes") // MkdirAll: ENOTDIR |
| 643 | refusedChild(t, m, host, GenChild(1), TierDefault, ErrLimitNotApplied) |
| 644 | noGenerationChild(t, parent) |
| 645 | } |
| 646 | |
| 647 | // F: one of the generation's four limit writes fails. All four are the |
| 648 | // child's limits (spec §7.3), so any failure refuses the start, and the |
| 649 | // process never lands in the child (no cgroup.procs write). The flat start |
| 650 | // keeps logging and spawning (TestStartLimitWriteFailureStillSpawns). |
| 651 | func TestAGenerationWhoseLimitsCannotBeWrittenIsRefused(t *testing.T) { |
| 652 | const host = "h.example.com" |
| 653 | for _, file := range []string{"memory.max", "memory.high", "memory.swap.max", "memory.oom.group"} { |
| 654 | t.Run(file, func(t *testing.T) { |
| 655 | m, parent := genManager(t, host) |
| 656 | g1 := filepath.Join(parent, "g1") |
| 657 | if err := os.MkdirAll(filepath.Join(g1, file), 0o755); err != nil { // WriteFile: EISDIR |
| 658 | t.Fatal(err) |
| 659 | } |
| 660 | refusedChild(t, m, host, GenChild(1), TierDefault, ErrLimitNotApplied) |
| 661 | if _, err := os.Stat(filepath.Join(g1, "cgroup.procs")); !os.IsNotExist(err) { |
| 662 | t.Errorf("a process was placed in the refused child: %v", err) |
| 663 | } |
| 664 | noChildBut(t, parent, "g1") |
| 665 | }) |
| 666 | } |
| 667 | // applyLimit reports the first failure; with every write landing it |
| 668 | // answers nil. |
| 669 | m, parent := genManager(t, host) |
| 670 | dir := filepath.Join(parent, "g9") |
| 671 | if err := os.Mkdir(dir, 0o755); err != nil { |
| 672 | t.Fatal(err) |
| 673 | } |
| 674 | for _, required := range []bool{false, true} { |
| 675 | if err := m.applyLimit(host, dir, TierDefault, required); err != nil { |
| 676 | t.Fatalf("applyLimit(required %v) with every write landing = %v", required, err) |
| 677 | } |
| 678 | if err := m.applyLimit(host, dir, Limit{}, required); err != nil { |
| 679 | t.Fatalf("applyLimit(required %v) with no limit = %v", required, err) |
| 680 | } |
| 681 | } |
| 682 | } |
| 683 | |
| 684 | // Fix round 1, finding 7: a generation whose memory.max cannot be written |
| 685 | // is refused, so the journal must not say it "runs uncapped". The required |
| 686 | // path logs one refusal line naming the file; the flat start's lines are |
| 687 | // unchanged. |
| 688 | func TestARefusedGenerationsLimitFailureLogsTheRefusalOnly(t *testing.T) { |
| 689 | const host = "h.example.com" |
| 690 | m, parent, lines := legacyParent(t, host, "") |
| 691 | if err := os.MkdirAll(filepath.Join(parent, "g1", "memory.max"), 0o755); err != nil { // WriteFile: EISDIR |
| 692 | t.Fatal(err) |
| 693 | } |
| 694 | refusedChild(t, m, host, GenChild(1), TierDefault, ErrLimitNotApplied) |
| 695 | var refusals int |
| 696 | for _, l := range *lines { |
| 697 | if strings.Contains(l, "uncapped") || strings.Contains(l, ": memory.max=") { // the parent's own "parent memory.max=" line is true |
| 698 | t.Errorf("a refused generation logged %q", l) |
| 699 | } |
| 700 | if strings.Contains(l, "g1: memory.max:") && strings.Contains(l, "generation start refused") { |
| 701 | refusals++ |
| 702 | } |
| 703 | } |
| 704 | if refusals != 1 { |
| 705 | t.Errorf("log lines = %q, want one refusal naming g1's memory.max", *lines) |
| 706 | } |
| 707 | // The flat start keeps its line. |
| 708 | *lines = nil |
| 709 | flat := m.hostDir("flat.example.com") |
| 710 | if err := os.MkdirAll(filepath.Join(flat, "memory.max"), 0o755); err != nil { |
| 711 | t.Fatal(err) |
| 712 | } |
| 713 | fakeStart(t, func(*exec.Cmd) error { return nil }) |
| 714 | if _, err := m.Start("flat.example.com", TierDefault, buildTrue); err != nil { |
| 715 | t.Fatal(err) |
| 716 | } |
| 717 | if !slices.ContainsFunc(*lines, func(l string) bool { |
| 718 | return strings.Contains(l, "memory.max:") && strings.Contains(l, "child runs uncapped") |
| 719 | }) { |
| 720 | t.Errorf("flat log lines = %q, want the unchanged \"child runs uncapped\" line", *lines) |
| 721 | } |
| 722 | } |
| 723 | |
| 724 | // G: with no shim in this process (a binary whose main lacks |
| 725 | // RunShimIfRequested) a generation could only be started and then moved by |
| 726 | // PID: running, for that window, in the agent's cgroup with no limit, and |
| 727 | // for good if the move fails. It is refused before the child is created |
| 728 | // or build() runs (ensureParent has already run). The flat start keeps the |
| 729 | // PID-move start. |
| 730 | func TestGenerationRefusedWithoutTheShim(t *testing.T) { |
| 731 | const host = "h.example.com" |
| 732 | m, parent := genFixture(t, host) |
| 733 | noShim(t) |
| 734 | for _, child := range []string{GenChild(1), RoleChild("seed")} { |
| 735 | own := Limit{} |
| 736 | if child == GenChild(1) { |
| 737 | own = TierDefault |
| 738 | } |
| 739 | refusedChild(t, m, host, child, own, ErrLimitNotApplied) |
| 740 | noGenerationChild(t, parent) |
| 741 | } |
| 742 | calls := fakeStart(t, func(*exec.Cmd) error { return nil }) |
| 743 | if cmd, err := m.Start("flat.example.com", TierDefault, buildTrue); err != nil || cmd == nil || len(*calls) != 1 { |
| 744 | t.Fatalf("flag-off Start without the shim = %v, %v after %d starts: want the PID-move start", cmd, err, len(*calls)) |
| 745 | } |
| 746 | if _, err := os.Stat(filepath.Join(m.hostDir("flat.example.com"), "cgroup.procs")); err != nil { |
| 747 | t.Fatalf("flag-off Start did not move the PID: %v", err) |
| 748 | } |
| 749 | } |
| 750 | |
| 751 | // noShim makes this test binary one without RunShimIfRequested until t's |
| 752 | // cleanup. |
| 753 | func noShim(t *testing.T) { |
| 754 | t.Helper() |
| 755 | hooked, forced := shimHooked.Swap(false), shimForcedForTest.Swap(false) |
| 756 | t.Cleanup(func() { |
| 757 | shimHooked.Store(hooked) |
| 758 | shimForcedForTest.Store(forced) |
| 759 | }) |
| 760 | } |
| 761 | |
| 762 | // stubPlacement swaps the placement seam until t's cleanup. |
| 763 | func stubPlacement(t *testing.T, f func(m *Manager, dir string, cmd *exec.Cmd, before func() error) (bool, error)) { |
| 764 | t.Helper() |
| 765 | orig := placeStart |
| 766 | placeStart = f |
| 767 | t.Cleanup(func() { placeStart = orig }) |
| 768 | } |
| 769 | |
| 770 | // The fixture's limit files are ordinary files, which keep a refused |
| 771 | // child's rmdir from succeeding (CleanupHost's comment). The removal tests |
| 772 | // below start their generation with no limit of its own, so the child is |
| 773 | // empty when it should go. |
| 774 | |
| 775 | // Review focus 5: a missing binary is the command's own failure. The shim |
| 776 | // answers fork/exec's own error; the start returns it as is, and it is not |
| 777 | // the box's condition, so it is not ErrLimitNotApplied. The empty child goes. |
| 778 | func TestGenerationMissingBinaryIsNotABoxCondition(t *testing.T) { |
| 779 | const host = "h.example.com" |
| 780 | m, parent := genManager(t, host) |
| 781 | missing := &os.PathError{Op: "fork/exec", Path: "/x", Err: syscall.ENOENT} |
| 782 | stubPlacement(t, func(*Manager, string, *exec.Cmd, func() error) (bool, error) { return false, missing }) |
| 783 | builds := 0 |
| 784 | cmd, err := m.StartChild(host, GenChild(1), TierDefault, Limit{}, func() *exec.Cmd { builds++; return buildTrue() }) |
| 785 | if err != missing || cmd != nil { |
| 786 | t.Fatalf("StartChild = %v, %v: want the command's own error, unwrapped, and no command", cmd, err) |
| 787 | } |
| 788 | if errors.Is(err, ErrLimitNotApplied) { |
| 789 | t.Fatal("a missing binary read as the box's condition") |
| 790 | } |
| 791 | if builds != 1 { |
| 792 | t.Fatalf("built %d times, want once: the command's own failure is never retried", builds) |
| 793 | } |
| 794 | if _, err := os.Stat(filepath.Join(parent, "g1")); !os.IsNotExist(err) { |
| 795 | t.Fatalf("the empty child survived a failed start: %v", err) |
| 796 | } |
| 797 | } |
| 798 | |
| 799 | // A placement failure is the box's condition: ErrLimitNotApplied, wrapping |
| 800 | // the shim's answer, with no retry (a generation is never started then |
| 801 | // moved) and the empty child removed. |
| 802 | func TestGenerationPlacementFailureIsLimitNotApplied(t *testing.T) { |
| 803 | const host = "h.example.com" |
| 804 | m, parent, lines := legacyParent(t, host, "") |
| 805 | failed := &placementError{errors.New("the shim exited before placement")} |
| 806 | stubPlacement(t, func(*Manager, string, *exec.Cmd, func() error) (bool, error) { return false, failed }) |
| 807 | calls := fakeStart(t, func(*exec.Cmd) error { return nil }) |
| 808 | builds := 0 |
| 809 | cmd, err := m.StartChild(host, GenChild(1), TierDefault, Limit{}, func() *exec.Cmd { builds++; return buildTrue() }) |
| 810 | if !errors.Is(err, ErrLimitNotApplied) || !errors.Is(err, failed) || cmd != nil { |
| 811 | t.Fatalf("StartChild = %v, %v: want ErrLimitNotApplied wrapping the placement error, and no command", cmd, err) |
| 812 | } |
| 813 | if builds != 1 || len(*calls) != 0 { |
| 814 | t.Fatalf("built %d, started %d: want one build and no start-and-move", builds, len(*calls)) |
| 815 | } |
| 816 | if _, err := os.Stat(filepath.Join(parent, "g1")); !os.IsNotExist(err) { |
| 817 | t.Fatalf("the empty child survived a failed placement: %v", err) |
| 818 | } |
| 819 | if !slices.ContainsFunc(*lines, func(l string) bool { |
| 820 | return strings.Contains(l, "placement before exec failed") && strings.Contains(l, "generation start refused") |
| 821 | }) { |
| 822 | t.Errorf("log lines = %q, want the refusal line", *lines) |
| 823 | } |
| 824 | } |
| 825 | |
| 826 | // Ruling 6: a sibling generation being placed is something alive under the |
| 827 | // parent, so a cold-parent reset (enable the controllers, write the tier) |
| 828 | // must not run: a swap may have raised that parent on purpose. |
| 829 | func TestEnsureParentCountsAPlacementBelowTheParent(t *testing.T) { |
| 830 | const host = "h.example.com" |
| 831 | m, parent := genManager(t, host) |
| 832 | fakeStart(t, func(*exec.Cmd) error { return nil }) |
| 833 | writeFixture(t, filepath.Join(parent, "cgroup.events"), "populated 0\nfrozen 0\n") |
| 834 | writeFixture(t, filepath.Join(parent, "cgroup.subtree_control"), "cpu memory io pids\n") |
| 835 | writeFixture(t, filepath.Join(parent, "memory.max"), "1073741824") |
| 836 | done := m.beginPlacement(filepath.Join(parent, GenChild(1))) |
| 837 | defer done() |
| 838 | if _, err := m.StartChildGated(host, GenChild(2), TierDefault, TierDefault, buildTrue, nil); err != nil { |
| 839 | t.Fatalf("StartChildGated beside a generation being placed: %v", err) |
| 840 | } |
| 841 | if got := readFixture(t, filepath.Join(parent, "memory.max")); got != "1073741824" { |
| 842 | t.Errorf("the parent's memory.max was rewritten to %q: a placement in flight was taken for a cold parent", got) |
| 843 | } |
| 844 | if got := readFixture(t, filepath.Join(parent, "cgroup.subtree_control")); got != "cpu memory io pids\n" { |
| 845 | t.Errorf("the parent's subtree_control was rewritten to %q", got) |
| 846 | } |
| 847 | } |
| 848 | |
| 849 | // Ruling 6: a flat start being placed into the host directory itself is a |
| 850 | // direct process of the parent, so a generation start is refused as for a |
| 851 | // legacy parent, before anything is built. |
| 852 | func TestEnsureParentCountsAPlacementAtTheParent(t *testing.T) { |
| 853 | const host = "h.example.com" |
| 854 | m, parent := genManager(t, host) |
| 855 | done := m.beginPlacement(parent) |
| 856 | defer done() |
| 857 | err := refusedChild(t, m, host, GenChild(1), TierDefault, ErrLegacyParentPopulated) |
| 858 | if !strings.Contains(err.Error(), "1 process") { |
| 859 | t.Errorf("refusal %q, want it to count the placement as one process", err) |
| 860 | } |
| 861 | noGenerationChild(t, parent) |
| 862 | } |
| 863 | |
| 864 | // A generation start whose exec.CommandContext context ends during the |
| 865 | // placement (a seed's wake timing out, the agent stopping) is the caller's |
| 866 | // condition, not the box's: the shim reports it as a placement failure, |
| 867 | // but the start must not come back as ErrLimitNotApplied. Two ways in: |
| 868 | // the context had ended before the shim could start (Start answers its |
| 869 | // error, which the placement error wraps), and it ends while the shim is |
| 870 | // being placed (exec.Cmd's Cancel kills the shim, whose answer names no |
| 871 | // context). Either way the start answers the context's own error, bare, |
| 872 | // as exec.Cmd.Start does for an ended context: context.Canceled or |
| 873 | // context.DeadlineExceeded as the context ended, with no placement text, |
| 874 | // which goes to the operator log (whole-branch review, finding 3). |
| 875 | func TestGenerationCancelledStartIsNotABoxCondition(t *testing.T) { |
| 876 | const host = "h.example.com" |
| 877 | ended := map[string]func(t *testing.T) context.Context{ |
| 878 | "cancelled": func(t *testing.T) context.Context { |
| 879 | ctx, cancel := context.WithCancel(t.Context()) |
| 880 | cancel() |
| 881 | return ctx |
| 882 | }, |
| 883 | "past its deadline": func(t *testing.T) context.Context { |
| 884 | ctx, cancel := context.WithDeadline(t.Context(), time.Unix(1, 0)) |
| 885 | t.Cleanup(cancel) |
| 886 | return ctx |
| 887 | }, |
| 888 | } |
| 889 | for name, place := range map[string]func(*Manager, string, *exec.Cmd, func() error) (bool, error){ |
| 890 | "ended before the shim started": func(_ *Manager, _ string, cmd *exec.Cmd, _ func() error) (bool, error) { |
| 891 | return false, &placementError{fmt.Errorf("start the shim: %w", cmd.Start())} |
| 892 | }, |
| 893 | "ended during the placement": func(_ *Manager, _ string, cmd *exec.Cmd, _ func() error) (bool, error) { |
| 894 | _ = cmd.Cancel() // what exec.Cmd's context watcher does |
| 895 | return false, &placementError{errors.New("the shim exited before placement")} |
| 896 | }, |
| 897 | } { |
| 898 | for ctxName, mkCtx := range ended { |
| 899 | t.Run(name+", "+ctxName, func(t *testing.T) { |
| 900 | m, parent, lines := legacyParent(t, host, "") |
| 901 | stubPlacement(t, place) |
| 902 | ctx := mkCtx(t) |
| 903 | cmd, err := m.StartChild(host, GenChild(1), TierDefault, Limit{}, func() *exec.Cmd { |
| 904 | c := exec.CommandContext(ctx, "/bin/true") |
| 905 | c.Cancel = func() error { return nil } // no process to kill here |
| 906 | return c |
| 907 | }) |
| 908 | if cmd != nil || err != ctx.Err() { |
| 909 | t.Fatalf("StartChild = %v, %v: want the context's own error, %v, bare", cmd, err, ctx.Err()) |
| 910 | } |
| 911 | if errors.Is(err, ErrLimitNotApplied) || isPlacementError(err) { |
| 912 | t.Fatalf("StartChild = %v: a cancelled start read as the box's condition", err) |
| 913 | } |
| 914 | if _, serr := os.Stat(filepath.Join(parent, "g1")); !os.IsNotExist(serr) { |
| 915 | t.Fatalf("the empty child survived: %v", serr) |
| 916 | } |
| 917 | logged := false |
| 918 | for _, l := range *lines { |
| 919 | if strings.Contains(l, "generation start refused") { |
| 920 | t.Errorf("a cancelled start logged a box condition: %q", l) |
| 921 | } |
| 922 | if strings.Contains(l, "the start's context ended during placement") && strings.Contains(l, "placement before exec") { |
| 923 | logged = true |
| 924 | } |
| 925 | } |
| 926 | if !logged { |
| 927 | t.Errorf("log lines = %q, want the placement's detail in the operator line", *lines) |
| 928 | } |
| 929 | }) |
| 930 | } |
| 931 | } |
| 932 | } |
| 933 | |
| 934 | // contextErr asks a copy of the command, taken before its start, for the |
| 935 | // context's error. The copy must never run anything, even with the |
| 936 | // original's files and attributes set, and must leave the original as it |
| 937 | // was. |
| 938 | func TestCancelWatchContextErrRunsNothing(t *testing.T) { |
| 939 | for _, tc := range []struct { |
| 940 | name string |
| 941 | ctx func(t *testing.T) context.Context |
| 942 | want error |
| 943 | }{ |
| 944 | {"cancelled", func(t *testing.T) context.Context { |
| 945 | ctx, cancel := context.WithCancel(t.Context()) |
| 946 | cancel() |
| 947 | return ctx |
| 948 | }, context.Canceled}, |
| 949 | {"past its deadline", func(t *testing.T) context.Context { |
| 950 | ctx, cancel := context.WithDeadline(t.Context(), time.Unix(1, 0)) |
| 951 | t.Cleanup(cancel) |
| 952 | return ctx |
| 953 | }, context.DeadlineExceeded}, |
| 954 | } { |
| 955 | t.Run(tc.name, func(t *testing.T) { |
| 956 | marker := filepath.Join(t.TempDir(), "ran") |
| 957 | r, w, err := os.Pipe() |
| 958 | if err != nil { |
| 959 | t.Fatal(err) |
| 960 | } |
| 961 | defer r.Close() |
| 962 | defer w.Close() |
| 963 | ctx := tc.ctx(t) |
| 964 | cmd := exec.CommandContext(ctx, "/bin/sh", "-c", `touch "$0"`, marker) |
| 965 | cmd.ExtraFiles = []*os.File{w} |
| 966 | cmd.Stdout = io.Discard |
| 967 | cmd.Cancel = func() error { return nil } // no process to kill here |
| 968 | watch := watchCancel(cmd) |
| 969 | _ = cmd.Cancel() // the context watcher's call |
| 970 | got, ended := watch.endedErr(&placementError{errors.New("the shim exited before placement")}) |
| 971 | if !ended || got != tc.want { |
| 972 | t.Fatalf("endedErr = %v, %v: want %v", got, ended, tc.want) |
| 973 | } |
| 974 | if _, err := os.Stat(marker); err == nil { |
| 975 | t.Fatal("the probe ran the command") |
| 976 | } |
| 977 | if cmd.Path != "/bin/sh" || len(cmd.ExtraFiles) != 1 || cmd.Stdout != io.Discard || cmd.Process != nil { |
| 978 | t.Fatalf("the probe changed the original command: %+v", cmd) |
| 979 | } |
| 980 | }) |
| 981 | } |
| 982 | // Without a context nothing is watched and nothing ended. |
| 983 | if _, ended := watchCancel(exec.Command("/bin/true")).endedErr(&placementError{errors.New("x")}); ended { |
| 984 | t.Fatal("a command without a context read as ended") |
| 985 | } |
| 986 | } |
| 987 | |
| 988 | // Whole-branch review, finding 1, at ensureParent: a sibling generation in |
| 989 | // flight when the parent's population is read lands, and ends its |
| 990 | // registration, before the registry is asked. Read in that order, the |
| 991 | // parent would look cold (the kernel read predates the process, the |
| 992 | // registry no longer lists it) and its limits, which a swap may have raised, |
| 993 | // would be reset. The hook ends the placement right after the kernel read. |
| 994 | func TestEnsureParentCountsASiblingThatLandsBetweenTheReads(t *testing.T) { |
| 995 | const host = "h.example.com" |
| 996 | m, parent := genManager(t, host) |
| 997 | fakeStart(t, func(*exec.Cmd) error { return nil }) |
| 998 | writeFixture(t, filepath.Join(parent, "cgroup.events"), "populated 0\nfrozen 0\n") |
| 999 | writeFixture(t, filepath.Join(parent, "cgroup.subtree_control"), "cpu memory io pids\n") |
| 1000 | writeFixture(t, filepath.Join(parent, "memory.max"), "1073741824") |
| 1001 | done := m.beginPlacement(filepath.Join(parent, GenChild(1))) |
| 1002 | defer done() |
| 1003 | hooked := false |
| 1004 | setPlacementReadHook(t, func(string) { |
| 1005 | if !hooked { |
| 1006 | hooked = true |
| 1007 | done() // g1's shim answered placed: the registry drops it |
| 1008 | } |
| 1009 | }) |
| 1010 | if _, err := m.StartChildGated(host, GenChild(2), TierDefault, TierDefault, buildTrue, nil); err != nil { |
| 1011 | t.Fatalf("StartChildGated beside a sibling that landed: %v", err) |
| 1012 | } |
| 1013 | if !hooked { |
| 1014 | t.Fatal("the read hook never ran: the test proves nothing") |
| 1015 | } |
| 1016 | if got := readFixture(t, filepath.Join(parent, "memory.max")); got != "1073741824" { |
| 1017 | t.Errorf("the parent's memory.max was rewritten to %q: a sibling that landed between the reads was taken for a cold parent", got) |
| 1018 | } |
| 1019 | if got := readFixture(t, filepath.Join(parent, "cgroup.subtree_control")); got != "cpu memory io pids\n" { |
| 1020 | t.Errorf("the parent's subtree_control was rewritten to %q", got) |
| 1021 | } |
| 1022 | } |
| 1023 | |
| 1024 | // Finding 1, the other schedule at ensureParent: a sibling placement that |
| 1025 | // tries to register after the parent was found cold must wait until the |
| 1026 | // cold parent's controllers and limits are written. Otherwise the reset |
| 1027 | // could land on a parent that already has a live generation under it. The |
| 1028 | // hook starts the sibling's registration right after the population read |
| 1029 | // and waits until the registration reports that a hold stopped it. |
| 1030 | func TestEnsureParentHoldsPlacementsUntilTheColdParentIsWritten(t *testing.T) { |
| 1031 | const host = "h.example.com" |
| 1032 | m, parent := genManager(t, host) |
| 1033 | fakeStart(t, func(*exec.Cmd) error { return nil }) |
| 1034 | writeFixture(t, filepath.Join(parent, "cgroup.events"), "populated 0\nfrozen 0\n") |
| 1035 | blocked := make(chan struct{}) |
| 1036 | var once sync.Once |
| 1037 | origHeld := placementHeldHook |
| 1038 | placementHeldHook = func(string) { once.Do(func() { close(blocked) }) } |
| 1039 | t.Cleanup(func() { placementHeldHook = origHeld }) |
| 1040 | |
| 1041 | type seen struct { |
| 1042 | max, subtree string |
| 1043 | done func() |
| 1044 | } |
| 1045 | registered := make(chan seen, 1) |
| 1046 | hooked := false |
| 1047 | setPlacementReadHook(t, func(string) { |
| 1048 | if hooked { |
| 1049 | return |
| 1050 | } |
| 1051 | hooked = true |
| 1052 | go func() { |
| 1053 | done := m.beginPlacement(filepath.Join(parent, GenChild(1))) |
| 1054 | // What the sibling's start would find once it is registered. |
| 1055 | max, _ := os.ReadFile(filepath.Join(parent, "memory.max")) |
| 1056 | sub, _ := os.ReadFile(filepath.Join(parent, "cgroup.subtree_control")) |
| 1057 | registered <- seen{string(max), string(sub), done} |
| 1058 | }() |
| 1059 | select { |
| 1060 | case <-blocked: |
| 1061 | case r := <-registered: |
| 1062 | r.done() |
| 1063 | t.Errorf("a sibling registered while the cold parent was being written (memory.max %q)", r.max) |
| 1064 | case <-time.After(5 * time.Second): |
| 1065 | t.Error("the sibling's registration neither registered nor waited on a hold") |
| 1066 | } |
| 1067 | }) |
| 1068 | if _, err := m.StartChildGated(host, GenChild(2), TierDefault, TierDefault, buildTrue, nil); err != nil { |
| 1069 | t.Fatalf("StartChildGated on a cold parent: %v", err) |
| 1070 | } |
| 1071 | if !hooked { |
| 1072 | t.Fatal("the read hook never ran: the test proves nothing") |
| 1073 | } |
| 1074 | select { |
| 1075 | case r := <-registered: |
| 1076 | defer r.done() |
| 1077 | if want := strconv.FormatInt(TierDefault.MemoryMax, 10); r.max != want { |
| 1078 | t.Errorf("the sibling registered with the parent's memory.max %q, want the cold parent's tier %q written first", r.max, want) |
| 1079 | } |
| 1080 | if !strings.Contains(r.subtree, "+memory") { |
| 1081 | t.Errorf("the sibling registered with the parent's subtree_control %q, want the controllers delegated first", r.subtree) |
| 1082 | } |
| 1083 | case <-time.After(5 * time.Second): |
| 1084 | t.Fatal("the sibling's registration was never released after the parent was written") |
| 1085 | } |
| 1086 | if n := m.placementsUnder(parent); n != 1 { |
| 1087 | t.Fatalf("%d placements registered, want the sibling's one", n) |
| 1088 | } |
| 1089 | } |
| 1090 | |
| 1091 | // setPlacementReadHook sets placementReadHook for one test. |
| 1092 | func setPlacementReadHook(t *testing.T, f func(dir string)) { |
| 1093 | t.Helper() |
| 1094 | orig := placementReadHook |
| 1095 | placementReadHook = f |
| 1096 | t.Cleanup(func() { placementReadHook = orig }) |
| 1097 | } |
| 1098 | |
amadan