| 1 | //go:build linux |
| 2 | |
| 3 | package cgroup |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "errors" |
| 8 | "fmt" |
| 9 | "os" |
| 10 | "os/exec" |
| 11 | "path/filepath" |
| 12 | "slices" |
| 13 | "strconv" |
| 14 | "strings" |
| 15 | "syscall" |
| 16 | "testing" |
| 17 | ) |
| 18 | |
| 19 | // Every Manager start places through the shim, and the shim keeps the |
| 20 | // SysProcAttr the caller built (Setpgid, Pdeathsig, ...). No start may set |
| 21 | // the clone3 cgroup fields: the shim would carry them to its own fork, and |
| 22 | // the shipped unit answers clone3 with ENOSYS (amadan #272). Only the |
| 23 | // strict claims path sets them (strict_linux.go). |
| 24 | func TestManagerStartsNeverSetTheCgroupFD(t *testing.T) { |
| 25 | const host = "h.example.com" |
| 26 | m, _ := genManager(t, host) |
| 27 | var seen []*syscall.SysProcAttr |
| 28 | stubPlacement(t, func(_ *Manager, _ string, cmd *exec.Cmd, _ func() error) (bool, error) { |
| 29 | seen = append(seen, cmd.SysProcAttr) |
| 30 | cmd.Process, _ = os.FindProcess(os.Getpid()) |
| 31 | return false, nil |
| 32 | }) |
| 33 | build := func() *exec.Cmd { |
| 34 | cmd := exec.Command("/bin/true") |
| 35 | cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} |
| 36 | return cmd |
| 37 | } |
| 38 | if _, err := m.Start("flat.example.com", Limit{}, build); err != nil { |
| 39 | t.Fatal(err) |
| 40 | } |
| 41 | if _, err := m.StartChild(host, GenChild(1), TierDefault, TierDefault, build); err != nil { |
| 42 | t.Fatal(err) |
| 43 | } |
| 44 | if len(seen) != 2 { |
| 45 | t.Fatalf("placed %d starts, want the flat and the generation start", len(seen)) |
| 46 | } |
| 47 | for i, attr := range seen { |
| 48 | if attr == nil || !attr.Setpgid { |
| 49 | t.Errorf("start %d: SysProcAttr = %+v, want build()'s Setpgid kept", i, attr) |
| 50 | } else if attr.UseCgroupFD || attr.CgroupFD != 0 { |
| 51 | t.Errorf("start %d: SysProcAttr = %+v, want no clone3 cgroup fd", i, attr) |
| 52 | } |
| 53 | } |
| 54 | } |
| 55 | |
| 56 | // Ruling 2: a flat start whose placement the shim proved failed before exec |
| 57 | // falls back, for that spawn, to starting the command and moving its PID. |
| 58 | // The fallback builds a fresh command (a Cmd cannot be reused) and asks |
| 59 | // before again, since ownership can change between the two attempts. |
| 60 | func TestFlatPlacementFailureFallsBackToPIDMove(t *testing.T) { |
| 61 | root := newDelegatedRoot(t) |
| 62 | m, err := SetupAt(root, t.Logf) |
| 63 | if err != nil { |
| 64 | t.Fatal(err) |
| 65 | } |
| 66 | quietOOM(t) |
| 67 | stubPlacement(t, func(_ *Manager, _ string, _ *exec.Cmd, before func() error) (bool, error) { |
| 68 | // The shim was placed and before agreed; then it died before exec. |
| 69 | if err := before(); err != nil { |
| 70 | return true, err |
| 71 | } |
| 72 | return false, &placementError{errors.New("the shim exited before exec")} |
| 73 | }) |
| 74 | calls := fakeStart(t, func(*exec.Cmd) error { return nil }) |
| 75 | builds, asked := 0, 0 |
| 76 | cmd, err := m.StartGated("h.example.com", Limit{}, func() *exec.Cmd { builds++; return buildTrue() }, func() error { asked++; return nil }) |
| 77 | if err != nil || cmd == nil { |
| 78 | t.Fatalf("StartGated = %v, %v: want the PID-move start", cmd, err) |
| 79 | } |
| 80 | if builds != 2 || asked != 2 || len(*calls) != 1 { |
| 81 | t.Fatalf("built %d, asked %d, started %d: want two builds, two asks, one start-and-move", builds, asked, len(*calls)) |
| 82 | } |
| 83 | if got := readFixture(t, filepath.Join(m.hostDir("h.example.com"), "cgroup.procs")); got != strconv.Itoa(os.Getpid()) { |
| 84 | t.Fatalf("cgroup.procs = %q, want the child's PID moved in", got) |
| 85 | } |
| 86 | } |
| 87 | |
| 88 | // The fallback asks before again, and that second answer stands on its |
| 89 | // own (whole-branch review triage): before agrees at the shim, the shim then |
| 90 | // fails before exec, and before refuses at the PID-move fallback. Nothing |
| 91 | // starts, the host dir this spawn made is removed, and the start answers |
| 92 | // before's own error, unjoined with the placement failure. |
| 93 | func TestFlatFallbackRefusedByBeforeStartsNothing(t *testing.T) { |
| 94 | root := newDelegatedRoot(t) |
| 95 | m, err := SetupAt(root, t.Logf) |
| 96 | if err != nil { |
| 97 | t.Fatal(err) |
| 98 | } |
| 99 | quietOOM(t) |
| 100 | placementFailed := &placementError{errors.New("the shim exited before exec")} |
| 101 | stubPlacement(t, func(_ *Manager, _ string, _ *exec.Cmd, before func() error) (bool, error) { |
| 102 | if err := before(); err != nil { |
| 103 | return true, err |
| 104 | } |
| 105 | return false, placementFailed |
| 106 | }) |
| 107 | calls := fakeStart(t, func(*exec.Cmd) error { return nil }) |
| 108 | refusal := errors.New("ownership changed between the two attempts") |
| 109 | builds, asked := 0, 0 |
| 110 | cmd, err := m.StartGated("h.example.com", Limit{}, func() *exec.Cmd { builds++; return buildTrue() }, func() error { |
| 111 | asked++ |
| 112 | if asked == 1 { |
| 113 | return nil |
| 114 | } |
| 115 | return refusal |
| 116 | }) |
| 117 | if cmd != nil || err != refusal { |
| 118 | t.Fatalf("StartGated = %v, %v: want no command and before's own error, unjoined", cmd, err) |
| 119 | } |
| 120 | if errors.Is(err, placementFailed) || isPlacementError(err) { |
| 121 | t.Fatalf("the refusal %v carries the placement failure", err) |
| 122 | } |
| 123 | if builds != 2 || asked != 2 || len(*calls) != 0 { |
| 124 | t.Fatalf("built %d, asked %d, started %d: want two builds, two asks and nothing started", builds, asked, len(*calls)) |
| 125 | } |
| 126 | if n := m.placementsUnder(root); n != 0 { |
| 127 | t.Fatalf("%d placements left registered", n) |
| 128 | } |
| 129 | if _, err := os.Stat(m.hostDir("h.example.com")); !os.IsNotExist(err) { |
| 130 | t.Fatalf("host dir after a refused fallback: %v, want it removed", err) |
| 131 | } |
| 132 | } |
| 133 | |
| 134 | // A failure that is the command's own (a missing binary) is returned as |
| 135 | // is, with no PID-move retry: a retry would fail the same way, and an |
| 136 | // uncertain start must never run a tenant twice. |
| 137 | func TestFlatCommandFailureIsNotRetried(t *testing.T) { |
| 138 | root := newDelegatedRoot(t) |
| 139 | m, err := SetupAt(root, t.Logf) |
| 140 | if err != nil { |
| 141 | t.Fatal(err) |
| 142 | } |
| 143 | missing := &os.PathError{Op: "fork/exec", Path: "/x", Err: syscall.ENOENT} |
| 144 | stubPlacement(t, func(*Manager, string, *exec.Cmd, func() error) (bool, error) { return false, missing }) |
| 145 | calls := fakeStart(t, func(*exec.Cmd) error { return nil }) |
| 146 | builds := 0 |
| 147 | cmd, err := m.Start("h.example.com", Limit{}, func() *exec.Cmd { builds++; return buildTrue() }) |
| 148 | if err != missing || cmd != nil { |
| 149 | t.Fatalf("Start = %v, %v: want the command's own error as is", cmd, err) |
| 150 | } |
| 151 | if builds != 1 || len(*calls) != 0 { |
| 152 | t.Fatalf("built %d, started %d: want one build and no retry", builds, len(*calls)) |
| 153 | } |
| 154 | if _, err := os.Stat(m.hostDir("h.example.com")); !os.IsNotExist(err) { |
| 155 | t.Fatalf("host dir after a failed start: %v, want it removed", err) |
| 156 | } |
| 157 | } |
| 158 | |
| 159 | // A flat start whose context is already cancelled (Task 1's review carry): |
| 160 | // the shim cannot start, which is a placement error by the protocol, so the |
| 161 | // fallback runs once, and its own Start answers the context's error at |
| 162 | // once. That error comes back as is, with no box condition beyond the one |
| 163 | // fallback line, and nothing ran. This uses the real placement: a temp dir |
| 164 | // is refused by the shim's checks before any fork, as a cancelled start is |
| 165 | // refused at its fork. |
| 166 | func TestFlatCancelledContextIsTheCommandsError(t *testing.T) { |
| 167 | root := newDelegatedRoot(t) |
| 168 | m, err := SetupAt(root, t.Logf) |
| 169 | if err != nil { |
| 170 | t.Fatal(err) |
| 171 | } |
| 172 | var lines []string |
| 173 | m.logf = func(format string, args ...any) { lines = append(lines, fmt.Sprintf(format, args...)) } |
| 174 | ctx, cancel := context.WithCancel(t.Context()) |
| 175 | cancel() |
| 176 | cmd, err := m.Start("h.example.com", Limit{}, func() *exec.Cmd { return exec.CommandContext(ctx, "/bin/true") }) |
| 177 | if !errors.Is(err, context.Canceled) || cmd != nil { |
| 178 | t.Fatalf("Start = %v, %v: want context.Canceled and no command", cmd, err) |
| 179 | } |
| 180 | if isPlacementError(err) || errors.Is(err, ErrLimitNotApplied) { |
| 181 | t.Fatalf("a cancelled start read as a placement failure: %v", err) |
| 182 | } |
| 183 | if len(lines) != 1 || !strings.Contains(lines[0], "using the PID-move fallback for this spawn") { |
| 184 | t.Fatalf("log lines = %q, want only the one fallback line", lines) |
| 185 | } |
| 186 | if _, err := os.Stat(m.hostDir("h.example.com")); !os.IsNotExist(err) { |
| 187 | t.Fatalf("host dir after a cancelled start: %v, want it removed", err) |
| 188 | } |
| 189 | } |
| 190 | |
| 191 | // startAndMoveIn's child runs in the agent's cgroup from its start until its |
| 192 | // PID is written, so its target is registered in flight for exactly that |
| 193 | // window: not while before is asked, through the start, and no longer once |
| 194 | // the PID write has returned. |
| 195 | func TestStartAndMoveRegistersItsTarget(t *testing.T) { |
| 196 | root := newDelegatedRoot(t) |
| 197 | m, err := SetupAt(root, t.Logf) |
| 198 | if err != nil { |
| 199 | t.Fatal(err) |
| 200 | } |
| 201 | quietOOM(t) |
| 202 | dir := m.hostDir("h.example.com") |
| 203 | if err := os.MkdirAll(dir, 0o755); err != nil { |
| 204 | t.Fatal(err) |
| 205 | } |
| 206 | var atBefore, atStart int |
| 207 | fakeStart(t, func(*exec.Cmd) error { atStart = m.placementsAt(dir); return nil }) |
| 208 | cmd, refused, err := m.startAndMoveIn("h.example.com", dir, buildTrue, func() error { atBefore = m.placementsAt(dir); return nil }) |
| 209 | if err != nil || refused || cmd == nil { |
| 210 | t.Fatalf("startAndMoveIn = %v, %v, %v", cmd, refused, err) |
| 211 | } |
| 212 | if atBefore != 0 || atStart != 1 || m.placementsAt(dir) != 0 { |
| 213 | t.Fatalf("in flight: %d at before, %d at the start, %d after the move; want 0, 1, 0", atBefore, atStart, m.placementsAt(dir)) |
| 214 | } |
| 215 | // A failed start ends its registration too. |
| 216 | fakeStart(t, func(*exec.Cmd) error { return errors.New("no such binary") }) |
| 217 | if _, _, err := m.startAndMoveIn("h.example.com", dir, buildTrue, nil); err == nil { |
| 218 | t.Fatal("a failing start succeeded") |
| 219 | } |
| 220 | if n := m.placementsUnder(root); n != 0 { |
| 221 | t.Fatalf("%d placements left in flight after a failed start", n) |
| 222 | } |
| 223 | } |
| 224 | |
| 225 | // A refusal before any start: nothing starts, no retry. The real placement |
| 226 | // cannot place in a temp dir (its checks refuse before any fork, so before |
| 227 | // is not asked there); the fallback then asks before, which refuses. |
| 228 | func TestStartGatedRefusedBeforeTheFirstStart(t *testing.T) { |
| 229 | root := newDelegatedRoot(t) |
| 230 | m, err := SetupAt(root, t.Logf) |
| 231 | if err != nil { |
| 232 | t.Fatal(err) |
| 233 | } |
| 234 | calls := fakeStart(t, func(*exec.Cmd) error { return nil }) |
| 235 | refused := errors.New("lease lost") |
| 236 | if cmd, err := m.StartGated("h.example.com", Limit{}, buildTrue, func() error { return refused }); err != refused || cmd != nil { |
| 237 | t.Fatalf("StartGated = %v, %v: want the hook's own error and no command", cmd, err) |
| 238 | } |
| 239 | if len(*calls) != 0 { |
| 240 | t.Fatalf("started %d times after the hook refused, want none", len(*calls)) |
| 241 | } |
| 242 | if _, err := os.Stat(filepath.Join(root, ExecName("h.example.com"))); !os.IsNotExist(err) { |
| 243 | t.Fatalf("host dir after a refused start: %v, want it removed", err) |
| 244 | } |
| 245 | } |
| 246 | |
| 247 | // A refusal through the shim (before said no after placement): nothing of |
| 248 | // the command ran, the refusal comes back as the hook's own error, and |
| 249 | // there is no PID-move retry, for a flat start and a generation alike. |
| 250 | func TestARefusalAtPlacementIsNotRetried(t *testing.T) { |
| 251 | const host = "h.example.com" |
| 252 | m, parent := genManager(t, host) |
| 253 | refused := errors.New("the attempt was superseded") |
| 254 | stubPlacement(t, func(_ *Manager, _ string, _ *exec.Cmd, before func() error) (bool, error) { |
| 255 | return true, before() |
| 256 | }) |
| 257 | calls := fakeStart(t, func(*exec.Cmd) error { return nil }) |
| 258 | if cmd, err := m.StartGated("flat.example.com", Limit{}, buildTrue, func() error { return refused }); err != refused || cmd != nil { |
| 259 | t.Fatalf("StartGated = %v, %v: want the hook's own error", cmd, err) |
| 260 | } |
| 261 | if cmd, err := m.StartChildGated(host, GenChild(1), TierDefault, Limit{}, buildTrue, func() error { return refused }); err != refused || cmd != nil { |
| 262 | t.Fatalf("StartChildGated = %v, %v: want the hook's own error", cmd, err) |
| 263 | } |
| 264 | if len(*calls) != 0 { |
| 265 | t.Fatalf("started %d times after a refusal at placement, want none", len(*calls)) |
| 266 | } |
| 267 | for _, dir := range []string{m.hostDir("flat.example.com"), filepath.Join(parent, "g1")} { |
| 268 | if _, err := os.Stat(dir); !os.IsNotExist(err) { |
| 269 | t.Errorf("%s after a refused start: %v, want it removed", dir, err) |
| 270 | } |
| 271 | } |
| 272 | } |
| 273 | |
| 274 | // FakePlacementForTest lets another package's fixture (a temp directory, |
| 275 | // where the shim cannot place) start a generation: the command starts |
| 276 | // directly behind its before hook, as if placed, and the shim counts as |
| 277 | // available even where the binary has no hook. Both are restored at the |
| 278 | // test's cleanup. |
| 279 | func TestFakePlacementForTestStartsWithoutTheShimAndRestores(t *testing.T) { |
| 280 | const host = "h.example.com" |
| 281 | m, _ := genFixture(t, host) |
| 282 | t.Run("faked", func(t *testing.T) { |
| 283 | noShim(t) // as in a test binary whose TestMain has no hook |
| 284 | FakePlacementForTest(t) |
| 285 | if !shimAvailable() { |
| 286 | t.Fatal("the fake did not make the shim available") |
| 287 | } |
| 288 | var asked int |
| 289 | calls := fakeStart(t, func(*exec.Cmd) error { return nil }) |
| 290 | if _, err := m.StartChildGated(host, GenChild(1), TierDefault, TierDefault, buildTrue, func() error { asked++; return nil }); err != nil { |
| 291 | t.Fatalf("StartChildGated under the fake = %v", err) |
| 292 | } |
| 293 | if len(*calls) != 1 || asked != 1 { |
| 294 | t.Fatalf("started %d times, asked %d: want one direct start behind one ask", len(*calls), asked) |
| 295 | } |
| 296 | if slices.ContainsFunc(*calls, func(c *exec.Cmd) bool { return slices.Contains(c.Args, ShimVerb) }) { |
| 297 | t.Fatal("the fake started a shim") |
| 298 | } |
| 299 | }) |
| 300 | if shimForcedForTest.Load() { |
| 301 | t.Fatal("the fake's availability override outlived its test") |
| 302 | } |
| 303 | calls := fakeStart(t, func(*exec.Cmd) error { return nil }) |
| 304 | if _, err := m.StartChild(host, GenChild(2), TierDefault, TierDefault, buildTrue); !errors.Is(err, ErrLimitNotApplied) || !isPlacementError(err) { |
| 305 | t.Fatalf("StartChild after the fake = %v: want the real shim's placement refusal on a temp dir", err) |
| 306 | } |
| 307 | if len(*calls) != 0 { |
| 308 | t.Fatal("the fake placement outlived its test") |
| 309 | } |
| 310 | } |
| 311 | |
| 312 | // The real shim cannot place a process in a temp directory (no cgroup2 |
| 313 | // filesystem). A generation start there is refused as the box's condition, |
| 314 | // having built the command once and started nothing. |
| 315 | func TestAGenerationTheShimCannotPlaceIsRefused(t *testing.T) { |
| 316 | const host = "h.example.com" |
| 317 | m, parent := genFixture(t, host) |
| 318 | calls := fakeStart(t, func(*exec.Cmd) error { return nil }) |
| 319 | builds := 0 |
| 320 | cmd, err := m.StartChild(host, RoleChild("seed"), TierDefault, Limit{}, func() *exec.Cmd { builds++; return buildTrue() }) |
| 321 | if !errors.Is(err, ErrLimitNotApplied) || !isPlacementError(err) || cmd != nil { |
| 322 | t.Fatalf("StartChild = %v, %v: want ErrLimitNotApplied from the shim's placement error", cmd, err) |
| 323 | } |
| 324 | if builds != 1 || len(*calls) != 0 { |
| 325 | t.Fatalf("built %d, started %d: want one build and nothing started", builds, len(*calls)) |
| 326 | } |
| 327 | noGenerationChild(t, parent) |
| 328 | } |
| 329 | |
amadan