amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1//go:build linux
2
3package cgroup
4
5import (
6 "context"
7 "errors"
8 "os"
9 "path/filepath"
10 "reflect"
11 "strconv"
12 "testing"
13 "time"
14)
15
16func cleanupFixture(t *testing.T) (*Manager, string, string) {
17 t.Helper()
18 host, root := "cleanup.example", t.TempDir()
19 dir := filepath.Join(root, ExecName(host))
20 if err := os.Mkdir(dir, 0700); err != nil {
21 t.Fatal(err)
22 }
23 for name, body := range map[string]string{"cgroup.events": "populated 1\n", "cgroup.procs": "123\n456\n", "cgroup.kill": ""} {
24 if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0600); err != nil {
25 t.Fatal(err)
26 }
27 }
28 return &Manager{root: root, logf: t.Logf}, host, dir
29}
30
31func fixtureCleanup(t *testing.T, m *Manager, host string) *Cleanup {
32 t.Helper()
33 c, err := m.openCleanup(host, func(*os.File) error { return nil })
34 if err != nil {
35 t.Fatal(err)
36 }
37 t.Cleanup(func() { _ = c.Close() })
38 return c
39}
40
41func TestCleanupReopensAndDrains(t *testing.T) {
42 m, host, dir := cleanupFixture(t)
43 c := fixtureCleanup(t, m, host)
44 if _, exists := reflect.TypeOf(c).MethodByName("Start"); exists {
45 t.Fatal("cleanup grants spawn")
46 }
47 if populated, err := c.Populated(); err != nil || !populated {
48 t.Fatal(populated, err)
49 }
50 if members, err := c.Members(); err != nil || !reflect.DeepEqual(members, []int{123, 456}) {
51 t.Fatal(members, err)
52 }
53 ctx, cancel := context.WithTimeout(t.Context(), 30*time.Millisecond)
54 defer cancel()
55 if err := c.KillAndEmpty(ctx); !errors.Is(err, context.DeadlineExceeded) {
56 t.Fatal(err)
57 }
58 if b, err := os.ReadFile(filepath.Join(dir, "cgroup.kill")); err != nil || string(b) != "1" {
59 t.Fatalf("kill: %q %v", b, err)
60 }
61 if err := os.WriteFile(filepath.Join(dir, "cgroup.events"), []byte("populated 0\n"), 0600); err != nil {
62 t.Fatal(err)
63 }
64 ctx2, cancel2 := context.WithTimeout(t.Context(), time.Second)
65 defer cancel2()
66 // Population alone is insufficient: the members still prevent success.
67 ctx3, cancel3 := context.WithTimeout(t.Context(), 30*time.Millisecond)
68 defer cancel3()
69 if err := c.KillAndEmpty(ctx3); !errors.Is(err, context.DeadlineExceeded) {
70 t.Fatal(err)
71 }
72 if err := os.WriteFile(filepath.Join(dir, "cgroup.procs"), nil, 0600); err != nil {
73 t.Fatal(err)
74 }
75 if err := c.KillAndEmpty(ctx2); err != nil {
76 t.Fatal(err)
77 }
78 if err := c.Close(); err != nil {
79 t.Fatal(err)
80 }
81 if _, err := c.Populated(); !errors.Is(err, ErrStrictState) {
82 t.Fatal(err)
83 }
84 if _, err := os.Stat(dir); err != nil {
85 t.Fatal("close removed group", err)
86 }
87}
88
89func TestCleanupRefusals(t *testing.T) {
90 var absent *Manager
91 if _, err := absent.OpenCleanup("cleanup.example"); !errors.Is(err, ErrNoDelegation) {
92 t.Fatal(err)
93 }
94 if _, err := (&Manager{}).OpenCleanup("cleanup.example"); !errors.Is(err, ErrNoDelegation) {
95 t.Fatal(err)
96 }
97 m, host, dir := cleanupFixture(t)
98 if _, err := m.OpenCleanup(host); !errors.Is(err, ErrStrictUnsupported) {
99 t.Fatal("ordinary filesystem admitted", err)
100 }
101 if _, err := m.openCleanup("../outside", func(*os.File) error { return nil }); !errors.Is(err, ErrStrictRefused) {
102 t.Fatal(err)
103 }
104 if _, err := m.openCleanup("missing.example", func(*os.File) error { return nil }); !errors.Is(err, ErrNoCgroup) {
105 t.Fatal(err)
106 }
107 if _, err := os.Stat(filepath.Join(m.root, ExecName("missing.example"))); !errors.Is(err, os.ErrNotExist) {
108 t.Fatal("created missing group", err)
109 }
110 c := fixtureCleanup(t, m, host)
111 for _, body := range []string{"", "populated 2\n", "populated 0\npopulated 0\n", "broken\n"} {
112 if err := os.WriteFile(filepath.Join(dir, "cgroup.events"), []byte(body), 0600); err != nil {
113 t.Fatal(err)
114 }
115 if _, err := c.Populated(); err == nil {
116 t.Fatal("malformed events accepted")
117 }
118 }
119 for _, body := range []string{"0\n", "-1\n", "oops\n", "9999999999999999999999999\n"} {
120 if err := os.WriteFile(filepath.Join(dir, "cgroup.procs"), []byte(body), 0600); err != nil {
121 t.Fatal(err)
122 }
123 if _, err := c.Members(); err == nil {
124 t.Fatal("malformed members accepted")
125 }
126 }
127 if err := c.KillAndEmpty(context.Background()); !errors.Is(err, ErrStrictDeadline) {
128 t.Fatal(err)
129 }
130 if err := c.KillAndEmpty(nil); !errors.Is(err, ErrStrictDeadline) {
131 t.Fatal(err)
132 }
133 if err := os.Rename(dir, dir+".old"); err != nil {
134 t.Fatal(err)
135 }
136 if err := os.Mkdir(dir, 0700); err != nil {
137 t.Fatal(err)
138 }
139 if _, err := c.Populated(); err == nil {
140 t.Fatal("replaced identity accepted")
141 }
142 if _, err := c.Members(); err == nil {
143 t.Fatal("replaced membership accepted")
144 }
145 if err := c.Close(); err != nil {
146 t.Fatal(err)
147 }
148}
149
150func TestCleanupDeadlineWhileLocked(t *testing.T) {
151 m, host, _ := cleanupFixture(t)
152 c := fixtureCleanup(t, m, host)
153 c.state.pin.mu.Lock()
154 ctx, cancel := context.WithTimeout(t.Context(), 30*time.Millisecond)
155 defer cancel()
156 done := make(chan error, 1)
157 go func() { done <- c.KillAndEmpty(ctx) }()
158 select {
159 case err := <-done:
160 c.state.pin.mu.Unlock()
161 if !errors.Is(err, context.DeadlineExceeded) {
162 t.Fatal(err)
163 }
164 case <-time.After(time.Second):
165 c.state.pin.mu.Unlock()
166 <-done
167 t.Fatal("cleanup exceeded its deadline waiting for a handle lock")
168 }
169}
170
171func TestCleanupRejectsMissingControlsAndSymlinks(t *testing.T) {
172 for _, name := range []string{"cgroup.events", "cgroup.procs", "cgroup.kill"} {
173 t.Run(name, func(t *testing.T) {
174 m, host, dir := cleanupFixture(t)
175 if err := os.Remove(filepath.Join(dir, name)); err != nil {
176 t.Fatal(err)
177 }
178 if c, err := m.openCleanup(host, func(*os.File) error { return nil }); err == nil || c != nil || errors.Is(err, ErrNoCgroup) {
179 t.Fatalf("missing control: %v %v", c, err)
180 }
181 if err := os.Symlink(filepath.Join(dir, "elsewhere"), filepath.Join(dir, name)); err != nil {
182 t.Fatal(err)
183 }
184 if c, err := m.openCleanup(host, func(*os.File) error { return nil }); err == nil || c != nil {
185 t.Fatalf("symlink control: %v %v", c, err)
186 }
187 })
188 }
189}
190
191// A kernel before 5.14 has no cgroup.kill. A cleanup opened there names
192// that (ErrNoKill), so a caller can fall back to the leader-only proof
193// instead of reading it as an unproven kill on every stop. A cgroup.kill
194// that is there but a symlink is not that case.
195func TestCleanupNamesAMissingKillFile(t *testing.T) {
196 m, host, dir := cleanupFixture(t)
197 if err := os.Remove(filepath.Join(dir, "cgroup.kill")); err != nil {
198 t.Fatal(err)
199 }
200 if c, err := m.openCleanup(host, func(*os.File) error { return nil }); c != nil || !errors.Is(err, ErrNoKill) {
201 t.Fatalf("missing cgroup.kill: %v %v, want ErrNoKill", c, err)
202 }
203 if err := os.Symlink(filepath.Join(dir, "elsewhere"), filepath.Join(dir, "cgroup.kill")); err != nil {
204 t.Fatal(err)
205 }
206 if c, err := m.openCleanup(host, func(*os.File) error { return nil }); c != nil || err == nil || errors.Is(err, ErrNoKill) {
207 t.Fatalf("symlinked cgroup.kill: %v %v, want a refusal that is not ErrNoKill", c, err)
208 }
209}
210
211func mkCgroupFixture(t *testing.T, dir, procs, events string) {
212 t.Helper()
213 if err := os.MkdirAll(dir, 0o700); err != nil {
214 t.Fatal(err)
215 }
216 for name, body := range map[string]string{"cgroup.events": events, "cgroup.procs": procs, "cgroup.kill": ""} {
217 if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o600); err != nil {
218 t.Fatal(err)
219 }
220 }
221}
222
223func fixtureCleanupAt(t *testing.T, root, host, child string) *Cleanup {
224 t.Helper()
225 c, err := (&Manager{root: root}).openCleanupAt(host, child, func(*os.File) error { return nil })
226 if err != nil {
227 t.Fatal(err)
228 }
229 t.Cleanup(func() { _ = c.Close() })
230 return c
231}
232
233// Stop evidence is taken on the parent (spec §7.3). A generation parent's
234// own cgroup.procs is always empty; its generations' are not.
235func TestCleanupMembersReachEveryGeneration(t *testing.T) {
236 host, root := "gens.example", t.TempDir()
237 parent := filepath.Join(root, ExecName(host))
238 mkCgroupFixture(t, parent, "", "populated 1\n")
239 mkCgroupFixture(t, filepath.Join(parent, "g1"), "11\n", "populated 1\n")
240 mkCgroupFixture(t, filepath.Join(parent, "g2"), "22\n", "populated 1\n")
241 // Review focus 3: a flat tenant runs as the agent's uid and can make
242 // cgroups of its own under its generation.
243 mkCgroupFixture(t, filepath.Join(parent, "g2", "tenant"), "33\n", "populated 1\n")
244 c := fixtureCleanupAt(t, root, host, "")
245 members, err := c.Members()
246 if err != nil || !reflect.DeepEqual(members, []int{11, 22, 33}) {
247 t.Fatalf("members = %v, %v; want every generation's", members, err)
248 }
249}
250
251func TestCleanupMembersRefuseATreeTooDeep(t *testing.T) {
252 host, root := "deep.example", t.TempDir()
253 dir := filepath.Join(root, ExecName(host))
254 mkCgroupFixture(t, dir, "", "populated 1\n")
255 for i := range maxCleanupDepth + 1 {
256 dir = filepath.Join(dir, "d"+strconv.Itoa(i))
257 mkCgroupFixture(t, dir, "", "populated 0\n")
258 }
259 c := fixtureCleanupAt(t, root, host, "")
260 if _, err := c.Members(); !errors.Is(err, ErrStrictRefused) {
261 t.Fatalf("a tree deeper than %d: %v, want ErrStrictRefused", maxCleanupDepth, err)
262 }
263}
264
265// The group budget is a width bound too, and it is enforced while the
266// directory is read: a tree one group too wide is refused before any child
267// is opened. Every child here is unreadable (mode 0), so a walk that
268// opened one before counting them all would fail with EACCES instead.
269func TestCleanupMembersRefuseATreeTooWide(t *testing.T) {
270 if os.Geteuid() == 0 {
271 t.Skip("root opens a mode-0 directory; the probe needs an ordinary user")
272 }
273 host, root := "wide.example", t.TempDir()
274 dir := filepath.Join(root, ExecName(host))
275 mkCgroupFixture(t, dir, "", "populated 1\n")
276 for i := range maxCleanupGroups + 1 {
277 if err := os.Mkdir(filepath.Join(dir, "w"+strconv.Itoa(i)), 0); err != nil {
278 t.Fatal(err)
279 }
280 }
281 c := fixtureCleanupAt(t, root, host, "")
282 if _, err := c.Members(); !errors.Is(err, ErrStrictRefused) {
283 t.Fatalf("%d groups at one level: %v, want ErrStrictRefused before any child is opened", maxCleanupGroups+1, err)
284 }
285}
286
287func TestCleanupMembersAcceptATreeExactlyAtTheBound(t *testing.T) {
288 host, root := "bound.example", t.TempDir()
289 dir := filepath.Join(root, ExecName(host))
290 mkCgroupFixture(t, dir, "", "populated 1\n")
291 for i := range maxCleanupGroups {
292 if err := os.Mkdir(filepath.Join(dir, "w"+strconv.Itoa(i)), 0o700); err != nil {
293 t.Fatal(err)
294 }
295 }
296 mkCgroupFixture(t, filepath.Join(dir, "w7"), "77\n", "populated 1\n")
297 c := fixtureCleanupAt(t, root, host, "")
298 if members, err := c.Members(); err != nil || !reflect.DeepEqual(members, []int{77}) {
299 t.Fatalf("%d groups: %v, %v; want [77]", maxCleanupGroups, members, err)
300 }
301}
302
303// KillAndEmpty on a generation acts only on that generation's child.
304func TestCleanupChildKillsOnlyItsGeneration(t *testing.T) {
305 host, root := "kill.example", t.TempDir()
306 parent := filepath.Join(root, ExecName(host))
307 mkCgroupFixture(t, parent, "", "populated 1\n")
308 mkCgroupFixture(t, filepath.Join(parent, "g1"), "11\n", "populated 1\n")
309 mkCgroupFixture(t, filepath.Join(parent, "g2"), "", "populated 0\n")
310 c := fixtureCleanupAt(t, root, host, "g2")
311 ctx, cancel := context.WithTimeout(t.Context(), time.Second)
312 defer cancel()
313 if err := c.KillAndEmpty(ctx); err != nil {
314 t.Fatal(err)
315 }
316 for dir, want := range map[string]string{filepath.Join(parent, "g2"): "1", filepath.Join(parent, "g1"): "", parent: ""} {
317 if got := readFixture(t, filepath.Join(dir, "cgroup.kill")); got != want {
318 t.Errorf("%s/cgroup.kill = %q, want %q", dir, got, want)
319 }
320 }
321}
322
323func TestOpenCleanupChildRefusesForeignNamesAndMissingChildren(t *testing.T) {
324 host, root := "names.example", t.TempDir()
325 mkCgroupFixture(t, filepath.Join(root, ExecName(host)), "", "populated 0\n")
326 m := &Manager{root: root}
327 for _, child := range []string{"child", "../x", "g0"} {
328 if _, err := m.openCleanupAt(host, child, func(*os.File) error { return nil }); !errors.Is(err, ErrStrictRefused) {
329 t.Errorf("child %q: %v, want ErrStrictRefused", child, err)
330 }
331 }
332 if _, err := m.openCleanupAt(host, "g9", func(*os.File) error { return nil }); !errors.Is(err, ErrNoCgroup) {
333 t.Errorf("missing g9: %v, want ErrNoCgroup", err)
334 }
335}
336
337// Review focus 4, Ruling 6: a process being placed under the host (here a
338// generation into g1) is a member the kernel cannot see yet. While it is in
339// flight the host's cleanup reports populated, refuses to list members,
340// and kills nothing; once it resolves, Populated reads cgroup.events again.
341func TestCleanupCountsPlacementInFlight(t *testing.T) {
342 m, host, dir := cleanupFixture(t)
343 writeFixture(t, filepath.Join(dir, "cgroup.events"), "populated 0\n")
344 writeFixture(t, filepath.Join(dir, "cgroup.procs"), "")
345 c := fixtureCleanup(t, m, host)
346 done := m.beginPlacement(filepath.Join(dir, "g1"))
347 defer done()
348 if populated, err := c.Populated(); err != nil || !populated {
349 t.Fatalf("Populated = %v, %v with a placement in flight: want true", populated, err)
350 }
351 if members, err := c.Members(); !errors.Is(err, ErrPlacementInFlight) {
352 t.Fatalf("Members = %v, %v with a placement in flight: want ErrPlacementInFlight", members, err)
353 }
354 ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond)
355 defer cancel()
356 if err := c.KillAndEmpty(ctx); err == nil {
357 t.Fatal("KillAndEmpty proved the host empty with a placement in flight")
358 }
359 if got := readFixture(t, filepath.Join(dir, "cgroup.kill")); got != "" {
360 t.Fatalf("cgroup.kill = %q: the kill was written before the placement resolved", got)
361 }
362 done()
363 if populated, err := c.Populated(); err != nil || populated {
364 t.Fatalf("Populated = %v, %v once the placement resolved: want cgroup.events' answer, false", populated, err)
365 }
366 if members, err := c.Members(); err != nil || len(members) != 0 {
367 t.Fatalf("Members = %v, %v once the placement resolved: want none", members, err)
368 }
369}
370
371// Ruling 6: a placement that registers after KillAndEmpty's wait, here
372// right after the kill is written, keeps the proof from finishing: every
373// emptiness check after the kill counts it. The fake cgroupfs says empty
374// throughout, so only the registry stands between the proof and a false
375// empty.
376func TestKillAndEmptySeesAPlacementRegisteredMidProof(t *testing.T) {
377 m, host, dir := cleanupFixture(t)
378 writeFixture(t, filepath.Join(dir, "cgroup.events"), "populated 0\n")
379 writeFixture(t, filepath.Join(dir, "cgroup.procs"), "")
380 c := fixtureCleanup(t, m, host)
381 var done func()
382 orig := cleanupKillWrite
383 cleanupKillWrite = func(f *os.File) error {
384 err := orig(f)
385 done = m.beginPlacement(filepath.Join(dir, "g1"))
386 return err
387 }
388 t.Cleanup(func() {
389 cleanupKillWrite = orig
390 if done != nil {
391 done()
392 }
393 })
394 ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
395 defer cancel()
396 if err := c.KillAndEmpty(ctx); err == nil {
397 t.Fatal("KillAndEmpty proved the host empty while a placement registered mid-proof was in flight")
398 }
399 if done == nil {
400 t.Fatal("the kill was never written: the test proves nothing")
401 }
402 // The control: with nothing in flight the same proof finishes.
403 done()
404 ctx2, cancel2 := context.WithTimeout(t.Context(), time.Second)
405 defer cancel2()
406 cleanupKillWrite = orig
407 if err := c.KillAndEmpty(ctx2); err != nil {
408 t.Fatalf("KillAndEmpty with nothing in flight = %v, want the proof", err)
409 }
410}
411
412// Whole-branch review, finding 1, first schedule: a placement registers
413// while the kernel is being read (the hook runs right after the read, before
414// the registry is asked again). The fake cgroupfs says empty, so only the
415// registry can stop a false empty: Populated must answer populated and
416// Members must refuse, even though nothing was in flight when either began.
417func TestCleanupCountsAPlacementRegisteredDuringTheRead(t *testing.T) {
418 m, host, dir := cleanupFixture(t)
419 writeFixture(t, filepath.Join(dir, "cgroup.events"), "populated 0\n")
420 writeFixture(t, filepath.Join(dir, "cgroup.procs"), "")
421 c := fixtureCleanup(t, m, host)
422 var dones []func()
423 t.Cleanup(func() {
424 for _, d := range dones {
425 d()
426 }
427 })
428 reads := 0
429 setPlacementReadHook(t, func(string) {
430 reads++
431 dones = append(dones, m.beginPlacement(filepath.Join(dir, "g1")))
432 })
433 if populated, err := c.Populated(); err != nil || !populated {
434 t.Fatalf("Populated = %v, %v with a placement registered during the read: want true", populated, err)
435 }
436 for _, d := range dones {
437 d()
438 }
439 if members, err := c.Members(); !errors.Is(err, ErrPlacementInFlight) {
440 t.Fatalf("Members = %v, %v with a placement registered during the read: want ErrPlacementInFlight", members, err)
441 }
442 if reads != 2 {
443 t.Fatalf("the read hook ran %d times, want 2: the test proves nothing", reads)
444 }
445 // The control: the same reads with nothing registering answer empty.
446 for _, d := range dones {
447 d()
448 }
449 placementReadHook = nil
450 if populated, err := c.Populated(); err != nil || populated {
451 t.Fatalf("Populated = %v, %v with nothing in flight: want false", populated, err)
452 }
453 if members, err := c.Members(); err != nil || len(members) != 0 {
454 t.Fatalf("Members = %v, %v with nothing in flight: want none", members, err)
455 }
456}
457
458// Finding 1, second schedule: a placement in flight when the kernel is read
459// lands and ends its registration between the kernel read and the
460// registry's. Asked in that order, the registry would read zero beside a
461// kernel view taken before the process arrived. Both answers must count it.
462func TestCleanupCountsAPlacementThatEndsBetweenTheReads(t *testing.T) {
463 m, host, dir := cleanupFixture(t)
464 writeFixture(t, filepath.Join(dir, "cgroup.events"), "populated 0\n")
465 writeFixture(t, filepath.Join(dir, "cgroup.procs"), "")
466 c := fixtureCleanup(t, m, host)
467 var done func()
468 setPlacementReadHook(t, func(string) {
469 if done != nil {
470 done()
471 }
472 })
473 done = m.beginPlacement(filepath.Join(dir, "g1"))
474 if populated, err := c.Populated(); err != nil || !populated {
475 t.Fatalf("Populated = %v, %v with a placement that ended after the kernel read: want true", populated, err)
476 }
477 done = m.beginPlacement(filepath.Join(dir, "g1"))
478 if members, err := c.Members(); !errors.Is(err, ErrPlacementInFlight) {
479 t.Fatalf("Members = %v, %v with a placement that ended after the kernel read: want ErrPlacementInFlight", members, err)
480 }
481 if m.placementsUnder(dir) != 0 {
482 t.Fatal("the hook did not end the placements: the test proves nothing")
483 }
484}
485
486// Finding 1 through KillAndEmpty: a placement registered during each
487// emptiness read after the kill keeps the proof from finishing, so it ends
488// unproven at ctx's end and never answers empty.
489func TestKillAndEmptyCountsAPlacementRegisteredDuringARead(t *testing.T) {
490 m, host, dir := cleanupFixture(t)
491 writeFixture(t, filepath.Join(dir, "cgroup.events"), "populated 0\n")
492 writeFixture(t, filepath.Join(dir, "cgroup.procs"), "")
493 c := fixtureCleanup(t, m, host)
494 var dones []func()
495 t.Cleanup(func() {
496 for _, d := range dones {
497 d()
498 }
499 })
500 setPlacementReadHook(t, func(string) {
501 // Each read sees one registration begin; the previous one ends, so
502 // a check that asked the registry only before its read would see
503 // nothing in flight.
504 for _, d := range dones {
505 d()
506 }
507 dones = append(dones, m.beginPlacement(filepath.Join(dir, "g1")))
508 })
509 ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
510 defer cancel()
511 if err := c.KillAndEmpty(ctx); !errors.Is(err, context.DeadlineExceeded) {
512 t.Fatalf("KillAndEmpty = %v, want unproven at ctx's end (context.DeadlineExceeded)", err)
513 }
514 if len(dones) == 0 {
515 t.Fatal("no emptiness read ran: the test proves nothing")
516 }
517}
518