amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1//go:build linux
2
3package cgroup
4
5import (
6 "bytes"
7 "errors"
8 "fmt"
9 "io"
10 "os"
11 "os/exec"
12 "os/signal"
13 "path/filepath"
14 "slices"
15 "strconv"
16 "strings"
17 "syscall"
18 "testing"
19 "time"
20 "unsafe"
21
22 "golang.org/x/sys/unix"
23)
24
25// The placement shim against a real kernel (zdd readiness 3, Task 1).
26// Opt-in: CARLOS_SHIM_KERNEL=1 inside a disposable unit with Delegate=yes:
27//
28// systemd-run --user --wait --pipe --collect -p Delegate=yes \
29// -E TMPDIR=/tmp/k/zddr3 -E CARLOS_SHIM_KERNEL=1 -E CARLOS_GEN_KERNEL=1 \
30// cgroup.test -test.run 'ShimKernel|ShimExecFailure' -test.v -test.count=1
31//
32// Without the variable every test here skips. With it, a missing delegation
33// fails the test: a run that was asked for cannot pass by skipping.
34
35// shimKernelManager is genKernelManager's setup, except that once
36// CARLOS_SHIM_KERNEL=1 asks for the run, no delegation is a failure.
37func shimKernelManager(t *testing.T) *Manager {
38 t.Helper()
39 if os.Getenv("CARLOS_SHIM_KERNEL") != "1" {
40 t.Skip("set CARLOS_SHIM_KERNEL=1 inside a unit with Delegate=yes (zdd readiness 3, Task 1)")
41 }
42 raw, err := os.ReadFile("/proc/self/cgroup")
43 if err != nil {
44 t.Fatalf("CARLOS_SHIM_KERNEL=1 but no cgroup: %v", err)
45 }
46 rel, ok := ParseSelfCgroup(string(raw))
47 if !ok {
48 t.Fatal("CARLOS_SHIM_KERNEL=1 but no unified cgroup path")
49 }
50 root := filepath.Join("/sys/fs/cgroup", rel)
51 if filepath.Base(root) == "supervise" {
52 root = filepath.Dir(root) // an earlier test in this process moved us
53 }
54 sup := filepath.Join(root, "supervise")
55 if err := os.MkdirAll(sup, 0o755); err != nil {
56 t.Fatalf("CARLOS_SHIM_KERNEL=1 but no delegation: %v", err)
57 }
58 if err := os.WriteFile(filepath.Join(sup, "cgroup.procs"), []byte(strconv.Itoa(os.Getpid())), 0o644); err != nil {
59 t.Fatalf("CARLOS_SHIM_KERNEL=1 but no delegation: %v", err)
60 }
61 if err := enableChildren(root); err != nil {
62 t.Fatalf("CARLOS_SHIM_KERNEL=1 but no delegation: %v", err)
63 }
64 return &Manager{root: root, logf: t.Logf}
65}
66
67// shimKernelTarget makes m.root/shim-<name> with the default tier's limits,
68// and removes it when the test ends.
69func shimKernelTarget(t *testing.T, m *Manager, name string) string {
70 t.Helper()
71 dir := filepath.Join(m.root, "shim-"+name)
72 if err := os.Mkdir(dir, 0o755); err != nil {
73 t.Fatal(err)
74 }
75 t.Cleanup(func() {
76 if err := os.Remove(dir); err != nil {
77 t.Errorf("remove %s: %v", dir, err)
78 }
79 })
80 if err := m.applyLimit(name, dir, TierDefault, true); err != nil {
81 t.Fatal(err)
82 }
83 return dir
84}
85
86// markerCmd is a tenant that touches marker first thing, then runs script.
87func markerCmd(marker, script string) *exec.Cmd {
88 return exec.Command("/bin/sh", "-c", `touch "$0"; `+script, marker)
89}
90
91func assertNoMarker(t *testing.T, marker string) {
92 t.Helper()
93 if _, err := os.Stat(marker); err == nil {
94 t.Fatal("tenant code ran")
95 }
96}
97
98func assertReaped(t *testing.T, cmd *exec.Cmd) {
99 t.Helper()
100 if cmd.Process == nil {
101 t.Fatal("no shim was started")
102 }
103 if _, err := os.Stat(fmt.Sprintf("/proc/%d", cmd.Process.Pid)); err == nil {
104 t.Fatalf("pid %d was not reaped", cmd.Process.Pid)
105 }
106}
107
108func assertNothingInFlight(t *testing.T, m *Manager) {
109 t.Helper()
110 if n := m.placementsUnder(m.root); n != 0 {
111 t.Fatalf("%d placements still registered", n)
112 }
113}
114
115func shimCgroupPath(dir string) string { return strings.TrimPrefix(dir, "/sys/fs/cgroup") }
116
117// setShimVar sets a package var for one test.
118func setShimVar[T any](t *testing.T, p *T, v T) {
119 orig := *p
120 *p = v
121 t.Cleanup(func() { *p = orig })
122}
123
124// shimDumpSep separates the dump's argv, env and cwd. It cannot be "\x00\x00":
125// an empty argv element joins to exactly that.
126const shimDumpSep = "\x00--shim-dump--\x00"
127
128// TestShimKernelChild is the synthetic tenant. Outside a child run it does
129// nothing, so a kernel run shows no skip for it.
130func TestShimKernelChild(t *testing.T) {
131 switch os.Getenv("CARLOS_SHIM_CHILD") {
132 case "dump":
133 wd, err := os.Getwd()
134 if err != nil {
135 os.Exit(3)
136 }
137 var b bytes.Buffer
138 b.WriteString(strings.Join(os.Args, "\x00"))
139 b.WriteString(shimDumpSep)
140 b.WriteString(strings.Join(os.Environ(), "\x00"))
141 b.WriteString(shimDumpSep)
142 b.WriteString(wd)
143 os.Stdout.Write(b.Bytes())
144 os.Exit(0)
145 case "state":
146 var sig int
147 if err := unix.Prctl(unix.PR_GET_PDEATHSIG, uintptr(unsafe.Pointer(&sig)), 0, 0, 0); err != nil {
148 os.Exit(3)
149 }
150 pgid, err := unix.Getpgid(0)
151 if err != nil {
152 os.Exit(3)
153 }
154 fmt.Printf("pdeathsig %d\npgid-leader %v\n", sig, pgid == os.Getpid())
155 os.Exit(0)
156 }
157}
158
159func TestShimKernelPlacesBeforeExec(t *testing.T) {
160 m := shimKernelManager(t)
161 dir := shimKernelTarget(t, m, "places")
162 cmd := exec.Command("/bin/sh", "-c", `cat /proc/self/cgroup; echo $$; read x || :`)
163 var out bytes.Buffer
164 cmd.Stdout = &out
165 stdin, err := cmd.StdinPipe()
166 if err != nil {
167 t.Fatal(err)
168 }
169 refused, err := m.startViaShim(dir, cmd, nil)
170 if refused || err != nil {
171 t.Fatalf("refused=%v err=%v", refused, err)
172 }
173 assertNothingInFlight(t, m)
174 want := "0::" + shimCgroupPath(dir)
175 raw, err := os.ReadFile(fmt.Sprintf("/proc/%d/cgroup", cmd.Process.Pid))
176 if err != nil {
177 t.Fatal(err)
178 }
179 if strings.TrimSpace(string(raw)) != want {
180 t.Fatalf("/proc/%d/cgroup = %q, want %q", cmd.Process.Pid, raw, want)
181 }
182 stdin.Close()
183 if err := cmd.Wait(); err != nil {
184 t.Fatal(err)
185 }
186 lines := strings.Split(strings.TrimSpace(out.String()), "\n")
187 if len(lines) != 2 || lines[0] != want || lines[1] != strconv.Itoa(cmd.Process.Pid) {
188 t.Fatalf("tenant printed %q, want %q and pid %d", out.String(), want, cmd.Process.Pid)
189 }
190}
191
192func TestShimKernelExitCodePassesThrough(t *testing.T) {
193 m := shimKernelManager(t)
194 dir := shimKernelTarget(t, m, "exitcode")
195 cmd := exec.Command("/bin/sh", "-c", "exit 7")
196 if _, err := m.startViaShim(dir, cmd, nil); err != nil {
197 t.Fatal(err)
198 }
199 err := cmd.Wait()
200 var ee *exec.ExitError
201 if !errors.As(err, &ee) || ee.ExitCode() != 7 {
202 t.Fatalf("Wait = %v, want exit status 7", err)
203 }
204}
205
206func TestShimKernelPassesArgvEnvExactly(t *testing.T) {
207 m := shimKernelManager(t)
208 dir := shimKernelTarget(t, m, "argvenv")
209 wd := t.TempDir()
210 mk := func(large string) *exec.Cmd {
211 return &exec.Cmd{
212 Path: os.Args[0],
213 Args: []string{os.Args[0], "-test.run=^TestShimKernelChild$", "", "two words", "line\nbreak"},
214 Env: []string{"CARLOS_SHIM_CHILD=dump", "EMPTY=", "EQ=a=b", "BAD=\xff\xfe", "LARGE=" + large},
215 Dir: wd,
216 }
217 }
218 run := func(cmd *exec.Cmd, viaShim bool) []byte {
219 t.Helper()
220 var out bytes.Buffer
221 cmd.Stdout = &out
222 cmd.Stderr = os.Stderr
223 var err error
224 if viaShim {
225 _, err = m.startViaShim(dir, cmd, nil)
226 } else {
227 err = cmd.Start()
228 }
229 if err != nil {
230 t.Fatalf("start (shim %v): %v", viaShim, err)
231 }
232 if err := cmd.Wait(); err != nil {
233 t.Fatalf("wait (shim %v): %v", viaShim, err)
234 }
235 return out.Bytes()
236 }
237 large := strings.Repeat("v", 100<<10)
238 want := mk(large)
239 viaShim := run(mk(large), true)
240 direct := run(mk(large), false)
241 if !bytes.Equal(viaShim, direct) {
242 t.Fatalf("the shim's tenant saw different argv, env or cwd from a direct start's")
243 }
244 parts := bytes.Split(viaShim, []byte(shimDumpSep))
245 if len(parts) != 3 {
246 t.Fatalf("dump has %d parts", len(parts))
247 }
248 if got := strings.Split(string(parts[0]), "\x00"); !slices.Equal(got, want.Args) {
249 t.Fatalf("argv = %q, want %q", got, want.Args)
250 }
251 if got := strings.Split(string(parts[1]), "\x00"); !slices.Equal(got, want.Env) {
252 t.Fatalf("env differs from cmd.Env (%d entries, want %d)", len(got), len(want.Env))
253 }
254 if string(parts[2]) != wd {
255 t.Fatalf("cwd = %q, want %q", parts[2], wd)
256 }
257
258 // Above Linux's per-string limit (32 pages): both fail with E2BIG, and
259 // the shim's is the command's own error, not a placement failure.
260 huge := strings.Repeat("v", 200<<10)
261 if err := mk(huge).Start(); !errors.Is(err, syscall.E2BIG) {
262 t.Fatalf("direct start = %v, want E2BIG", err)
263 }
264 cmd := mk(huge)
265 refused, err := m.startViaShim(dir, cmd, nil)
266 var pe *os.PathError
267 if refused || isPlacementError(err) || !errors.As(err, &pe) || pe.Op != "fork/exec" || !errors.Is(err, syscall.E2BIG) {
268 t.Fatalf("shim start = %v, want *os.PathError fork/exec E2BIG", err)
269 }
270 assertReaped(t, cmd)
271 assertNothingInFlight(t, m)
272}
273
274// stateScript prints the tenant's inherited state from a shell, which,
275// unlike a Go program, leaves its signal mask and RLIMIT_NOFILE as it found
276// them, then execs the Go child for what only prctl can read. ls lists the
277// shell's descriptors, not its own.
278const stateScript = `echo fds-begin; ls /proc/$$/fd; echo fds-end
279grep -E '^(SigBlk|SigIgn|CapInh|CapPrm|CapEff|CapBnd|CapAmb|NoNewPrivs):' /proc/$$/status
280echo nofile $(ulimit -Sn) $(ulimit -Hn)
281exec "$0" "$@"`
282
283func TestShimKernelInheritedStateMatchesADirectStart(t *testing.T) {
284 m := shimKernelManager(t)
285 dir := shimKernelTarget(t, m, "state")
286 // SIGHUP is the case Go's runtime keeps ignored by itself; SIGTERM and
287 // SIGUSR1 are ones the shim's runtime would replace with its handlers,
288 // so the tenant's exec would reset them to default unless the shim
289 // ignores them again (whole-branch review, finding 2).
290 ignored := []syscall.Signal{syscall.SIGHUP, syscall.SIGTERM, syscall.SIGUSR1}
291 for _, sig := range ignored {
292 signal.Ignore(sig)
293 }
294 t.Cleanup(func() {
295 for _, sig := range ignored {
296 signal.Reset(sig)
297 }
298 })
299 opened, err := syscall.Open("/dev/null", syscall.O_RDONLY, 0) // no O_CLOEXEC
300 if err != nil {
301 t.Fatal(err)
302 }
303 // Moved to 100 or above (F_DUPFD leaves close-on-exec clear), so the
304 // tenant's ExtraFiles at 3 and 4 cannot cover it in a direct start.
305 leaked, err := unix.FcntlInt(uintptr(opened), unix.F_DUPFD, 100)
306 syscall.Close(opened)
307 if err != nil {
308 t.Fatal(err)
309 }
310 t.Cleanup(func() { syscall.Close(leaked) })
311
312 run := func(viaShim bool) (fds []string, rest []string) {
313 t.Helper()
314 r1, w1, err := os.Pipe()
315 if err != nil {
316 t.Fatal(err)
317 }
318 defer r1.Close()
319 defer w1.Close()
320 r2, w2, err := os.Pipe()
321 if err != nil {
322 t.Fatal(err)
323 }
324 defer r2.Close()
325 defer w2.Close()
326 cmd := exec.Command("/bin/sh", "-c", stateScript, os.Args[0], "-test.run=^TestShimKernelChild$")
327 cmd.Env = []string{"CARLOS_SHIM_CHILD=state"}
328 cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true, Pdeathsig: syscall.SIGKILL}
329 cmd.ExtraFiles = []*os.File{r1, w2}
330 var out bytes.Buffer
331 cmd.Stdout = &out
332 cmd.Stderr = os.Stderr
333 if viaShim {
334 _, err = m.startViaShim(dir, cmd, nil)
335 } else {
336 err = cmd.Start()
337 }
338 if err != nil {
339 t.Fatalf("start (shim %v): %v", viaShim, err)
340 }
341 if err := cmd.Wait(); err != nil {
342 t.Fatalf("wait (shim %v): %v", viaShim, err)
343 }
344 inFDs := false
345 for _, l := range strings.Split(strings.TrimSpace(out.String()), "\n") {
346 switch {
347 case l == "fds-begin":
348 inFDs = true
349 case l == "fds-end":
350 inFDs = false
351 case inFDs:
352 fds = append(fds, l)
353 default:
354 rest = append(rest, l)
355 }
356 }
357 return fds, rest
358 }
359 shimFDs, shimState := run(true)
360 directFDs, directState := run(false)
361 t.Logf("shim tenant: fds %v %q", shimFDs, shimState)
362 t.Logf("direct tenant: fds %v %q", directFDs, directState)
363
364 if !slices.Equal(shimFDs, []string{"0", "1", "2", "3", "4"}) {
365 t.Fatalf("the shim's tenant has fds %v, want exactly 0 1 2 3 4", shimFDs)
366 }
367 // The control: a direct start does inherit the leaked descriptor, so the
368 // shim's clean listing is the shim's doing and not an absent leak.
369 if !slices.Contains(directFDs, strconv.Itoa(leaked)) {
370 t.Fatalf("the direct start's fds %v lack the leaked fd %d: the control is broken", directFDs, leaked)
371 }
372 if !slices.Equal(shimState, directState) {
373 t.Fatalf("inherited state differs:\nshim %q\ndirect %q", shimState, directState)
374 }
375 // And the compared state is not vacuous.
376 for _, want := range []string{"pdeathsig 9", "pgid-leader true"} {
377 if !slices.Contains(shimState, want) {
378 t.Fatalf("shim state %q lacks %q", shimState, want)
379 }
380 }
381 sawSigIgn := false
382 for _, l := range shimState {
383 if v, ok := strings.CutPrefix(l, "SigIgn:\t"); ok {
384 sawSigIgn = true
385 mask, err := strconv.ParseUint(v, 16, 64)
386 if err != nil {
387 t.Fatalf("SigIgn %q: %v", v, err)
388 }
389 for _, sig := range ignored {
390 if mask&(1<<(uint(sig)-1)) == 0 {
391 t.Fatalf("SigIgn %q does not carry the ignored %v", v, sig)
392 }
393 }
394 }
395 }
396 if !sawSigIgn {
397 t.Fatal("the shim tenant's state has no SigIgn line")
398 }
399}
400
401func TestShimKernelChdirFailureIsTheCommands(t *testing.T) {
402 m := shimKernelManager(t)
403 dir := shimKernelTarget(t, m, "chdir")
404 marker := filepath.Join(t.TempDir(), "ran")
405 cmd := markerCmd(marker, "")
406 missing := filepath.Join(t.TempDir(), "missing")
407 cmd.Dir = missing
408 refused, err := m.startViaShim(dir, cmd, nil)
409 var pe *os.PathError
410 if refused || isPlacementError(err) || !errors.As(err, &pe) || pe.Op != "chdir" || pe.Path != missing || !errors.Is(err, syscall.ENOENT) {
411 t.Fatalf("refused=%v err=%v, want *os.PathError chdir ENOENT", refused, err)
412 }
413 assertReaped(t, cmd)
414 assertNoMarker(t, marker)
415 assertNothingInFlight(t, m)
416}
417
418func TestShimExecFailureIsTheCommands(t *testing.T) {
419 m := shimKernelManager(t)
420 dir := shimKernelTarget(t, m, "execfail")
421 cmd := &exec.Cmd{Path: "/nonexistent/app", Args: []string{"app"}}
422 refused, err := m.startViaShim(dir, cmd, nil)
423 var pe *os.PathError
424 if refused || isPlacementError(err) || !errors.As(err, &pe) || pe.Op != "fork/exec" || pe.Path != "/nonexistent/app" || !errors.Is(err, syscall.ENOENT) {
425 t.Fatalf("refused=%v err=%v, want *os.PathError fork/exec ENOENT", refused, err)
426 }
427 t.Logf("the command's own error: %v", err)
428 assertReaped(t, cmd)
429 assertNothingInFlight(t, m)
430}
431
432func TestShimKernelPlacementFailureNeverRunsTenant(t *testing.T) {
433 m := shimKernelManager(t)
434 dir := shimKernelTarget(t, m, "noprocs")
435 procs := filepath.Join(dir, "cgroup.procs")
436 if err := os.Chmod(procs, 0o444); err != nil {
437 t.Fatal(err)
438 }
439 t.Cleanup(func() { _ = os.Chmod(procs, 0o644) })
440 marker := filepath.Join(t.TempDir(), "ran")
441 cmd := markerCmd(marker, "")
442 refused, err := m.startViaShim(dir, cmd, nil)
443 if refused || !isPlacementError(err) || !strings.Contains(err.Error(), "procs") {
444 t.Fatalf("refused=%v err=%v, want the shim's procs refusal as a placement error", refused, err)
445 }
446 t.Logf("placement error: %v", err)
447 assertReaped(t, cmd)
448 assertNoMarker(t, marker)
449 assertNothingInFlight(t, m)
450}
451
452func TestShimKernelBeforeRunsAfterPlacement(t *testing.T) {
453 m := shimKernelManager(t)
454 dir := shimKernelTarget(t, m, "before")
455 var events string
456 inFlight := -1
457 before := func() error {
458 b, err := os.ReadFile(filepath.Join(dir, "cgroup.events"))
459 events = string(b)
460 inFlight = m.placementsUnder(dir)
461 return err
462 }
463 cmd := exec.Command("/bin/true")
464 if refused, err := m.startViaShim(dir, cmd, before); refused || err != nil {
465 t.Fatalf("refused=%v err=%v", refused, err)
466 }
467 if err := cmd.Wait(); err != nil {
468 t.Fatal(err)
469 }
470 if !slices.Contains(strings.Split(events, "\n"), "populated 1") {
471 t.Fatalf("before saw cgroup.events %q: the shim was not in place when it was asked", events)
472 }
473 if inFlight != 0 {
474 t.Fatalf("before ran with %d placements registered; the registry covers only fork to placement", inFlight)
475 }
476
477 refusal := errors.New("not yours any more")
478 marker := filepath.Join(t.TempDir(), "ran")
479 cmd = markerCmd(marker, "")
480 refused, err := m.startViaShim(dir, cmd, func() error { return refusal })
481 if !refused || err != refusal {
482 t.Fatalf("refused=%v err=%v, want before's own error", refused, err)
483 }
484 assertReaped(t, cmd)
485 assertNoMarker(t, marker)
486 assertNothingInFlight(t, m)
487}
488
489// waitShimHeld waits until the shim has marked its spec fd close-on-exec
490// (its step 3), which it does just before the step-1 test hold.
491func waitShimHeld(pid, specFD int) bool {
492 path := fmt.Sprintf("/proc/%d/fdinfo/%d", pid, specFD)
493 for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(time.Millisecond) {
494 b, err := os.ReadFile(path)
495 if err != nil {
496 continue
497 }
498 for _, l := range strings.Split(string(b), "\n") {
499 if v, ok := strings.CutPrefix(l, "flags:"); ok {
500 flags, err := strconv.ParseUint(strings.TrimSpace(v), 8, 64)
501 if err == nil && flags&unix.O_CLOEXEC != 0 {
502 return true
503 }
504 }
505 }
506 }
507 return false
508}
509
510func TestShimKernelSignalBeforePlacement(t *testing.T) {
511 m := shimKernelManager(t)
512 dir := shimKernelTarget(t, m, "sigplace")
513 setShimVar(t, &shimTestHold, 1)
514 for _, sig := range []syscall.Signal{syscall.SIGTERM, syscall.SIGKILL} {
515 orig := startCmd
516 held := make(chan bool, 1)
517 startCmd = func(cmd *exec.Cmd) error {
518 if err := orig(cmd); err != nil {
519 return err
520 }
521 // The spec fd is the first of the shim's three, after the
522 // tenant's ExtraFiles: 3 + (len(cmd.ExtraFiles) - 3).
523 specFD := len(cmd.ExtraFiles)
524 go func() {
525 held <- waitShimHeld(cmd.Process.Pid, specFD)
526 _ = cmd.Process.Signal(sig)
527 }()
528 return nil
529 }
530 marker := filepath.Join(t.TempDir(), "ran")
531 cmd := markerCmd(marker, "")
532 _, err := m.startViaShim(dir, cmd, nil)
533 startCmd = orig
534 if !<-held {
535 t.Fatalf("%v: the shim never reached its hold", sig)
536 }
537 if !isPlacementError(err) || !strings.Contains(err.Error(), "exited before placement") {
538 t.Fatalf("%v: err=%v, want a placement error, exited before placement", sig, err)
539 }
540 t.Logf("%v: %v", sig, err)
541 assertReaped(t, cmd)
542 assertNoMarker(t, marker)
543 assertNothingInFlight(t, m)
544 }
545}
546
547func TestShimKernelSignalBeforeExec(t *testing.T) {
548 m := shimKernelManager(t)
549 dir := shimKernelTarget(t, m, "sigexec")
550 setShimVar(t, &shimTestHold, 2)
551 marker := filepath.Join(t.TempDir(), "ran")
552 cmd := markerCmd(marker, "")
553 setShimVar(t, &shimAfterRelease, func() { _ = cmd.Process.Kill() })
554 _, err := m.startViaShim(dir, cmd, nil)
555 if !isPlacementError(err) || !strings.Contains(err.Error(), "exited before exec") {
556 t.Fatalf("err=%v, want a placement error, exited before exec", err)
557 }
558 assertReaped(t, cmd)
559 assertNoMarker(t, marker)
560 assertNothingInFlight(t, m)
561}
562
563// TestShimKernelExecBeforeAcknowledgement pins the rule that an exec that
564// happened is never reported as a failure before exec: the tenant runs, the
565// parent's deadline passes before it reads R, and the start is still
566// reported started.
567func TestShimKernelExecBeforeAcknowledgement(t *testing.T) {
568 m := shimKernelManager(t)
569 dir := shimKernelTarget(t, m, "lateack")
570 setShimVar(t, &shimDeadline, shimDeadline)
571 // The short deadline goes in once the shim is started, so it governs the
572 // release-to-exec read the test is about, and a loaded box's slow shim
573 // start cannot fail the placement read instead.
574 orig := startCmd
575 t.Cleanup(func() { startCmd = orig })
576 startCmd = func(cmd *exec.Cmd) error {
577 shimDeadline = 200 * time.Millisecond
578 return orig(cmd)
579 }
580 marker := filepath.Join(t.TempDir(), "ran")
581 cmd := markerCmd(marker, "")
582 setShimVar(t, &shimAfterRelease, func() {
583 for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(time.Millisecond) {
584 if _, err := os.Stat(marker); err == nil {
585 break
586 }
587 }
588 time.Sleep(400 * time.Millisecond)
589 })
590 refused, err := m.startViaShim(dir, cmd, nil)
591 if refused || err != nil {
592 t.Fatalf("refused=%v err=%v, want started", refused, err)
593 }
594 if err := cmd.Wait(); err != nil {
595 t.Fatalf("Wait = %v", err)
596 }
597 if _, err := os.Stat(marker); err != nil {
598 t.Fatal("the tenant did not run")
599 }
600}
601
602func TestShimKernelDeadlineBeforePlacement(t *testing.T) {
603 m := shimKernelManager(t)
604 dir := shimKernelTarget(t, m, "deadline")
605 setShimVar(t, &shimTestHold, 1)
606 setShimVar(t, &shimDeadline, 300*time.Millisecond)
607 marker := filepath.Join(t.TempDir(), "ran")
608 cmd := markerCmd(marker, "")
609 _, err := m.startViaShim(dir, cmd, nil)
610 if !isPlacementError(err) || !strings.Contains(err.Error(), "no answer from the shim within 300ms") {
611 t.Fatalf("err=%v, want a placement error naming the deadline", err)
612 }
613 assertReaped(t, cmd)
614 assertNoMarker(t, marker)
615 assertNothingInFlight(t, m)
616}
617
618// TestShimKernelSurvivesDeletedExecutable: a roll replaces the carlos binary
619// on disk under a running agent; /proc/self/exe still execs the build the
620// agent is running.
621func TestShimKernelSurvivesDeletedExecutable(t *testing.T) {
622 shimKernelManager(t)
623 self, err := os.Executable()
624 if err != nil {
625 t.Fatal(err)
626 }
627 copyPath := filepath.Join(t.TempDir(), "copy.test")
628 src, err := os.Open(self)
629 if err != nil {
630 t.Fatal(err)
631 }
632 dst, err := os.OpenFile(copyPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o755)
633 if err != nil {
634 t.Fatal(err)
635 }
636 if _, err := io.Copy(dst, src); err != nil {
637 t.Fatal(err)
638 }
639 src.Close()
640 if err := dst.Close(); err != nil {
641 t.Fatal(err)
642 }
643 child := exec.Command(copyPath, "-test.run=^TestShimKernelDeletedChild$", "-test.v", "-test.count=1")
644 child.Env = []string{
645 "CARLOS_SHIM_KERNEL=1", "CARLOS_SHIM_DELETED_CHILD=1", "CARLOS_GEN_KERNEL=1",
646 "TMPDIR=" + os.TempDir(), "PATH=" + os.Getenv("PATH"),
647 }
648 out, err := child.CombinedOutput()
649 t.Logf("child:\n%s", out)
650 if err != nil || !strings.Contains(string(out), "--- PASS: TestShimKernelDeletedChild") {
651 t.Fatalf("the child with a deleted executable failed: %v", err)
652 }
653}
654
655// TestShimKernelDeletedChild is TestShimKernelSurvivesDeletedExecutable's
656// child. Outside that run it does nothing.
657func TestShimKernelDeletedChild(t *testing.T) {
658 if os.Getenv("CARLOS_SHIM_DELETED_CHILD") != "1" {
659 return
660 }
661 if err := os.Remove(os.Args[0]); err != nil {
662 t.Fatal(err)
663 }
664 exe, err := os.Readlink("/proc/self/exe")
665 if err != nil || !strings.HasSuffix(exe, " (deleted)") {
666 t.Fatalf("/proc/self/exe = %q (%v), want a deleted file", exe, err)
667 }
668 t.Logf("running from %s", exe)
669 m := shimKernelManager(t)
670 dir := shimKernelTarget(t, m, "deleted")
671 cmd := exec.Command("/bin/sh", "-c", "cat /proc/self/cgroup")
672 var out bytes.Buffer
673 cmd.Stdout = &out
674 if refused, err := m.startViaShim(dir, cmd, nil); refused || err != nil {
675 t.Fatalf("refused=%v err=%v", refused, err)
676 }
677 if err := cmd.Wait(); err != nil {
678 t.Fatal(err)
679 }
680 if got, want := strings.TrimSpace(out.String()), "0::"+shimCgroupPath(dir); got != want {
681 t.Fatalf("tenant ran in %q, want %q", got, want)
682 }
683}
684
685// TestShimKernelOverhead is informational: it never fails on time.
686func TestShimKernelOverhead(t *testing.T) {
687 m := shimKernelManager(t)
688 dir := shimKernelTarget(t, m, "overhead")
689 const n = 20
690 median := func(viaShim bool) time.Duration {
691 var d []time.Duration
692 for range n {
693 cmd := exec.Command("/bin/true")
694 start := time.Now()
695 var err error
696 if viaShim {
697 _, err = m.startViaShim(dir, cmd, nil)
698 } else {
699 err = cmd.Start()
700 }
701 if err != nil {
702 t.Fatal(err)
703 }
704 if err := cmd.Wait(); err != nil {
705 t.Fatal(err)
706 }
707 d = append(d, time.Since(start))
708 }
709 slices.Sort(d)
710 return d[n/2]
711 }
712 shim := median(true)
713 direct := median(false)
714 t.Logf("start-to-exit of /bin/true, median of %d: shim %v, direct %v, overhead %v", n, shim, direct, shim-direct)
715}
716