| 1 | //go:build linux |
| 2 | |
| 3 | package cgroup |
| 4 | |
| 5 | import ( |
| 6 | "errors" |
| 7 | "fmt" |
| 8 | "os" |
| 9 | "os/exec" |
| 10 | "path/filepath" |
| 11 | "strings" |
| 12 | "syscall" |
| 13 | "testing" |
| 14 | "time" |
| 15 | ) |
| 16 | |
| 17 | // These run without a delegated cgroup: a temp directory is not cgroupfs, |
| 18 | // so they exercise the failure paths. The kernel file covers the rest. |
| 19 | |
| 20 | // TestShimRefusesANonCgroupDirectory: the parent's own check (step 2) |
| 21 | // refuses before anything starts, as a placement error. |
| 22 | func TestShimRefusesANonCgroupDirectory(t *testing.T) { |
| 23 | m := &Manager{root: t.TempDir(), logf: t.Logf} |
| 24 | marker := filepath.Join(t.TempDir(), "ran") |
| 25 | cmd := exec.Command("/bin/sh", "-c", "touch "+marker) |
| 26 | refused, err := m.startViaShim(m.root, cmd, nil) |
| 27 | if refused || !isPlacementError(err) { |
| 28 | t.Fatalf("refused=%v err=%v, want a placement error", refused, err) |
| 29 | } |
| 30 | if cmd.Process != nil { |
| 31 | t.Fatal("a process was started") |
| 32 | } |
| 33 | if _, err := os.Stat(marker); err == nil { |
| 34 | t.Fatal("tenant code ran") |
| 35 | } |
| 36 | } |
| 37 | |
| 38 | // TestShimRefusesAPathOutsideCgroupfs: a cgroup2 directory is not enough; |
| 39 | // the path must name it under /sys/fs/cgroup/, or the shim could not check |
| 40 | // where it landed. |
| 41 | func TestShimRefusesAPathOutsideCgroupfs(t *testing.T) { |
| 42 | if _, err := os.Stat("/sys/fs/cgroup/cgroup.controllers"); err != nil { |
| 43 | t.Skip("no cgroup2 mount at /sys/fs/cgroup") |
| 44 | } |
| 45 | m := &Manager{root: "/sys/fs/cgroup", logf: t.Logf} |
| 46 | cmd := exec.Command("/bin/true") |
| 47 | _, err := m.startViaShim("/sys/fs/cgroup", cmd, nil) |
| 48 | if !isPlacementError(err) || cmd.Process != nil { |
| 49 | t.Fatalf("err=%v process=%v, want a placement error and no process", err, cmd.Process) |
| 50 | } |
| 51 | } |
| 52 | |
| 53 | // TestShimUnavailableWithoutTheHook: a process whose main never ran the |
| 54 | // hook never starts a shim. |
| 55 | func TestShimUnavailableWithoutTheHook(t *testing.T) { |
| 56 | if !shimHooked.Load() { |
| 57 | t.Fatal("TestMain did not run the hook") |
| 58 | } |
| 59 | shimHooked.Store(false) |
| 60 | t.Cleanup(func() { shimHooked.Store(true) }) |
| 61 | m := &Manager{root: t.TempDir(), logf: t.Logf} |
| 62 | cmd := exec.Command("/bin/true") |
| 63 | refused, err := m.startViaShim(m.root, cmd, nil) |
| 64 | if refused || !isPlacementError(err) { |
| 65 | t.Fatalf("refused=%v err=%v, want a placement error", refused, err) |
| 66 | } |
| 67 | if cmd.Process != nil { |
| 68 | t.Fatal("a process was started") |
| 69 | } |
| 70 | } |
| 71 | |
| 72 | // TestShimLookPathErrorIsTheCommands: a command whose LookPath failed |
| 73 | // answers that error, as exec.Cmd.Start would, and starts nothing. |
| 74 | func TestShimLookPathErrorIsTheCommands(t *testing.T) { |
| 75 | m := &Manager{root: t.TempDir(), logf: t.Logf} |
| 76 | cmd := exec.Command("carlos-no-such-command-anywhere") |
| 77 | if cmd.Err == nil { |
| 78 | t.Skip("LookPath found the command") |
| 79 | } |
| 80 | _, err := m.startViaShim(m.root, cmd, nil) |
| 81 | if err != cmd.Err || isPlacementError(err) || cmd.Process != nil { |
| 82 | t.Fatalf("err=%v, want the LookPath error %v and no process", err, cmd.Err) |
| 83 | } |
| 84 | } |
| 85 | |
| 86 | // TestShimPlacementFailureIsReported runs a real shim (this test binary, |
| 87 | // through TestMain's hook) against a temp directory, calling the core with |
| 88 | // a directory fd the prologue would have refused. The shim's own fstatfs |
| 89 | // check (shim step 4) answers "P". |
| 90 | func TestShimPlacementFailureIsReported(t *testing.T) { |
| 91 | m := &Manager{root: t.TempDir(), logf: t.Logf} |
| 92 | f, err := os.Open(m.root) |
| 93 | if err != nil { |
| 94 | t.Fatal(err) |
| 95 | } |
| 96 | defer f.Close() |
| 97 | marker := filepath.Join(t.TempDir(), "ran") |
| 98 | cmd := exec.Command("/bin/sh", "-c", "touch "+marker) |
| 99 | _, err = m.startShimCore(m.root, f, "/nowhere", cmd, nil, time.Time{}) |
| 100 | if !isPlacementError(err) { |
| 101 | t.Fatalf("err=%v, want a placement error", err) |
| 102 | } |
| 103 | if !strings.Contains(err.Error(), "statfs") { |
| 104 | t.Fatalf("err=%v, want the shim's statfs refusal", err) |
| 105 | } |
| 106 | if _, err := os.Stat(fmt.Sprintf("/proc/%d", cmd.Process.Pid)); err == nil { |
| 107 | t.Fatal("the shim was not reaped") |
| 108 | } |
| 109 | if _, err := os.Stat(marker); err == nil { |
| 110 | t.Fatal("tenant code ran") |
| 111 | } |
| 112 | if m.placementsUnder(m.root) != 0 { |
| 113 | t.Fatal("the placement is still registered") |
| 114 | } |
| 115 | } |
| 116 | |
| 117 | // TestShimBeforeRefusalRegistersNothing: before is asked only once the |
| 118 | // process is placed, so a start that cannot be placed never asks it, starts |
| 119 | // nothing of the tenant, and leaves nothing registered. A refusal after a |
| 120 | // real placement is TestShimKernelBeforeRunsAfterPlacement. |
| 121 | func TestShimBeforeRefusalRegistersNothing(t *testing.T) { |
| 122 | m := &Manager{root: t.TempDir(), logf: t.Logf} |
| 123 | asked := 0 |
| 124 | before := func() error { asked++; return errors.New("not yours any more") } |
| 125 | marker := filepath.Join(t.TempDir(), "ran") |
| 126 | |
| 127 | cmd := exec.Command("/bin/sh", "-c", "touch "+marker) |
| 128 | refused, err := m.startViaShim(m.root, cmd, before) |
| 129 | if refused || !isPlacementError(err) || cmd.Process != nil { |
| 130 | t.Fatalf("prologue: refused=%v err=%v process=%v", refused, err, cmd.Process) |
| 131 | } |
| 132 | |
| 133 | f, err := os.Open(m.root) |
| 134 | if err != nil { |
| 135 | t.Fatal(err) |
| 136 | } |
| 137 | defer f.Close() |
| 138 | cmd = exec.Command("/bin/sh", "-c", "touch "+marker) |
| 139 | refused, err = m.startShimCore(m.root, f, "/nowhere", cmd, before, time.Time{}) |
| 140 | if refused || !isPlacementError(err) { |
| 141 | t.Fatalf("core: refused=%v err=%v, want a placement error", refused, err) |
| 142 | } |
| 143 | if asked != 0 { |
| 144 | t.Fatalf("before was asked %d times for a process that was never placed", asked) |
| 145 | } |
| 146 | if _, err := os.Stat(marker); err == nil { |
| 147 | t.Fatal("tenant code ran") |
| 148 | } |
| 149 | if m.placementsUnder(m.root) != 0 { |
| 150 | t.Fatal("the placement is still registered") |
| 151 | } |
| 152 | } |
| 153 | |
| 154 | // TestShimRefusesNULInEnv: Start refuses an environment entry that holds a |
| 155 | // NUL (Go issue #56284), where cmd.Environ() would silently drop it. The |
| 156 | // core answers Start's own error, not a placement error, and starts and |
| 157 | // registers nothing. With Env nil the NUL can arrive only through Dir, as |
| 158 | // the PWD Start adds. |
| 159 | func TestShimRefusesNULInEnv(t *testing.T) { |
| 160 | const want = "exec: environment variable contains NUL" |
| 161 | for _, c := range []struct { |
| 162 | name string |
| 163 | env []string |
| 164 | dir string |
| 165 | }{ |
| 166 | {"env entry", []string{"A=1", "BAD=x\x00y", "B=2"}, ""}, |
| 167 | {"env key", []string{"B\x00AD=x"}, ""}, |
| 168 | {"PWD from dir", nil, "/no\x00where"}, |
| 169 | } { |
| 170 | t.Run(c.name, func(t *testing.T) { |
| 171 | marker := filepath.Join(t.TempDir(), "ran") |
| 172 | newCmd := func() *exec.Cmd { |
| 173 | cmd := exec.Command("/bin/sh", "-c", "touch "+marker) |
| 174 | cmd.Env = c.env |
| 175 | cmd.Dir = c.dir |
| 176 | return cmd |
| 177 | } |
| 178 | // The control: a direct start answers exactly this. |
| 179 | direct := newCmd() |
| 180 | if err := direct.Start(); err == nil || err.Error() != want { |
| 181 | t.Fatalf("direct Start: err=%v, want %q", err, want) |
| 182 | } |
| 183 | |
| 184 | m := &Manager{root: t.TempDir(), logf: t.Logf} |
| 185 | f, err := os.Open(m.root) |
| 186 | if err != nil { |
| 187 | t.Fatal(err) |
| 188 | } |
| 189 | defer f.Close() |
| 190 | cmd := newCmd() |
| 191 | refused, err := m.startShimCore(m.root, f, "/nowhere", cmd, nil, time.Time{}) |
| 192 | if refused || err == nil || err.Error() != want || isPlacementError(err) { |
| 193 | t.Fatalf("refused=%v err=%v, want the command's error %q", refused, err, want) |
| 194 | } |
| 195 | if cmd.Process != nil { |
| 196 | t.Fatal("a process was started") |
| 197 | } |
| 198 | if cmd.Path != "/bin/sh" { |
| 199 | t.Fatalf("cmd was rewritten to %s", cmd.Path) |
| 200 | } |
| 201 | if _, err := os.Stat(marker); err == nil { |
| 202 | t.Fatal("tenant code ran") |
| 203 | } |
| 204 | if m.placementsUnder(m.root) != 0 { |
| 205 | t.Fatal("a placement was registered") |
| 206 | } |
| 207 | }) |
| 208 | } |
| 209 | } |
| 210 | |
| 211 | func TestShimStatusClassification(t *testing.T) { |
| 212 | for _, c := range []struct { |
| 213 | in string |
| 214 | timedOut bool |
| 215 | placed bool |
| 216 | }{ |
| 217 | {"A\n", false, true}, |
| 218 | {"P procs: permission denied\n", false, false}, |
| 219 | {"", false, false}, // died before placement |
| 220 | {"", true, false}, // deadline before placement |
| 221 | {"garbage\n", false, false}, |
| 222 | {"A", false, false}, // no newline: not the shim's answer |
| 223 | } { |
| 224 | placed, err := classifyPlacement([]byte(c.in), c.timedOut, shimDeadline) |
| 225 | if placed != c.placed { |
| 226 | t.Fatalf("classifyPlacement(%q, %v) placed=%v, want %v", c.in, c.timedOut, placed, c.placed) |
| 227 | } |
| 228 | if !placed && !isPlacementError(err) { |
| 229 | t.Fatalf("classifyPlacement(%q, %v) err=%v, want a placement error", c.in, c.timedOut, err) |
| 230 | } |
| 231 | if placed && err != nil { |
| 232 | t.Fatalf("classifyPlacement(%q, %v) err=%v", c.in, c.timedOut, err) |
| 233 | } |
| 234 | } |
| 235 | if _, err := classifyPlacement([]byte("P procs: permission denied\n"), false, shimDeadline); !strings.Contains(err.Error(), "procs: permission denied") { |
| 236 | t.Fatalf("the shim's reason is lost: %v", err) |
| 237 | } |
| 238 | if _, err := classifyPlacement(nil, true, shimDeadline); !strings.Contains(err.Error(), "no answer") { |
| 239 | t.Fatalf("the deadline is not named: %v", err) |
| 240 | } |
| 241 | if _, err := classifyPlacement(nil, false, shimDeadline); !strings.Contains(err.Error(), "exited before placement") { |
| 242 | t.Fatalf("the early exit is not named: %v", err) |
| 243 | } |
| 244 | |
| 245 | for _, c := range []struct { |
| 246 | in string |
| 247 | killed bool |
| 248 | started bool |
| 249 | op string // "" for a placement error, else the *os.PathError's Op |
| 250 | errno syscall.Errno |
| 251 | }{ |
| 252 | {"R\n", false, true, "", 0}, |
| 253 | {"R\n", true, true, "", 0}, // R then killed: may have exec'd, so started |
| 254 | {"", false, false, "", 0}, // died after release, before R: placement error |
| 255 | {"", true, false, "", 0}, // killed before R: placement error |
| 256 | {"E chdir 2\n", false, false, "chdir", syscall.ENOENT}, |
| 257 | {"R\nE exec 2\n", false, false, "fork/exec", syscall.ENOENT}, |
| 258 | {"R\nE exec 7\n", false, false, "fork/exec", syscall.E2BIG}, |
| 259 | {"garbage\n", false, false, "", 0}, |
| 260 | {"R\ngarbage\n", false, true, "", 0}, // R and no E after it: started |
| 261 | {"R", true, true, "", 0}, // R without its newline still counts |
| 262 | } { |
| 263 | started, err := classifyExec("/bin/app", "/wd", []byte(c.in), c.killed, shimDeadline) |
| 264 | if started != c.started { |
| 265 | t.Fatalf("classifyExec(%q, %v) started=%v, want %v", c.in, c.killed, started, c.started) |
| 266 | } |
| 267 | if started { |
| 268 | if err != nil { |
| 269 | t.Fatalf("classifyExec(%q, %v) err=%v", c.in, c.killed, err) |
| 270 | } |
| 271 | continue |
| 272 | } |
| 273 | if c.op == "" { |
| 274 | if !isPlacementError(err) { |
| 275 | t.Fatalf("classifyExec(%q, %v) err=%v, want a placement error", c.in, c.killed, err) |
| 276 | } |
| 277 | continue |
| 278 | } |
| 279 | var pe *os.PathError |
| 280 | if !errors.As(err, &pe) || isPlacementError(err) { |
| 281 | t.Fatalf("classifyExec(%q, %v) err=%v, want a *os.PathError", c.in, c.killed, err) |
| 282 | } |
| 283 | wantPath := "/bin/app" |
| 284 | if c.op == "chdir" { |
| 285 | wantPath = "/wd" |
| 286 | } |
| 287 | if pe.Op != c.op || pe.Path != wantPath || pe.Err != c.errno { |
| 288 | t.Fatalf("classifyExec(%q, %v) = %#v, want Op %q Path %q Err %v", c.in, c.killed, pe, c.op, wantPath, c.errno) |
| 289 | } |
| 290 | } |
| 291 | if _, err := classifyExec("/bin/app", "/wd", nil, false, shimDeadline); !strings.Contains(err.Error(), "exited before exec") { |
| 292 | t.Fatalf("the early exit is not named: %v", err) |
| 293 | } |
| 294 | } |
| 295 | |
| 296 | // TestShimWaitCapsAtBy: with by unset or later than shimDeadline from now, |
| 297 | // each wait is shimDeadline, as for every tenant start (Ruling 9); with by |
| 298 | // sooner, the wait ends at by and the error names what was left of it. |
| 299 | func TestShimWaitCapsAtBy(t *testing.T) { |
| 300 | before := time.Now() |
| 301 | d, within := shimWait(time.Time{}) |
| 302 | if within != shimDeadline || d.Before(before.Add(shimDeadline)) || d.After(time.Now().Add(shimDeadline)) { |
| 303 | t.Fatalf("no cap: deadline %v within %v, want shimDeadline from now", d.Sub(before), within) |
| 304 | } |
| 305 | late := time.Now().Add(shimDeadline + time.Hour) |
| 306 | if d, within := shimWait(late); within != shimDeadline || !d.Before(late) { |
| 307 | t.Fatalf("a later by: deadline %v within %v, want shimDeadline from now", d, within) |
| 308 | } |
| 309 | soon := time.Now().Add(time.Second) |
| 310 | if d, within := shimWait(soon); !d.Equal(soon) || within <= 0 || within > time.Second { |
| 311 | t.Fatalf("a sooner by: deadline %v within %v, want by and at most 1s", d, within) |
| 312 | } |
| 313 | past := time.Now().Add(-time.Second) |
| 314 | if d, within := shimWait(past); !d.Equal(past) || within != 0 { |
| 315 | t.Fatalf("a past by: deadline %v within %v, want by and 0", d, within) |
| 316 | } |
| 317 | if _, err := classifyPlacement(nil, true, 1500*time.Millisecond); !strings.Contains(err.Error(), "within 1.5s") { |
| 318 | t.Fatalf("the capped wait is not named: %v", err) |
| 319 | } |
| 320 | if _, err := classifyExec("/bin/app", "/wd", nil, true, 1500*time.Millisecond); !strings.Contains(err.Error(), "within 1.5s") { |
| 321 | t.Fatalf("the capped wait is not named: %v", err) |
| 322 | } |
| 323 | } |
| 324 | |
amadan