| 1 | //go:build linux |
| 2 | |
| 3 | package cgroup |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "errors" |
| 8 | "os" |
| 9 | "os/exec" |
| 10 | "path/filepath" |
| 11 | "sync/atomic" |
| 12 | "syscall" |
| 13 | "testing" |
| 14 | "time" |
| 15 | |
| 16 | "golang.org/x/sys/unix" |
| 17 | ) |
| 18 | |
| 19 | // Filesystem fixtures test handle state only. PrepareStrict must reject them; |
| 20 | // the delegated-kernel suite separately proves actual containment and controls. |
| 21 | func strictFixture(t *testing.T) *Prepared { |
| 22 | t.Helper() |
| 23 | forgetKillsAfter(t) |
| 24 | root := t.TempDir() |
| 25 | host := "fixture.example" |
| 26 | dir := filepath.Join(root, ExecName(host)) |
| 27 | if e := os.Mkdir(dir, 0700); e != nil { |
| 28 | t.Fatal(e) |
| 29 | } |
| 30 | for name, value := range map[string]string{"cgroup.events": "populated 0\nfrozen 0\n", "cgroup.kill": "", "memory.max": "4096", "memory.high": "4096", "memory.swap.max": "4096", "memory.oom.group": "1"} { |
| 31 | if e := os.WriteFile(filepath.Join(dir, name), []byte(value), 0600); e != nil { |
| 32 | t.Fatal(e) |
| 33 | } |
| 34 | } |
| 35 | open := func(path string) *os.File { |
| 36 | flags := os.O_RDWR |
| 37 | if info, e := os.Stat(path); e == nil && info.IsDir() { |
| 38 | flags = os.O_RDONLY |
| 39 | } |
| 40 | f, e := os.OpenFile(path, flags, 0) |
| 41 | if e != nil { |
| 42 | t.Fatal(e) |
| 43 | } |
| 44 | return f |
| 45 | } |
| 46 | p := &Prepared{strictPreparedState: &strictPreparedState{root: open(root), dir: open(dir), host: host, limits: map[string]*os.File{}, expected: map[string]string{}, start: func(*exec.Cmd) error { return syscall.ENOSYS }}} |
| 47 | p.kill = open(filepath.Join(dir, "cgroup.kill")) |
| 48 | p.events = open(filepath.Join(dir, "cgroup.events")) |
| 49 | p.identity, _ = strictID(p.dir) |
| 50 | for _, name := range []string{"memory.max", "memory.high", "memory.swap.max", "memory.oom.group"} { |
| 51 | p.limits[name] = open(filepath.Join(dir, name)) |
| 52 | p.expected[name], _ = strictRead(p.limits[name]) |
| 53 | } |
| 54 | strictReservations.Lock() |
| 55 | strictReservations.held[p.identity] = true |
| 56 | strictReservations.Unlock() |
| 57 | t.Cleanup(func() { |
| 58 | p.closeFiles() |
| 59 | strictReservations.Lock() |
| 60 | delete(strictReservations.held, p.identity) |
| 61 | strictReservations.Unlock() |
| 62 | }) |
| 63 | return p |
| 64 | } |
| 65 | func strictContext(t *testing.T, d time.Duration) context.Context { |
| 66 | t.Helper() |
| 67 | ctx, cancel := context.WithTimeout(t.Context(), d) |
| 68 | t.Cleanup(cancel) |
| 69 | return ctx |
| 70 | } |
| 71 | |
| 72 | func TestStrictPopulationUsesRetainedIdentity(t *testing.T) { |
| 73 | p := strictFixture(t) |
| 74 | for _, populated := range []bool{false, true} { |
| 75 | value := "populated 0\nfrozen 0\n" |
| 76 | if populated { |
| 77 | value = "populated 1\nfrozen 0\n" |
| 78 | } |
| 79 | if e := strictWrite(p.events, value); e != nil { |
| 80 | t.Fatal(e) |
| 81 | } |
| 82 | if got, e := p.Populated(); e != nil || got != populated { |
| 83 | t.Fatal("incorrect population", got, e) |
| 84 | } |
| 85 | } |
| 86 | if e := os.Rename(p.dir.Name(), p.dir.Name()+".old"); e != nil { |
| 87 | t.Fatal(e) |
| 88 | } |
| 89 | if _, e := p.Populated(); e == nil { |
| 90 | t.Fatal("replaced cgroup accepted") |
| 91 | } |
| 92 | if _, e := (&Prepared{}).Populated(); e == nil { |
| 93 | t.Fatal("empty handle accepted") |
| 94 | } |
| 95 | } |
| 96 | func TestStrictRejectsUnavailableManagerHostAndTier(t *testing.T) { |
| 97 | for _, m := range []*Manager{nil, {root: t.TempDir()}} { |
| 98 | for _, host := range []string{"ok.example", "../outside", "Upper.example", "", "a/b"} { |
| 99 | if p, e := m.PrepareStrict(host, TierDefault); e == nil || p != nil { |
| 100 | t.Fatal("unsupported filesystem or host accepted") |
| 101 | } |
| 102 | } |
| 103 | } |
| 104 | m := &Manager{root: "/sys/fs/cgroup"} |
| 105 | for _, limit := range []Limit{{}, {MemoryMax: -1}, {MemoryMax: 1}} { |
| 106 | if p, e := m.PrepareStrict("x.example", limit); e == nil || p != nil { |
| 107 | t.Fatal("missing or foreign tier accepted") |
| 108 | } |
| 109 | } |
| 110 | } |
| 111 | func TestStrictStartConsumesOneAttemptWithoutFallback(t *testing.T) { |
| 112 | p := strictFixture(t) |
| 113 | var calls atomic.Int32 |
| 114 | p.start = func(cmd *exec.Cmd) error { |
| 115 | calls.Add(1) |
| 116 | if cmd.SysProcAttr == nil || !cmd.SysProcAttr.UseCgroupFD || cmd.SysProcAttr.CgroupFD != int(p.dir.Fd()) || cmd.SysProcAttr.Pdeathsig != syscall.SIGKILL || !cmd.SysProcAttr.Setpgid { |
| 117 | t.Error("strict spawn attributes missing or clobbered") |
| 118 | } |
| 119 | return syscall.ENOSYS |
| 120 | } |
| 121 | copied := *p |
| 122 | cmd := exec.Command("/not-executed") |
| 123 | cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} |
| 124 | if e := p.Start(cmd); !errors.Is(e, ErrStrictUnsupported) { |
| 125 | t.Fatal("unsupported clone accepted", e) |
| 126 | } |
| 127 | if e := p.Start(exec.Command("/not-executed")); !errors.Is(e, ErrStrictState) { |
| 128 | t.Fatal("second spawn accepted", e) |
| 129 | } |
| 130 | if e := copied.Start(exec.Command("/not-executed")); !errors.Is(e, ErrStrictState) { |
| 131 | t.Fatal("copied handle restored spawn authority", e) |
| 132 | } |
| 133 | if calls.Load() != 1 { |
| 134 | t.Fatal("strict start retried", calls.Load()) |
| 135 | } |
| 136 | if raw, e := strictRead(p.kill); e != nil || raw != "" { |
| 137 | t.Fatal("failed spawn invented cleanup", raw, e) |
| 138 | } |
| 139 | } |
| 140 | func TestStrictCleanupRequiresDeadlineAndSurvivesTimeout(t *testing.T) { |
| 141 | p := strictFixture(t) |
| 142 | for _, ctx := range []context.Context{nil, context.Background(), strictContext(t, -time.Second)} { |
| 143 | if e := p.KillAndEmpty(ctx); !errors.Is(e, ErrStrictDeadline) { |
| 144 | t.Fatal("unbounded/expired cleanup accepted", e) |
| 145 | } |
| 146 | } |
| 147 | if e := os.WriteFile(filepath.Join(p.root.Name(), ExecName(p.host), "cgroup.events"), []byte("populated 1\n"), 0600); e != nil { |
| 148 | t.Fatal(e) |
| 149 | } |
| 150 | if e := p.Close(); !errors.Is(e, ErrStrictRefused) { |
| 151 | t.Fatal("populated group handle discarded", e) |
| 152 | } |
| 153 | if e := p.KillAndEmpty(strictContext(t, 15*time.Millisecond)); !errors.Is(e, ErrStrictDeadline) { |
| 154 | t.Fatal("populated group claimed empty", e) |
| 155 | } |
| 156 | if e := p.Start(exec.Command("/not-executed")); !errors.Is(e, ErrStrictState) { |
| 157 | t.Fatal("cleanup timeout restored spawn authority", e) |
| 158 | } |
| 159 | if e := os.WriteFile(filepath.Join(p.root.Name(), ExecName(p.host), "cgroup.events"), []byte("populated 0\n"), 0600); e != nil { |
| 160 | t.Fatal(e) |
| 161 | } |
| 162 | if e := p.KillAndEmpty(strictContext(t, time.Second)); e != nil { |
| 163 | t.Fatal("same-handle cleanup retry failed", e) |
| 164 | } |
| 165 | if e := p.KillAndEmpty(strictContext(t, time.Second)); !errors.Is(e, ErrStrictState) { |
| 166 | t.Fatal("second cleanup reused proof", e) |
| 167 | } |
| 168 | if e := p.Close(); e != nil { |
| 169 | t.Fatal(e) |
| 170 | } |
| 171 | if e := p.Close(); !errors.Is(e, ErrStrictState) { |
| 172 | t.Fatal("double close accepted") |
| 173 | } |
| 174 | if e := p.Start(exec.Command("/not-executed")); !errors.Is(e, ErrStrictState) { |
| 175 | t.Fatal("closed spawn accepted") |
| 176 | } |
| 177 | if e := p.KillAndEmpty(strictContext(t, time.Second)); !errors.Is(e, ErrStrictState) { |
| 178 | t.Fatal("closed cleanup accepted") |
| 179 | } |
| 180 | } |
| 181 | func TestStrictCleanupMutexWaitHonorsDeadline(t *testing.T) { |
| 182 | p := strictFixture(t) |
| 183 | p.mu.Lock() |
| 184 | defer p.mu.Unlock() |
| 185 | start := time.Now() |
| 186 | if e := p.KillAndEmpty(strictContext(t, 15*time.Millisecond)); !errors.Is(e, ErrStrictDeadline) { |
| 187 | t.Fatal(e) |
| 188 | } |
| 189 | if time.Since(start) > 250*time.Millisecond { |
| 190 | t.Fatal("mutex wait escaped caller deadline") |
| 191 | } |
| 192 | } |
| 193 | func TestStrictStaleHandleCannotKillReplacement(t *testing.T) { |
| 194 | p := strictFixture(t) |
| 195 | old := filepath.Join(p.root.Name(), ExecName(p.host)) |
| 196 | if e := os.Rename(old, old+"-old"); e != nil { |
| 197 | t.Fatal(e) |
| 198 | } |
| 199 | if e := os.Mkdir(old, 0700); e != nil { |
| 200 | t.Fatal(e) |
| 201 | } |
| 202 | kill := filepath.Join(old, "cgroup.kill") |
| 203 | if e := os.WriteFile(kill, []byte("replacement"), 0600); e != nil { |
| 204 | t.Fatal(e) |
| 205 | } |
| 206 | if e := p.KillAndEmpty(strictContext(t, time.Second)); !errors.Is(e, ErrStrictRefused) { |
| 207 | t.Fatal("wrong identity accepted", e) |
| 208 | } |
| 209 | if e := p.Close(); !errors.Is(e, ErrStrictRefused) { |
| 210 | t.Fatal("stale close claimed proof", e) |
| 211 | } |
| 212 | raw, e := os.ReadFile(kill) |
| 213 | if e != nil || string(raw) != "replacement" { |
| 214 | t.Fatal("replacement group touched", e) |
| 215 | } |
| 216 | if e := p.Start(exec.Command("/not-executed")); !errors.Is(e, ErrStrictState) { |
| 217 | t.Fatal("stale spawn accepted", e) |
| 218 | } |
| 219 | } |
| 220 | func TestStrictChangedLimitsAndMalformedEventsRefuse(t *testing.T) { |
| 221 | for _, name := range []string{"memory.max", "memory.high", "memory.swap.max", "memory.oom.group", "cgroup.events"} { |
| 222 | t.Run(name, func(t *testing.T) { |
| 223 | p := strictFixture(t) |
| 224 | if e := os.WriteFile(filepath.Join(p.root.Name(), ExecName(p.host), name), []byte("missing\n"), 0600); e != nil { |
| 225 | t.Fatal(e) |
| 226 | } |
| 227 | called := false |
| 228 | p.start = func(*exec.Cmd) error { called = true; return nil } |
| 229 | if e := p.Start(exec.Command("/not-executed")); e == nil || called { |
| 230 | t.Fatal("unchecked controls reached spawn", e) |
| 231 | } |
| 232 | if name == "cgroup.events" { |
| 233 | if e := p.Close(); e == nil { |
| 234 | t.Fatal("unknown emptiness discarded handle") |
| 235 | } |
| 236 | } |
| 237 | }) |
| 238 | } |
| 239 | } |
| 240 | |
| 241 | func TestStrictConcurrentAndInvalidStartsConsumeAttempt(t *testing.T) { |
| 242 | p := strictFixture(t) |
| 243 | var calls atomic.Int32 |
| 244 | p.start = func(*exec.Cmd) error { calls.Add(1); return syscall.ENOSYS } |
| 245 | results := make(chan error, 16) |
| 246 | for range 16 { |
| 247 | go func() { results <- p.Start(exec.Command("/not-executed")) }() |
| 248 | } |
| 249 | unsupported := 0 |
| 250 | for range 16 { |
| 251 | switch err := <-results; err { |
| 252 | case ErrStrictUnsupported: |
| 253 | unsupported++ |
| 254 | case ErrStrictState: |
| 255 | default: |
| 256 | t.Fatal(err) |
| 257 | } |
| 258 | } |
| 259 | if calls.Load() != 1 || unsupported != 1 { |
| 260 | t.Fatal("concurrent starts repeated attempt") |
| 261 | } |
| 262 | q := strictFixture(t) |
| 263 | if err := q.Start(nil); err != ErrStrictState { |
| 264 | t.Fatal(err) |
| 265 | } |
| 266 | if err := q.Start(exec.Command("/not-executed")); err != ErrStrictState { |
| 267 | t.Fatal("invalid command restored attempt", err) |
| 268 | } |
| 269 | for _, zero := range []*Prepared{nil, {}} { |
| 270 | if zero.Start(nil) != ErrStrictState || zero.Close() != ErrStrictState || zero.KillAndEmpty(context.Background()) != ErrStrictState { |
| 271 | t.Fatal("zero handle accepted") |
| 272 | } |
| 273 | } |
| 274 | } |
| 275 | |
| 276 | // fakeSelfCgroup points the strict preflight's /proc/self/cgroup read at a |
| 277 | // file holding line, so a test can place the agent wherever it needs. |
| 278 | func fakeSelfCgroup(t *testing.T, line string) { |
| 279 | t.Helper() |
| 280 | path := filepath.Join(t.TempDir(), "cgroup") |
| 281 | if e := os.WriteFile(path, []byte(line+"\n"), 0o644); e != nil { |
| 282 | t.Fatal(e) |
| 283 | } |
| 284 | orig := selfCgroupFile |
| 285 | selfCgroupFile = path |
| 286 | t.Cleanup(func() { selfCgroupFile = orig }) |
| 287 | } |
| 288 | |
| 289 | // TestStrictPreflightIsPrepareStrictsRuntimeCheck: StrictPreflight answers |
| 290 | // what PrepareStrict's host-independent runtime checks answer, because it |
| 291 | // is the same function. Delegation present but the agent placed outside |
| 292 | // the delegated root or its supervise child is refused by both; placed in |
| 293 | // either, the preflight passes on a real cgroup2 mount. |
| 294 | func TestStrictPreflightIsPrepareStrictsRuntimeCheck(t *testing.T) { |
| 295 | var nilManager *Manager |
| 296 | if e := nilManager.StrictPreflight(); !errors.Is(e, ErrStrictUnsupported) { |
| 297 | t.Fatalf("nil manager: %v", e) |
| 298 | } |
| 299 | var fs unix.Statfs_t |
| 300 | if e := unix.Statfs("/sys/fs/cgroup", &fs); e != nil || uint64(fs.Type) != uint64(unix.CGROUP2_SUPER_MAGIC) { |
| 301 | t.Skip("no cgroup2 mount at /sys/fs/cgroup") |
| 302 | } |
| 303 | m := &Manager{root: "/sys/fs/cgroup/carlos.slice/carlos-edge.service"} |
| 304 | fakeSelfCgroup(t, "0::/user.slice/elsewhere.service") |
| 305 | if e := m.StrictPreflight(); !errors.Is(e, ErrStrictRefused) { |
| 306 | t.Fatalf("misplaced agent: preflight %v, want ErrStrictRefused", e) |
| 307 | } |
| 308 | if p, e := m.PrepareStrict("x.example", TierDefault); p != nil || !errors.Is(e, ErrStrictRefused) { |
| 309 | t.Fatalf("misplaced agent: PrepareStrict %v, want the preflight's ErrStrictRefused", e) |
| 310 | } |
| 311 | // The test's own cgroup stands in for the delegated root: a real |
| 312 | // cgroup2 directory the agent is placed in, or in its supervise child. |
| 313 | raw, e := os.ReadFile("/proc/self/cgroup") |
| 314 | rel, ok := ParseSelfCgroup(string(raw)) |
| 315 | if e != nil || !ok { |
| 316 | t.Skip("no cgroup v2 placement for this test process") |
| 317 | } |
| 318 | own := &Manager{root: filepath.Join("/sys/fs/cgroup", rel)} |
| 319 | for _, line := range []string{"0::/" + rel, "0::/" + rel + "/supervise"} { |
| 320 | fakeSelfCgroup(t, line) |
| 321 | if e := own.StrictPreflight(); e != nil { |
| 322 | t.Fatalf("agent at %q: preflight %v, want nil", line, e) |
| 323 | } |
| 324 | } |
| 325 | fakeSelfCgroup(t, "0::/"+rel) |
| 326 | if e := (&Manager{root: t.TempDir()}).StrictPreflight(); !errors.Is(e, ErrStrictRefused) { |
| 327 | t.Fatalf("root outside the agent's placement: %v", e) |
| 328 | } |
| 329 | } |
| 330 | |
amadan