| 1 | //go:build linux |
| 2 | |
| 3 | package stopevidence |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "errors" |
| 8 | "os" |
| 9 | "os/exec" |
| 10 | "path/filepath" |
| 11 | "reflect" |
| 12 | "strconv" |
| 13 | "testing" |
| 14 | "time" |
| 15 | |
| 16 | "github.com/carlosframework/platform/internal/cgroup" |
| 17 | "github.com/carlosframework/platform/internal/moves" |
| 18 | ) |
| 19 | |
| 20 | // §14.1: Reconstruct against a real cgroup v2 hierarchy with a forked |
| 21 | // descendant that closes its descriptors and outlives its parent (witness |
| 22 | // wEvidenceWholeTree). Needs a delegated cgroup root: CARLOS_CGROUP_TEST=1 |
| 23 | // and CARLOS_CGROUP_ROOT=<writable cgroup2 dir>, for instance from inside |
| 24 | // `systemd-run --user --scope -p Delegate=yes`. |
| 25 | func TestReconstructStopEvidenceKillsTheWholeTree(t *testing.T) { |
| 26 | root := os.Getenv("CARLOS_CGROUP_ROOT") |
| 27 | if os.Getenv("CARLOS_CGROUP_TEST") != "1" || root == "" { |
| 28 | t.Skip("needs CARLOS_CGROUP_TEST=1 and a delegated cgroup v2 root") |
| 29 | } |
| 30 | m, err := cgroup.SetupAt(root, t.Logf) |
| 31 | if err != nil { |
| 32 | t.Skip(err) |
| 33 | } |
| 34 | host := "ev.go.example.com" |
| 35 | // A parent that forks a child which detaches and sleeps, then exits. |
| 36 | cmd, err := m.Start(host, cgroup.Limit{}, func() *exec.Cmd { |
| 37 | return exec.Command("sh", "-c", "(sleep 300 </dev/null >/dev/null 2>&1 &) ; exit 0") |
| 38 | }) |
| 39 | if err != nil { |
| 40 | t.Fatal(err) |
| 41 | } |
| 42 | cmd.Wait() |
| 43 | c, err := m.OpenCleanup(host) |
| 44 | if err != nil { |
| 45 | t.Fatal(err) |
| 46 | } |
| 47 | defer c.Close() |
| 48 | if pop, err := c.Populated(); err != nil || !pop { |
| 49 | t.Fatalf("the descendant should still be alive before Reconstruct: populated %v err %v", pop, err) |
| 50 | } |
| 51 | // The real /proc entry of this process, projected into an otherwise |
| 52 | // empty scan root (as TestFDHoldersRealProc does): the whole-machine |
| 53 | // scan needs every uid's /proc, which the agent has under |
| 54 | // ProtectProc=invisible and a test usually does not. |
| 55 | procRoot := projectedProc(t) |
| 56 | db := filepath.Join(t.TempDir(), "ev.go.example.com.db") |
| 57 | rep, err := Reconstruct(context.Background(), ReconstructInputs{Host: host, Cgroups: m, DBPaths: []string{db}, ProcRoot: procRoot, |
| 58 | LeaseFree: func(context.Context) (bool, error) { return true, nil }}) |
| 59 | if err != nil { |
| 60 | t.Fatal(err) |
| 61 | } |
| 62 | if !rep.Steps[moves.StepTree].OK || !rep.TreeStopped() || rep.Complete() { |
| 63 | t.Fatalf("tree evidence after the parent exited: %+v (Complete needs the caller's replica step)", rep) |
| 64 | } |
| 65 | if pop, err := c.Populated(); err != nil || pop { |
| 66 | t.Fatalf("the descendant survived: populated %v err %v", pop, err) |
| 67 | } |
| 68 | } |
| 69 | |
| 70 | // Step 2 against the running kernel: the leaf /proc/self/cgroup names is |
| 71 | // found under the cgroup2 mount /proc/self/mountinfo names, and its |
| 72 | // cgroup.procs holds this process. Whether StepStray passes depends on |
| 73 | // what else shares the test's cgroup (alone in a fresh scope, it passes; |
| 74 | // on a CI runner the leaf holds other processes, some listed as pid 0 |
| 75 | // because they are outside this namespace, and it fails). The read itself |
| 76 | // must succeed either way, and the step's verdict must agree with Stray. |
| 77 | func TestReconstructReadsTheRealOwnLeaf(t *testing.T) { |
| 78 | if _, ok := cgroup.ParseSelfCgroup(readOrSkip(t, "/proc/self/cgroup")); !ok { |
| 79 | t.Skip("no cgroup v2 leaf for this process") |
| 80 | } |
| 81 | db := filepath.Join(t.TempDir(), "h.example.com.db") |
| 82 | rep, err := reconstruct(context.Background(), ReconstructInputs{Host: "h.example.com", DBPaths: []string{db}, ProcRoot: projectedProc(t), |
| 83 | LeaseFree: leaseFree}, opener(&fakeTree{}, nil)) |
| 84 | if err != nil { |
| 85 | t.Fatal(err) |
| 86 | } |
| 87 | if _, ok := rep.Steps[moves.StepStray]; !ok { |
| 88 | t.Fatal("no stray step") |
| 89 | } |
| 90 | for _, p := range rep.Stray { |
| 91 | if p == os.Getpid() { |
| 92 | t.Fatal("the agent counted itself as a stray") |
| 93 | } |
| 94 | } |
| 95 | // A pass means the agent is alone; a failure must be accounted for by |
| 96 | // the other members the read found, 0 marking those it cannot name. |
| 97 | if c := rep.Steps[moves.StepStray]; c.OK != (len(rep.Stray) == 0) { |
| 98 | t.Fatalf("stray step %+v disagrees with the %d other members read: %v", c, len(rep.Stray), rep.Stray) |
| 99 | } |
| 100 | t.Logf("stray step %+v, %d others in the leaf", rep.Steps[moves.StepStray], len(rep.Stray)) |
| 101 | } |
| 102 | |
| 103 | func readOrSkip(t *testing.T, path string) string { |
| 104 | t.Helper() |
| 105 | b, err := os.ReadFile(path) |
| 106 | if err != nil { |
| 107 | t.Skip(err) |
| 108 | } |
| 109 | return string(b) |
| 110 | } |
| 111 | |
| 112 | func TestReconstructStopEvidenceRefusesAStrayProcess(t *testing.T) { |
| 113 | // A fake self-cgroup listing with a second pid refuses with StepStray. |
| 114 | rep := strayCheck([]int{os.Getpid(), 99999}, os.Getpid()) |
| 115 | if rep.OK { |
| 116 | t.Fatal("a stray pid passed") |
| 117 | } |
| 118 | if rep.Reason != "pid 99999 in the agent's cgroup is not the agent" { |
| 119 | t.Fatalf("reason %q", rep.Reason) |
| 120 | } |
| 121 | if got := strayCheck([]int{os.Getpid()}, os.Getpid()); !got.OK { |
| 122 | t.Fatalf("only the agent: %+v", got) |
| 123 | } |
| 124 | } |
| 125 | |
| 126 | // projectedProc is a scan root holding this process's real /proc entry, |
| 127 | // under its pid and as self. |
| 128 | func projectedProc(t *testing.T) string { |
| 129 | t.Helper() |
| 130 | root := t.TempDir() |
| 131 | pid := strconv.Itoa(os.Getpid()) |
| 132 | for _, name := range []string{pid, "self"} { |
| 133 | if err := os.Symlink(filepath.Join("/proc", pid), filepath.Join(root, name)); err != nil { |
| 134 | t.Fatal(err) |
| 135 | } |
| 136 | } |
| 137 | return root |
| 138 | } |
| 139 | |
| 140 | // boxFixture is a fake box: a proc root whose self/cgroup names the |
| 141 | // agent's leaf, a self/mountinfo mounting a cgroup2 hierarchy at a temp |
| 142 | // directory (never /sys/fs/cgroup), and that leaf's cgroup.procs. |
| 143 | type boxFixture struct { |
| 144 | proc, mount, leafDir, db string |
| 145 | } |
| 146 | |
| 147 | func newBoxFixture(t *testing.T, leafPIDs ...int) *boxFixture { |
| 148 | t.Helper() |
| 149 | dir := t.TempDir() |
| 150 | f := &boxFixture{proc: filepath.Join(dir, "proc"), mount: filepath.Join(dir, "cg fs"), db: filepath.Join(dir, "data", "h.example.com.db")} |
| 151 | f.leafDir = filepath.Join(f.mount, "system.slice", "carlos-edge.service", "supervise") |
| 152 | for _, d := range []string{filepath.Join(f.proc, "self"), f.leafDir, filepath.Dir(f.db)} { |
| 153 | if err := os.MkdirAll(d, 0o755); err != nil { |
| 154 | t.Fatal(err) |
| 155 | } |
| 156 | } |
| 157 | write(t, filepath.Join(f.proc, "self", "cgroup"), "0::/system.slice/carlos-edge.service/supervise\n") |
| 158 | write(t, filepath.Join(f.proc, "self", "mountinfo"), |
| 159 | "22 1 0:21 / /proc rw,nosuid,nodev,noexec,relatime shared:5 - proc proc rw\n"+ |
| 160 | "32 24 0:27 / "+escapeMount(f.mount)+" rw,nosuid,nodev,noexec,relatime shared:8 - cgroup2 cgroup2 rw,nsdelegate\n") |
| 161 | if leafPIDs == nil { |
| 162 | leafPIDs = []int{os.Getpid()} |
| 163 | } |
| 164 | f.setLeaf(t, leafPIDs...) |
| 165 | return f |
| 166 | } |
| 167 | |
| 168 | func (f *boxFixture) setLeaf(t *testing.T, pids ...int) { |
| 169 | t.Helper() |
| 170 | var body string |
| 171 | for _, p := range pids { |
| 172 | body += strconv.Itoa(p) + "\n" |
| 173 | } |
| 174 | write(t, filepath.Join(f.leafDir, "cgroup.procs"), body) |
| 175 | } |
| 176 | |
| 177 | // holdDB makes pid a process with the path open as descriptor 7. |
| 178 | func (f *boxFixture) holdDB(t *testing.T, pid int, path string) { |
| 179 | t.Helper() |
| 180 | fd := filepath.Join(f.proc, strconv.Itoa(pid), "task", strconv.Itoa(pid), "fd") |
| 181 | if err := os.MkdirAll(fd, 0o755); err != nil { |
| 182 | t.Fatal(err) |
| 183 | } |
| 184 | if err := os.Symlink(path, filepath.Join(fd, "7")); err != nil { |
| 185 | t.Fatal(err) |
| 186 | } |
| 187 | } |
| 188 | |
| 189 | func escapeMount(p string) string { |
| 190 | out := "" |
| 191 | for _, r := range p { |
| 192 | if r == ' ' { |
| 193 | out += `\040` |
| 194 | continue |
| 195 | } |
| 196 | out += string(r) |
| 197 | } |
| 198 | return out |
| 199 | } |
| 200 | |
| 201 | func write(t *testing.T, path, body string) { |
| 202 | t.Helper() |
| 203 | if err := os.WriteFile(path, []byte(body), 0o644); err != nil { |
| 204 | t.Fatal(err) |
| 205 | } |
| 206 | } |
| 207 | |
| 208 | // fakeTree is the host cgroup as reconstruct sees it. |
| 209 | type fakeTree struct { |
| 210 | killErr error |
| 211 | deadline bool |
| 212 | killed bool |
| 213 | closed bool |
| 214 | } |
| 215 | |
| 216 | func (g *fakeTree) KillAndEmpty(ctx context.Context) error { |
| 217 | g.killed = true |
| 218 | _, g.deadline = ctx.Deadline() |
| 219 | return g.killErr |
| 220 | } |
| 221 | func (g *fakeTree) Close() error { g.closed = true; return nil } |
| 222 | |
| 223 | func opener(g *fakeTree, err error) func(string) (treeGroup, error) { |
| 224 | return func(string) (treeGroup, error) { |
| 225 | if err != nil { |
| 226 | return nil, err |
| 227 | } |
| 228 | return g, nil |
| 229 | } |
| 230 | } |
| 231 | |
| 232 | func leaseFree(context.Context) (bool, error) { return true, nil } |
| 233 | |
| 234 | func (f *boxFixture) inputs() ReconstructInputs { |
| 235 | return ReconstructInputs{Host: "h.example.com", DBPaths: []string{f.db}, ProcRoot: f.proc, LeaseFree: leaseFree} |
| 236 | } |
| 237 | |
| 238 | func TestReconstructAllStepsPass(t *testing.T) { |
| 239 | f := newBoxFixture(t) |
| 240 | g := &fakeTree{} |
| 241 | rep, err := reconstruct(context.Background(), f.inputs(), opener(g, nil)) |
| 242 | if err != nil { |
| 243 | t.Fatal(err) |
| 244 | } |
| 245 | for _, step := range []string{moves.StepTree, moves.StepStray, moves.StepFDs, moves.StepLease} { |
| 246 | if c := rep.Steps[step]; !c.OK || c.Reason != "" { |
| 247 | t.Fatalf("%s: %+v", step, c) |
| 248 | } |
| 249 | } |
| 250 | if !rep.TreeStopped() || rep.Complete() || len(rep.Stray) != 0 || rep.V != 1 || rep.Host != "h.example.com" || rep.CheckedAt.IsZero() { |
| 251 | t.Fatalf("report %+v", rep) |
| 252 | } |
| 253 | if !g.killed || !g.closed { |
| 254 | t.Fatalf("the tree was not killed and closed: %+v", g) |
| 255 | } |
| 256 | // KillAndEmpty refuses a context with no deadline, so Reconstruct |
| 257 | // bounds the drain itself when its caller did not. |
| 258 | if !g.deadline { |
| 259 | t.Fatal("the drain ran without a deadline") |
| 260 | } |
| 261 | } |
| 262 | |
| 263 | func TestReconstructNoHostCgroupIsAnEmptyTree(t *testing.T) { |
| 264 | f := newBoxFixture(t) |
| 265 | rep, err := reconstruct(context.Background(), f.inputs(), opener(nil, cgroup.ErrNoCgroup)) |
| 266 | if err != nil || !rep.Steps[moves.StepTree].OK || !rep.TreeStopped() { |
| 267 | t.Fatalf("no cgroup: %+v %v", rep, err) |
| 268 | } |
| 269 | } |
| 270 | |
| 271 | func TestReconstructTreeThatWillNotEmpty(t *testing.T) { |
| 272 | f := newBoxFixture(t) |
| 273 | g := &fakeTree{killErr: context.DeadlineExceeded} |
| 274 | rep, err := reconstruct(context.Background(), f.inputs(), opener(g, nil)) |
| 275 | if err != nil { |
| 276 | t.Fatal(err) |
| 277 | } |
| 278 | if c := rep.Steps[moves.StepTree]; c.OK || c.Reason != "the instance's processes could not be stopped" || rep.TreeStopped() { |
| 279 | t.Fatalf("tree: %+v", rep) |
| 280 | } |
| 281 | if !g.closed { |
| 282 | t.Fatal("the cleanup handle leaked") |
| 283 | } |
| 284 | // The later steps still ran, so the report says everything it saw. |
| 285 | if !rep.Steps[moves.StepStray].OK || !rep.Steps[moves.StepFDs].OK || !rep.Steps[moves.StepLease].OK { |
| 286 | t.Fatalf("later steps: %+v", rep.Steps) |
| 287 | } |
| 288 | } |
| 289 | |
| 290 | func TestReconstructOpenFailureIsAnError(t *testing.T) { |
| 291 | f := newBoxFixture(t) |
| 292 | boom := errors.New("boom") |
| 293 | if _, err := reconstruct(context.Background(), f.inputs(), opener(nil, boom)); !errors.Is(err, boom) { |
| 294 | t.Fatalf("err %v", err) |
| 295 | } |
| 296 | // A kill that fails for any reason but the drain running out of time |
| 297 | // could not observe the tree: an error, not a recorded step. |
| 298 | g := &fakeTree{killErr: boom} |
| 299 | if _, err := reconstruct(context.Background(), f.inputs(), opener(g, nil)); !errors.Is(err, boom) || !g.closed { |
| 300 | t.Fatalf("kill error: %v, closed %v", err, g.closed) |
| 301 | } |
| 302 | } |
| 303 | |
| 304 | func TestReconstructCallerCancelledIsAnError(t *testing.T) { |
| 305 | f := newBoxFixture(t) |
| 306 | ctx, cancel := context.WithCancel(context.Background()) |
| 307 | g := &fakeTree{} |
| 308 | g.killErr = context.Canceled |
| 309 | open := func(string) (treeGroup, error) { cancel(); return g, nil } |
| 310 | if _, err := reconstruct(ctx, f.inputs(), open); !errors.Is(err, context.Canceled) { |
| 311 | t.Fatalf("err %v", err) |
| 312 | } |
| 313 | } |
| 314 | |
| 315 | func TestReconstructStrayInTheAgentsLeaf(t *testing.T) { |
| 316 | f := newBoxFixture(t, os.Getpid(), 4243, 4244) |
| 317 | rep, err := reconstruct(context.Background(), f.inputs(), opener(&fakeTree{}, nil)) |
| 318 | if err != nil { |
| 319 | t.Fatal(err) |
| 320 | } |
| 321 | if c := rep.Steps[moves.StepStray]; c.OK || c.Reason != "pid 4243 in the agent's cgroup is not the agent" || rep.TreeStopped() { |
| 322 | t.Fatalf("stray: %+v", rep) |
| 323 | } |
| 324 | if !reflect.DeepEqual(rep.Stray, []int{4243, 4244}) { |
| 325 | t.Fatalf("Stray %v", rep.Stray) |
| 326 | } |
| 327 | } |
| 328 | |
| 329 | // The kernel lists pid 0 in cgroup.procs for a member the reader's pid |
| 330 | // namespace cannot name (a CI runner's cgroup holds such members). It is |
| 331 | // a process this box cannot account for: it fails StepStray with no error, |
| 332 | // is never taken for the agent, and is marked in Stray by the 0 entry. |
| 333 | func TestReconstructUnidentifiableMemberIsAStray(t *testing.T) { |
| 334 | self := os.Getpid() |
| 335 | if c := strayCheck([]int{self, 0}, self); c.OK || c.Reason != "1 process(es) in the agent's cgroup cannot be identified" { |
| 336 | t.Fatalf("strayCheck with the agent and a 0: %+v", c) |
| 337 | } |
| 338 | if c := strayCheck([]int{0, self, 0}, self); c.OK || c.Reason != "2 process(es) in the agent's cgroup cannot be identified" { |
| 339 | t.Fatalf("strayCheck with two 0s: %+v", c) |
| 340 | } |
| 341 | f := newBoxFixture(t, self, 0) |
| 342 | rep, err := reconstruct(context.Background(), f.inputs(), opener(&fakeTree{}, nil)) |
| 343 | if err != nil { |
| 344 | t.Fatalf("a pid 0 entry is not a parse error: %v", err) |
| 345 | } |
| 346 | if c := rep.Steps[moves.StepStray]; c.OK || c.Reason == "" || rep.TreeStopped() { |
| 347 | t.Fatalf("stray: %+v", rep) |
| 348 | } |
| 349 | if !reflect.DeepEqual(rep.Stray, []int{0}) { |
| 350 | t.Fatalf("Stray %v", rep.Stray) |
| 351 | } |
| 352 | // Never the agent itself: a leaf of only 0s does not name the agent. |
| 353 | f.setLeaf(t, 0) |
| 354 | if _, err := reconstruct(context.Background(), f.inputs(), opener(&fakeTree{}, nil)); err == nil { |
| 355 | t.Fatal("a leaf of only pid 0 passed as the agent's") |
| 356 | } |
| 357 | } |
| 358 | |
| 359 | // Genuinely malformed entries are still refused. |
| 360 | func TestReadProcsRefusesMalformedEntries(t *testing.T) { |
| 361 | for _, body := range []string{"abc\n", "-1\n", "-0\n", "12x\n", "1 2\n"} { |
| 362 | file := filepath.Join(t.TempDir(), "cgroup.procs") |
| 363 | write(t, file, body) |
| 364 | if _, err := readProcs(file); err == nil { |
| 365 | t.Errorf("%q accepted", body) |
| 366 | } |
| 367 | } |
| 368 | file := filepath.Join(t.TempDir(), "cgroup.procs") |
| 369 | write(t, file, "7\n0\n\n9\n") |
| 370 | if got, err := readProcs(file); err != nil || !reflect.DeepEqual(got, []int{7, 0, 9}) { |
| 371 | t.Fatalf("got %v, %v", got, err) |
| 372 | } |
| 373 | } |
| 374 | |
| 375 | // The leaf is read where /proc/self/mountinfo mounts the hierarchy, so a |
| 376 | // leaf that names the agent but is not the agent's is an error, not a pass. |
| 377 | func TestReconstructRefusesALeafWithoutTheAgent(t *testing.T) { |
| 378 | f := newBoxFixture(t, 4243) |
| 379 | if _, err := reconstruct(context.Background(), f.inputs(), opener(&fakeTree{}, nil)); err == nil { |
| 380 | t.Fatal("a leaf without the agent passed") |
| 381 | } |
| 382 | } |
| 383 | |
| 384 | func TestReconstructOwnCgroupUnreadable(t *testing.T) { |
| 385 | for name, mutate := range map[string]func(f *boxFixture){ |
| 386 | "no v2 line": func(f *boxFixture) { |
| 387 | os.WriteFile(filepath.Join(f.proc, "self", "cgroup"), []byte("1:name=systemd:/x\n"), 0o644) |
| 388 | }, |
| 389 | "root cgroup": func(f *boxFixture) { os.WriteFile(filepath.Join(f.proc, "self", "cgroup"), []byte("0::/\n"), 0o644) }, |
| 390 | "no mount": func(f *boxFixture) { os.WriteFile(filepath.Join(f.proc, "self", "mountinfo"), []byte(""), 0o644) }, |
| 391 | "no procs": func(f *boxFixture) { os.Remove(filepath.Join(f.leafDir, "cgroup.procs")) }, |
| 392 | "garbled pid": func(f *boxFixture) { os.WriteFile(filepath.Join(f.leafDir, "cgroup.procs"), []byte("12x\n"), 0o644) }, |
| 393 | "no mountinfo": func(f *boxFixture) { os.Remove(filepath.Join(f.proc, "self", "mountinfo")) }, |
| 394 | } { |
| 395 | f := newBoxFixture(t) |
| 396 | mutate(f) |
| 397 | if _, err := reconstruct(context.Background(), f.inputs(), opener(&fakeTree{}, nil)); err == nil { |
| 398 | t.Fatalf("%s: passed", name) |
| 399 | } |
| 400 | } |
| 401 | } |
| 402 | |
| 403 | func TestReconstructDatabaseHolders(t *testing.T) { |
| 404 | for _, path := range []string{"", "-wal", "-shm"} { |
| 405 | f := newBoxFixture(t) |
| 406 | f.holdDB(t, 5150, f.db+path) |
| 407 | rep, err := reconstruct(context.Background(), f.inputs(), opener(&fakeTree{}, nil)) |
| 408 | if err != nil { |
| 409 | t.Fatal(err) |
| 410 | } |
| 411 | if c := rep.Steps[moves.StepFDs]; c.OK || c.Reason != "1 process(es) hold the database open" || rep.TreeStopped() { |
| 412 | t.Fatalf("holder of %q: %+v", path, rep.Steps) |
| 413 | } |
| 414 | } |
| 415 | // A descriptor on some other file is not a holder. |
| 416 | f := newBoxFixture(t) |
| 417 | other := filepath.Join(filepath.Dir(f.db), "other.db") |
| 418 | write(t, other, "x") |
| 419 | f.holdDB(t, 5150, other) |
| 420 | if rep, err := reconstruct(context.Background(), f.inputs(), opener(&fakeTree{}, nil)); err != nil || !rep.Steps[moves.StepFDs].OK { |
| 421 | t.Fatalf("other file: %+v %v", rep.Steps, err) |
| 422 | } |
| 423 | } |
| 424 | |
| 425 | // A descriptor whose file cannot be identified (its stat says ENOENT while |
| 426 | // its entry remains: here a dangling fixture link) should not pass step 3. |
| 427 | // Main's FDHolders lstats the entry after an ENOENT stat (9cf9f18f), so an |
| 428 | // entry that remains but cannot be followed is ErrIncomplete, seen here |
| 429 | // through Reconstruct. |
| 430 | func TestReconstructUnidentifiedDescriptorIsNotAPass(t *testing.T) { |
| 431 | f := newBoxFixture(t) |
| 432 | // The database exists, so FDHolders compares identities; a descriptor |
| 433 | // whose target cannot be followed is then unreadable, not closed. |
| 434 | if err := os.WriteFile(f.db, nil, 0o600); err != nil { |
| 435 | t.Fatal(err) |
| 436 | } |
| 437 | f.holdDB(t, 5150, filepath.Join(filepath.Dir(f.db), "vanished.db")) |
| 438 | rep, err := reconstruct(context.Background(), f.inputs(), opener(&fakeTree{}, nil)) |
| 439 | if !errors.Is(err, ErrIncomplete) { |
| 440 | t.Fatalf("unidentifiable descriptor: %+v %v", rep.Steps, err) |
| 441 | } |
| 442 | } |
| 443 | |
| 444 | // Step 3 compares file identity, not path text: a holder of a hard link |
| 445 | // to the database (another name, same file) fails StepFDs. |
| 446 | func TestReconstructDatabaseHeldThroughAHardLink(t *testing.T) { |
| 447 | f := newBoxFixture(t) |
| 448 | write(t, f.db, "SQLite format 3") |
| 449 | alias := filepath.Join(filepath.Dir(f.db), "..", "alias.db") |
| 450 | if err := os.Link(f.db, alias); err != nil { |
| 451 | t.Skip("no hard links here:", err) |
| 452 | } |
| 453 | f.holdDB(t, 5150, filepath.Clean(alias)) |
| 454 | rep, err := reconstruct(context.Background(), f.inputs(), opener(&fakeTree{}, nil)) |
| 455 | if err != nil { |
| 456 | t.Fatal(err) |
| 457 | } |
| 458 | if c := rep.Steps[moves.StepFDs]; c.OK || c.Reason != "1 process(es) hold the database open" || rep.TreeStopped() { |
| 459 | t.Fatalf("hard-link holder passed: %+v", rep.Steps) |
| 460 | } |
| 461 | } |
| 462 | |
| 463 | func TestReconstructLeaseStep(t *testing.T) { |
| 464 | f := newBoxFixture(t) |
| 465 | in := f.inputs() |
| 466 | in.LeaseFree = func(context.Context) (bool, error) { return false, nil } |
| 467 | rep, err := reconstruct(context.Background(), in, opener(&fakeTree{}, nil)) |
| 468 | if err != nil { |
| 469 | t.Fatal(err) |
| 470 | } |
| 471 | if c := rep.Steps[moves.StepLease]; c.OK || c.Reason != "lock.json still held" || rep.TreeStopped() { |
| 472 | t.Fatalf("lease held: %+v", rep.Steps) |
| 473 | } |
| 474 | boom := errors.New("boom") |
| 475 | in.LeaseFree = func(context.Context) (bool, error) { return true, boom } |
| 476 | if _, err := reconstruct(context.Background(), in, opener(&fakeTree{}, nil)); !errors.Is(err, boom) { |
| 477 | t.Fatalf("lease read error: %v", err) |
| 478 | } |
| 479 | } |
| 480 | |
| 481 | func TestReconstructInvalidInputs(t *testing.T) { |
| 482 | f := newBoxFixture(t) |
| 483 | for name, mutate := range map[string]func(*ReconstructInputs){ |
| 484 | "bad host": func(in *ReconstructInputs) { in.Host = "../etc" }, |
| 485 | "no lease reader": func(in *ReconstructInputs) { in.LeaseFree = nil }, |
| 486 | "no database": func(in *ReconstructInputs) { in.DBPaths = nil }, |
| 487 | "relative db": func(in *ReconstructInputs) { in.DBPaths = []string{"h.example.com.db"} }, |
| 488 | "relative proc": func(in *ReconstructInputs) { in.ProcRoot = "proc" }, |
| 489 | "empty db in list": func(in *ReconstructInputs) { in.DBPaths = []string{f.db, ""} }, |
| 490 | } { |
| 491 | in := f.inputs() |
| 492 | mutate(&in) |
| 493 | g := &fakeTree{} |
| 494 | if _, err := reconstruct(context.Background(), in, opener(g, nil)); !errors.Is(err, ErrInvalidInput) { |
| 495 | t.Fatalf("%s: err %v", name, err) |
| 496 | } |
| 497 | if g.killed { |
| 498 | t.Fatalf("%s: killed the tree before refusing the input", name) |
| 499 | } |
| 500 | } |
| 501 | in := f.inputs() |
| 502 | if _, err := Reconstruct(context.Background(), in); !errors.Is(err, ErrInvalidInput) { |
| 503 | t.Fatalf("nil manager: err %v", err) |
| 504 | } |
| 505 | if _, err := reconstruct(nil, f.inputs(), opener(&fakeTree{}, nil)); !errors.Is(err, ErrInvalidInput) { //nolint:staticcheck // a nil context is the input under test |
| 506 | t.Fatalf("nil context: err %v", err) |
| 507 | } |
| 508 | } |
| 509 | |
| 510 | func TestCgroupLeafDir(t *testing.T) { |
| 511 | line := func(root, mnt, fstype string) string { |
| 512 | return "40 30 0:27 " + root + " " + mnt + " rw,relatime shared:8 - " + fstype + " " + fstype + " rw\n" |
| 513 | } |
| 514 | for _, c := range []struct { |
| 515 | name, mountinfo, leaf, want string |
| 516 | ok bool |
| 517 | }{ |
| 518 | {"root mount", line("/", "/sys/fs/cgroup", "cgroup2"), "a.service/supervise", "/sys/fs/cgroup/a.service/supervise", true}, |
| 519 | {"mounted elsewhere", line("/", "/run/cg", "cgroup2"), "a.service/supervise", "/run/cg/a.service/supervise", true}, |
| 520 | {"escaped space", line("/", `/run/c\040g`, "cgroup2"), "a/b", "/run/c g/a/b", true}, |
| 521 | {"subtree bind", line("/a.service", "/mnt/a", "cgroup2"), "a.service/supervise", "/mnt/a/supervise", true}, |
| 522 | {"subtree is the leaf", line("/a.service/supervise", "/mnt/s", "cgroup2"), "a.service/supervise", "/mnt/s", true}, |
| 523 | {"longest root wins", line("/", "/sys/fs/cgroup", "cgroup2") + line("/a.service", "/mnt/a", "cgroup2"), "a.service/x", "/mnt/a/x", true}, |
| 524 | {"first of equal roots", line("/", "/sys/fs/cgroup", "cgroup2") + line("/", "/other", "cgroup2"), "a/b", "/sys/fs/cgroup/a/b", true}, |
| 525 | {"sibling subtree only", line("/a.serviceX", "/mnt/x", "cgroup2"), "a.service/supervise", "", false}, |
| 526 | {"v1 only", line("/", "/sys/fs/cgroup/memory", "cgroup"), "a/b", "", false}, |
| 527 | {"no mounts", "", "a/b", "", false}, |
| 528 | {"outside the namespace", line("/", "/sys/fs/cgroup", "cgroup2"), "../../x", "", false}, |
| 529 | {"garbled line", "40 30 0:27 / /sys/fs/cgroup\n", "a/b", "", false}, |
| 530 | } { |
| 531 | got, err := cgroupLeafDir(c.mountinfo, c.leaf) |
| 532 | if c.ok != (err == nil) || got != c.want { |
| 533 | t.Fatalf("%s: %q %v, want %q ok=%v", c.name, got, err, c.want, c.ok) |
| 534 | } |
| 535 | } |
| 536 | } |
| 537 | |
| 538 | // The drain bound never extends the caller's own deadline. |
| 539 | func TestReconstructKeepsTheCallersDeadline(t *testing.T) { |
| 540 | f := newBoxFixture(t) |
| 541 | ctx, cancel := context.WithTimeout(context.Background(), time.Minute) |
| 542 | defer cancel() |
| 543 | want, _ := ctx.Deadline() |
| 544 | var got time.Time |
| 545 | open := func(string) (treeGroup, error) { |
| 546 | return deadlineTree(func(d time.Time) { got = d }), nil |
| 547 | } |
| 548 | if _, err := reconstruct(ctx, f.inputs(), open); err != nil { |
| 549 | t.Fatal(err) |
| 550 | } |
| 551 | if got.After(want) || got.IsZero() { |
| 552 | t.Fatalf("drain deadline %v, caller's %v", got, want) |
| 553 | } |
| 554 | } |
| 555 | |
| 556 | type deadlineTree func(time.Time) |
| 557 | |
| 558 | func (d deadlineTree) KillAndEmpty(ctx context.Context) error { |
| 559 | dl, _ := ctx.Deadline() |
| 560 | d(dl) |
| 561 | return nil |
| 562 | } |
| 563 | func (deadlineTree) Close() error { return nil } |
| 564 | |
| 565 | // The agent's leaf is read again after the descriptor scan and the lease |
| 566 | // read, so a process that joins it while those run fails StepStray for |
| 567 | // this pass. LeaseFree runs between the two reads, so it is the hook that |
| 568 | // adds the member: deterministic, no timing. |
| 569 | func TestReconstructStrayJoiningDuringTheScanFails(t *testing.T) { |
| 570 | f := newBoxFixture(t) |
| 571 | in := f.inputs() |
| 572 | in.LeaseFree = func(context.Context) (bool, error) { |
| 573 | f.setLeaf(t, os.Getpid(), 4321) |
| 574 | return true, nil |
| 575 | } |
| 576 | rep, err := reconstruct(context.Background(), in, opener(&fakeTree{}, nil)) |
| 577 | if err != nil { |
| 578 | t.Fatal(err) |
| 579 | } |
| 580 | if c := rep.Steps[moves.StepStray]; c.OK || c.Reason != "pid 4321 in the agent's cgroup is not the agent" || rep.TreeStopped() { |
| 581 | t.Fatalf("a pid that joined during the scan passed: %+v", rep.Steps) |
| 582 | } |
| 583 | if !reflect.DeepEqual(rep.Stray, []int{4321}) { |
| 584 | t.Fatalf("Stray %v", rep.Stray) |
| 585 | } |
| 586 | // A stray in both reads is listed once. |
| 587 | f = newBoxFixture(t, os.Getpid(), 4243) |
| 588 | in = f.inputs() |
| 589 | in.LeaseFree = func(context.Context) (bool, error) { |
| 590 | f.setLeaf(t, os.Getpid(), 4243, 4321) |
| 591 | return true, nil |
| 592 | } |
| 593 | rep, err = reconstruct(context.Background(), in, opener(&fakeTree{}, nil)) |
| 594 | if err != nil || rep.Steps[moves.StepStray].OK || !reflect.DeepEqual(rep.Stray, []int{4243, 4321}) { |
| 595 | t.Fatalf("stray in both reads: %+v %v %v", rep.Steps, rep.Stray, err) |
| 596 | } |
| 597 | } |
| 598 | |
| 599 | // The second read is held to the first's standard: unreadable, or without |
| 600 | // the agent, is an error. |
| 601 | func TestReconstructSecondLeafReadFailsClosed(t *testing.T) { |
| 602 | for name, mutate := range map[string]func(f *boxFixture){ |
| 603 | "leaf gone": func(f *boxFixture) { os.Remove(filepath.Join(f.leafDir, "cgroup.procs")) }, |
| 604 | "agent gone": func(f *boxFixture) { f.setLeaf(t, 4321) }, |
| 605 | "garbled listing": func(f *boxFixture) { write(t, filepath.Join(f.leafDir, "cgroup.procs"), "x\n") }, |
| 606 | } { |
| 607 | f := newBoxFixture(t) |
| 608 | in := f.inputs() |
| 609 | in.LeaseFree = func(context.Context) (bool, error) { mutate(f); return true, nil } |
| 610 | if _, err := reconstruct(context.Background(), in, opener(&fakeTree{}, nil)); err == nil { |
| 611 | t.Fatalf("%s: passed", name) |
| 612 | } |
| 613 | } |
| 614 | } |
| 615 | |
amadan