| 1 | //go:build linux |
| 2 | |
| 3 | package cgroup |
| 4 | |
| 5 | import ( |
| 6 | "bytes" |
| 7 | "context" |
| 8 | "errors" |
| 9 | "fmt" |
| 10 | "os" |
| 11 | "os/exec" |
| 12 | "path/filepath" |
| 13 | "strconv" |
| 14 | "strings" |
| 15 | "syscall" |
| 16 | "time" |
| 17 | |
| 18 | "golang.org/x/sys/unix" |
| 19 | ) |
| 20 | |
| 21 | // placementCheckName is the scratch child of the delegated root that |
| 22 | // VerifyPlacement places its probe in. It is no carlos-exec@ name, so no |
| 23 | // host's cleanup or the stale sweep ever takes it for a host's cgroup. |
| 24 | const placementCheckName = "carlos-placement-check" |
| 25 | |
| 26 | // placementProbeArgs is the probe tenant's argv after shimExe. Only tests |
| 27 | // change it, to stand in a probe that misreports or never exits. |
| 28 | var placementProbeArgs = []string{ProbeVerb} |
| 29 | |
| 30 | // placementCheckBudget bounds VerifyPlacement's waits, the shim's two and |
| 31 | // the probe's run, so a decline is decided in time for the restarted edge |
| 32 | // to answer `carlos system update`'s 20 s gate (systemGateWait) with the |
| 33 | // rest of its boot work. What runs after the budget is the reap of a |
| 34 | // SIGKILLed shim or probe and an rmdir. That is milliseconds on the |
| 35 | // assumption that the kernel ends a killed process promptly, which the |
| 36 | // second of headroom under 5 s covers. It is an assumption, not a bound: |
| 37 | // SIGKILL does not end a process in uninterruptible sleep (stuck kernel |
| 38 | // I/O) until that sleep ends, and the reap waits for it (reap, in |
| 39 | // shim_linux.go; awaitProbe). A box stalled that way can take longer than |
| 40 | // 4 s to decline (whole-branch review, finding 4). |
| 41 | const placementCheckBudget = 4 * time.Second |
| 42 | |
| 43 | // VerifyPlacement does one real placement, through the shim, into a scratch |
| 44 | // child cgroup with the default tier's limits, and answers nil only if the |
| 45 | // placed process saw itself in that cgroup and the limit reads back. It is |
| 46 | // the arm-time question for CARLOS_SWAP=overlap (#272): the second dev-box |
| 47 | // check found a box that armed and then failed every generation start, |
| 48 | // because no placement was possible under carlos-edge.service's hardening. |
| 49 | // |
| 50 | // It calls the shim itself (startViaShimBy), never the placeStart seam, so a |
| 51 | // test's FakePlacementForTest cannot make it pass. Its waits end at ctx's |
| 52 | // deadline or after placementCheckBudget, whichever is sooner; the shim's |
| 53 | // waits are capped at the same instant. The reap of a process it killed |
| 54 | // then follows, outside that budget (placementCheckBudget says why). The |
| 55 | // scratch cgroup is removed once the probe is reaped; a removal failure is |
| 56 | // logged, not returned. |
| 57 | func (m *Manager) VerifyPlacement(ctx context.Context) error { |
| 58 | if m == nil { |
| 59 | return ErrNoDelegation |
| 60 | } |
| 61 | if !shimAvailable() { |
| 62 | return errors.New("cgroup: placement check: the placement shim is not available in this process") |
| 63 | } |
| 64 | ctx, cancel := context.WithTimeout(ctx, placementCheckBudget) |
| 65 | defer cancel() |
| 66 | by, _ := ctx.Deadline() |
| 67 | dir := filepath.Join(m.root, placementCheckName) |
| 68 | // A leftover from an earlier run is empty: the unit's restart killed |
| 69 | // its members. One that will not go fails the Mkdir below. |
| 70 | _ = os.Remove(dir) |
| 71 | // Nothing is created under a root that is no cgroup: the limit files |
| 72 | // would be plain files there, and the directory could not be removed. |
| 73 | var st unix.Statfs_t |
| 74 | if err := unix.Statfs(m.root, &st); err != nil { |
| 75 | return fmt.Errorf("cgroup: placement check: statfs %s: %w", m.root, err) |
| 76 | } |
| 77 | if st.Type != unix.CGROUP2_SUPER_MAGIC { |
| 78 | return fmt.Errorf("cgroup: placement check: %s is not a cgroup2 directory", m.root) |
| 79 | } |
| 80 | if err := os.Mkdir(dir, 0o755); err != nil { |
| 81 | return fmt.Errorf("cgroup: placement check: %w", err) |
| 82 | } |
| 83 | // Every path below reaps any probe it started before it returns, so |
| 84 | // this runs on an empty cgroup. |
| 85 | defer func() { |
| 86 | if err := os.Remove(dir); err != nil { |
| 87 | m.logf("cgroup: placement check: %s not removed: %v", dir, err) |
| 88 | } |
| 89 | }() |
| 90 | if err := m.applyLimit("placement check", dir, TierDefault, true); err != nil { |
| 91 | return fmt.Errorf("cgroup: placement check: %w", err) |
| 92 | } |
| 93 | |
| 94 | var out bytes.Buffer |
| 95 | cmd := exec.Command(shimExe, placementProbeArgs...) |
| 96 | cmd.Env = []string{} |
| 97 | cmd.Stdout = &out |
| 98 | if _, err := m.startViaShimBy(dir, cmd, nil, by); err != nil { |
| 99 | // A placement error or not, the check failed; the shim call has |
| 100 | // reaped anything it started. |
| 101 | return fmt.Errorf("cgroup: placement check: %w", err) |
| 102 | } |
| 103 | if err := awaitProbe(ctx, cmd); err != nil { |
| 104 | return fmt.Errorf("cgroup: placement check: %w", err) |
| 105 | } |
| 106 | // What the probe saw in /proc/self/cgroup: dir as startViaShim names it |
| 107 | // to the shim, below /sys/fs/cgroup. |
| 108 | expected := strings.TrimPrefix(dir, "/sys/fs/cgroup") |
| 109 | if got := out.String(); got != expected+"\n" { |
| 110 | return fmt.Errorf("cgroup: placement check: the probe reported cgroup %q, want %q", got, expected+"\n") |
| 111 | } |
| 112 | raw, err := os.ReadFile(filepath.Join(dir, "memory.max")) |
| 113 | if err != nil { |
| 114 | return fmt.Errorf("cgroup: placement check: %w", err) |
| 115 | } |
| 116 | if got, want := strings.TrimSpace(string(raw)), strconv.FormatInt(TierDefault.MemoryMax, 10); got != want { |
| 117 | return fmt.Errorf("cgroup: placement check: memory.max reads %q, want %q", got, want) |
| 118 | } |
| 119 | return nil |
| 120 | } |
| 121 | |
| 122 | // awaitProbe reaps the started probe: nil when it exited 0, its exit as the |
| 123 | // error otherwise. When ctx ends first the probe is SIGKILLed and reaped |
| 124 | // before this returns, and the check fails on ctx's error. The judgement is |
| 125 | // the reaped status, never which channel was ready first: a probe that had |
| 126 | // already exited when ctx ended is a zombie the kill cannot touch, so it is |
| 127 | // judged by its own exit, not declined as one that hung. |
| 128 | func awaitProbe(ctx context.Context, cmd *exec.Cmd) error { |
| 129 | waited := make(chan error, 1) |
| 130 | go func() { waited <- cmd.Wait() }() |
| 131 | var werr error |
| 132 | select { |
| 133 | case werr = <-waited: |
| 134 | case <-ctx.Done(): |
| 135 | _ = cmd.Process.Kill() |
| 136 | // Bounded once the kernel has ended the killed probe, which holds no |
| 137 | // other process; not by ctx (placementCheckBudget). |
| 138 | werr = <-waited |
| 139 | var ee *exec.ExitError |
| 140 | if errors.As(werr, &ee) { |
| 141 | if ws, ok := ee.Sys().(syscall.WaitStatus); ok && ws.Signaled() && ws.Signal() == syscall.SIGKILL { |
| 142 | return fmt.Errorf("the probe did not exit: %w", ctx.Err()) |
| 143 | } |
| 144 | } |
| 145 | } |
| 146 | if werr != nil { |
| 147 | return fmt.Errorf("the probe failed: %w", werr) |
| 148 | } |
| 149 | return nil |
| 150 | } |
| 151 | |
amadan