| 1 | //go:build linux |
| 2 | |
| 3 | package stopevidence |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "errors" |
| 8 | "fmt" |
| 9 | "os" |
| 10 | "path" |
| 11 | "path/filepath" |
| 12 | "slices" |
| 13 | "sort" |
| 14 | "strconv" |
| 15 | "strings" |
| 16 | "time" |
| 17 | |
| 18 | "github.com/carlosframework/platform/internal/cgroup" |
| 19 | "github.com/carlosframework/platform/internal/controlname" |
| 20 | "github.com/carlosframework/platform/internal/moves" |
| 21 | ) |
| 22 | |
| 23 | // treeDrainLimit bounds step 1's kill-and-drain when the caller's context |
| 24 | // has no earlier deadline (KillAndEmpty refuses a context without one). It |
| 25 | // matches the activator's strictDrainTimeout. |
| 26 | const treeDrainLimit = 30 * time.Second |
| 27 | |
| 28 | // treeGroup is the host cgroup as step 1 uses it: *cgroup.Cleanup, or a |
| 29 | // fake in tests. |
| 30 | type treeGroup interface { |
| 31 | KillAndEmpty(context.Context) error |
| 32 | Close() error |
| 33 | } |
| 34 | |
| 35 | // Reconstruct is §5.3's evidence from durable, observable state: it reads |
| 36 | // nothing from the agent's memory, so a restarted agent gives the same |
| 37 | // answer. It runs steps 1 to 4 and records each in the report's Steps; the |
| 38 | // caller does step 5 (the final checkpoint and the digests, then |
| 39 | // StepReplica), since only it knows the replica it closes. Role, Box, |
| 40 | // Incarnation, ClaimGen and OpID are the caller's to fill in too. |
| 41 | // |
| 42 | // A step that observed a failure is a Check with OK false and a reason; |
| 43 | // the pass retries until its stop evidence budget runs out. A step that |
| 44 | // could not observe (an unreadable /proc or cgroup file, a lease read |
| 45 | // error, the caller's context ending) is a returned error, and the report |
| 46 | // beside it is partial and must not be written. |
| 47 | // |
| 48 | // Reconstruct takes neither per-host gate. schedule.HostGate is the timers |
| 49 | // drain's gate (the runner, the converger and moveCfg.TimersDrained); the |
| 50 | // activator's strictGates (ExecBackend.gate) serialise a strict spawn with |
| 51 | // its own exit cleanup. By the time the cut calls this, the registry hold |
| 52 | // and the activator's stop fence have closed admission, so nothing spawns |
| 53 | // for the host; the kill works on the group whoever last owned it, and a |
| 54 | // strict owner's concurrent cleanup only kills and drains the same group. |
| 55 | func Reconstruct(ctx context.Context, in ReconstructInputs) (moves.StoppedReport, error) { |
| 56 | if in.Cgroups == nil { |
| 57 | return moves.StoppedReport{V: 1, Host: in.Host}, ErrInvalidInput |
| 58 | } |
| 59 | m := in.Cgroups |
| 60 | return reconstruct(ctx, in, func(host string) (treeGroup, error) { |
| 61 | c, err := m.OpenCleanup(host) |
| 62 | if err != nil { |
| 63 | return nil, err |
| 64 | } |
| 65 | return c, nil |
| 66 | }) |
| 67 | } |
| 68 | |
| 69 | func reconstruct(ctx context.Context, in ReconstructInputs, open func(string) (treeGroup, error)) (moves.StoppedReport, error) { |
| 70 | rep := moves.StoppedReport{V: 1, Host: in.Host, Steps: map[string]moves.Check{}} |
| 71 | if ctx == nil || open == nil || in.LeaseFree == nil || !controlname.ValidHostShape(in.Host) { |
| 72 | return rep, ErrInvalidInput |
| 73 | } |
| 74 | procRoot := in.ProcRoot |
| 75 | if procRoot == "" { |
| 76 | procRoot = "/proc" |
| 77 | } |
| 78 | if !filepath.IsAbs(procRoot) { |
| 79 | return rep, ErrInvalidInput |
| 80 | } |
| 81 | dbFiles, err := dbPathsWithSidecars(in.DBPaths) |
| 82 | if err != nil { |
| 83 | return rep, err |
| 84 | } |
| 85 | if err := ctx.Err(); err != nil { |
| 86 | return rep, err |
| 87 | } |
| 88 | |
| 89 | // 1. Kill and drain the tree: cgroup.kill on carlos-exec@<host>, then |
| 90 | // wait for it to empty. Phase 1 moves flat exec instances only (S1 |
| 91 | // refuses a nested one), so this is the whole of step 1. |
| 92 | tree, err := stopTree(ctx, in.Host, open) |
| 93 | if err != nil { |
| 94 | return rep, err |
| 95 | } |
| 96 | rep.Steps[moves.StepTree] = tree |
| 97 | |
| 98 | // 2. No stray process in the agent's own leaf. A spawn that escaped its |
| 99 | // host cgroup (best-effort placement before the move went strict) lands |
| 100 | // here, which is why an absent host cgroup counts as an empty tree only |
| 101 | // alongside this step. So does any short-lived child the agent forks |
| 102 | // for itself: it fails StepStray for this pass, and the cut's retry |
| 103 | // until its deadline is the intended behaviour. Only a repeat is worth |
| 104 | // the alert (M-C20). The leaf is read again after steps 3 and 4 (below). |
| 105 | self := os.Getpid() |
| 106 | first, err := agentLeaf(procRoot, self) |
| 107 | if err != nil { |
| 108 | return rep, err |
| 109 | } |
| 110 | rep.Steps[moves.StepStray] = strayCheck(first, self) |
| 111 | |
| 112 | // 3. No open database. scanFDs is FDHolders with the pass's context: it |
| 113 | // matches by path text and by device and inode, so a hard link, a |
| 114 | // symlinked path or another mount namespace's bind path is the same |
| 115 | // file. The scan covers every process /proc shows. On a box the |
| 116 | // agent runs under ProtectProc=invisible (carlos-edge.service), so /proc |
| 117 | // shows only the carlos uid's processes: §5.3's "restricted to the |
| 118 | // carlos uid". A process it can see but not read makes the scan |
| 119 | // incomplete, which is an error, never a pass. |
| 120 | holders, err := scanFDs(ctx, procRoot, dbFiles) |
| 121 | if err != nil { |
| 122 | return rep, err |
| 123 | } |
| 124 | if len(holders) == 0 { |
| 125 | rep.Steps[moves.StepFDs] = moves.Check{OK: true} |
| 126 | } else { |
| 127 | rep.Steps[moves.StepFDs] = moves.Check{Reason: fmt.Sprintf("%d process(es) hold the database open", len(holders))} |
| 128 | } |
| 129 | |
| 130 | // 4. Lease released. |
| 131 | free, err := in.LeaseFree(ctx) |
| 132 | if err != nil { |
| 133 | return rep, err |
| 134 | } |
| 135 | if free { |
| 136 | rep.Steps[moves.StepLease] = moves.Check{OK: true} |
| 137 | } else { |
| 138 | rep.Steps[moves.StepLease] = moves.Check{Reason: "lock.json still held"} |
| 139 | } |
| 140 | |
| 141 | // 2, again. The first read is a snapshot taken before the descriptor |
| 142 | // scan and the lease read, which take time; a process that joined the |
| 143 | // agent's leaf meanwhile and holds no database descriptor would pass |
| 144 | // both. Any pid in the second read but the agent fails StepStray for |
| 145 | // this pass, and the cut's retry covers it. Rep.Stray lists the strays |
| 146 | // of both reads. |
| 147 | second, err := agentLeaf(procRoot, self) |
| 148 | if err != nil { |
| 149 | return rep, err |
| 150 | } |
| 151 | if rep.Steps[moves.StepStray].OK { |
| 152 | rep.Steps[moves.StepStray] = strayCheck(second, self) |
| 153 | } |
| 154 | rep.Stray = strays(self, first, second) |
| 155 | |
| 156 | // The evidence time is when the last step was observed, not when the |
| 157 | // pass began: §5.3's late-completion horizon runs from it, and a later |
| 158 | // time is the safe direction. |
| 159 | rep.CheckedAt = time.Now().UTC() |
| 160 | return rep, nil |
| 161 | } |
| 162 | |
| 163 | // stopTree is step 1. A group that never existed, or that its last strict |
| 164 | // owner removed, is an empty tree. A drain that runs out of time is an |
| 165 | // observed failure; any other failure to open, kill or read the group is |
| 166 | // an error. |
| 167 | func stopTree(ctx context.Context, host string, open func(string) (treeGroup, error)) (moves.Check, error) { |
| 168 | c, err := open(host) |
| 169 | if errors.Is(err, cgroup.ErrNoCgroup) { |
| 170 | return moves.Check{OK: true}, nil |
| 171 | } |
| 172 | if err != nil { |
| 173 | return moves.Check{}, err |
| 174 | } |
| 175 | defer c.Close() |
| 176 | drain, cancel := context.WithTimeout(ctx, treeDrainLimit) |
| 177 | defer cancel() |
| 178 | err = c.KillAndEmpty(drain) |
| 179 | switch { |
| 180 | case err == nil: |
| 181 | return moves.Check{OK: true}, nil |
| 182 | case ctx.Err() != nil: |
| 183 | return moves.Check{}, ctx.Err() |
| 184 | case errors.Is(err, context.DeadlineExceeded): |
| 185 | return moves.Check{Reason: "the instance's processes could not be stopped"}, nil |
| 186 | } |
| 187 | return moves.Check{}, err |
| 188 | } |
| 189 | |
| 190 | // strayCheck is StepStray's verdict on one read of the agent's leaf. A pid |
| 191 | // other than the agent's fails it with that pid. A 0 entry (the kernel's |
| 192 | // name for a member outside this process's pid namespace) fails it too: it |
| 193 | // is a process the box cannot identify, so it is never taken for the agent |
| 194 | // and never for nothing. |
| 195 | func strayCheck(pids []int, self int) moves.Check { |
| 196 | unknown := 0 |
| 197 | for _, p := range pids { |
| 198 | switch { |
| 199 | case p == 0: |
| 200 | unknown++ |
| 201 | case p != self: |
| 202 | return moves.Check{Reason: fmt.Sprintf("pid %d in the agent's cgroup is not the agent", p)} |
| 203 | } |
| 204 | } |
| 205 | if unknown > 0 { |
| 206 | return moves.Check{Reason: fmt.Sprintf("%d process(es) in the agent's cgroup cannot be identified", unknown)} |
| 207 | } |
| 208 | return moves.Check{OK: true} |
| 209 | } |
| 210 | |
| 211 | // agentLeaf reads the agent's own leaf and requires the agent in it: a |
| 212 | // listing without the agent is the wrong directory, an error. |
| 213 | func agentLeaf(procRoot string, self int) ([]int, error) { |
| 214 | pids, err := ownLeafProcs(procRoot) |
| 215 | if err != nil { |
| 216 | return nil, err |
| 217 | } |
| 218 | if !slices.Contains(pids, self) { |
| 219 | return nil, fmt.Errorf("stopevidence: the agent (pid %d) is not in the cgroup its /proc entry names", self) |
| 220 | } |
| 221 | return pids, nil |
| 222 | } |
| 223 | |
| 224 | // strays is every pid but self across the reads, sorted, each once. A 0 |
| 225 | // is the marker for members the box could not identify (once, however many |
| 226 | // there were; the step's reason carries the count). |
| 227 | func strays(self int, reads ...[]int) []int { |
| 228 | var out []int |
| 229 | for _, pids := range reads { |
| 230 | for _, p := range pids { |
| 231 | if p != self && !slices.Contains(out, p) { |
| 232 | out = append(out, p) |
| 233 | } |
| 234 | } |
| 235 | } |
| 236 | sort.Ints(out) |
| 237 | return out |
| 238 | } |
| 239 | |
| 240 | // dbPathsWithSidecars is every database file with its -wal and -shm, clean, |
| 241 | // sorted and without repeats. Each path must be absolute, and there must be |
| 242 | // at least one: a cut always has a database path, so an empty list is a |
| 243 | // caller's mistake, not "nothing to hold open". |
| 244 | func dbPathsWithSidecars(paths []string) ([]string, error) { |
| 245 | if len(paths) == 0 { |
| 246 | return nil, ErrInvalidInput |
| 247 | } |
| 248 | seen := map[string]bool{} |
| 249 | for _, p := range paths { |
| 250 | if !filepath.IsAbs(p) { |
| 251 | return nil, ErrInvalidInput |
| 252 | } |
| 253 | p = filepath.Clean(p) |
| 254 | seen[p], seen[p+"-wal"], seen[p+"-shm"] = true, true, true |
| 255 | } |
| 256 | out := make([]string, 0, len(seen)) |
| 257 | for p := range seen { |
| 258 | out = append(out, p) |
| 259 | } |
| 260 | sort.Strings(out) |
| 261 | return out, nil |
| 262 | } |
| 263 | |
| 264 | // ownLeafProcs reads the pids in the agent's own cgroup leaf: the v2 path |
| 265 | // <procRoot>/self/cgroup names, under the cgroup2 mount |
| 266 | // <procRoot>/self/mountinfo names (ruling G32: never a hard-coded |
| 267 | // /sys/fs/cgroup). |
| 268 | func ownLeafProcs(procRoot string) ([]int, error) { |
| 269 | self, err := os.ReadFile(filepath.Join(procRoot, "self", "cgroup")) |
| 270 | if err != nil { |
| 271 | return nil, err |
| 272 | } |
| 273 | leaf, ok := cgroup.ParseSelfCgroup(string(self)) |
| 274 | if !ok { |
| 275 | return nil, errors.New("stopevidence: own cgroup unreadable") |
| 276 | } |
| 277 | mounts, err := os.ReadFile(filepath.Join(procRoot, "self", "mountinfo")) |
| 278 | if err != nil { |
| 279 | return nil, err |
| 280 | } |
| 281 | dir, err := cgroupLeafDir(string(mounts), leaf) |
| 282 | if err != nil { |
| 283 | return nil, err |
| 284 | } |
| 285 | return readProcs(filepath.Join(dir, "cgroup.procs")) |
| 286 | } |
| 287 | |
| 288 | // cgroupLeafDir is the directory of the v2 cgroup leaf (as |
| 289 | // ParseSelfCgroup returns it, relative to the hierarchy's root) under the |
| 290 | // cgroup2 mount that contains it. A mount's root field is the part of the |
| 291 | // hierarchy it shows; the mount whose root holds the leaf most closely |
| 292 | // wins, and of equal roots the first listed. A leaf that climbs out of |
| 293 | // the hierarchy (a cgroup outside the agent's cgroup namespace shows as |
| 294 | // "/../..") or that no cgroup2 mount shows is an error. |
| 295 | func cgroupLeafDir(mountinfo, leaf string) (string, error) { |
| 296 | abs := "/" + leaf |
| 297 | if path.Clean(abs) != abs { |
| 298 | return "", fmt.Errorf("stopevidence: own cgroup %q is outside the hierarchy", leaf) |
| 299 | } |
| 300 | best, bestRoot := "", -1 |
| 301 | for _, line := range strings.Split(mountinfo, "\n") { |
| 302 | fields := strings.Fields(line) |
| 303 | if len(fields) < 7 { |
| 304 | continue |
| 305 | } |
| 306 | sep := slices.Index(fields[6:], "-") |
| 307 | if sep < 0 || 6+sep+1 >= len(fields) || fields[6+sep+1] != "cgroup2" { |
| 308 | continue |
| 309 | } |
| 310 | root, mnt := unescapeMount(fields[3]), unescapeMount(fields[4]) |
| 311 | if !filepath.IsAbs(mnt) { |
| 312 | continue |
| 313 | } |
| 314 | var rel string |
| 315 | switch { |
| 316 | case root == "/": |
| 317 | rel = abs |
| 318 | case abs == root: |
| 319 | rel = "/" |
| 320 | case strings.HasPrefix(abs, root+"/"): |
| 321 | rel = strings.TrimPrefix(abs, root) |
| 322 | default: |
| 323 | continue |
| 324 | } |
| 325 | if len(root) > bestRoot { |
| 326 | best, bestRoot = filepath.Join(mnt, rel), len(root) |
| 327 | } |
| 328 | } |
| 329 | if bestRoot < 0 { |
| 330 | return "", fmt.Errorf("stopevidence: no cgroup2 mount shows own cgroup %q", leaf) |
| 331 | } |
| 332 | return best, nil |
| 333 | } |
| 334 | |
| 335 | // unescapeMount undoes mountinfo's octal escapes (\040 for a space, \011, |
| 336 | // \012, \134). |
| 337 | func unescapeMount(s string) string { |
| 338 | if !strings.Contains(s, `\`) { |
| 339 | return s |
| 340 | } |
| 341 | var b strings.Builder |
| 342 | for i := 0; i < len(s); i++ { |
| 343 | if s[i] == '\\' && i+4 <= len(s) { |
| 344 | if v, err := strconv.ParseUint(s[i+1:i+4], 8, 8); err == nil { |
| 345 | b.WriteByte(byte(v)) |
| 346 | i += 3 |
| 347 | continue |
| 348 | } |
| 349 | } |
| 350 | b.WriteByte(s[i]) |
| 351 | } |
| 352 | return b.String() |
| 353 | } |
| 354 | |
| 355 | // readProcs parses cgroup.procs: one pid per line. The kernel lists pid 0 |
| 356 | // for a member that is not visible in the reader's pid namespace; it is |
| 357 | // returned as 0, for the caller to treat as an unidentifiable member. Any |
| 358 | // other entry that is not a plain non-negative number is an error. |
| 359 | |
| 360 | func readProcs(file string) ([]int, error) { |
| 361 | raw, err := os.ReadFile(file) |
| 362 | if err != nil { |
| 363 | return nil, err |
| 364 | } |
| 365 | var pids []int |
| 366 | for _, line := range strings.Split(string(raw), "\n") { |
| 367 | line = strings.TrimSpace(line) |
| 368 | if line == "" { |
| 369 | continue |
| 370 | } |
| 371 | pid, err := strconv.Atoi(line) |
| 372 | if err != nil || pid < 0 || line[0] == '+' || line[0] == '-' { |
| 373 | return nil, fmt.Errorf("stopevidence: %s: invalid pid %q", file, line) |
| 374 | } |
| 375 | pids = append(pids, pid) |
| 376 | } |
| 377 | return pids, nil |
| 378 | } |
| 379 | |
amadan