| 1 | //go:build linux |
| 2 | |
| 3 | package stopevidence |
| 4 | |
| 5 | import ( |
| 6 | "bufio" |
| 7 | "context" |
| 8 | "errors" |
| 9 | "fmt" |
| 10 | "io/fs" |
| 11 | "os" |
| 12 | "os/exec" |
| 13 | "path/filepath" |
| 14 | "reflect" |
| 15 | "strconv" |
| 16 | "strings" |
| 17 | "syscall" |
| 18 | "testing" |
| 19 | "time" |
| 20 | ) |
| 21 | |
| 22 | // heldAndQuiet ends every holder's script: the shell says it has finished |
| 23 | // starting, then blocks in its own read on a pipe nothing is written to. |
| 24 | // |
| 25 | // A scan must not meet a holder that is still starting. A process that has |
| 26 | // just been exec'd opens and closes the linker cache, libc and the locale |
| 27 | // files in turn, each on the first free descriptor. A scan that reads one of |
| 28 | // them between its close and the next open sees a descriptor whose stat says |
| 29 | // gone and whose entry is there, which is ErrIncomplete by design |
| 30 | // (TestFDHoldersStatGoneWhileTheEntryRemains). The holders here used to be |
| 31 | // `sleep 60`, scanned as soon as it was started, and the hard-link test |
| 32 | // failed that way once under load (#274). |
| 33 | const heldAndQuiet = "echo ready && read wait" |
| 34 | |
| 35 | // startHolder starts cmd, a shell whose script ends in heldAndQuiet, and |
| 36 | // returns its pid once it has said ready. An error means it never did. |
| 37 | func startHolder(t *testing.T, cmd *exec.Cmd) (int, error) { |
| 38 | t.Helper() |
| 39 | // Wait closes this end, and until then the child's read has nothing. |
| 40 | if _, err := cmd.StdinPipe(); err != nil { |
| 41 | t.Fatal(err) |
| 42 | } |
| 43 | r, w, err := os.Pipe() |
| 44 | if err != nil { |
| 45 | t.Fatal(err) |
| 46 | } |
| 47 | defer r.Close() |
| 48 | cmd.Stdout = w |
| 49 | err = cmd.Start() |
| 50 | w.Close() |
| 51 | if err != nil { |
| 52 | return 0, err |
| 53 | } |
| 54 | t.Cleanup(func() { cmd.Process.Kill(); cmd.Wait() }) |
| 55 | r.SetReadDeadline(time.Now().Add(10 * time.Second)) |
| 56 | if line, err := bufio.NewReader(r).ReadString('\n'); line != "ready\n" { |
| 57 | return 0, fmt.Errorf("the holder said %q before it stopped: %v", line, err) |
| 58 | } |
| 59 | return cmd.Process.Pid, nil |
| 60 | } |
| 61 | |
| 62 | // holdOpen starts a child that holds path open as descriptor 3 and opens |
| 63 | // nothing more. |
| 64 | func holdOpen(t *testing.T, path string) int { |
| 65 | t.Helper() |
| 66 | f, err := os.Open(path) |
| 67 | if err != nil { |
| 68 | t.Fatal(err) |
| 69 | } |
| 70 | defer f.Close() |
| 71 | cmd := exec.Command("sh", "-c", heldAndQuiet) |
| 72 | cmd.ExtraFiles = []*os.File{f} |
| 73 | pid, err := startHolder(t, cmd) |
| 74 | if err != nil { |
| 75 | t.Fatal(err) |
| 76 | } |
| 77 | return pid |
| 78 | } |
| 79 | |
| 80 | // projectPIDs is a scan root holding only these processes' real /proc |
| 81 | // entries, so the scan needs no access to other uids. |
| 82 | func projectPIDs(t *testing.T, pids ...int) string { |
| 83 | t.Helper() |
| 84 | root := t.TempDir() |
| 85 | for _, pid := range pids { |
| 86 | p := strconv.Itoa(pid) |
| 87 | if err := os.Symlink(filepath.Join("/proc", p), filepath.Join(root, p)); err != nil { |
| 88 | t.Fatal(err) |
| 89 | } |
| 90 | } |
| 91 | return root |
| 92 | } |
| 93 | |
| 94 | func newDB(t *testing.T, dir string) string { |
| 95 | t.Helper() |
| 96 | if err := os.MkdirAll(dir, 0o755); err != nil { |
| 97 | t.Fatal(err) |
| 98 | } |
| 99 | db := filepath.Join(dir, "h.example.com.db") |
| 100 | if err := os.WriteFile(db, []byte("SQLite format 3\x00"), 0o644); err != nil { |
| 101 | t.Fatal(err) |
| 102 | } |
| 103 | return db |
| 104 | } |
| 105 | |
| 106 | func dbFilesOf(t *testing.T, db string) []string { |
| 107 | t.Helper() |
| 108 | files, err := dbPathsWithSidecars([]string{db}) |
| 109 | if err != nil { |
| 110 | t.Fatal(err) |
| 111 | } |
| 112 | return files |
| 113 | } |
| 114 | |
| 115 | // Regression tests for FDHolders' identity matching (main 8e9ac1b8), the |
| 116 | // scan Reconstruct's step 3 uses. Each holder is a real child process. |
| 117 | |
| 118 | // A hard link is the same file under another name: a holder that opened |
| 119 | // the link holds the database. |
| 120 | func TestFDIdentityFindsAHardLinkHeldByAChild(t *testing.T) { |
| 121 | dir := t.TempDir() |
| 122 | db := newDB(t, filepath.Join(dir, "data")) |
| 123 | alias := filepath.Join(dir, "elsewhere", "alias.db") |
| 124 | os.MkdirAll(filepath.Dir(alias), 0o755) |
| 125 | if err := os.Link(db, alias); err != nil { |
| 126 | t.Skip("no hard links here:", err) |
| 127 | } |
| 128 | pid := holdOpen(t, alias) |
| 129 | root := projectPIDs(t, pid) |
| 130 | got, err := FDHolders(root, dbFilesOf(t, db)) |
| 131 | if err != nil || !reflect.DeepEqual(got, []int{pid}) { |
| 132 | t.Fatalf("identity scan: %v %v, want [%d]", got, err, pid) |
| 133 | } |
| 134 | } |
| 135 | |
| 136 | // The database path the caller passes may run through a symlink (a data |
| 137 | // directory linked elsewhere) while the holder's descriptor names the real |
| 138 | // path. |
| 139 | func TestFDIdentityFindsAHolderThroughASymlinkedDBPath(t *testing.T) { |
| 140 | dir := t.TempDir() |
| 141 | db := newDB(t, filepath.Join(dir, "real")) |
| 142 | link := filepath.Join(dir, "data") |
| 143 | if err := os.Symlink(filepath.Join(dir, "real"), link); err != nil { |
| 144 | t.Fatal(err) |
| 145 | } |
| 146 | pid := holdOpen(t, db) |
| 147 | root := projectPIDs(t, pid) |
| 148 | viaLink := filepath.Join(link, filepath.Base(db)) |
| 149 | got, err := FDHolders(root, dbFilesOf(t, viaLink)) |
| 150 | if err != nil || !reflect.DeepEqual(got, []int{pid}) { |
| 151 | t.Fatalf("identity scan through the symlinked path: %v %v, want [%d]", got, err, pid) |
| 152 | } |
| 153 | } |
| 154 | |
| 155 | // A holder in another mount namespace sees the database through a bind |
| 156 | // mount at a different path; its descriptor's link text is that path. An |
| 157 | // unprivileged user and mount namespace builds it; where the box forbids |
| 158 | // that, the test skips. |
| 159 | func TestFDIdentityFindsAHolderInAnotherMountNamespace(t *testing.T) { |
| 160 | if _, err := exec.LookPath("unshare"); err != nil { |
| 161 | t.Skip("no unshare:", err) |
| 162 | } |
| 163 | dir := t.TempDir() |
| 164 | db := newDB(t, filepath.Join(dir, "real")) |
| 165 | alias := filepath.Join(dir, "alias") |
| 166 | if err := os.MkdirAll(alias, 0o755); err != nil { |
| 167 | t.Fatal(err) |
| 168 | } |
| 169 | aliased := filepath.Join(alias, filepath.Base(db)) |
| 170 | // The shell opens the database on descriptor 3 itself, after the mount, |
| 171 | // and says ready only then: until that point descriptor 3 can be some |
| 172 | // other file (the dynamic loader's libc, while unshare or sh starts). |
| 173 | cmd := exec.Command("unshare", "--user", "--map-root-user", "--mount", "sh", "-c", |
| 174 | `mount --bind "$1" "$2" && exec 3<"$3" && `+heldAndQuiet, "sh", filepath.Join(dir, "real"), alias, aliased) |
| 175 | pid, err := startHolder(t, cmd) |
| 176 | if err != nil { |
| 177 | t.Skip("the namespace holder never came up holding the database (unprivileged user namespaces refused?):", err) |
| 178 | } |
| 179 | if target, _ := os.Readlink(filepath.Join("/proc", strconv.Itoa(pid), "fd", "3")); target != aliased { |
| 180 | t.Logf("descriptor text %q (expected the bind path %q)", target, aliased) |
| 181 | } |
| 182 | root := projectPIDs(t, pid) |
| 183 | got, err := FDHolders(root, dbFilesOf(t, db)) |
| 184 | if err != nil || !reflect.DeepEqual(got, []int{pid}) { |
| 185 | t.Fatalf("identity scan across a mount namespace: %v %v, want [%d]", got, err, pid) |
| 186 | } |
| 187 | } |
| 188 | |
| 189 | // A writer still holding a database that was deleted and replaced holds a |
| 190 | // file that is no longer at the path; its link text names the path with |
| 191 | // " (deleted)", and that is a hold, never a pass. |
| 192 | func TestFDIdentityDeletedAndReplacedDatabaseIsAHold(t *testing.T) { |
| 193 | db := newDB(t, t.TempDir()) |
| 194 | pid := holdOpen(t, db) |
| 195 | if err := os.Remove(db); err != nil { |
| 196 | t.Fatal(err) |
| 197 | } |
| 198 | newDB(t, filepath.Dir(db)) |
| 199 | root := projectPIDs(t, pid) |
| 200 | got, err := FDHolders(root, dbFilesOf(t, db)) |
| 201 | if err != nil || !reflect.DeepEqual(got, []int{pid}) { |
| 202 | t.Fatalf("deleted and replaced: %v %v, want [%d]", got, err, pid) |
| 203 | } |
| 204 | } |
| 205 | |
| 206 | // A process holding something else is not a holder, and a database with |
| 207 | // no WAL or SHM on disk is checked by the files that exist. |
| 208 | func TestFDIdentityOtherFilesAreNotHolders(t *testing.T) { |
| 209 | dir := t.TempDir() |
| 210 | db := newDB(t, dir) |
| 211 | other := filepath.Join(dir, "other.db") |
| 212 | os.WriteFile(other, []byte("x"), 0o644) |
| 213 | pid := holdOpen(t, other) |
| 214 | got, err := FDHolders(projectPIDs(t, pid), dbFilesOf(t, db)) |
| 215 | if err != nil || len(got) != 0 { |
| 216 | t.Fatalf("other file: %v %v", got, err) |
| 217 | } |
| 218 | } |
| 219 | |
| 220 | type fakeInfo struct{ fs.FileInfo } |
| 221 | |
| 222 | func (fakeInfo) Sys() any { return nil } |
| 223 | |
| 224 | // aliasFaultFixture is a proc root with one descriptor on an unrelated |
| 225 | // real file, and the database's files. |
| 226 | func aliasFaultFixture(t *testing.T) (root, fdLink, db string, files []string) { |
| 227 | t.Helper() |
| 228 | dir := t.TempDir() |
| 229 | db = newDB(t, filepath.Join(dir, "data")) |
| 230 | other := filepath.Join(dir, "other") |
| 231 | os.WriteFile(other, []byte("x"), 0o644) |
| 232 | root = filepath.Join(dir, "proc") |
| 233 | fdDir := filepath.Join(root, "20", "task", "20", "fd") |
| 234 | os.MkdirAll(fdDir, 0o755) |
| 235 | fdLink = filepath.Join(fdDir, "0") |
| 236 | if err := os.Symlink(other, fdLink); err != nil { |
| 237 | t.Fatal(err) |
| 238 | } |
| 239 | return root, fdLink, db, dbFilesOf(t, db) |
| 240 | } |
| 241 | |
| 242 | // Every way the scan could fail to see a descriptor is an error, never a |
| 243 | // pass. |
| 244 | func TestFDIdentityFailsClosed(t *testing.T) { |
| 245 | root, fdLink, db, files := aliasFaultFixture(t) |
| 246 | real := realFDOps() |
| 247 | for _, c := range []struct { |
| 248 | name string |
| 249 | mutate func(*fdOps) |
| 250 | wantErr bool |
| 251 | }{ |
| 252 | {"fd stat refused", func(o *fdOps) { |
| 253 | o.stat = func(p string) (os.FileInfo, error) { |
| 254 | if p == fdLink { |
| 255 | return nil, syscall.EACCES |
| 256 | } |
| 257 | return real.stat(p) |
| 258 | } |
| 259 | }, true}, |
| 260 | {"fd closed between the reads", func(o *fdOps) { |
| 261 | // The descriptor is gone: both the stat through the link and the |
| 262 | // entry itself answer ENOENT. |
| 263 | o.stat = func(p string) (os.FileInfo, error) { |
| 264 | if p == fdLink { |
| 265 | return nil, syscall.ENOENT |
| 266 | } |
| 267 | return real.stat(p) |
| 268 | } |
| 269 | o.lstat = func(p string) (os.FileInfo, error) { |
| 270 | if p == fdLink { |
| 271 | return nil, syscall.ENOENT |
| 272 | } |
| 273 | return real.lstat(p) |
| 274 | } |
| 275 | }, false}, |
| 276 | {"fd link refused", func(o *fdOps) { |
| 277 | o.readlink = func(string) (string, error) { return "", syscall.EACCES } |
| 278 | }, true}, |
| 279 | {"fd closed before the link read", func(o *fdOps) { |
| 280 | o.readlink = func(string) (string, error) { return "", syscall.ENOENT } |
| 281 | }, false}, |
| 282 | {"fd with no identity", func(o *fdOps) { |
| 283 | o.stat = func(p string) (os.FileInfo, error) { |
| 284 | fi, err := real.stat(p) |
| 285 | if p == fdLink && err == nil { |
| 286 | return fakeInfo{fi}, nil |
| 287 | } |
| 288 | return fi, err |
| 289 | } |
| 290 | }, true}, |
| 291 | {"database stat refused", func(o *fdOps) { |
| 292 | o.stat = func(p string) (os.FileInfo, error) { |
| 293 | if p == db { |
| 294 | return nil, syscall.EACCES |
| 295 | } |
| 296 | return real.stat(p) |
| 297 | } |
| 298 | }, true}, |
| 299 | {"database with no identity", func(o *fdOps) { |
| 300 | o.stat = func(p string) (os.FileInfo, error) { |
| 301 | fi, err := real.stat(p) |
| 302 | if p == db && err == nil { |
| 303 | return fakeInfo{fi}, nil |
| 304 | } |
| 305 | return fi, err |
| 306 | } |
| 307 | }, true}, |
| 308 | } { |
| 309 | ops := realFDOps() |
| 310 | c.mutate(&ops) |
| 311 | got, err := fdHolders(context.Background(), root, files, ops) |
| 312 | if c.wantErr { |
| 313 | if !errors.Is(err, ErrIncomplete) || got != nil { |
| 314 | t.Fatalf("%s: %v %v, want ErrIncomplete", c.name, got, err) |
| 315 | } |
| 316 | continue |
| 317 | } |
| 318 | if err != nil || len(got) != 0 { |
| 319 | t.Fatalf("%s: %v %v, want no holder", c.name, got, err) |
| 320 | } |
| 321 | } |
| 322 | } |
| 323 | |
| 324 | // A descriptor whose stat through /proc answers ENOENT while its entry |
| 325 | // remains is unreadable, not closed: main's FDHolders lstats the entry after |
| 326 | // such a stat (9cf9f18f) and fails the scan with ErrIncomplete. |
| 327 | func TestFDHoldersStatGoneWhileTheEntryRemains(t *testing.T) { |
| 328 | root, fdLink, _, files := aliasFaultFixture(t) |
| 329 | ops := realFDOps() |
| 330 | real := ops.stat |
| 331 | ops.stat = func(p string) (os.FileInfo, error) { |
| 332 | if p == fdLink { |
| 333 | return nil, syscall.ENOENT |
| 334 | } |
| 335 | return real(p) |
| 336 | } |
| 337 | got, err := fdHolders(context.Background(), root, files, ops) |
| 338 | if !errors.Is(err, ErrIncomplete) || got != nil { |
| 339 | t.Fatalf("%v %v, want ErrIncomplete", got, err) |
| 340 | } |
| 341 | } |
| 342 | |
| 343 | // Descriptors that are not paths (sockets, pipes, anonymous inodes) are |
| 344 | // never followed: they cannot be the database, and a stat could block. |
| 345 | func TestFDIdentitySkipsNonPathDescriptors(t *testing.T) { |
| 346 | db := newDB(t, t.TempDir()) |
| 347 | root := t.TempDir() |
| 348 | fdDir := filepath.Join(root, "20", "task", "20", "fd") |
| 349 | os.MkdirAll(fdDir, 0o755) |
| 350 | for n, target := range []string{"socket:[123]", "pipe:[9]", "anon_inode:[eventfd]"} { |
| 351 | os.Symlink(target, filepath.Join(fdDir, strconv.Itoa(n))) |
| 352 | } |
| 353 | ops := realFDOps() |
| 354 | real := ops.stat |
| 355 | ops.stat = func(p string) (os.FileInfo, error) { |
| 356 | if strings.HasPrefix(p, root) && !strings.HasSuffix(p, "/20") && !strings.HasSuffix(p, "/task/20") { |
| 357 | t.Fatalf("stat of a non-path descriptor %s", p) |
| 358 | } |
| 359 | return real(p) |
| 360 | } |
| 361 | if got, err := fdHolders(context.Background(), root, dbFilesOf(t, db), ops); err != nil || len(got) != 0 { |
| 362 | t.Fatalf("%v %v", got, err) |
| 363 | } |
| 364 | } |
| 365 | |
| 366 | func TestFDIdentityInvalidInput(t *testing.T) { |
| 367 | db := newDB(t, t.TempDir()) |
| 368 | ctx := context.Background() |
| 369 | for name, c := range map[string]struct { |
| 370 | ctx context.Context |
| 371 | root string |
| 372 | paths []string |
| 373 | }{ |
| 374 | "nil context": {nil, "/proc", []string{db}}, |
| 375 | "relative root": {ctx, "proc", []string{db}}, |
| 376 | "no paths": {ctx, "/proc", nil}, |
| 377 | "relative path": {ctx, "/proc", []string{"h.db"}}, |
| 378 | } { |
| 379 | if _, err := fdHolders(c.ctx, c.root, c.paths, realFDOps()); !errors.Is(err, ErrInvalidInput) { |
| 380 | t.Fatalf("%s: %v", name, err) |
| 381 | } |
| 382 | } |
| 383 | } |
| 384 | |
amadan