| 1 | //go:build linux |
| 2 | |
| 3 | package host |
| 4 | |
| 5 | import ( |
| 6 | "errors" |
| 7 | "fmt" |
| 8 | "os" |
| 9 | "os/user" |
| 10 | "path/filepath" |
| 11 | "strconv" |
| 12 | "syscall" |
| 13 | "time" |
| 14 | |
| 15 | "github.com/carlosframework/platform/internal/safefile" |
| 16 | "golang.org/x/sys/unix" |
| 17 | ) |
| 18 | |
| 19 | // movesDir is <DataDir>/.moves, opened with O_NOFOLLOW below a data dir |
| 20 | // root trusts, so nothing root reads or writes through it can be a |
| 21 | // symlink the sandboxed agent planted. Every operation on the floor files |
| 22 | // goes through one: the lock and the writes are all relative to its fd. |
| 23 | type movesDir struct { |
| 24 | fd int |
| 25 | path string |
| 26 | lock *os.File |
| 27 | lockName string |
| 28 | } |
| 29 | |
| 30 | // floorLockFile serialises every writer of the floor files (root's apply, |
| 31 | // the agent's refresh) with every reader whose answer must not change under |
| 32 | // it (root's judgement of carlos.prev and its activation guard). |
| 33 | const floorLockFile = "floor.lock" |
| 34 | |
| 35 | const floorLockPoll = 100 * time.Millisecond |
| 36 | |
| 37 | // openMovesDir opens <root>/.moves. With create it makes the directory |
| 38 | // when missing; without, a missing directory is os.ErrNotExist. |
| 39 | // |
| 40 | // Crash ordering (fix round 2): with create, the parent (root) is fsynced |
| 41 | // EVERY time, after .moves is known to exist, whoever made it. A writer |
| 42 | // never relies on having created the directory itself: another writer may |
| 43 | // have made it and not yet synced the parent, or failed to. Only then is |
| 44 | // any file written inside, each renamed into place and followed by an fsync |
| 45 | // of .moves. So a floor write that returned nil survives a power loss, |
| 46 | // directory entry and all. A failed fsync is the open's error, which the |
| 47 | // writers report as a floor that could not be written (errFloorWrite, which |
| 48 | // defers an apply) and the activation guard as a lock it could not take. |
| 49 | // The cost is one fsync of the data dir per write. |
| 50 | func openMovesDir(root string, create bool) (*movesDir, error) { |
| 51 | name := movesDirName |
| 52 | rootDir, err := safefile.OpenDir(root) |
| 53 | if err != nil { |
| 54 | return nil, fmt.Errorf("open %s: %w", root, err) |
| 55 | } |
| 56 | defer rootDir.Close() |
| 57 | rfd := int(rootDir.Fd()) |
| 58 | flags := syscall.O_RDONLY | syscall.O_DIRECTORY | syscall.O_NOFOLLOW | syscall.O_CLOEXEC |
| 59 | fd, err := syscall.Openat(rfd, name, flags, 0) |
| 60 | if errors.Is(err, syscall.ENOENT) && create { |
| 61 | if err := syscall.Mkdirat(rfd, name, 0o755); err != nil && !errors.Is(err, syscall.EEXIST) { |
| 62 | return nil, fmt.Errorf("mkdir %s: %w", filepath.Join(root, name), err) |
| 63 | } |
| 64 | if fd, err = syscall.Openat(rfd, name, flags, 0); err == nil { |
| 65 | fchownToService(fd) |
| 66 | } |
| 67 | } |
| 68 | if err != nil { |
| 69 | return nil, fmt.Errorf("open %s: %w", filepath.Join(root, name), err) |
| 70 | } |
| 71 | if create { |
| 72 | if err := syncParent(rfd); err != nil { |
| 73 | syscall.Close(fd) |
| 74 | return nil, fmt.Errorf("fsync %s: %w", root, err) |
| 75 | } |
| 76 | } |
| 77 | return &movesDir{fd: fd, path: filepath.Join(root, name)}, nil |
| 78 | } |
| 79 | |
| 80 | // syncParent fsyncs the data dir. A variable so a test can count the calls. |
| 81 | var syncParent = func(fd int) error { return syscall.Fsync(fd) } |
| 82 | |
| 83 | // Close releases the lock (closing its fd) and the directory. |
| 84 | func (d *movesDir) Close() { |
| 85 | if d.lock != nil { |
| 86 | d.lock.Close() |
| 87 | d.lock = nil |
| 88 | } |
| 89 | syscall.Close(d.fd) |
| 90 | } |
| 91 | |
| 92 | // Lock takes floor.lock, waiting up to floorLockWait. |
| 93 | func (d *movesDir) Lock() error { return d.lockNamed(floorLockFile, floorLockWait, errFloorLockBusy) } |
| 94 | |
| 95 | // lockNamed takes an exclusive flock on the file name in the directory, |
| 96 | // waiting up to wait and then failing with busy. Opened read-only (a |
| 97 | // read-only fd can be flocked), so the agent can lock a file root created |
| 98 | // and root a file the agent created; O_NOFOLLOW and O_NONBLOCK so a symlink |
| 99 | // or a FIFO planted in its place cannot redirect or park root. The fd is |
| 100 | // kept until Close (or unlockRemoving), which releases the lock. One lock |
| 101 | // per movesDir. |
| 102 | // |
| 103 | // A holder may remove the file it holds before releasing it |
| 104 | // (unlockRemoving, the held-stage lock). A waiter that opened that file |
| 105 | // would then lock a file no longer at the name, beside whoever has locked |
| 106 | // the new one there. So once it has the flock it checks that the name still |
| 107 | // names the file it locked (fstat of the fd against an fstatat of the name) |
| 108 | // and, when the name is gone or names another file, closes it and starts |
| 109 | // again on whatever is there now. The same removal can land inside the |
| 110 | // open itself (the name vanishes, or the file opened has no link left); |
| 111 | // that is retried too, while a symlink, a FIFO or a second link at the name |
| 112 | // is still refused. Each retry pauses (lockNamedRetryPause, capped at what |
| 113 | // is left of the wait), and none starts once the wait has passed: the call |
| 114 | // fails with busy then, an open race included, and holds nothing. |
| 115 | // |
| 116 | // The deadline bounds the wait; correctness does not depend on it. A flock |
| 117 | // taken just after the deadline (the check and the syscall are not one |
| 118 | // step) is still a valid, exclusive lock, and is kept. |
| 119 | func (d *movesDir) lockNamed(name string, wait time.Duration, busy error) error { |
| 120 | deadline := time.Now().Add(wait) |
| 121 | timedOut := func() error { return fmt.Errorf("lock %s: %w", filepath.Join(d.path, name), busy) } |
| 122 | for { |
| 123 | if time.Now().After(deadline) { |
| 124 | return timedOut() |
| 125 | } |
| 126 | f, err := lockNamedOpenAt(d.fd, name, os.O_RDONLY|os.O_CREATE, 0o644) |
| 127 | if err != nil { |
| 128 | if !d.removedDuringOpen(name, err) { |
| 129 | return fmt.Errorf("open %s: %w", filepath.Join(d.path, name), err) |
| 130 | } |
| 131 | // A holder's removal landed inside the open: start again. |
| 132 | lockNamedPause(deadline) |
| 133 | continue |
| 134 | } |
| 135 | lockNamedOpened(name) |
| 136 | if err := flockWithin(f, deadline); err != nil { |
| 137 | f.Close() |
| 138 | if errors.Is(err, syscall.EWOULDBLOCK) { |
| 139 | return timedOut() |
| 140 | } |
| 141 | return fmt.Errorf("lock %s: %w", filepath.Join(d.path, name), err) |
| 142 | } |
| 143 | same, err := d.names(name, f) |
| 144 | if err != nil { |
| 145 | f.Close() |
| 146 | return fmt.Errorf("lock %s: %w", filepath.Join(d.path, name), err) |
| 147 | } |
| 148 | if same { |
| 149 | d.lock, d.lockName = f, name |
| 150 | return nil |
| 151 | } |
| 152 | // The holder before us removed this file as it released it. Start |
| 153 | // again, within the wait: the publication waits inside the |
| 154 | // registry's freeze, so a name that keeps changing ends in busy. |
| 155 | f.Close() |
| 156 | lockNamedPause(deadline) |
| 157 | } |
| 158 | } |
| 159 | |
| 160 | // lockNamedPause sleeps lockNamedRetryPause, or what is left of the wait |
| 161 | // when that is less. |
| 162 | func lockNamedPause(deadline time.Time) { |
| 163 | if left := time.Until(deadline); left > 0 { |
| 164 | time.Sleep(min(lockNamedRetryPause, left)) |
| 165 | } |
| 166 | } |
| 167 | |
| 168 | // flockWithin takes an exclusive flock on f, polling until deadline, and |
| 169 | // once the deadline has passed stops trying and returns EWOULDBLOCK. A |
| 170 | // flock that lands just after the deadline is kept: it is a valid, |
| 171 | // exclusive lock, and the deadline only bounds the wait. |
| 172 | func flockWithin(f *os.File, deadline time.Time) error { |
| 173 | for { |
| 174 | if time.Now().After(deadline) { |
| 175 | return syscall.EWOULDBLOCK |
| 176 | } |
| 177 | err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) |
| 178 | if err == nil || !errors.Is(err, syscall.EWOULDBLOCK) { |
| 179 | return err |
| 180 | } |
| 181 | time.Sleep(floorLockPoll) |
| 182 | } |
| 183 | } |
| 184 | |
| 185 | // writeHook, when set, is called with each file name WriteDurable is about |
| 186 | // to write: a test seam for the order the floor files are written in. |
| 187 | var writeHook func(name string) |
| 188 | |
| 189 | // names reports whether name in the directory is still the file f has |
| 190 | // open: a regular file at the name with f's device and inode. A name that |
| 191 | // is gone is false, not an error. |
| 192 | func (d *movesDir) names(name string, f *os.File) (bool, error) { |
| 193 | var held, now unix.Stat_t |
| 194 | if err := unix.Fstat(int(f.Fd()), &held); err != nil { |
| 195 | return false, err |
| 196 | } |
| 197 | err := unix.Fstatat(d.fd, name, &now, unix.AT_SYMLINK_NOFOLLOW) |
| 198 | if errors.Is(err, unix.ENOENT) { |
| 199 | return false, nil |
| 200 | } |
| 201 | if err != nil { |
| 202 | return false, err |
| 203 | } |
| 204 | return now.Dev == held.Dev && now.Ino == held.Ino, nil |
| 205 | } |
| 206 | |
| 207 | // removedDuringOpen reports whether err, from opening name, is the race |
| 208 | // with a holder removing it: not an errno other than ENOENT (a real refusal, |
| 209 | // such as EACCES or ELOOP), and the name is now gone or a regular file with |
| 210 | // one link (a fresh lock file). Anything else at the name keeps the open's |
| 211 | // refusal, and so does a .moves that has itself been removed (no create |
| 212 | // can succeed in it, so waiting out the lock's wait would only delay the |
| 213 | // error). |
| 214 | func (d *movesDir) removedDuringOpen(name string, err error) bool { |
| 215 | var errno syscall.Errno |
| 216 | if errors.As(err, &errno) && errno != syscall.ENOENT { |
| 217 | return false |
| 218 | } |
| 219 | var st unix.Stat_t |
| 220 | if derr := unix.Fstat(d.fd, &st); derr != nil || st.Nlink == 0 { |
| 221 | return false |
| 222 | } |
| 223 | serr := unix.Fstatat(d.fd, name, &st, unix.AT_SYMLINK_NOFOLLOW) |
| 224 | if errors.Is(serr, unix.ENOENT) { |
| 225 | return true |
| 226 | } |
| 227 | return serr == nil && st.Mode&unix.S_IFMT == unix.S_IFREG && st.Nlink == 1 |
| 228 | } |
| 229 | |
| 230 | // unlockRemoving removes the lock file while still holding its flock, then |
| 231 | // releases the lock (closing its fd). The name is removed only while it |
| 232 | // still names the file held, so another holder's file is never taken. A |
| 233 | // waiter on the removed file finds the name gone after its flock and starts |
| 234 | // again (lockNamed), so two holders never exist at once. The lock is |
| 235 | // released whatever the removal does; an error is the removal's. |
| 236 | func (d *movesDir) unlockRemoving() error { |
| 237 | if d.lock == nil { |
| 238 | return nil |
| 239 | } |
| 240 | f, name := d.lock, d.lockName |
| 241 | d.lock, d.lockName = nil, "" |
| 242 | defer f.Close() |
| 243 | same, err := d.names(name, f) |
| 244 | if err != nil || !same { |
| 245 | return err |
| 246 | } |
| 247 | if err := unix.Unlinkat(d.fd, name, 0); err != nil && !errors.Is(err, unix.ENOENT) { |
| 248 | return fmt.Errorf("remove %s: %w", filepath.Join(d.path, name), err) |
| 249 | } |
| 250 | return nil |
| 251 | } |
| 252 | |
| 253 | // lockNamedRetryPause is the pause between lockNamed's retries after a |
| 254 | // holder's removal, capped at what is left of the wait. |
| 255 | const lockNamedRetryPause = 10 * time.Millisecond |
| 256 | |
| 257 | // lockNamedOpenAt opens the lock file for lockNamed. A test seam: a test |
| 258 | // can make every open meet the race with a holder's removal. |
| 259 | var lockNamedOpenAt = safefile.OpenFileAt |
| 260 | |
| 261 | // lockNamedOpened runs in lockNamed after each open of the lock file and |
| 262 | // before its flock. A test seam: the window in which the file a waiter has |
| 263 | // open can be removed by the holder before it and replaced by a new one. |
| 264 | var lockNamedOpened = func(name string) {} |
| 265 | |
| 266 | // WriteDurable writes data to .moves/name the way writeDurable does (a |
| 267 | // temp file, fsync, rename, fsync of the directory), all relative to the |
| 268 | // directory's fd. The temp name is random, so two writers never share one. |
| 269 | func (d *movesDir) WriteDurable(name string, data []byte, mode os.FileMode) error { |
| 270 | if writeHook != nil { |
| 271 | writeHook(name) |
| 272 | } |
| 273 | return safefile.WriteAt(d.fd, name, data, mode) |
| 274 | } |
| 275 | |
| 276 | // fchownToService hands a directory root just created to the service user, |
| 277 | // best-effort: a process that is not root, or a box with no such user (a |
| 278 | // test), keeps the owner it has. |
| 279 | func fchownToService(fd int) { |
| 280 | if os.Geteuid() != 0 { |
| 281 | return |
| 282 | } |
| 283 | u, err := user.Lookup(serviceUser) |
| 284 | if err != nil { |
| 285 | return |
| 286 | } |
| 287 | uid, uerr := strconv.Atoi(u.Uid) |
| 288 | gid, gerr := strconv.Atoi(u.Gid) |
| 289 | if uerr != nil || gerr != nil { |
| 290 | return |
| 291 | } |
| 292 | syscall.Fchown(fd, uid, gid) |
| 293 | } |
| 294 | |
| 295 | // floorLockCannotExist reports whether err, from opening or locking the |
| 296 | // floor lock, means the lock cannot exist on this box at all: .moves or |
| 297 | // floor.lock cannot be created (a read-only, full or unwritable data dir). |
| 298 | // Then no writer can arm a floor either, which is what makes the F15 |
| 299 | // fallback safe. Contention (errFloorLockBusy) is never this: a writer is |
| 300 | // active. |
| 301 | func floorLockCannotExist(err error) bool { |
| 302 | if errors.Is(err, errFloorLockBusy) { |
| 303 | return false |
| 304 | } |
| 305 | for _, e := range []syscall.Errno{syscall.EROFS, syscall.ENOSPC, syscall.EACCES, syscall.EPERM, syscall.EDQUOT} { |
| 306 | if errors.Is(err, e) { |
| 307 | return true |
| 308 | } |
| 309 | } |
| 310 | return false |
| 311 | } |
| 312 | |
amadan