amadan

carlos / platform Public

hostfiles: SyncDir stages a directory's differences, and never follows a link

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Paul Campbell pushed by claude a550cbdca19d5b7949ed8572e52cab97d8f5d9a9 parent 4517c98
6 files changed, +840 −0
  • internal/hostfiles/dir_other.go +26 −0
  • internal/hostfiles/dir_unix.go +172 −0
  • internal/hostfiles/sync.go +125 −0
  • internal/hostfiles/sync_test.go +190 −0
  • internal/hostfiles/sync_unix_test.go +254 −0
  • internal/hostfiles/walk_unix.go +73 −0
diff --git a/internal/hostfiles/dir_other.go b/internal/hostfiles/dir_other.go
new file mode 100644
index 000000000..d56a54449
--- /dev/null
+++ b/internal/hostfiles/dir_other.go
@@ -0,0 +1,26 @@
+//go:build !unix
+
+package hostfiles
+
+import "errors"
+
+// The publisher reads a files directory by descriptor, without following
+// links, which needs openat. The agent runs on Linux; elsewhere this
+// package builds, restores and reads manifests, and does not publish.
+var errNoPublish = errors.New("hostfiles: publishing a directory is not supported on this OS")
+
+type dirHandle struct{}
+
+func openTree(path string) (*dirHandle, error) { return nil, errNoPublish }
+func (d *dirHandle) Close() error { return nil }
+func (d *dirHandle) Name() string { return "" }
+
+func syncTree(d *dirHandle, p *Publisher, skip func(string, bool) bool, visit func(*dirHandle, string, string) error) error {
+ return errNoPublish
+}
+
+func (d *dirHandle) descend(rel string) (*dirHandle, bool, error) { return nil, false, errNoPublish }
+
+func readStable(d *dirHandle, name string) ([]byte, uint32, fileState, error) {
+ return nil, 0, fileUnreadable, errNoPublish
+}
diff --git a/internal/hostfiles/dir_unix.go b/internal/hostfiles/dir_unix.go
new file mode 100644
index 000000000..0ed6696f7
--- /dev/null
+++ b/internal/hostfiles/dir_unix.go
@@ -0,0 +1,172 @@
+//go:build unix
+
+package hostfiles
+
+import (
+ "errors"
+ "io"
+ "os"
+ "sort"
+ "strings"
+
+ "golang.org/x/sys/unix"
+)
+
+// dirHandle is a directory held open by descriptor. Everything beneath it
+// is reached one path component at a time with openat and O_NOFOLLOW, so
+// no symlink is ever followed, inside the tree or out of it, and a
+// directory swapped for something else after it was looked at is refused
+// rather than entered. Every open is non-blocking, so a FIFO swapped in
+// for a file or a directory cannot hang the publisher.
+type dirHandle struct{ f *os.File }
+
+// openTree opens the root of a files directory. The path to the root is
+// the platform's own and may contain symlinks; nothing beneath it may.
+func openTree(path string) (*dirHandle, error) {
+ fd, err := unix.Open(path, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_CLOEXEC|unix.O_NONBLOCK, 0)
+ if err != nil {
+ return nil, &os.PathError{Op: "open", Path: path, Err: err}
+ }
+ return &dirHandle{os.NewFile(uintptr(fd), path)}, nil
+}
+
+func (d *dirHandle) Close() error { return d.f.Close() }
+func (d *dirHandle) Name() string { return d.f.Name() }
+
+// openDir opens the directory name inside d. A name that is not a real
+// directory (a symlink, a file, a FIFO) fails with ENOTDIR or ELOOP
+// without blocking; a missing one with ENOENT.
+func (d *dirHandle) openDir(name string) (*dirHandle, error) {
+ fd, err := unix.Openat(int(d.f.Fd()), name, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_NOFOLLOW|unix.O_CLOEXEC|unix.O_NONBLOCK, 0)
+ if err != nil {
+ return nil, &os.PathError{Op: "openat", Path: name, Err: err}
+ }
+ return &dirHandle{os.NewFile(uintptr(fd), name)}, nil
+}
+
+// names lists d's entries, sorted.
+func (d *dirHandle) names() ([]string, error) {
+ if _, err := d.f.Seek(0, io.SeekStart); err != nil {
+ return nil, err
+ }
+ names, err := d.f.Readdirnames(-1)
+ if err != nil {
+ return nil, err
+ }
+ sort.Strings(names)
+ return names, nil
+}
+
+// look inspects name inside d without following a symlink.
+func (d *dirHandle) look(name string) (unix.Stat_t, error) {
+ var st unix.Stat_t
+ err := unix.Fstatat(int(d.f.Fd()), name, &st, unix.AT_SYMLINK_NOFOLLOW)
+ return st, err
+}
+
+func isDir(st unix.Stat_t) bool { return st.Mode&unix.S_IFMT == unix.S_IFDIR }
+func isRegular(st unix.Stat_t) bool { return st.Mode&unix.S_IFMT == unix.S_IFREG }
+
+func sameFile(a, b unix.Stat_t) bool { return a.Dev == b.Dev && a.Ino == b.Ino }
+
+func sameImage(a, b unix.Stat_t) bool {
+ return sameFile(a, b) && a.Size == b.Size && a.Mtim == b.Mtim
+}
+
+// notADirectory reports the errors openDir gives for a name that exists
+// but is not a real directory.
+func notADirectory(err error) bool {
+ return errors.Is(err, unix.ENOTDIR) || errors.Is(err, unix.ELOOP)
+}
+
+// descend opens the directory rel ("" or "a/b") beneath d, one component
+// at a time. gone is true when some component is missing or is not a real
+// directory: nothing of ours lives at a path under it.
+func (d *dirHandle) descend(rel string) (sub *dirHandle, gone bool, err error) {
+ cur, owned := d, false
+ if rel == "" || rel == "." {
+ return d, false, nil
+ }
+ for _, part := range strings.Split(rel, "/") {
+ betweenLookAndOpen()
+ next, err := cur.openDir(part)
+ if owned {
+ cur.Close()
+ }
+ if err != nil {
+ if errors.Is(err, unix.ENOENT) || notADirectory(err) {
+ return nil, true, nil
+ }
+ return nil, false, err
+ }
+ cur, owned = next, true
+ }
+ return cur, false, nil
+}
+
+// readStable reads the file name inside d for publishing.
+//
+// It reads only what it opened without following a link and found to be
+// the same regular file it had just inspected. A file over MaxFileBytes is
+// refused on its size alone, before any read. What it reports:
+//
+// - fileGone: nothing by that name.
+// - fileNotRegular: something by that name that is not a regular file.
+// - fileUnreadable: a regular file that could not be opened or read. It
+// still exists, so the caller must keep what the manifest has for it.
+// - fileRetry: the regular file was replaced by another regular file
+// between the look and the open. Nothing was read; ask again.
+// - fileTorn: read, but its size, mtime or identity changed meanwhile.
+// The bytes are a possibly inconsistent image.
+func readStable(d *dirHandle, name string) ([]byte, uint32, fileState, error) {
+ before, err := d.look(name)
+ if errors.Is(err, unix.ENOENT) {
+ return nil, 0, fileGone, nil
+ }
+ if err != nil {
+ return nil, 0, fileUnreadable, nil
+ }
+ if !isRegular(before) {
+ return nil, 0, fileNotRegular, nil
+ }
+ if before.Size > MaxFileBytes {
+ return nil, 0, fileTooLarge, nil
+ }
+ betweenLookAndOpen()
+ fd, err := unix.Openat(int(d.f.Fd()), name, unix.O_RDONLY|unix.O_NOFOLLOW|unix.O_CLOEXEC|unix.O_NONBLOCK, 0)
+ switch {
+ case errors.Is(err, unix.ENOENT):
+ return nil, 0, fileGone, nil
+ case errors.Is(err, unix.ELOOP):
+ return nil, 0, fileNotRegular, nil // it became a symlink
+ case err != nil:
+ return nil, 0, fileUnreadable, nil
+ }
+ f := os.NewFile(uintptr(fd), name)
+ defer f.Close()
+ var opened unix.Stat_t
+ if err := unix.Fstat(fd, &opened); err != nil {
+ return nil, 0, fileUnreadable, nil
+ }
+ if !isRegular(opened) {
+ return nil, 0, fileNotRegular, nil // swapped for a FIFO, a device, a directory
+ }
+ if !sameFile(before, opened) {
+ return nil, 0, fileRetry, nil
+ }
+ if opened.Size > MaxFileBytes {
+ return nil, 0, fileTooLarge, nil
+ }
+ body, err := readAll(io.LimitReader(f, MaxFileBytes+1))
+ if err != nil {
+ return nil, 0, fileUnreadable, nil
+ }
+ if int64(len(body)) > MaxFileBytes {
+ return nil, 0, fileTooLarge, nil // it grew past the cap while being read
+ }
+ state := fileOK
+ if after, err := d.look(name); err != nil || !sameImage(before, after) {
+ state = fileTorn
+ }
+ return body, uint32(before.Mode & 0o777), state, nil
+}
diff --git a/internal/hostfiles/sync.go b/internal/hostfiles/sync.go
new file mode 100644
index 000000000..8604a1e1f
--- /dev/null
+++ b/internal/hostfiles/sync.go
@@ -0,0 +1,125 @@
+package hostfiles
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "path"
+ "sort"
+)
+
+type fileState int
+
+const (
+ fileOK fileState = iota
+ fileTorn
+ fileGone
+ fileNotRegular
+ fileTooLarge
+ fileUnreadable
+ fileRetry
+)
+
+// Test seams. betweenLookAndOpen runs after something has been inspected
+// and before it is opened; readAll is the one place file contents are read.
+var (
+ betweenLookAndOpen = func() {}
+ readAll = io.ReadAll
+)
+
+type SyncStats struct {
+ Published, Deleted int
+ TooLarge, NotRegular, Unreadable []string
+}
+
+// skipped reports whether rel, or any directory above it, is excluded.
+func skipped(skip func(string, bool) bool, rel string, isDir bool) bool {
+ if skip == nil {
+ return false
+ }
+ if skip(rel, isDir) {
+ return true
+ }
+ for dir := path.Dir(rel); dir != "."; dir = path.Dir(dir) {
+ if skip(dir, true) {
+ return true
+ }
+ }
+ return false
+}
+
+// SyncDir stages every difference between dir and the manifest: changed
+// and new regular files, and a tombstone for each manifest path that is
+// no longer there. It does not commit. skip, when non-nil, excludes a
+// relative path and, for a directory, everything under it; an excluded
+// path is left exactly as the manifest has it.
+//
+// A tombstone needs proof of absence. A file that exists but could not be
+// read keeps its manifest entry and is reported in Unreadable. A root
+// that cannot be opened is an error and stages nothing: an unreadable
+// directory is not an empty one.
+func SyncDir(ctx context.Context, p *Publisher, dir string, skip func(rel string, isDir bool) bool) (SyncStats, error) {
+ root, err := openTree(dir)
+ if err != nil {
+ return SyncStats{}, fmt.Errorf("hostfiles: sync %s: %w", dir, err)
+ }
+ defer root.Close()
+ return syncRoot(ctx, p, root, skip)
+}
+
+func syncRoot(ctx context.Context, p *Publisher, root *dirHandle, skip func(rel string, isDir bool) bool) (SyncStats, error) {
+ var st SyncStats
+ seen := map[string]bool{}
+ err := syncTree(root, p, skip, func(d *dirHandle, name, rel string) error {
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ body, mode, state, err := readStable(d, name)
+ if err != nil {
+ return err
+ }
+ switch state {
+ case fileGone:
+ return nil
+ case fileNotRegular:
+ st.NotRegular = append(st.NotRegular, rel)
+ return nil
+ case fileUnreadable:
+ seen[rel] = true // it exists: keep what the manifest has
+ st.Unreadable = append(st.Unreadable, rel)
+ return nil
+ case fileRetry:
+ seen[rel] = true // replaced mid-look: the next scan reads it
+ return nil
+ case fileTooLarge:
+ seen[rel] = true
+ st.TooLarge = append(st.TooLarge, rel)
+ return p.StageSkipped(rel, "too-large")
+ }
+ seen[rel] = true
+ before := p.pending()
+ if err := p.Stage(ctx, rel, body, mode, state == fileTorn); err != nil {
+ return err
+ }
+ if p.pending() != before {
+ st.Published++
+ }
+ return nil
+ })
+ if err != nil {
+ return st, fmt.Errorf("hostfiles: sync %s: %w", root.Name(), err)
+ }
+ for _, rel := range p.Paths() {
+ if seen[rel] || skipped(skip, rel, false) {
+ continue
+ }
+ if err := p.StageDelete(rel); err != nil {
+ return st, err
+ }
+ st.Deleted++
+ }
+ sort.Strings(st.TooLarge)
+ sort.Strings(st.NotRegular)
+ sort.Strings(st.Unreadable)
+ return st, nil
+}
diff --git a/internal/hostfiles/sync_test.go b/internal/hostfiles/sync_test.go
new file mode 100644
index 000000000..9d79c771f
--- /dev/null
+++ b/internal/hostfiles/sync_test.go
@@ -0,0 +1,190 @@
+//go:build unix
+
+package hostfiles
+
+import (
+ "context"
+ "io"
+ "os"
+ "path/filepath"
+ "reflect"
+ "testing"
+
+ "github.com/carlosframework/platform/internal/release"
+)
+
+func write(t *testing.T, dir, rel, body string) {
+ t.Helper()
+ full := filepath.Join(dir, rel)
+ if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(full, []byte(body), 0o640); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestSyncDirPublishesNewChangedAndDeleted(t *testing.T) {
+ ctx, s, src := context.Background(), release.NewDirStore(t.TempDir()), t.TempDir()
+ p := open(t, s)
+ write(t, src, "instance.key", "k")
+ write(t, src, "uploads/a.txt", "a")
+ st, err := SyncDir(ctx, p, src, nil)
+ if err != nil || st.Published != 2 {
+ t.Fatalf("first sync: %+v, %v", st, err)
+ }
+ commit(t, p)
+
+ write(t, src, "uploads/a.txt", "a2")
+ if err := os.Remove(filepath.Join(src, "instance.key")); err != nil {
+ t.Fatal(err)
+ }
+ write(t, src, "new.txt", "n")
+ st, err = SyncDir(ctx, p, src, nil)
+ if err != nil || st.Published != 2 || st.Deleted != 1 {
+ t.Fatalf("second sync: %+v, %v", st, err)
+ }
+ commit(t, p)
+
+ dst := t.TempDir()
+ if _, _, err := Restore(ctx, s, pfx, dst); err != nil {
+ t.Fatal(err)
+ }
+ if read(t, dst, "uploads/a.txt") != "a2" || read(t, dst, "new.txt") != "n" {
+ t.Fatal("content differs")
+ }
+ if _, err := os.Stat(filepath.Join(dst, "instance.key")); !os.IsNotExist(err) {
+ t.Fatal("deleted file restored")
+ }
+ fi, _ := os.Stat(filepath.Join(dst, "new.txt"))
+ if fi.Mode().Perm() != 0o640 {
+ t.Fatalf("mode = %v, want 0640", fi.Mode().Perm())
+ }
+}
+
+func TestSyncDirOfAnUnchangedDirectoryStagesNothing(t *testing.T) {
+ ctx, s, src := context.Background(), release.NewDirStore(t.TempDir()), t.TempDir()
+ p := open(t, s)
+ write(t, src, "a", "a")
+ if _, err := SyncDir(ctx, p, src, nil); err != nil {
+ t.Fatal(err)
+ }
+ commit(t, p)
+ st, err := SyncDir(ctx, p, src, nil)
+ if err != nil || st.Published != 0 || st.Deleted != 0 || p.Dirty() {
+ t.Fatalf("second sync: %+v dirty=%v err=%v", st, p.Dirty(), err)
+ }
+}
+
+// Review Focus 2: never follow or open what is not a regular file.
+func TestSyncDirSkipsWhatIsNotARegularFile(t *testing.T) {
+ ctx, s, src := context.Background(), release.NewDirStore(t.TempDir()), t.TempDir()
+ outside := filepath.Join(t.TempDir(), "secret")
+ if err := os.WriteFile(outside, []byte("do not publish"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ write(t, src, "real.txt", "r")
+ if err := os.Symlink(outside, filepath.Join(src, "link")); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Symlink(filepath.Dir(outside), filepath.Join(src, "linkdir")); err != nil {
+ t.Fatal(err)
+ }
+ p := open(t, s)
+ st, err := SyncDir(ctx, p, src, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if st.Published != 1 || !reflect.DeepEqual(st.NotRegular, []string{"link", "linkdir"}) {
+ t.Fatalf("stats = %+v", st)
+ }
+ if got := p.Paths(); !reflect.DeepEqual(got, []string{"real.txt"}) {
+ t.Fatalf("Paths = %v", got)
+ }
+}
+
+// Review Focus 3: too large is recorded, never read whole, never dropped.
+func TestSyncDirRecordsAFileOverTheCapWithoutReadingIt(t *testing.T) {
+ ctx, s, src := context.Background(), release.NewDirStore(t.TempDir()), t.TempDir()
+ big := filepath.Join(src, "huge.bin")
+ f, err := os.Create(big)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := f.Truncate(MaxFileBytes + 1); err != nil { // sparse: costs no disk
+ t.Fatal(err)
+ }
+ f.Close()
+ reads := 0
+ orig := readAll
+ readAll = func(r io.Reader) ([]byte, error) { reads++; return orig(r) }
+ t.Cleanup(func() { readAll = orig })
+ p := open(t, s)
+ st, err := SyncDir(ctx, p, src, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !reflect.DeepEqual(st.TooLarge, []string{"huge.bin"}) || st.Published != 0 {
+ t.Fatalf("stats = %+v", st)
+ }
+ if reads != 0 {
+ t.Fatalf("the oversized file was read %d times; it must be refused on its size alone", reads)
+ }
+ commit(t, p)
+ _, rs, err := Restore(ctx, s, pfx, t.TempDir())
+ if err != nil || !reflect.DeepEqual(rs.Skipped, []string{"huge.bin"}) {
+ t.Fatalf("restore skipped %v, err %v", rs.Skipped, err)
+ }
+}
+
+func TestSyncDirHonoursSkip(t *testing.T) {
+ ctx, s, src := context.Background(), release.NewDirStore(t.TempDir()), t.TempDir()
+ write(t, src, "keep.txt", "k")
+ write(t, src, ".files-gen", "marker")
+ write(t, src, "cache/x", "x")
+ p := open(t, s)
+ skip := func(rel string, isDir bool) bool { return rel == ".files-gen" || (isDir && rel == "cache") }
+ if _, err := SyncDir(ctx, p, src, skip); err != nil {
+ t.Fatal(err)
+ }
+ if got := p.Paths(); !reflect.DeepEqual(got, []string{"keep.txt"}) {
+ t.Fatalf("Paths = %v", got)
+ }
+}
+
+func TestSyncDirOfAMissingDirectoryTombstonesNothing(t *testing.T) {
+ ctx, s, src := context.Background(), release.NewDirStore(t.TempDir()), t.TempDir()
+ p := open(t, s)
+ write(t, src, "a", "a")
+ if _, err := SyncDir(ctx, p, src, nil); err != nil {
+ t.Fatal(err)
+ }
+ commit(t, p)
+ if _, err := SyncDir(ctx, p, filepath.Join(src, "does-not-exist"), nil); err == nil {
+ t.Fatal("a missing root must be an error, not an empty directory")
+ }
+ if p.Dirty() {
+ t.Fatal("a missing root staged tombstones")
+ }
+}
+
+// An excluded directory's files, published before the exclusion applied,
+// are left as the manifest has them: not republished, not tombstoned.
+func TestSyncDirDoesNotTombstoneWhatASkippedDirectoryHolds(t *testing.T) {
+ ctx, s, src := context.Background(), release.NewDirStore(t.TempDir()), t.TempDir()
+ write(t, src, "cache/x", "x")
+ write(t, src, "keep.txt", "k")
+ p := open(t, s)
+ if _, err := SyncDir(ctx, p, src, nil); err != nil {
+ t.Fatal(err)
+ }
+ commit(t, p)
+ skip := func(rel string, isDir bool) bool { return isDir && rel == "cache" }
+ st, err := SyncDir(ctx, p, src, skip)
+ if err != nil || st.Deleted != 0 || p.Dirty() {
+ t.Fatalf("stats = %+v dirty=%v err=%v", st, p.Dirty(), err)
+ }
+ if got := p.Paths(); !reflect.DeepEqual(got, []string{"cache/x", "keep.txt"}) {
+ t.Fatalf("Paths = %v", got)
+ }
+}
diff --git a/internal/hostfiles/sync_unix_test.go b/internal/hostfiles/sync_unix_test.go
new file mode 100644
index 000000000..c9c9c974a
--- /dev/null
+++ b/internal/hostfiles/sync_unix_test.go
@@ -0,0 +1,254 @@
+//go:build unix
+
+package hostfiles
+
+import (
+ "context"
+ "net"
+ "os"
+ "path/filepath"
+ "reflect"
+ "syscall"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// within fails the test if fn has not returned in two seconds: the way a
+// blocked open shows up.
+func within(t *testing.T, what string, fn func()) {
+ t.Helper()
+ done := make(chan struct{})
+ go func() { defer close(done); fn() }()
+ select {
+ case <-done:
+ case <-time.After(2 * time.Second):
+ t.Fatalf("%s did not return: something blocked", what)
+ }
+}
+
+// swapAt arranges for fn to run once, at the nth pause between a look and
+// an open (1 is the first).
+func swapAt(t *testing.T, n int, fn func()) {
+ t.Helper()
+ orig, calls := betweenLookAndOpen, 0
+ t.Cleanup(func() { betweenLookAndOpen = orig })
+ betweenLookAndOpen = func() {
+ calls++
+ if calls == n {
+ fn()
+ }
+ }
+}
+
+func tree(t *testing.T, dir string) *dirHandle {
+ t.Helper()
+ d, err := openTree(dir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { d.Close() })
+ return d
+}
+
+func fifo(t *testing.T, path string) {
+ t.Helper()
+ if err := syscall.Mkfifo(path, 0o600); err != nil {
+ t.Skipf("no FIFOs here: %v", err)
+ }
+}
+
+// Review Focus 2: special files are skipped, never opened for reading.
+func TestSyncDirSkipsAFIFO(t *testing.T) {
+ ctx, s, src := context.Background(), release.NewDirStore(t.TempDir()), t.TempDir()
+ write(t, src, "real.txt", "r")
+ fifo(t, filepath.Join(src, "pipe"))
+ p := open(t, s)
+ var st SyncStats
+ within(t, "SyncDir", func() { st, _ = SyncDir(ctx, p, src, nil) })
+ if st.Published != 1 || !reflect.DeepEqual(st.NotRegular, []string{"pipe"}) {
+ t.Fatalf("stats = %+v", st)
+ }
+}
+
+func TestSyncDirSkipsASocket(t *testing.T) {
+ ctx, s, src := context.Background(), release.NewDirStore(t.TempDir()), t.TempDir()
+ l, err := net.Listen("unix", filepath.Join(src, "sock"))
+ if err != nil {
+ t.Skipf("no Unix sockets here: %v", err)
+ }
+ defer l.Close()
+ p := open(t, s)
+ var st SyncStats
+ within(t, "SyncDir", func() { st, _ = SyncDir(ctx, p, src, nil) })
+ if !reflect.DeepEqual(st.NotRegular, []string{"sock"}) || len(p.Paths()) != 0 {
+ t.Fatalf("stats = %+v, paths = %v", st, p.Paths())
+ }
+}
+
+func TestSyncDirSkipsADevice(t *testing.T) {
+ if os.Getuid() != 0 {
+ t.Skip("making a device node needs root")
+ }
+ ctx, s, src := context.Background(), release.NewDirStore(t.TempDir()), t.TempDir()
+ if err := syscall.Mknod(filepath.Join(src, "null"), syscall.S_IFCHR|0o600, 1<<8|3); err != nil {
+ t.Skipf("mknod: %v", err)
+ }
+ p := open(t, s)
+ st, err := SyncDir(ctx, p, src, nil)
+ if err != nil || !reflect.DeepEqual(st.NotRegular, []string{"null"}) {
+ t.Fatalf("stats = %+v, err = %v", st, err)
+ }
+}
+
+// A symlinked directory is never descended into, wherever it points.
+func TestSyncDirDoesNotDescendASymlinkedDirectory(t *testing.T) {
+ ctx, s, src, outside := context.Background(), release.NewDirStore(t.TempDir()), t.TempDir(), t.TempDir()
+ if err := os.WriteFile(filepath.Join(outside, "secret"), []byte("s"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ write(t, src, "inside/own", "o")
+ for link, target := range map[string]string{"out": outside, "in": "inside"} {
+ if err := os.Symlink(target, filepath.Join(src, link)); err != nil {
+ t.Fatal(err)
+ }
+ }
+ p := open(t, s)
+ st, err := SyncDir(ctx, p, src, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got := p.Paths(); !reflect.DeepEqual(got, []string{"inside/own"}) {
+ t.Fatalf("Paths = %v", got)
+ }
+ if !reflect.DeepEqual(st.NotRegular, []string{"in", "out"}) {
+ t.Fatalf("NotRegular = %v", st.NotRegular)
+ }
+}
+
+// A regular file swapped for a FIFO after it was inspected must neither
+// block the open nor be read.
+func TestReadStableDoesNotBlockOnAFIFOSwappedIn(t *testing.T) {
+ dir := t.TempDir()
+ write(t, dir, "f", "regular")
+ d := tree(t, dir)
+ swapAt(t, 1, func() {
+ full := filepath.Join(dir, "f")
+ if err := os.Remove(full); err != nil {
+ t.Error(err)
+ }
+ if err := syscall.Mkfifo(full, 0o600); err != nil {
+ t.Error(err)
+ }
+ })
+ var body []byte
+ var state fileState
+ within(t, "readStable", func() { body, _, state, _ = readStable(d, "f") })
+ if state != fileNotRegular || body != nil {
+ t.Fatalf("state = %v, %d bytes; want fileNotRegular and nothing read", state, len(body))
+ }
+}
+
+// A directory swapped for a FIFO between being listed and being entered
+// must not hang the scan.
+func TestSyncDirDoesNotBlockOnADirectorySwappedForAFIFO(t *testing.T) {
+ ctx, s, src := context.Background(), release.NewDirStore(t.TempDir()), t.TempDir()
+ write(t, src, "d/child", "c")
+ write(t, src, "z.txt", "z")
+ swapAt(t, 1, func() { // the first pause is before entering "d"
+ if err := os.RemoveAll(filepath.Join(src, "d")); err != nil {
+ t.Error(err)
+ }
+ if err := syscall.Mkfifo(filepath.Join(src, "d"), 0o600); err != nil {
+ t.Error(err)
+ }
+ })
+ p := open(t, s)
+ var st SyncStats
+ var err error
+ within(t, "SyncDir", func() { st, err = SyncDir(ctx, p, src, nil) })
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !reflect.DeepEqual(st.NotRegular, []string{"d"}) || !reflect.DeepEqual(p.Paths(), []string{"z.txt"}) {
+ t.Fatalf("stats = %+v, paths = %v", st, p.Paths())
+ }
+}
+
+// Plan round 2, finding 2: a file that still exists is never tombstoned
+// just because it could not be read.
+func TestSyncDirKeepsAnUnreadableFilesPublishedEntry(t *testing.T) {
+ if os.Getuid() == 0 {
+ t.Skip("root can read anything")
+ }
+ ctx, s, src := context.Background(), release.NewDirStore(t.TempDir()), t.TempDir()
+ write(t, src, "locked", "v1")
+ p := open(t, s)
+ if _, err := SyncDir(ctx, p, src, nil); err != nil {
+ t.Fatal(err)
+ }
+ commit(t, p)
+ if err := os.Chmod(filepath.Join(src, "locked"), 0); err != nil {
+ t.Fatal(err)
+ }
+ st, err := SyncDir(ctx, p, src, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if st.Deleted != 0 || p.Dirty() || !reflect.DeepEqual(st.Unreadable, []string{"locked"}) {
+ t.Fatalf("stats = %+v dirty = %v", st, p.Dirty())
+ }
+}
+
+// A regular file replaced by another regular file between the look and
+// the open is neither read nor declared gone: the caller asks again.
+func TestReadStableAsksAgainWhenARegularFileIsReplacedMidLook(t *testing.T) {
+ dir := t.TempDir()
+ write(t, dir, "f", "v1")
+ d := tree(t, dir)
+ swapAt(t, 1, func() {
+ tmp := filepath.Join(dir, "f.new")
+ if err := os.WriteFile(tmp, []byte("v2"), 0o640); err != nil {
+ t.Error(err)
+ }
+ if err := os.Rename(tmp, filepath.Join(dir, "f")); err != nil {
+ t.Error(err)
+ }
+ })
+ body, _, state, err := readStable(d, "f")
+ if err != nil || state != fileRetry || body != nil {
+ t.Fatalf("state = %v, body = %q, err = %v; want fileRetry and nothing read", state, body, err)
+ }
+ body, _, state, err = readStable(d, "f")
+ if err != nil || state != fileOK || string(body) != "v2" {
+ t.Fatalf("second look: state = %v, body = %q, err = %v", state, body, err)
+ }
+}
+
+// And through a scan: the entry the manifest already has is kept.
+func TestSyncDirKeepsAnEntryWhoseFileWasReplacedMidLook(t *testing.T) {
+ ctx, s, src := context.Background(), release.NewDirStore(t.TempDir()), t.TempDir()
+ write(t, src, "f", "v1")
+ p := open(t, s)
+ if _, err := SyncDir(ctx, p, src, nil); err != nil {
+ t.Fatal(err)
+ }
+ commit(t, p)
+ swapAt(t, 1, func() {
+ tmp := filepath.Join(src, "f.new")
+ if err := os.WriteFile(tmp, []byte("v2"), 0o640); err != nil {
+ t.Error(err)
+ }
+ if err := os.Rename(tmp, filepath.Join(src, "f")); err != nil {
+ t.Error(err)
+ }
+ })
+ st, err := SyncDir(ctx, p, src, nil)
+ if err != nil || st.Deleted != 0 || p.Dirty() {
+ t.Fatalf("stats = %+v dirty = %v err = %v; the replaced file must be left for the next scan", st, p.Dirty(), err)
+ }
+ if _, err := SyncDir(ctx, p, src, nil); err != nil || !p.Dirty() {
+ t.Fatalf("the next scan did not pick the replacement up: dirty = %v err = %v", p.Dirty(), err)
+ }
+}
diff --git a/internal/hostfiles/walk_unix.go b/internal/hostfiles/walk_unix.go
new file mode 100644
index 000000000..861075714
--- /dev/null
+++ b/internal/hostfiles/walk_unix.go
@@ -0,0 +1,73 @@
+//go:build unix
+
+package hostfiles
+
+import (
+ "errors"
+
+ "golang.org/x/sys/unix"
+)
+
+// syncTree walks the tree beneath root by descriptor and calls visit for
+// every entry that is not a directory, with the open directory that holds
+// it, its name there, and its path relative to root. It never follows a
+// symlink and never blocks on a FIFO: a directory is entered only through
+// openDir.
+func syncTree(root *dirHandle, p *Publisher, skip func(string, bool) bool, visit func(d *dirHandle, name, rel string) error) error {
+ return walkDir(root, "", skip, visit)
+}
+
+func walkDir(d *dirHandle, prefix string, skip func(string, bool) bool, visit func(d *dirHandle, name, rel string) error) error {
+ names, err := d.names()
+ if err != nil {
+ return err
+ }
+ for _, name := range names {
+ rel := name
+ if prefix != "" {
+ rel = prefix + "/" + name
+ }
+ st, err := d.look(name)
+ if errors.Is(err, unix.ENOENT) {
+ continue
+ }
+ if err != nil {
+ return err
+ }
+ if !isDir(st) {
+ if skipped(skip, rel, false) {
+ continue
+ }
+ if err := visit(d, name, rel); err != nil {
+ return err
+ }
+ continue
+ }
+ if skipped(skip, rel, true) {
+ continue
+ }
+ betweenLookAndOpen()
+ sub, err := d.openDir(name)
+ if err != nil {
+ if errors.Is(err, unix.ENOENT) {
+ continue
+ }
+ if notADirectory(err) {
+ // It was a directory a moment ago and is something else
+ // now. Let readStable say what: it will not follow or
+ // block on it either.
+ if err := visit(d, name, rel); err != nil {
+ return err
+ }
+ continue
+ }
+ return err
+ }
+ err = walkDir(sub, rel, skip, visit)
+ sub.Close()
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+}