amadan

carlos / platform Public

rootfsimage: OCI layouts, save archives and layer flattening (rootfs Plan 1 Task 4)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude e9918836d0950707089d4c2968839e9dca84cd78 parent ecb1b1d
9 files changed, +2559 −0
  • approved_rootfs_copy_test.go +2 −0
  • internal/rootfsimage/archive.go +160 −0
  • internal/rootfsimage/errors.go +89 −0
  • internal/rootfsimage/flatten.go +270 −0
  • internal/rootfsimage/flatten_test.go +431 −0
  • internal/rootfsimage/oci.go +475 −0
  • internal/rootfsimage/oci_test.go +377 −0
  • internal/rootfsimage/testimage_test.go +426 −0
  • internal/rootfsimage/tree.go +329 −0
diff --git a/approved_rootfs_copy_test.go b/approved_rootfs_copy_test.go
index a71e43a6b..eaa9c6240 100644
--- a/approved_rootfs_copy_test.go
+++ b/approved_rootfs_copy_test.go
@@ -150,6 +150,8 @@ var rootfsCopyBuilt = []string{
"RFS-1-C01", "RFS-1-C02", "RFS-1-C03", "RFS-1-C04", "RFS-1-C05", "RFS-1-C06", "RFS-1-C07", "RFS-1-C08", "RFS-1-C09", "RFS-1-C10", "RFS-1-C11", "RFS-1-C12", "RFS-1-C13", "RFS-1-C14", "RFS-1-C15", "RFS-1-C16", "RFS-1-C17",
// Task 3: kind and hibernation fixed at ship (internal/release/rootfs_ship.go)
"RFS-1-C18", "RFS-1-C19", "RFS-1-C20", "RFS-1-C21", "RFS-1-C80", "RFS-1-C81", "RFS-1-C82", "RFS-1-C83",
+ // Task 4: image sources and flattening (internal/rootfsimage/errors.go)
+ "RFS-1-C45", "RFS-1-C46", "RFS-1-C50", "RFS-1-C51", "RFS-1-C53", "RFS-1-C55", "RFS-1-C78",
}
func mustParseRootfsRecord(t *testing.T) []copyRow {
diff --git a/internal/rootfsimage/archive.go b/internal/rootfsimage/archive.go
new file mode 100644
index 000000000..3c46f0eb3
--- /dev/null
+++ b/internal/rootfsimage/archive.go
@@ -0,0 +1,160 @@
+package rootfsimage
+
+import (
+ "archive/tar"
+ "encoding/json"
+ "fmt"
+ "io"
+ "os"
+ "path"
+ "strings"
+)
+
+// maxArchiveEntries bounds the entries of a save archive's outer tar.
+const maxArchiveEntries = 1 << 16
+
+// OpenArchive flattens the image of a docker save or podman save archive
+// into a new tree spooled under spool. The archive is indexed once (each
+// regular entry's name, data offset and size) and its files are then read
+// in place as sections of the one file: nothing is extracted, so no name in
+// the archive ever reaches the disk. With oci-layout and index.json it is an
+// OCI layout in a tar (Docker 25 and later); otherwise its manifest.json is
+// the legacy form, whose layers carry no digests.
+func OpenArchive(archivePath, spool string) (*Tree, error) {
+ fi, err := os.Stat(archivePath)
+ if err != nil {
+ return nil, err
+ }
+ if !fi.Mode().IsRegular() {
+ return nil, NotAnImageError{Source: archivePath}
+ }
+ f, err := os.Open(archivePath)
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+ st, err := indexArchive(f, archivePath)
+ if err != nil {
+ return nil, err
+ }
+ switch {
+ case st.has("oci-layout") && st.has("index.json"):
+ return openOCI(st, archivePath, spool)
+ case st.has("manifest.json"):
+ return openLegacy(st, archivePath, spool)
+ }
+ return nil, NotAnImageError{Source: archivePath}
+}
+
+type section struct{ off, size int64 }
+
+// archiveStore serves an archive's regular entries as sections of the file.
+type archiveStore struct {
+ f *os.File
+ files map[string]section
+}
+
+func (a *archiveStore) open(name string) (io.ReadCloser, int64, error) {
+ s, ok := a.files[name]
+ if !ok {
+ return nil, 0, fmt.Errorf("rootfsimage: the archive has no %s", shown(name))
+ }
+ return io.NopCloser(io.NewSectionReader(a.f, s.off, s.size)), s.size, nil
+}
+
+func (a *archiveStore) has(name string) bool {
+ _, ok := a.files[name]
+ return ok
+}
+
+// indexArchive reads the outer tar's headers once. tar.Reader reads the
+// file in whole blocks with no read-ahead and seeks past each entry's data,
+// so the file offset after Next is where that entry's data starts. A file
+// that is not a tar at all is NotAnImageError.
+func indexArchive(f *os.File, source string) (*archiveStore, error) {
+ st := &archiveStore{f: f, files: map[string]section{}}
+ tr := tar.NewReader(f)
+ for n := 0; ; n++ {
+ hdr, err := tr.Next()
+ if err == io.EOF {
+ if n == 0 {
+ return nil, NotAnImageError{Source: source}
+ }
+ return st, nil
+ }
+ if err != nil {
+ if n == 0 {
+ return nil, NotAnImageError{Source: source}
+ }
+ return nil, fmt.Errorf("rootfsimage: reading %s: %w", shown(source), err)
+ }
+ if n >= maxArchiveEntries {
+ return nil, fmt.Errorf("%w: the archive has more than %d entries", errTooLarge, maxArchiveEntries)
+ }
+ if hdr.Typeflag != tar.TypeReg {
+ continue
+ }
+ off, err := f.Seek(0, io.SeekCurrent)
+ if err != nil {
+ return nil, err
+ }
+ name := archiveName(hdr.Name)
+ if _, dup := st.files[name]; dup {
+ return nil, fmt.Errorf("rootfsimage: %s holds %s twice", shown(source), shown(name))
+ }
+ st.files[name] = section{off: off, size: hdr.Size}
+ }
+}
+
+// archiveName is an entry's name as a key of the index. It is only ever
+// compared with the names a manifest gives (through the same function),
+// never opened.
+func archiveName(name string) string {
+ return path.Clean(strings.TrimLeft(name, "/"))
+}
+
+// openLegacy flattens the one image of a legacy save archive's
+// manifest.json: [{"Config": "<hash>.json", "Layers": ["<id>/layer.tar", ...]}].
+func openLegacy(st *archiveStore, source, spool string) (*Tree, error) {
+ raw, err := readFile(st, "manifest.json")
+ if err != nil {
+ return nil, err
+ }
+ var ms []struct {
+ Config string
+ Layers []string
+ }
+ if err := json.Unmarshal(raw, &ms); err != nil {
+ return nil, NotAnImageError{Source: source}
+ }
+ switch {
+ case len(ms) == 0:
+ return nil, NotAnImageError{Source: source}
+ case len(ms) > 1:
+ return nil, ManyImagesError{N: len(ms)}
+ }
+ raw, err = readFile(st, archiveName(ms[0].Config))
+ if err != nil {
+ return nil, err
+ }
+ var cfg imageConfig
+ if err := json.Unmarshal(raw, &cfg); err != nil {
+ return nil, fmt.Errorf("rootfsimage: %s: config: %w", shown(source), err)
+ }
+ t := NewTree(spool)
+ if t.Config, err = checkConfig(cfg, source); err != nil {
+ return nil, err
+ }
+ for _, l := range ms[0].Layers {
+ rc, _, err := st.open(archiveName(l))
+ if err != nil {
+ return nil, err
+ }
+ err = applyStream(t, rc, nil)
+ rc.Close()
+ if err != nil {
+ return nil, err
+ }
+ }
+ return t, nil
+}
diff --git a/internal/rootfsimage/errors.go b/internal/rootfsimage/errors.go
new file mode 100644
index 000000000..3e3ab2259
--- /dev/null
+++ b/internal/rootfsimage/errors.go
@@ -0,0 +1,89 @@
+package rootfsimage
+
+import (
+ "errors"
+ "fmt"
+ "strconv"
+ "strings"
+ "unicode"
+)
+
+// The package's refusals. Each Error() is its approved sentence from the
+// rootfs copy deck (docs/superpowers/plans/2026-10-24-rootfs-apps-copy.md),
+// so carlos ship can print it as it is.
+
+// ArchError refuses an image whose config names a platform other than
+// linux/arm64 (RFS-1-C45).
+type ArchError struct{ Platform string }
+
+func (e ArchError) Error() string {
+ return fmt.Sprintf("this image is built for %s, and boxes run linux/arm64. rebuild it for linux/arm64, for example with docker buildx build --platform linux/arm64", shown(e.Platform))
+}
+
+// NoArm64Error refuses an image index none of whose images is linux/arm64
+// (RFS-1-C46). Platforms are the ones it has, in the index's order.
+type NoArm64Error struct{ Platforms []string }
+
+func (e NoArm64Error) Error() string {
+ ps := make([]string, len(e.Platforms))
+ for i, p := range e.Platforms {
+ ps[i] = shown(p)
+ }
+ return fmt.Sprintf("this image has no linux/arm64 variant (it has %s). rebuild it for linux/arm64, for example with docker buildx build --platform linux/arm64", strings.Join(ps, ", "))
+}
+
+// NotAnImageError refuses a source that is neither an OCI layout, a save
+// archive nor a directory (RFS-1-C50).
+type NotAnImageError struct{ Source string }
+
+func (e NotAnImageError) Error() string {
+ return fmt.Sprintf("%s is not an OCI image, a docker save archive or a directory", shown(e.Source))
+}
+
+// UnsafePathError refuses a layer entry whose name leaves the root
+// (RFS-1-C51). Path is the name as the layer gives it.
+type UnsafePathError struct{ Path string }
+
+func (e UnsafePathError) Error() string {
+ return fmt.Sprintf("the image has an entry outside its root (%s), so it was not shipped", shown(e.Path))
+}
+
+// LinkError refuses a hardlink whose target is not a regular file of the
+// tree, and a path whose symlinks do not resolve within maxLinks (RFS-1-C53).
+type LinkError struct{ Path, Target string }
+
+func (e LinkError) Error() string {
+ return fmt.Sprintf("the image links %s to %s, which is not a file in the image", shown(e.Path), shown(e.Target))
+}
+
+// ManyImagesError refuses a source holding more than one image (RFS-1-C55).
+type ManyImagesError struct{ N int }
+
+func (e ManyImagesError) Error() string {
+ return fmt.Sprintf("the archive holds %d images. save one image and ship that", e.N)
+}
+
+// DigestError refuses a blob whose bytes do not match its descriptor's
+// digest or size (RFS-1-C78). Digest is "sha256:<hex>".
+type DigestError struct{ Digest string }
+
+func (e DigestError) Error() string {
+ return fmt.Sprintf("the image's layer %s does not match its digest, so it was not shipped", shown(e.Digest))
+}
+
+// errTooLarge is wrapped by the refusal of a tree past its bounds (maxBytes,
+// maxHeaders) and of a JSON document past maxJSONBytes. carlos ship's own
+// size refusal (RFS-1-C43) speaks for the built image; this one stops a
+// decompression bomb before the spool fills the disk.
+var errTooLarge = errors.New("rootfsimage: the image is too large to flatten")
+
+// shown is s as the sentences print it: as it is when every rune is
+// printable, else quoted, so a name in an image cannot drive the terminal.
+func shown(s string) string {
+ for _, r := range s {
+ if !unicode.IsPrint(r) {
+ return strconv.Quote(s)
+ }
+ }
+ return s
+}
diff --git a/internal/rootfsimage/flatten.go b/internal/rootfsimage/flatten.go
new file mode 100644
index 000000000..ffcacf009
--- /dev/null
+++ b/internal/rootfsimage/flatten.go
@@ -0,0 +1,270 @@
+package rootfsimage
+
+import (
+ "archive/tar"
+ "errors"
+ "fmt"
+ "io"
+ "io/fs"
+ "math"
+ "path"
+ "strings"
+ "time"
+)
+
+// The whiteout names of the OCI layer format.
+const (
+ whiteoutPrefix = ".wh."
+ whiteoutMeta = ".wh..wh."
+ whiteoutOpaque = ".wh..wh..opq"
+)
+
+// ApplyLayer applies one uncompressed layer tar on top of the tree, as an
+// extracting runtime would: whiteouts delete, an opaque marker hides the
+// lower layers' entries of its directory, a directory replaced by anything
+// else takes its subtree with it, and a name written beneath a symlinked
+// directory lands where the symlink points (clamped to the root). Device
+// files, sockets and pipes are counted in Skipped and left out.
+//
+// A name that leaves the root is UnsafePathError, a hardlink to anything but
+// a regular file of the tree, or a path needing more than 40 symlinks, is
+// LinkError. On an error the tree is part-applied and must be discarded.
+func (t *Tree) ApplyLayer(r io.Reader) error {
+ // The decoded stream is bounded as well as the files, so a layer of
+ // skipped entries cannot be an endless decompression either.
+ lr := &boundedReader{r: r, n: t.maxBytes - t.usedBytes + int64(t.maxHeaders-t.headers+1)*headerOverhead}
+ tr := tar.NewReader(lr)
+ added := map[string]bool{}
+ for {
+ hdr, err := tr.Next()
+ if err == io.EOF {
+ return nil
+ }
+ if err != nil {
+ return fmt.Errorf("rootfsimage: reading the layer: %w", err)
+ }
+ if hdr.Typeflag == tar.TypeXGlobalHeader {
+ continue
+ }
+ t.headers++
+ if t.headers > t.maxHeaders {
+ return fmt.Errorf("%w: it has more than %d entries", errTooLarge, t.maxHeaders)
+ }
+ if err := t.applyEntry(hdr, tr, added); err != nil {
+ return err
+ }
+ }
+}
+
+func (t *Tree) applyEntry(hdr *tar.Header, body io.Reader, added map[string]bool) error {
+ name, err := cleanName(hdr.Name)
+ if err != nil {
+ return err
+ }
+ if name == "" {
+ if hdr.Typeflag != tar.TypeDir {
+ return fmt.Errorf("rootfsimage: the layer's root entry is not a directory")
+ }
+ m, err := metaOf(hdr)
+ if err != nil {
+ return err
+ }
+ t.Root.Mode, t.Root.UID, t.Root.GID, t.Root.ModTime = m.Mode, m.UID, m.GID, m.ModTime
+ return nil
+ }
+ dir, base := parentOf(name), path.Base(name)
+ rdir, _, err := t.resolve(dir, true)
+ if err != nil {
+ return err
+ }
+
+ if strings.HasPrefix(base, whiteoutPrefix) {
+ switch {
+ case base == whiteoutOpaque:
+ t.opaque(rdir, added)
+ case strings.HasPrefix(base, whiteoutMeta):
+ // aufs bookkeeping (.wh..wh.plnk and the like): no entry.
+ default:
+ gone := strings.TrimPrefix(base, whiteoutPrefix)
+ if gone == "." || gone == ".." {
+ return UnsafePathError{Path: hdr.Name}
+ }
+ t.remove(joinRel(rdir, gone))
+ }
+ return nil
+ }
+
+ target := joinRel(rdir, base)
+ switch hdr.Typeflag {
+ case tar.TypeChar, tar.TypeBlock, tar.TypeFifo:
+ // What a device replaces is gone, as it would be on a disk.
+ t.Skipped++
+ t.remove(target)
+ return nil
+ case tar.TypeLink:
+ return t.applyHardlink(hdr, name, target, added)
+ case tar.TypeDir, tar.TypeReg, tar.TypeSymlink:
+ default:
+ return fmt.Errorf("rootfsimage: %s: unsupported tar entry type %q", shown(name), hdr.Typeflag)
+ }
+
+ e, err := metaOf(hdr)
+ if err != nil {
+ return err
+ }
+ e.Path = target
+ switch hdr.Typeflag {
+ case tar.TypeDir:
+ e.Type = TypeDir
+ case tar.TypeSymlink:
+ if err := checkName(hdr.Linkname); err != nil {
+ return fmt.Errorf("rootfsimage: %s: symlink target: %w", shown(name), err)
+ }
+ e.Type, e.LinkTarget = TypeSymlink, hdr.Linkname
+ case tar.TypeReg:
+ e.Type, e.Size = TypeFile, hdr.Size
+ }
+ if err := t.makeParents(rdir, e.ModTime, added); err != nil {
+ return err
+ }
+ if e.Type == TypeFile {
+ if e.src, err = t.writeBlob(body, hdr.Size); err != nil {
+ return err
+ }
+ }
+ t.add(e)
+ added[target] = true
+ return nil
+}
+
+// applyHardlink adds target (the resolved form of the entry's clean name) as
+// a new name for the regular file the header links to, sharing its bytes.
+// The link's target is resolved like any name of the tree as it stands,
+// except for its last component, which link(2) does not follow either.
+func (t *Tree) applyHardlink(hdr *tar.Header, name, target string, added map[string]bool) error {
+ refuse := LinkError{Path: name, Target: hdr.Linkname}
+ ln, err := cleanName(hdr.Linkname)
+ if err != nil || ln == "" {
+ return refuse
+ }
+ ldir, _, err := t.resolve(parentOf(ln), true)
+ if err != nil {
+ return refuse
+ }
+ src, ok := t.entries[joinRel(ldir, path.Base(ln))]
+ if !ok || src.Type != TypeFile {
+ return refuse
+ }
+ e := *src
+ e.Path = target
+ if err := t.makeParents(parentOf(target), e.ModTime, added); err != nil {
+ return err
+ }
+ t.add(&e)
+ added[target] = true
+ return nil
+}
+
+// makeParents makes sure dir (a resolved path, "" for the root) is a
+// directory, creating each missing component as a 0:0 0755 directory with
+// the time of the entry that needs it. A component that is not a directory
+// is refused: a layer cannot write beneath a file.
+func (t *Tree) makeParents(dir string, mtime time.Time, added map[string]bool) error {
+ p := ""
+ for _, c := range splitPath(dir) {
+ p = joinRel(p, c)
+ added[p] = true
+ e, ok := t.entries[p]
+ if !ok {
+ t.add(&Entry{Path: p, Type: TypeDir, Mode: 0o755, ModTime: mtime})
+ continue
+ }
+ if e.Type != TypeDir {
+ return fmt.Errorf("rootfsimage: the layer writes beneath %s, which is not a directory", shown(p))
+ }
+ }
+ return nil
+}
+
+// opaque deletes every entry strictly under dir that this layer did not
+// add. A directory this layer added, or created as a parent of an entry it
+// added, stays, but its lower-layer contents go with the rest.
+func (t *Tree) opaque(dir string, added map[string]bool) {
+ var walk func(d string)
+ walk = func(d string) {
+ for k := range t.kids[d] {
+ if !added[k] {
+ t.remove(k)
+ continue
+ }
+ if e := t.entries[k]; e != nil && e.Type == TypeDir {
+ walk(k)
+ }
+ }
+ }
+ walk(dir)
+}
+
+// cleanName is a layer entry's name as a tree path ("" for the root): a
+// leading "/" or "./" stripped and the rest cleaned. A name that climbs out
+// of the root is UnsafePathError, and an empty or too long one is refused as
+// malformed. (archive/tar never yields a NUL byte in a name: it ends a USTAR
+// field, and its PAX reader refuses one in a path.)
+func cleanName(name string) (string, error) {
+ if err := checkName(name); err != nil {
+ return "", fmt.Errorf("rootfsimage: an entry's name: %w", err)
+ }
+ p := path.Clean(strings.TrimLeft(name, "/"))
+ if p == ".." || strings.HasPrefix(p, "../") {
+ return "", UnsafePathError{Path: name}
+ }
+ if p == "." {
+ return "", nil
+ }
+ return p, nil
+}
+
+// checkName refuses a name or link target that is empty or longer than
+// maxPathBytes.
+func checkName(s string) error {
+ switch {
+ case s == "":
+ return errors.New("empty")
+ case len(s) > maxPathBytes:
+ return fmt.Errorf("over %d bytes", maxPathBytes)
+ }
+ return nil
+}
+
+// metaOf is the entry metadata a header carries: the permission bits with
+// setuid, setgid and sticky, the owner and the time to the second.
+func metaOf(hdr *tar.Header) (*Entry, error) {
+ if hdr.Uid < 0 || hdr.Gid < 0 || hdr.Uid > math.MaxUint32 || hdr.Gid > math.MaxUint32 {
+ return nil, fmt.Errorf("rootfsimage: %s: owner %d:%d out of range", shown(hdr.Name), hdr.Uid, hdr.Gid)
+ }
+ mode := hdr.FileInfo().Mode() & (fs.ModePerm | fs.ModeSetuid | fs.ModeSetgid | fs.ModeSticky)
+ return &Entry{
+ Mode: mode,
+ UID: uint32(hdr.Uid),
+ GID: uint32(hdr.Gid),
+ ModTime: hdr.ModTime.Truncate(time.Second).UTC(),
+ }, nil
+}
+
+// boundedReader reads at most n bytes of r, then fails.
+type boundedReader struct {
+ r io.Reader
+ n int64
+}
+
+func (b *boundedReader) Read(p []byte) (int, error) {
+ if b.n <= 0 {
+ return 0, fmt.Errorf("%w: its layers decode to too many bytes", errTooLarge)
+ }
+ if int64(len(p)) > b.n {
+ p = p[:b.n]
+ }
+ n, err := b.r.Read(p)
+ b.n -= int64(n)
+ return n, err
+}
diff --git a/internal/rootfsimage/flatten_test.go b/internal/rootfsimage/flatten_test.go
new file mode 100644
index 000000000..c8c28c7c4
--- /dev/null
+++ b/internal/rootfsimage/flatten_test.go
@@ -0,0 +1,431 @@
+package rootfsimage
+
+import (
+ "archive/tar"
+ "bytes"
+ "errors"
+ "fmt"
+ "io/fs"
+ "strings"
+ "testing"
+ "time"
+)
+
+func TestFlattenWhiteoutsAndOpaqueDirs(t *testing.T) {
+ tr := NewTree(t.TempDir())
+ apply(t, tr, layerTar(t,
+ tEntry{name: "etc/", typ: tar.TypeDir, mode: 0o755},
+ tEntry{name: "etc/old.conf", typ: tar.TypeReg, mode: 0o644, body: "old"},
+ tEntry{name: "var/cache/", typ: tar.TypeDir, mode: 0o755},
+ tEntry{name: "var/cache/a", typ: tar.TypeReg, mode: 0o644, body: "a"},
+ tEntry{name: "var/cache/sub/b", typ: tar.TypeReg, mode: 0o644, body: "b"},
+ ))
+ apply(t, tr, layerTar(t,
+ tEntry{name: "etc/.wh.old.conf", typ: tar.TypeReg},
+ tEntry{name: "var/cache/new", typ: tar.TypeReg, mode: 0o644, body: "new"}, // before the opaque marker in the stream
+ tEntry{name: "var/cache/.wh..wh..opq", typ: tar.TypeReg},
+ ))
+ for _, gone := range []string{"etc/old.conf", "var/cache/a", "var/cache/sub", "var/cache/sub/b", "etc/.wh.old.conf", "var/cache/.wh..wh..opq"} {
+ if _, ok := tr.Lookup(gone); ok {
+ t.Errorf("%s survived", gone)
+ }
+ }
+ if e, ok := tr.Lookup("var/cache/new"); !ok || readAll(t, tr, e) != "new" {
+ t.Error("the opaque dir dropped an entry of its own layer")
+ }
+ if _, ok := tr.Lookup("var/cache"); !ok {
+ t.Error("the opaque dir itself went")
+ }
+}
+
+func TestFlattenOpaqueDirKeepsALowerParentOfItsOwnEntries(t *testing.T) {
+ tr := NewTree(t.TempDir())
+ apply(t, tr, layerTar(t,
+ tEntry{name: "srv/", typ: tar.TypeDir, mode: 0o755},
+ tEntry{name: "srv/sub/", typ: tar.TypeDir, mode: 0o750},
+ tEntry{name: "srv/sub/old", typ: tar.TypeReg, mode: 0o644, body: "old"},
+ ))
+ // srv/sub/new comes with no header of its own for srv/sub.
+ apply(t, tr, layerTar(t,
+ tEntry{name: "srv/.wh..wh..opq", typ: tar.TypeReg},
+ tEntry{name: "srv/sub/new", typ: tar.TypeReg, mode: 0o644, body: "new"},
+ ))
+ if _, ok := tr.Lookup("srv/sub/old"); ok {
+ t.Error("srv/sub/old survived the opaque dir")
+ }
+ if e, ok := tr.Lookup("srv/sub/new"); !ok || readAll(t, tr, e) != "new" {
+ t.Error("srv/sub/new missing")
+ }
+}
+
+func TestFlattenKeepsOwnershipModesAndTimes(t *testing.T) {
+ tr := NewTree(t.TempDir())
+ apply(t, tr, layerTar(t,
+ tEntry{name: "./", typ: tar.TypeDir, mode: 0o750, uid: 0, gid: 0, mtime: 1_600_000_009},
+ tEntry{name: "home/shop/", typ: tar.TypeDir, mode: 0o750, uid: 1000, gid: 1000, mtime: 1_600_000_001},
+ tEntry{name: "usr/bin/su", typ: tar.TypeReg, mode: 0o4755, body: "su", mtime: 1_600_000_002},
+ tEntry{name: "var/mail/", typ: tar.TypeDir, mode: 0o2755, gid: 8, mtime: 1_600_000_003},
+ tEntry{name: "tmp/", typ: tar.TypeDir, mode: 0o1777, mtime: 1_600_000_004},
+ tEntry{name: "bin", typ: tar.TypeSymlink, mode: 0o777, link: "usr/bin", mtime: 1_600_000_005},
+ ))
+ cases := []struct {
+ path string
+ typ EntryType
+ mode fs.FileMode
+ uid, gid uint32
+ mtime int64
+ }{
+ {"home/shop", TypeDir, 0o750, 1000, 1000, 1_600_000_001},
+ {"usr/bin/su", TypeFile, fs.ModeSetuid | 0o755, 0, 0, 1_600_000_002},
+ {"var/mail", TypeDir, fs.ModeSetgid | 0o755, 0, 8, 1_600_000_003},
+ {"tmp", TypeDir, fs.ModeSticky | 0o777, 0, 0, 1_600_000_004},
+ {"bin", TypeSymlink, 0o777, 0, 0, 1_600_000_005},
+ }
+ for _, c := range cases {
+ e := mustLookup(t, tr, c.path)
+ if e.Type != c.typ || e.Mode != c.mode || e.UID != c.uid || e.GID != c.gid || e.ModTime.Unix() != c.mtime {
+ t.Errorf("%s = type %d mode %v %d:%d %d, want type %d mode %v %d:%d %d",
+ c.path, e.Type, e.Mode, e.UID, e.GID, e.ModTime.Unix(), c.typ, c.mode, c.uid, c.gid, c.mtime)
+ }
+ }
+ if e := mustLookup(t, tr, "bin"); e.LinkTarget != "usr/bin" {
+ t.Errorf("bin -> %q", e.LinkTarget)
+ }
+ if tr.Root.Mode != 0o750 || tr.Root.ModTime.Unix() != 1_600_000_009 {
+ t.Errorf("root = %v %d, want the ./ entry's 0750 1600000009", tr.Root.Mode, tr.Root.ModTime.Unix())
+ }
+ if got := tr.Newest().Unix(); got != 1_600_000_009 {
+ t.Errorf("Newest = %d, want the root's 1600000009", got)
+ }
+ if tr.Files() != 1 {
+ t.Errorf("Files = %d, want 1", tr.Files())
+ }
+}
+
+func TestFlattenTruncatesTimesToTheSecond(t *testing.T) {
+ var buf bytes.Buffer
+ tw := tar.NewWriter(&buf)
+ h := &tar.Header{Name: "f", Typeflag: tar.TypeReg, Mode: 0o644, Format: tar.FormatPAX}
+ h.ModTime = time.Unix(1_600_000_000, 700_000_000)
+ if err := tw.WriteHeader(h); err != nil {
+ t.Fatal(err)
+ }
+ tw.Close()
+ tr := NewTree(t.TempDir())
+ apply(t, tr, buf.Bytes())
+ if e := mustLookup(t, tr, "f"); e.ModTime.Nanosecond() != 0 || e.ModTime.Unix() != 1_600_000_000 {
+ t.Errorf("mtime = %v, want 1600000000 exactly", e.ModTime)
+ }
+}
+
+func TestFlattenNewTreeRoot(t *testing.T) {
+ tr := NewTree(t.TempDir())
+ if tr.Root.Type != TypeDir || tr.Root.Mode != 0o755 || tr.Root.UID != 0 || tr.Root.GID != 0 || tr.Root.Path != "." {
+ t.Errorf("root = %+v, want a 0:0 0755 dir", tr.Root)
+ }
+ if e, ok := tr.Lookup("."); !ok || e != &tr.Root {
+ t.Error(`Lookup(".") is not the root`)
+ }
+}
+
+func TestFlattenReplacesADirWithAFileAndBack(t *testing.T) {
+ tr := NewTree(t.TempDir())
+ apply(t, tr, layerTar(t,
+ tEntry{name: "opt/app/", typ: tar.TypeDir, mode: 0o755},
+ tEntry{name: "opt/app/a", typ: tar.TypeReg, mode: 0o644, body: "a"},
+ tEntry{name: "opt/app/sub/b", typ: tar.TypeReg, mode: 0o644, body: "b"},
+ tEntry{name: "srv/data", typ: tar.TypeReg, mode: 0o644, body: "file"},
+ ))
+ apply(t, tr, layerTar(t,
+ tEntry{name: "opt/app", typ: tar.TypeReg, mode: 0o644, body: "now a file"},
+ tEntry{name: "srv/data/", typ: tar.TypeDir, mode: 0o700},
+ tEntry{name: "srv/data/x", typ: tar.TypeReg, mode: 0o600, body: "x"},
+ ))
+ for _, gone := range []string{"opt/app/a", "opt/app/sub", "opt/app/sub/b"} {
+ if _, ok := tr.Lookup(gone); ok {
+ t.Errorf("%s survived its dir becoming a file", gone)
+ }
+ }
+ if e := mustLookup(t, tr, "opt/app"); e.Type != TypeFile || readAll(t, tr, e) != "now a file" {
+ t.Errorf("opt/app = %+v", e)
+ }
+ if e := mustLookup(t, tr, "srv/data"); e.Type != TypeDir || e.Mode != 0o700 {
+ t.Errorf("srv/data = %+v, want a 0700 dir", e)
+ }
+ if e := mustLookup(t, tr, "srv/data/x"); readAll(t, tr, e) != "x" {
+ t.Error("srv/data/x lost its bytes")
+ }
+ // A dir over a dir keeps its children and takes the new metadata.
+ apply(t, tr, layerTar(t, tEntry{name: "srv/data/", typ: tar.TypeDir, mode: 0o755, uid: 7}))
+ if e := mustLookup(t, tr, "srv/data"); e.Mode != 0o755 || e.UID != 7 {
+ t.Errorf("srv/data = %+v, want 0755 uid 7", e)
+ }
+ mustLookup(t, tr, "srv/data/x")
+}
+
+func TestFlattenHardlinksShareBytes(t *testing.T) {
+ tr := NewTree(t.TempDir())
+ apply(t, tr, layerTar(t,
+ tEntry{name: "usr/bin/ruby", typ: tar.TypeReg, mode: 0o755, body: "ruby 3.3"},
+ tEntry{name: "usr/bin/ruby2", typ: tar.TypeLink, link: "usr/bin/ruby"},
+ tEntry{name: "usr/bin/ruby3", typ: tar.TypeLink, link: "/usr/bin/ruby"},
+ ))
+ ruby, ruby2 := mustLookup(t, tr, "usr/bin/ruby"), mustLookup(t, tr, "usr/bin/ruby2")
+ if ruby.src != ruby2.src || ruby2.Type != TypeFile || ruby2.Mode != 0o755 || ruby2.Size != ruby.Size {
+ t.Errorf("ruby2 = %+v, want ruby's file %+v", ruby2, ruby)
+ }
+ if mustLookup(t, tr, "usr/bin/ruby3").src != ruby.src {
+ t.Error("an absolute hardlink name did not share the bytes")
+ }
+ apply(t, tr, layerTar(t, tEntry{name: "usr/bin/ruby", typ: tar.TypeReg, mode: 0o755, body: "ruby 3.4"}))
+ if got := readAll(t, tr, mustLookup(t, tr, "usr/bin/ruby2")); got != "ruby 3.3" {
+ t.Errorf("ruby2 = %q after ruby was replaced, want the old bytes", got)
+ }
+ if got := readAll(t, tr, mustLookup(t, tr, "usr/bin/ruby")); got != "ruby 3.4" {
+ t.Errorf("ruby = %q", got)
+ }
+
+ for _, c := range []struct {
+ name string
+ layer []tEntry
+ path string
+ target string
+ }{
+ {"missing", []tEntry{{name: "a", typ: tar.TypeLink, link: "nowhere"}}, "a", "nowhere"},
+ {"a dir", []tEntry{{name: "d/", typ: tar.TypeDir, mode: 0o755}, {name: "a", typ: tar.TypeLink, link: "d"}}, "a", "d"},
+ {"a symlink", []tEntry{
+ {name: "f", typ: tar.TypeReg, mode: 0o644, body: "f"},
+ {name: "s", typ: tar.TypeSymlink, link: "f"},
+ {name: "a", typ: tar.TypeLink, link: "s"},
+ }, "a", "s"},
+ {"outside the root", []tEntry{{name: "a", typ: tar.TypeLink, link: "../../etc/shadow"}}, "a", "../../etc/shadow"},
+ {"the root", []tEntry{{name: "a", typ: tar.TypeLink, link: "./"}}, "a", "./"},
+ } {
+ t.Run(c.name, func(t *testing.T) {
+ tr := NewTree(t.TempDir())
+ err := tr.ApplyLayer(bytes.NewReader(layerTar(t, c.layer...)))
+ var le LinkError
+ if !errors.As(err, &le) || le != (LinkError{Path: c.path, Target: c.target}) {
+ t.Fatalf("err = %v, want LinkError{%s, %s}", err, c.path, c.target)
+ }
+ if _, ok := tr.Lookup("a"); ok {
+ t.Error("the refused link was added")
+ }
+ })
+ }
+}
+
+func TestFlattenFollowsASymlinkedParentInsideTheRoot(t *testing.T) {
+ tr := NewTree(t.TempDir())
+ apply(t, tr, layerTar(t,
+ tEntry{name: "usr/lib/", typ: tar.TypeDir, mode: 0o755},
+ tEntry{name: "lib", typ: tar.TypeSymlink, link: "usr/lib"},
+ tEntry{name: "up", typ: tar.TypeSymlink, link: "../../../etc"},
+ tEntry{name: "abs", typ: tar.TypeSymlink, link: "/../../srv"},
+ ))
+ apply(t, tr, layerTar(t,
+ tEntry{name: "lib/libshop.so", typ: tar.TypeReg, mode: 0o644, body: "elf"},
+ tEntry{name: "up/passwd", typ: tar.TypeReg, mode: 0o644, body: "root:x:0:0"},
+ tEntry{name: "abs/www/index.html", typ: tar.TypeReg, mode: 0o644, body: "hi"},
+ ))
+ for p, body := range map[string]string{
+ "usr/lib/libshop.so": "elf",
+ "etc/passwd": "root:x:0:0",
+ "srv/www/index.html": "hi",
+ } {
+ if e, ok := tr.Lookup(p); !ok || readAll(t, tr, e) != body {
+ t.Errorf("%s missing or wrong", p)
+ }
+ }
+ if e := mustLookup(t, tr, "lib"); e.Type != TypeSymlink {
+ t.Error("lib stopped being a symlink")
+ }
+ for _, c := range []struct{ in, out string }{
+ {"lib/libshop.so", "usr/lib/libshop.so"},
+ {"up/passwd", "etc/passwd"},
+ {"/up/../../lib", "usr/lib"},
+ {"lib/../..", "."},
+ } {
+ got, ok := tr.Resolve(c.in)
+ if !ok || got != c.out {
+ t.Errorf("Resolve(%q) = %q, %v; want %q, true", c.in, got, ok, c.out)
+ }
+ }
+ if got, ok := tr.Resolve("lib/none"); ok || got != "usr/lib/none" {
+ t.Errorf("Resolve(lib/none) = %q, %v; want usr/lib/none, false", got, ok)
+ }
+
+ // 41 links in a chain: l0 -> l1 -> ... -> l40 -> d.
+ var chain []tEntry
+ chain = append(chain, tEntry{name: "d/", typ: tar.TypeDir, mode: 0o755})
+ for i := 0; i <= 40; i++ {
+ target := fmt.Sprintf("l%d", i+1)
+ if i == 40 {
+ target = "d"
+ }
+ chain = append(chain, tEntry{name: fmt.Sprintf("l%d", i), typ: tar.TypeSymlink, link: target})
+ }
+ long := NewTree(t.TempDir())
+ apply(t, long, layerTar(t, chain...))
+ err := long.ApplyLayer(bytes.NewReader(layerTar(t, tEntry{name: "l0/x", typ: tar.TypeReg, mode: 0o644, body: "x"})))
+ var le LinkError
+ if !errors.As(err, &le) || le.Path == "" {
+ t.Fatalf("41 links: err = %v, want LinkError", err)
+ }
+ if _, ok := long.Resolve("l0/x"); ok {
+ t.Error("Resolve followed 41 links")
+ }
+ // 40 is the most: l1 -> ... -> l40 -> d resolves.
+ if got, _ := long.Resolve("l1"); got != "d" {
+ t.Errorf("Resolve(l1) = %q, want d", got)
+ }
+ // A loop is the same refusal.
+ loop := NewTree(t.TempDir())
+ apply(t, loop, layerTar(t,
+ tEntry{name: "a", typ: tar.TypeSymlink, link: "b"},
+ tEntry{name: "b", typ: tar.TypeSymlink, link: "a"},
+ ))
+ if err := loop.ApplyLayer(bytes.NewReader(layerTar(t, tEntry{name: "a/x", typ: tar.TypeReg}))); !errors.As(err, &le) {
+ t.Fatalf("loop: err = %v, want LinkError", err)
+ }
+}
+
+func TestFlattenRefusesAWriteThroughAFile(t *testing.T) {
+ tr := NewTree(t.TempDir())
+ apply(t, tr, layerTar(t, tEntry{name: "etc/passwd", typ: tar.TypeReg, mode: 0o644, body: "p"}))
+ if err := tr.ApplyLayer(bytes.NewReader(layerTar(t, tEntry{name: "etc/passwd/x", typ: tar.TypeReg}))); err == nil {
+ t.Fatal("a file under a file was accepted")
+ }
+}
+
+func TestFlattenRefusesAnEntryOutsideTheRoot(t *testing.T) {
+ for _, name := range []string{"../x", "a/../../x", "..", "x/.wh..."} {
+ t.Run(name, func(t *testing.T) {
+ tr := NewTree(t.TempDir())
+ err := tr.ApplyLayer(bytes.NewReader(layerTar(t, tEntry{name: name, typ: tar.TypeReg, mode: 0o644, body: "x"})))
+ var ue UnsafePathError
+ if !errors.As(err, &ue) || ue.Path != name {
+ t.Fatalf("err = %v, want UnsafePathError{%q}", err, name)
+ }
+ if len(tr.Entries()) != 0 {
+ t.Errorf("entries = %d, want none", len(tr.Entries()))
+ }
+ })
+ }
+ tr := NewTree(t.TempDir())
+ apply(t, tr, layerTar(t,
+ tEntry{name: "/abs/x", typ: tar.TypeReg, mode: 0o644, body: "abs"},
+ tEntry{name: "./rel/x", typ: tar.TypeReg, mode: 0o644, body: "rel"},
+ tEntry{name: "//dbl/./x", typ: tar.TypeReg, mode: 0o644, body: "dbl"},
+ ))
+ for p, body := range map[string]string{"abs/x": "abs", "rel/x": "rel", "dbl/x": "dbl"} {
+ if e, ok := tr.Lookup(p); !ok || readAll(t, tr, e) != body {
+ t.Errorf("%s missing", p)
+ }
+ }
+ for _, e := range tr.Entries() {
+ if strings.HasPrefix(e.Path, "/") || strings.HasPrefix(e.Path, ".") || e.Path == "" {
+ t.Errorf("unclean path %q", e.Path)
+ }
+ }
+}
+
+func TestFlattenSkipsDeviceFiles(t *testing.T) {
+ tr := NewTree(t.TempDir())
+ apply(t, tr, layerTar(t,
+ tEntry{name: "dev/null", typ: tar.TypeReg, mode: 0o644, body: "lower"},
+ ))
+ apply(t, tr, layerTar(t,
+ tEntry{name: "dev/null", typ: tar.TypeChar, mode: 0o666},
+ tEntry{name: "dev/sda", typ: tar.TypeBlock, mode: 0o660},
+ tEntry{name: "run/pipe", typ: tar.TypeFifo, mode: 0o600},
+ ))
+ if tr.Skipped != 3 {
+ t.Errorf("Skipped = %d, want 3", tr.Skipped)
+ }
+ for _, gone := range []string{"dev/null", "dev/sda", "run/pipe"} {
+ if _, ok := tr.Lookup(gone); ok {
+ t.Errorf("%s is in the tree", gone)
+ }
+ }
+}
+
+func TestFlattenCreatesMissingParents(t *testing.T) {
+ tr := NewTree(t.TempDir())
+ apply(t, tr, layerTar(t, tEntry{name: "a/b/c", typ: tar.TypeReg, mode: 0o600, uid: 5, body: "c", mtime: 1_650_000_000}))
+ for _, p := range []string{"a", "a/b"} {
+ e := mustLookup(t, tr, p)
+ if e.Type != TypeDir || e.Mode != 0o755 || e.UID != 0 || e.GID != 0 {
+ t.Errorf("%s = %+v, want a 0:0 0755 dir", p, e)
+ }
+ }
+ var got []string
+ for _, e := range tr.Entries() {
+ got = append(got, e.Path)
+ }
+ if strings.Join(got, ",") != "a,a/b,a/b/c" {
+ t.Errorf("Entries = %v", got)
+ }
+}
+
+func TestFlattenKeepsPathsThatDifferOnlyInCase(t *testing.T) {
+ tr := NewTree(t.TempDir())
+ apply(t, tr, layerTar(t,
+ tEntry{name: "etc/Foo", typ: tar.TypeReg, mode: 0o644, body: "upper"},
+ tEntry{name: "etc/foo", typ: tar.TypeReg, mode: 0o644, body: "lower"},
+ ))
+ if got := readAll(t, tr, mustLookup(t, tr, "etc/Foo")); got != "upper" {
+ t.Errorf("etc/Foo = %q", got)
+ }
+ if got := readAll(t, tr, mustLookup(t, tr, "etc/foo")); got != "lower" {
+ t.Errorf("etc/foo = %q", got)
+ }
+ if tr.Files() != 2 {
+ t.Errorf("Files = %d, want 2", tr.Files())
+ }
+}
+
+func TestFlattenRefusesMalformedLayers(t *testing.T) {
+ long := strings.Repeat("a/", maxPathBytes/2+1)
+ for _, c := range []struct {
+ name string
+ layer []byte
+ }{
+ {"not a tar", bytes.Repeat([]byte("hello "), 200)},
+ {"a truncated file", layerTar(t, tEntry{name: "f", typ: tar.TypeReg, mode: 0o644, body: strings.Repeat("x", 2000)})[:1024]},
+ {"a path too long", layerTar(t, tEntry{name: long + "f", typ: tar.TypeReg, mode: 0o644})},
+ {"an empty symlink", layerTar(t, tEntry{name: "s", typ: tar.TypeSymlink})},
+ {"a negative uid", layerTar(t, tEntry{name: "f", typ: tar.TypeReg, mode: 0o644, uid: -1})},
+ {"a file as the root", layerTar(t, tEntry{name: "./", typ: tar.TypeSymlink, link: "x"})},
+ } {
+ t.Run(c.name, func(t *testing.T) {
+ if err := NewTree(t.TempDir()).ApplyLayer(bytes.NewReader(c.layer)); err == nil {
+ t.Fatal("accepted")
+ }
+ })
+ }
+}
+
+func TestFlattenBoundsTheTreeBytes(t *testing.T) {
+ tr := NewTree(t.TempDir())
+ tr.maxBytes = 10
+ apply(t, tr, layerTar(t, tEntry{name: "a", typ: tar.TypeReg, mode: 0o644, body: "123456"}))
+ err := tr.ApplyLayer(bytes.NewReader(layerTar(t, tEntry{name: "b", typ: tar.TypeReg, mode: 0o644, body: "123456"})))
+ if !errors.Is(err, errTooLarge) {
+ t.Fatalf("err = %v, want errTooLarge", err)
+ }
+}
+
+func TestFlattenBoundsTheEntries(t *testing.T) {
+ tr := NewTree(t.TempDir())
+ tr.maxHeaders = 3
+ err := tr.ApplyLayer(bytes.NewReader(layerTar(t,
+ tEntry{name: "a", typ: tar.TypeReg},
+ tEntry{name: "b", typ: tar.TypeReg},
+ tEntry{name: "c", typ: tar.TypeReg},
+ tEntry{name: "d", typ: tar.TypeReg},
+ )))
+ if !errors.Is(err, errTooLarge) {
+ t.Fatalf("err = %v, want errTooLarge", err)
+ }
+}
diff --git a/internal/rootfsimage/oci.go b/internal/rootfsimage/oci.go
new file mode 100644
index 000000000..5ea3bc714
--- /dev/null
+++ b/internal/rootfsimage/oci.go
@@ -0,0 +1,475 @@
+package rootfsimage
+
+import (
+ "bufio"
+ "bytes"
+ "compress/gzip"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "hash"
+ "io"
+ "os"
+ "path/filepath"
+ "regexp"
+ "slices"
+
+ "github.com/klauspost/compress/zstd"
+)
+
+// The media types an OCI layout or a docker save archive uses for indexes
+// and image manifests. Anything else an index lists (signatures, other
+// artifacts) is not an image and is passed over.
+const (
+ mediaOCIIndex = "application/vnd.oci.image.index.v1+json"
+ mediaDockerList = "application/vnd.docker.distribution.manifest.list.v2+json"
+ mediaOCIManifest = "application/vnd.oci.image.manifest.v1+json"
+ mediaDockerManifest = "application/vnd.docker.distribution.manifest.v2+json"
+)
+
+// The reader's bounds on what an index can make it read.
+const (
+ // maxJSONBytes bounds index.json, every index, manifest and config.
+ maxJSONBytes = 4 << 20
+ // maxIndexDepth is how many indexes deep index.json may nest.
+ maxIndexDepth = 4
+ // maxDescriptors bounds the descriptors read over every index.
+ maxDescriptors = 1024
+ // maxZstdWindow is the zstd window the decoder accepts, the zstd
+ // tool's own default limit.
+ maxZstdWindow = 1 << 27
+)
+
+var sha256Digest = regexp.MustCompile(`^sha256:([0-9a-f]{64})$`)
+
+// blobStore reads the files of an OCI layout, wherever it is: a directory
+// (layoutStore) or the entries of an archive (archiveStore).
+type blobStore interface {
+ // open reads the regular file at name, slash-separated and relative,
+ // answering its size.
+ open(name string) (io.ReadCloser, int64, error)
+ has(name string) bool
+}
+
+type descriptor struct {
+ MediaType string `json:"mediaType"`
+ Digest string `json:"digest"`
+ Size int64 `json:"size"`
+ Platform *platform `json:"platform,omitempty"`
+ Annotations map[string]string `json:"annotations,omitempty"`
+}
+
+type platform struct {
+ Architecture string `json:"architecture"`
+ OS string `json:"os"`
+ Variant string `json:"variant,omitempty"`
+}
+
+func (p platform) String() string {
+ s := p.OS + "/" + p.Architecture
+ if p.Variant != "" {
+ s += "/" + p.Variant
+ }
+ return s
+}
+
+func (p platform) isArm64() bool {
+ return p.OS == "linux" && p.Architecture == "arm64" && (p.Variant == "" || p.Variant == "v8")
+}
+
+type index struct {
+ MediaType string `json:"mediaType"`
+ Manifests []descriptor `json:"manifests"`
+}
+
+type manifest struct {
+ MediaType string `json:"mediaType"`
+ Config descriptor `json:"config"`
+ Layers []descriptor `json:"layers"`
+}
+
+type imageConfig struct {
+ Architecture string `json:"architecture"`
+ OS string `json:"os"`
+ Variant string `json:"variant"`
+ Config struct {
+ Env []string `json:"Env"`
+ WorkingDir string `json:"WorkingDir"`
+ } `json:"config"`
+}
+
+// OpenLayout flattens the linux/arm64 image of an OCI layout directory into
+// a new tree spooled under spool. Every file of the layout is read through
+// os.Root, so a blob that is a symlink out of the layout is refused.
+func OpenLayout(dir, spool string) (*Tree, error) {
+ root, err := os.OpenRoot(dir)
+ if err != nil {
+ return nil, fmt.Errorf("rootfsimage: %w", err)
+ }
+ defer root.Close()
+ st := layoutStore{root}
+ if !st.has("index.json") {
+ return nil, NotAnImageError{Source: dir}
+ }
+ return openOCI(st, dir, spool)
+}
+
+// openOCI flattens the image an OCI layout's index.json leads to: nested
+// indexes followed at most maxIndexDepth deep, attestation manifests
+// (buildx's unknown/unknown entries) passed over, and of the images left
+// the linux/arm64 one picked. Every blob is checked against its
+// descriptor's size and sha256 digest (DigestError).
+func openOCI(st blobStore, source, spool string) (*Tree, error) {
+ raw, err := readFile(st, "index.json")
+ if err != nil {
+ return nil, err
+ }
+ var idx index
+ if err := json.Unmarshal(raw, &idx); err != nil {
+ return nil, NotAnImageError{Source: source}
+ }
+ o := &ociReader{st: st, source: source}
+ if err := o.collect(idx, 0); err != nil {
+ return nil, err
+ }
+ pick, err := o.pick()
+ if err != nil {
+ return nil, err
+ }
+ var m manifest
+ if err := o.readJSON(pick, &m); err != nil {
+ return nil, err
+ }
+ if m.Config.Digest == "" {
+ return nil, NotAnImageError{Source: source}
+ }
+ var cfg imageConfig
+ if err := o.readJSON(m.Config, &cfg); err != nil {
+ return nil, err
+ }
+ t := NewTree(spool)
+ if t.Config, err = checkConfig(cfg, source); err != nil {
+ return nil, err
+ }
+ for _, l := range m.Layers {
+ v, err := o.blob(l, -1)
+ if err != nil {
+ return nil, err
+ }
+ err = applyStream(t, v, v.drain)
+ v.Close()
+ if err != nil {
+ return nil, err
+ }
+ }
+ return t, nil
+}
+
+// checkConfig is the ImageConfig of a config that is linux/arm64, ArchError
+// for one that names another platform, and NotAnImageError for one that
+// names none (an artifact, not an image).
+func checkConfig(cfg imageConfig, source string) (ImageConfig, error) {
+ if cfg.OS == "" && cfg.Architecture == "" {
+ return ImageConfig{}, NotAnImageError{Source: source}
+ }
+ p := platform{OS: cfg.OS, Architecture: cfg.Architecture, Variant: cfg.Variant}
+ if p.OS != "linux" || p.Architecture != "arm64" {
+ return ImageConfig{}, ArchError{Platform: p.String()}
+ }
+ return ImageConfig{
+ Architecture: cfg.Architecture,
+ OS: cfg.OS,
+ Variant: cfg.Variant,
+ Env: cfg.Config.Env,
+ WorkingDir: cfg.Config.WorkingDir,
+ }, nil
+}
+
+type ociReader struct {
+ st blobStore
+ source string
+ descriptors int
+ seen map[string]bool
+ cands []descriptor
+}
+
+// collect gathers the image manifests idx leads to, deduplicated by digest.
+func (o *ociReader) collect(idx index, depth int) error {
+ if o.seen == nil {
+ o.seen = map[string]bool{}
+ }
+ for _, d := range idx.Manifests {
+ o.descriptors++
+ if o.descriptors > maxDescriptors {
+ return fmt.Errorf("%w: its indexes list more than %d entries", errTooLarge, maxDescriptors)
+ }
+ if isAttestation(d) || o.seen[d.Digest] {
+ continue
+ }
+ o.seen[d.Digest] = true
+ kind := d.MediaType
+ var raw []byte
+ if kind == "" {
+ // A descriptor that does not say what it is: the blob does.
+ var err error
+ if raw, err = o.readBlob(d, maxJSONBytes); err != nil {
+ return err
+ }
+ kind = sniffMediaType(raw)
+ }
+ switch kind {
+ case mediaOCIIndex, mediaDockerList:
+ if depth >= maxIndexDepth {
+ return fmt.Errorf("rootfsimage: %s: its indexes nest more than %d deep", shown(o.source), maxIndexDepth)
+ }
+ if raw == nil {
+ var err error
+ if raw, err = o.readBlob(d, maxJSONBytes); err != nil {
+ return err
+ }
+ }
+ var sub index
+ if err := json.Unmarshal(raw, &sub); err != nil {
+ return fmt.Errorf("rootfsimage: index %s: %w", d.Digest, err)
+ }
+ if err := o.collect(sub, depth+1); err != nil {
+ return err
+ }
+ case mediaOCIManifest, mediaDockerManifest:
+ o.cands = append(o.cands, d)
+ }
+ }
+ return nil
+}
+
+// pick is the one image to flatten. A manifest without a platform is a
+// whole image whose platform only its config tells; one with a platform is
+// a variant of a multi-platform image. So the pick is the one linux/arm64
+// variant, or the one image without a platform (its config is checked
+// next); more than one of those is ManyImagesError, and none is
+// NoArm64Error naming the platforms there are.
+func (o *ociReader) pick() (descriptor, error) {
+ var arm, bare []descriptor
+ var others []string
+ for _, d := range o.cands {
+ switch {
+ case d.Platform == nil:
+ bare = append(bare, d)
+ case d.Platform.isArm64():
+ arm = append(arm, d)
+ default:
+ s := d.Platform.String()
+ if !slices.Contains(others, s) {
+ others = append(others, s)
+ }
+ }
+ }
+ switch {
+ case len(arm)+len(bare) > 1:
+ return descriptor{}, ManyImagesError{N: len(arm) + len(bare)}
+ case len(arm) == 1:
+ return arm[0], nil
+ case len(bare) == 1:
+ return bare[0], nil
+ case len(others) > 0:
+ return descriptor{}, NoArm64Error{Platforms: others}
+ }
+ return descriptor{}, NotAnImageError{Source: o.source}
+}
+
+// isAttestation reports whether d is buildx's attestation manifest (an
+// in-toto statement about an image, not an image).
+func isAttestation(d descriptor) bool {
+ if d.Annotations["vnd.docker.reference.type"] == "attestation-manifest" {
+ return true
+ }
+ return d.Platform != nil && d.Platform.OS == "unknown" && d.Platform.Architecture == "unknown"
+}
+
+func sniffMediaType(raw []byte) string {
+ var probe struct {
+ MediaType string `json:"mediaType"`
+ Manifests json.RawMessage `json:"manifests"`
+ Layers json.RawMessage `json:"layers"`
+ Config json.RawMessage `json:"config"`
+ }
+ if json.Unmarshal(raw, &probe) != nil {
+ return ""
+ }
+ switch {
+ case probe.MediaType != "":
+ return probe.MediaType
+ case probe.Manifests != nil:
+ return mediaOCIIndex
+ case probe.Layers != nil || probe.Config != nil:
+ return mediaOCIManifest
+ }
+ return ""
+}
+
+func (o *ociReader) readJSON(d descriptor, v any) error {
+ raw, err := o.readBlob(d, maxJSONBytes)
+ if err != nil {
+ return err
+ }
+ if err := json.Unmarshal(raw, v); err != nil {
+ return fmt.Errorf("rootfsimage: blob %s: %w", d.Digest, err)
+ }
+ return nil
+}
+
+// readBlob is a whole blob of at most max bytes, verified.
+func (o *ociReader) readBlob(d descriptor, max int64) ([]byte, error) {
+ v, err := o.blob(d, max)
+ if err != nil {
+ return nil, err
+ }
+ defer v.Close()
+ return io.ReadAll(v)
+}
+
+// blob opens the blob d names, refusing one whose size is not the
+// descriptor's or past max (when max >= 0). The reader answers DigestError
+// in place of io.EOF when the bytes do not hash to the digest.
+func (o *ociReader) blob(d descriptor, max int64) (*verifier, error) {
+ m := sha256Digest.FindStringSubmatch(d.Digest)
+ if m == nil {
+ return nil, fmt.Errorf("rootfsimage: %s: malformed or unsupported digest %s", shown(o.source), shown(d.Digest))
+ }
+ if d.Size < 0 {
+ return nil, fmt.Errorf("rootfsimage: %s: blob %s has a negative size", shown(o.source), d.Digest)
+ }
+ if max >= 0 && d.Size > max {
+ return nil, fmt.Errorf("%w: blob %s is %d bytes, over %d", errTooLarge, d.Digest, d.Size, max)
+ }
+ rc, size, err := o.st.open("blobs/sha256/" + m[1])
+ if err != nil {
+ return nil, err
+ }
+ if size != d.Size {
+ rc.Close()
+ return nil, DigestError{Digest: d.Digest}
+ }
+ want, _ := hex.DecodeString(m[1])
+ return &verifier{rc: rc, r: io.LimitReader(rc, size), h: sha256.New(), want: want, digest: d.Digest}, nil
+}
+
+// verifier hashes what it reads and, at the end, answers DigestError
+// instead of io.EOF when the hash is not the digest's.
+type verifier struct {
+ rc io.ReadCloser
+ r io.Reader
+ h hash.Hash
+ want []byte
+ digest string
+ done error
+}
+
+func (v *verifier) Read(p []byte) (int, error) {
+ if v.done != nil {
+ return 0, v.done
+ }
+ n, err := v.r.Read(p)
+ v.h.Write(p[:n])
+ if err == io.EOF {
+ v.done = io.EOF
+ if !bytes.Equal(v.h.Sum(nil), v.want) {
+ v.done = DigestError{Digest: v.digest}
+ }
+ return n, v.done
+ }
+ return n, err
+}
+
+// drain reads what is left of the blob, answering DigestError if its bytes
+// did not match.
+func (v *verifier) drain() error {
+ _, err := io.Copy(io.Discard, v)
+ return err
+}
+
+func (v *verifier) Close() error { return v.rc.Close() }
+
+// applyStream applies one layer blob to t, decompressing it by its magic
+// bytes (gzip, zstd, or a plain tar). drain, when set, reads the rest of the
+// blob and checks it; a blob that fails that check is DigestError whatever
+// else went wrong, since its corruption explains the rest.
+func applyStream(t *Tree, r io.Reader, drain func() error) error {
+ br := bufio.NewReader(r)
+ magic, _ := br.Peek(4)
+ var layer io.Reader = br
+ var err error
+ switch {
+ case len(magic) >= 2 && magic[0] == 0x1f && magic[1] == 0x8b:
+ var zr *gzip.Reader
+ if zr, err = gzip.NewReader(br); err == nil {
+ defer zr.Close()
+ layer = zr
+ }
+ case bytes.Equal(magic, []byte{0x28, 0xb5, 0x2f, 0xfd}):
+ var zd *zstd.Decoder
+ if zd, err = zstd.NewReader(br, zstd.WithDecoderConcurrency(1), zstd.WithDecoderMaxWindow(maxZstdWindow)); err == nil {
+ defer zd.Close()
+ layer = zd
+ }
+ }
+ if err != nil {
+ err = fmt.Errorf("rootfsimage: decompressing a layer: %w", err)
+ } else {
+ err = t.ApplyLayer(layer)
+ }
+ if drain != nil {
+ if derr := drain(); derr != nil {
+ var de DigestError
+ if errors.As(derr, &de) || err == nil {
+ return derr
+ }
+ }
+ }
+ return err
+}
+
+// readFile is a whole file of the store, at most maxJSONBytes.
+func readFile(st blobStore, name string) ([]byte, error) {
+ rc, size, err := st.open(name)
+ if err != nil {
+ return nil, err
+ }
+ defer rc.Close()
+ if size > maxJSONBytes {
+ return nil, fmt.Errorf("%w: %s is %d bytes, over %d", errTooLarge, name, size, maxJSONBytes)
+ }
+ return io.ReadAll(io.LimitReader(rc, size))
+}
+
+// layoutStore reads a layout directory through os.Root: a name, or a
+// symlink on the way to it, that leaves the directory fails to open, and
+// only regular files are read (a FIFO would block the open).
+type layoutStore struct{ root *os.Root }
+
+func (s layoutStore) open(name string) (io.ReadCloser, int64, error) {
+ name = filepath.FromSlash(name)
+ fi, err := s.root.Stat(name)
+ if err != nil {
+ return nil, 0, fmt.Errorf("rootfsimage: %w", err)
+ }
+ if !fi.Mode().IsRegular() {
+ return nil, 0, fmt.Errorf("rootfsimage: %s in the layout is not a regular file", shown(name))
+ }
+ f, err := s.root.Open(name)
+ if err != nil {
+ return nil, 0, fmt.Errorf("rootfsimage: %w", err)
+ }
+ if fi, err = f.Stat(); err != nil || !fi.Mode().IsRegular() {
+ f.Close()
+ return nil, 0, fmt.Errorf("rootfsimage: %s in the layout is not a regular file", shown(name))
+ }
+ return f, fi.Size(), nil
+}
+
+func (s layoutStore) has(name string) bool {
+ fi, err := s.root.Stat(filepath.FromSlash(name))
+ return err == nil && fi.Mode().IsRegular()
+}
diff --git a/internal/rootfsimage/oci_test.go b/internal/rootfsimage/oci_test.go
new file mode 100644
index 000000000..be725aa55
--- /dev/null
+++ b/internal/rootfsimage/oci_test.go
@@ -0,0 +1,377 @@
+package rootfsimage
+
+import (
+ "archive/tar"
+ "encoding/json"
+ "errors"
+ "os"
+ "path/filepath"
+ "reflect"
+ "strings"
+ "testing"
+)
+
+func shopLayer(t *testing.T, body string) []byte {
+ return layerTar(t,
+ tEntry{name: "app/", typ: tar.TypeDir, mode: 0o755},
+ tEntry{name: "app/which", typ: tar.TypeReg, mode: 0o644, body: body},
+ )
+}
+
+func TestOpenLayoutPicksArm64AndReadsTheConfig(t *testing.T) {
+ dir := t.TempDir()
+ writeLayout(t, dir, true,
+ tImage{platform: "linux/amd64", env: []string{"ARCH=amd64"}, layers: [][]byte{shopLayer(t, "amd64")}},
+ tImage{platform: "linux/arm64/v8", env: []string{"PATH=/usr/local/bin:/usr/bin", "RAILS_ENV=production"}, workdir: "/app",
+ layers: [][]byte{shopLayer(t, "arm64"), layerTar(t, tEntry{name: "app/second", typ: tar.TypeReg, mode: 0o644, body: "2"})}},
+ )
+ tr, err := OpenLayout(dir, t.TempDir())
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got := readAll(t, tr, mustLookup(t, tr, "app/which")); got != "arm64" {
+ t.Errorf("app/which = %q, want the arm64 image's", got)
+ }
+ mustLookup(t, tr, "app/second")
+ want := ImageConfig{Architecture: "arm64", OS: "linux", Variant: "v8", Env: []string{"PATH=/usr/local/bin:/usr/bin", "RAILS_ENV=production"}, WorkingDir: "/app"}
+ if !reflect.DeepEqual(tr.Config, want) {
+ t.Errorf("Config = %+v, want %+v", tr.Config, want)
+ }
+}
+
+func TestOpenLayoutRefusesAnImageWithNoArm64(t *testing.T) {
+ dir := t.TempDir()
+ writeLayout(t, dir, true, tImage{platform: "linux/amd64", layers: [][]byte{shopLayer(t, "amd64")}})
+ _, err := OpenLayout(dir, t.TempDir())
+ var ne NoArm64Error
+ if !errors.As(err, &ne) || !reflect.DeepEqual(ne.Platforms, []string{"linux/amd64"}) {
+ t.Fatalf("err = %v, want NoArm64Error{[linux/amd64]}", err)
+ }
+
+ dir = t.TempDir()
+ writeLayout(t, dir, false,
+ tImage{platform: "linux/amd64", layers: [][]byte{shopLayer(t, "amd64")}},
+ tImage{platform: "linux/arm/v7", layers: [][]byte{shopLayer(t, "arm")}},
+ )
+ _, err = OpenLayout(dir, t.TempDir())
+ if !errors.As(err, &ne) || !reflect.DeepEqual(ne.Platforms, []string{"linux/amd64", "linux/arm/v7"}) {
+ t.Fatalf("err = %v, want NoArm64Error{[linux/amd64 linux/arm/v7]}", err)
+ }
+
+ dir = t.TempDir()
+ writeLayout(t, dir, false, tImage{config: "linux/amd64", layers: [][]byte{shopLayer(t, "amd64")}})
+ _, err = OpenLayout(dir, t.TempDir())
+ var ae ArchError
+ if !errors.As(err, &ae) || ae != (ArchError{Platform: "linux/amd64"}) {
+ t.Fatalf("err = %v, want ArchError{linux/amd64}", err)
+ }
+
+ // The index says arm64 but the config says otherwise: the config wins.
+ dir = t.TempDir()
+ writeLayout(t, dir, false, tImage{platform: "linux/arm64", config: "linux/amd64", layers: [][]byte{shopLayer(t, "amd64")}})
+ if _, err := OpenLayout(dir, t.TempDir()); !errors.As(err, &ae) || ae.Platform != "linux/amd64" {
+ t.Fatalf("err = %v, want ArchError{linux/amd64}", err)
+ }
+}
+
+func TestOpenLayoutRefusesTwoArm64Images(t *testing.T) {
+ dir := t.TempDir()
+ writeLayout(t, dir, false,
+ tImage{platform: "linux/arm64", layers: [][]byte{shopLayer(t, "one")}},
+ tImage{platform: "linux/arm64", layers: [][]byte{shopLayer(t, "two")}},
+ )
+ _, err := OpenLayout(dir, t.TempDir())
+ var me ManyImagesError
+ if !errors.As(err, &me) || me.N != 2 {
+ t.Fatalf("err = %v, want ManyImagesError{2}", err)
+ }
+ // The same image listed twice (two tags) is one image.
+ dir = t.TempDir()
+ img := tImage{platform: "linux/arm64", layers: [][]byte{shopLayer(t, "one")}}
+ writeLayout(t, dir, false, img, img)
+ if _, err := OpenLayout(dir, t.TempDir()); err != nil {
+ t.Fatalf("one image under two tags: %v", err)
+ }
+}
+
+func TestOpenLayoutReadsGzipZstdAndPlainLayers(t *testing.T) {
+ dir := t.TempDir()
+ writeLayout(t, dir, false, tImage{platform: "linux/arm64", layers: [][]byte{
+ gzipped(t, layerTar(t, tEntry{name: "gz", typ: tar.TypeReg, mode: 0o644, body: "gzip"})),
+ zstded(t, layerTar(t, tEntry{name: "zst", typ: tar.TypeReg, mode: 0o644, body: "zstd"})),
+ layerTar(t, tEntry{name: "plain", typ: tar.TypeReg, mode: 0o644, body: "tar"}),
+ }})
+ tr, err := OpenLayout(dir, t.TempDir())
+ if err != nil {
+ t.Fatal(err)
+ }
+ for p, body := range map[string]string{"gz": "gzip", "zst": "zstd", "plain": "tar"} {
+ if got := readAll(t, tr, mustLookup(t, tr, p)); got != body {
+ t.Errorf("%s = %q, want %q", p, got, body)
+ }
+ }
+}
+
+// layerBlob is the path of the image's only layer blob in a writeLayout dir.
+func layerBlob(t *testing.T, dir string) string {
+ t.Helper()
+ var idx struct{ Manifests []tDescriptor }
+ readJSONFile(t, filepath.Join(dir, "index.json"), &idx)
+ var m struct{ Layers []tDescriptor }
+ readJSONFile(t, filepath.Join(dir, "blobs", "sha256", strings.TrimPrefix(idx.Manifests[0].Digest, "sha256:")), &m)
+ return filepath.Join(dir, "blobs", "sha256", strings.TrimPrefix(m.Layers[0].Digest, "sha256:"))
+}
+
+func readJSONFile(t *testing.T, path string, v any) {
+ t.Helper()
+ b, err := os.ReadFile(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := json.Unmarshal(b, v); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestOpenLayoutRefusesABlobThatDoesNotMatchItsDigest(t *testing.T) {
+ for _, c := range []struct {
+ name string
+ layer []byte
+ }{
+ {"plain", shopLayer(t, "arm64")},
+ {"gzip", gzipped(t, shopLayer(t, "arm64"))},
+ } {
+ t.Run(c.name, func(t *testing.T) {
+ dir := t.TempDir()
+ writeLayout(t, dir, false, tImage{platform: "linux/arm64", layers: [][]byte{c.layer}})
+ blob := layerBlob(t, dir)
+ b, err := os.ReadFile(blob)
+ if err != nil {
+ t.Fatal(err)
+ }
+ // The last byte: tar end padding, or the gzip trailer's size.
+ b[len(b)-1] ^= 0x01
+ if err := os.WriteFile(blob, b, 0o644); err != nil {
+ t.Fatal(err)
+ }
+ _, err = OpenLayout(dir, t.TempDir())
+ var de DigestError
+ want := "sha256:" + filepath.Base(blob)
+ if !errors.As(err, &de) || de.Digest != want {
+ t.Fatalf("err = %v, want DigestError{%s}", err, want)
+ }
+ })
+ }
+}
+
+func TestOpenLayoutRefusesABlobOutsideTheLayout(t *testing.T) {
+ dir := t.TempDir()
+ writeLayout(t, dir, false, tImage{platform: "linux/arm64", layers: [][]byte{shopLayer(t, "arm64")}})
+ blob := layerBlob(t, dir)
+ outside := filepath.Join(t.TempDir(), "elsewhere")
+ b, err := os.ReadFile(blob)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(outside, b, 0o644); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Remove(blob); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Symlink(outside, blob); err != nil {
+ t.Skipf("no symlinks here: %v", err)
+ }
+ if _, err := OpenLayout(dir, t.TempDir()); err == nil {
+ t.Fatal("a blob symlinked out of the layout was read")
+ }
+}
+
+func TestOpenLayoutRefusesAMalformedDigest(t *testing.T) {
+ dir := t.TempDir()
+ writeLayout(t, dir, false, tImage{platform: "linux/arm64", layers: [][]byte{shopLayer(t, "arm64")}})
+ idx := []byte(`{"schemaVersion":2,"manifests":[{"mediaType":"` + mtOCIManifest + `","digest":"sha256:../../../etc/passwd","size":10,"platform":{"os":"linux","architecture":"arm64"}}]}`)
+ if err := os.WriteFile(filepath.Join(dir, "index.json"), idx, 0o644); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := OpenLayout(dir, t.TempDir()); err == nil || !strings.Contains(err.Error(), "digest") {
+ t.Fatalf("err = %v, want a malformed digest", err)
+ }
+}
+
+func TestOpenArchivePicksTheArm64ImageOverItsAttestation(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "shop.tar")
+ writeOCIArchive(t, path, true, tImage{platform: "linux/arm64", env: []string{"PORT=3000"}, layers: [][]byte{gzipped(t, shopLayer(t, "arm64"))}})
+ tr, err := OpenArchive(path, t.TempDir())
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got := readAll(t, tr, mustLookup(t, tr, "app/which")); got != "arm64" {
+ t.Errorf("app/which = %q", got)
+ }
+ if tr.Config.Architecture != "arm64" || !reflect.DeepEqual(tr.Config.Env, []string{"PORT=3000"}) {
+ t.Errorf("Config = %+v", tr.Config)
+ }
+
+ // A multi-platform save: amd64 and arm64, each with its attestation.
+ path = filepath.Join(t.TempDir(), "multi.tar")
+ writeOCIArchive(t, path, true,
+ tImage{platform: "linux/amd64", layers: [][]byte{shopLayer(t, "amd64")}},
+ tImage{platform: "linux/arm64", layers: [][]byte{shopLayer(t, "arm64")}},
+ )
+ tr, err = Open(path, t.TempDir())
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got := readAll(t, tr, mustLookup(t, tr, "app/which")); got != "arm64" {
+ t.Errorf("app/which = %q", got)
+ }
+}
+
+func TestOpenArchiveReadsTheLegacyForm(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "legacy.tar")
+ writeLegacyArchive(t, path, tImage{env: []string{"A=1"}, workdir: "/srv", layers: [][]byte{
+ shopLayer(t, "legacy"),
+ layerTar(t, tEntry{name: "app/.wh.which", typ: tar.TypeReg}, tEntry{name: "app/next", typ: tar.TypeReg, mode: 0o644, body: "n"}),
+ }})
+ tr, err := OpenArchive(path, t.TempDir())
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, ok := tr.Lookup("app/which"); ok {
+ t.Error("the second layer's whiteout was not applied")
+ }
+ mustLookup(t, tr, "app/next")
+ want := ImageConfig{Architecture: "arm64", OS: "linux", Env: []string{"A=1"}, WorkingDir: "/srv"}
+ if !reflect.DeepEqual(tr.Config, want) {
+ t.Errorf("Config = %+v, want %+v", tr.Config, want)
+ }
+
+ path = filepath.Join(t.TempDir(), "two.tar")
+ writeLegacyArchive(t, path,
+ tImage{layers: [][]byte{shopLayer(t, "one")}},
+ tImage{layers: [][]byte{shopLayer(t, "two")}},
+ )
+ _, err = OpenArchive(path, t.TempDir())
+ var me ManyImagesError
+ if !errors.As(err, &me) || me != (ManyImagesError{N: 2}) {
+ t.Fatalf("err = %v, want ManyImagesError{2}", err)
+ }
+
+ path = filepath.Join(t.TempDir(), "amd64.tar")
+ writeLegacyArchive(t, path, tImage{config: "linux/amd64", layers: [][]byte{shopLayer(t, "amd64")}})
+ var ae ArchError
+ if _, err := OpenArchive(path, t.TempDir()); !errors.As(err, &ae) || ae.Platform != "linux/amd64" {
+ t.Fatalf("err = %v, want ArchError{linux/amd64}", err)
+ }
+}
+
+func TestOpenRefusesWhatIsNotAnImage(t *testing.T) {
+ dir := t.TempDir()
+ text := filepath.Join(dir, "notes.txt")
+ if err := os.WriteFile(text, []byte(strings.Repeat("not an image\n", 100)), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ short := filepath.Join(dir, "short.txt")
+ if err := os.WriteFile(short, []byte("hi\n"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ empty := filepath.Join(dir, "empty.tar")
+ if err := os.WriteFile(empty, nil, 0o644); err != nil {
+ t.Fatal(err)
+ }
+ other := filepath.Join(dir, "other.tar")
+ writeTarFile(t, other, []tarFile{{"README", []byte("hello"), false}})
+ for _, src := range []string{text, short, empty, other} {
+ _, err := Open(src, t.TempDir())
+ var ne NotAnImageError
+ if !errors.As(err, &ne) || ne.Source != src {
+ t.Errorf("%s: err = %v, want NotAnImageError", src, err)
+ }
+ }
+}
+
+func TestOpenArchiveRefusesALayerOutsideTheArchive(t *testing.T) {
+ // A legacy manifest naming a path that is not an entry is refused, never
+ // read from the disk beside the archive.
+ dir := t.TempDir()
+ if err := os.WriteFile(filepath.Join(dir, "layer.tar"), shopLayer(t, "beside"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ cfg := []byte(`{"architecture":"arm64","os":"linux"}`)
+ mj := []byte(`[{"Config":"c.json","Layers":["../layer.tar"]}]`)
+ path := filepath.Join(dir, "sub", "evil.tar")
+ if err := os.Mkdir(filepath.Dir(path), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ writeTarFile(t, path, []tarFile{{"c.json", cfg, false}, {"manifest.json", mj, false}})
+ if _, err := OpenArchive(path, t.TempDir()); err == nil {
+ t.Fatal("a layer outside the archive was read")
+ }
+}
+
+func TestErrorsSpeakTheDeck(t *testing.T) {
+ for _, c := range []struct {
+ err error
+ want string
+ }{
+ {ArchError{"linux/amd64"}, "this image is built for linux/amd64, and boxes run linux/arm64. rebuild it for linux/arm64, for example with docker buildx build --platform linux/arm64"},
+ {NoArm64Error{[]string{"linux/amd64", "linux/arm/v7"}}, "this image has no linux/arm64 variant (it has linux/amd64, linux/arm/v7). rebuild it for linux/arm64, for example with docker buildx build --platform linux/arm64"},
+ {NotAnImageError{"shop.tar"}, "shop.tar is not an OCI image, a docker save archive or a directory"},
+ {UnsafePathError{"../x"}, "the image has an entry outside its root (../x), so it was not shipped"},
+ {LinkError{"usr/bin/ruby2", "usr/bin/ruby"}, "the image links usr/bin/ruby2 to usr/bin/ruby, which is not a file in the image"},
+ {ManyImagesError{2}, "the archive holds 2 images. save one image and ship that"},
+ {DigestError{"sha256:ab"}, "the image's layer sha256:ab does not match its digest, so it was not shipped"},
+ // A name with control bytes is quoted, so it cannot drive the terminal.
+ {UnsafePathError{"../\x1b[2Jx"}, `the image has an entry outside its root ("../\x1b[2Jx"), so it was not shipped`},
+ } {
+ if got := c.err.Error(); got != c.want {
+ t.Errorf("%T: %q\nwant %q", c.err, got, c.want)
+ }
+ }
+}
+
+func TestOpenLayoutRefusesAnOversizedManifest(t *testing.T) {
+ dir := t.TempDir()
+ writeLayout(t, dir, false, tImage{platform: "linux/arm64", layers: [][]byte{shopLayer(t, "arm64")}})
+ var idx struct{ Manifests []tDescriptor }
+ readJSONFile(t, filepath.Join(dir, "index.json"), &idx)
+ idx.Manifests[0].Size = 1 << 40
+ if err := os.WriteFile(filepath.Join(dir, "index.json"), indexJSON(t, idx.Manifests), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := OpenLayout(dir, t.TempDir()); !errors.Is(err, errTooLarge) {
+ t.Fatalf("err = %v, want errTooLarge", err)
+ }
+}
+
+func TestOpenLayoutRefusesIndexesNestedTooDeep(t *testing.T) {
+ for _, c := range []struct {
+ depth int
+ ok bool
+ }{{maxIndexDepth, true}, {maxIndexDepth + 1, false}} {
+ dir := t.TempDir()
+ w := newBlobWriter(t)
+ ds, _ := w.manifests(false, []tImage{{platform: "linux/arm64", layers: [][]byte{shopLayer(t, "deep")}}})
+ for i := 0; i < c.depth; i++ {
+ ds = []tDescriptor{w.add(mtOCIIndex, indexJSON(t, ds))}
+ }
+ files := map[string][]byte{"oci-layout": []byte(ociLayoutFile), "index.json": indexJSON(t, ds)}
+ for h, b := range w.blobs {
+ files["blobs/sha256/"+h] = b
+ }
+ for name, b := range files {
+ p := filepath.Join(dir, filepath.FromSlash(name))
+ if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(p, b, 0o644); err != nil {
+ t.Fatal(err)
+ }
+ }
+ _, err := OpenLayout(dir, t.TempDir())
+ if (err == nil) != c.ok {
+ t.Errorf("%d nested indexes: err = %v, want ok %v", c.depth, err, c.ok)
+ }
+ }
+}
diff --git a/internal/rootfsimage/testimage_test.go b/internal/rootfsimage/testimage_test.go
new file mode 100644
index 000000000..6daa07d78
--- /dev/null
+++ b/internal/rootfsimage/testimage_test.go
@@ -0,0 +1,426 @@
+package rootfsimage
+
+import (
+ "archive/tar"
+ "bytes"
+ "compress/gzip"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/klauspost/compress/zstd"
+)
+
+// Builders for the tests: layer tars, OCI layouts and the two docker save
+// archive forms, written with archive/tar and encoding/json only.
+
+// defaultMtime is the time an entry gets when its tEntry leaves mtime 0.
+const defaultMtime int64 = 1_700_000_000
+
+type tEntry struct {
+ name string
+ typ byte // tar.TypeReg, TypeDir, TypeSymlink, TypeLink, TypeChar, TypeFifo
+ mode int64 // with c_ISUID etc. as tar writes them
+ uid, gid int
+ body string
+ link string
+ mtime int64
+}
+
+// layerTar is one uncompressed layer holding es, in order.
+func layerTar(t *testing.T, es ...tEntry) []byte {
+ t.Helper()
+ var buf bytes.Buffer
+ tw := tar.NewWriter(&buf)
+ for _, e := range es {
+ mt := e.mtime
+ if mt == 0 {
+ mt = defaultMtime
+ }
+ h := &tar.Header{
+ Name: e.name,
+ Typeflag: e.typ,
+ Mode: e.mode,
+ Uid: e.uid,
+ Gid: e.gid,
+ Linkname: e.link,
+ ModTime: time.Unix(mt, 0),
+ Format: tar.FormatPAX,
+ }
+ if e.typ == tar.TypeReg {
+ h.Size = int64(len(e.body))
+ }
+ if err := tw.WriteHeader(h); err != nil {
+ t.Fatalf("header %s: %v", e.name, err)
+ }
+ if e.typ == tar.TypeReg {
+ if _, err := io.WriteString(tw, e.body); err != nil {
+ t.Fatal(err)
+ }
+ }
+ }
+ if err := tw.Close(); err != nil {
+ t.Fatal(err)
+ }
+ return buf.Bytes()
+}
+
+func gzipped(t *testing.T, b []byte) []byte {
+ t.Helper()
+ var buf bytes.Buffer
+ zw := gzip.NewWriter(&buf)
+ if _, err := zw.Write(b); err != nil {
+ t.Fatal(err)
+ }
+ if err := zw.Close(); err != nil {
+ t.Fatal(err)
+ }
+ return buf.Bytes()
+}
+
+func zstded(t *testing.T, b []byte) []byte {
+ t.Helper()
+ zw, err := zstd.NewWriter(nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer zw.Close()
+ return zw.EncodeAll(b, nil)
+}
+
+type tImage struct {
+ platform string // "linux/arm64"; "" = no platform in the index
+ config string // the config's os/arch when it differs from platform ("" = platform, or linux/arm64)
+ env []string
+ workdir string
+ layers [][]byte // already compressed or not, as the test wants
+}
+
+const (
+ mtOCIIndex = "application/vnd.oci.image.index.v1+json"
+ mtOCIManifest = "application/vnd.oci.image.manifest.v1+json"
+ mtOCIConfig = "application/vnd.oci.image.config.v1+json"
+ mtOCILayer = "application/vnd.oci.image.layer.v1.tar"
+ mtInToto = "application/vnd.in-toto+json"
+)
+
+type tDescriptor struct {
+ MediaType string `json:"mediaType"`
+ Digest string `json:"digest"`
+ Size int64 `json:"size"`
+ Platform *tPlatform `json:"platform,omitempty"`
+ Annotations map[string]string `json:"annotations,omitempty"`
+}
+
+type tPlatform struct {
+ Architecture string `json:"architecture"`
+ OS string `json:"os"`
+ Variant string `json:"variant,omitempty"`
+}
+
+func parsePlatform(s string) *tPlatform {
+ if s == "" {
+ return nil
+ }
+ parts := strings.Split(s, "/")
+ p := &tPlatform{OS: parts[0], Architecture: parts[1]}
+ if len(parts) > 2 {
+ p.Variant = parts[2]
+ }
+ return p
+}
+
+// blobWriter collects a layout's blobs by digest.
+type blobWriter struct {
+ t *testing.T
+ blobs map[string][]byte // hex -> bytes
+ order []string
+}
+
+func newBlobWriter(t *testing.T) *blobWriter {
+ return &blobWriter{t: t, blobs: map[string][]byte{}}
+}
+
+func (w *blobWriter) add(mediaType string, b []byte) tDescriptor {
+ sum := sha256.Sum256(b)
+ h := hex.EncodeToString(sum[:])
+ if _, ok := w.blobs[h]; !ok {
+ w.order = append(w.order, h)
+ }
+ w.blobs[h] = b
+ return tDescriptor{MediaType: mediaType, Digest: "sha256:" + h, Size: int64(len(b))}
+}
+
+func (w *blobWriter) addJSON(mediaType string, v any) tDescriptor {
+ b, err := json.Marshal(v)
+ if err != nil {
+ w.t.Fatal(err)
+ }
+ return w.add(mediaType, b)
+}
+
+// image writes one image's config, layers and manifest, returning the
+// manifest's descriptor with the index platform set from img.platform, and
+// the blob paths of its config and layers (what a docker manifest.json names).
+func (w *blobWriter) image(img tImage) (tDescriptor, string, []string) {
+ cp := img.config
+ if cp == "" {
+ cp = img.platform
+ }
+ if cp == "" {
+ cp = "linux/arm64"
+ }
+ p := parsePlatform(cp)
+ cfg := map[string]any{
+ "architecture": p.Architecture,
+ "os": p.OS,
+ "config": map[string]any{"Env": img.env, "WorkingDir": img.workdir},
+ "rootfs": map[string]any{"type": "layers", "diff_ids": []string{}},
+ }
+ if p.Variant != "" {
+ cfg["variant"] = p.Variant
+ }
+ cd := w.addJSON(mtOCIConfig, cfg)
+ var layers []tDescriptor
+ var layerPaths []string
+ for _, l := range img.layers {
+ ld := w.add(mtOCILayer, l)
+ layers = append(layers, ld)
+ layerPaths = append(layerPaths, blobPath(ld))
+ }
+ md := w.addJSON(mtOCIManifest, map[string]any{
+ "schemaVersion": 2,
+ "mediaType": mtOCIManifest,
+ "config": cd,
+ "layers": layers,
+ })
+ md.Platform = parsePlatform(img.platform)
+ return md, blobPath(cd), layerPaths
+}
+
+func blobPath(d tDescriptor) string {
+ return "blobs/sha256/" + strings.TrimPrefix(d.Digest, "sha256:")
+}
+
+// attestation writes an attestation manifest for the image whose manifest
+// is md, as buildx puts beside every image.
+func (w *blobWriter) attestation(md tDescriptor) tDescriptor {
+ cd := w.addJSON(mtOCIConfig, map[string]any{"architecture": "unknown", "os": "unknown", "rootfs": map[string]any{"type": "layers"}})
+ ld := w.add(mtInToto, []byte(`{"_type":"https://in-toto.io/Statement/v0.1"}`))
+ ad := w.addJSON(mtOCIManifest, map[string]any{
+ "schemaVersion": 2,
+ "mediaType": mtOCIManifest,
+ "config": cd,
+ "layers": []tDescriptor{ld},
+ })
+ ad.Platform = &tPlatform{Architecture: "unknown", OS: "unknown"}
+ ad.Annotations = map[string]string{
+ "vnd.docker.reference.digest": md.Digest,
+ "vnd.docker.reference.type": "attestation-manifest",
+ }
+ return ad
+}
+
+// manifests writes every image (and its attestation when attest is set),
+// returning the index descriptors and the docker manifest.json rows.
+func (w *blobWriter) manifests(attest bool, imgs []tImage) ([]tDescriptor, []map[string]any) {
+ var ds []tDescriptor
+ var legacy []map[string]any
+ for i, img := range imgs {
+ md, cfg, layers := w.image(img)
+ ds = append(ds, md)
+ if attest {
+ ds = append(ds, w.attestation(md))
+ }
+ legacy = append(legacy, map[string]any{
+ "Config": cfg,
+ "RepoTags": []string{fmt.Sprintf("shop:%d", i)},
+ "Layers": layers,
+ })
+ }
+ return ds, legacy
+}
+
+func indexJSON(t *testing.T, ds []tDescriptor) []byte {
+ t.Helper()
+ b, err := json.Marshal(map[string]any{"schemaVersion": 2, "mediaType": mtOCIIndex, "manifests": ds})
+ if err != nil {
+ t.Fatal(err)
+ }
+ return b
+}
+
+const ociLayoutFile = `{"imageLayoutVersion":"1.0.0"}`
+
+// writeLayout writes an OCI layout directory: oci-layout, index.json listing
+// each image's manifest, and blobs/sha256/...; attest adds an unknown/unknown
+// attestation manifest per image.
+func writeLayout(t *testing.T, dir string, attest bool, imgs ...tImage) {
+ t.Helper()
+ w := newBlobWriter(t)
+ ds, _ := w.manifests(attest, imgs)
+ files := map[string][]byte{
+ "oci-layout": []byte(ociLayoutFile),
+ "index.json": indexJSON(t, ds),
+ }
+ for h, b := range w.blobs {
+ files["blobs/sha256/"+h] = b
+ }
+ for name, b := range files {
+ p := filepath.Join(dir, filepath.FromSlash(name))
+ if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(p, b, 0o644); err != nil {
+ t.Fatal(err)
+ }
+ }
+}
+
+// writeOCIArchive writes a layout inside a tar as Docker 25+ docker save
+// does: index.json names one nested image index that lists the images (and
+// their attestations), and a docker manifest.json sits beside it.
+func writeOCIArchive(t *testing.T, path string, attest bool, imgs ...tImage) {
+ t.Helper()
+ w := newBlobWriter(t)
+ ds, legacy := w.manifests(attest, imgs)
+ nested := w.add(mtOCIIndex, indexJSON(t, ds))
+ nested.Annotations = map[string]string{"io.containerd.image.name": "docker.io/library/shop:latest"}
+ mj, err := json.Marshal(legacy)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var entries []tarFile
+ entries = append(entries, tarFile{"blobs/", nil, true}, tarFile{"blobs/sha256/", nil, true})
+ for _, h := range w.order {
+ entries = append(entries, tarFile{"blobs/sha256/" + h, w.blobs[h], false})
+ }
+ entries = append(entries,
+ tarFile{"index.json", indexJSON(t, []tDescriptor{nested}), false},
+ tarFile{"manifest.json", mj, false},
+ tarFile{"oci-layout", []byte(ociLayoutFile), false},
+ )
+ writeTarFile(t, path, entries)
+}
+
+// writeLegacyArchive writes the pre-OCI docker save form: manifest.json,
+// <id>/layer.tar per layer and <hash>.json per config.
+func writeLegacyArchive(t *testing.T, path string, imgs ...tImage) {
+ t.Helper()
+ var entries []tarFile
+ var manifest []map[string]any
+ written := map[string]bool{} // two images with one config share its file
+ for i, img := range imgs {
+ cp := img.config
+ if cp == "" {
+ cp = img.platform
+ }
+ if cp == "" {
+ cp = "linux/arm64"
+ }
+ p := parsePlatform(cp)
+ cfg, err := json.Marshal(map[string]any{
+ "architecture": p.Architecture,
+ "os": p.OS,
+ "config": map[string]any{"Env": img.env, "WorkingDir": img.workdir},
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ sum := sha256.Sum256(cfg)
+ cfgName := hex.EncodeToString(sum[:]) + ".json"
+ if !written[cfgName] {
+ written[cfgName] = true
+ entries = append(entries, tarFile{cfgName, cfg, false})
+ }
+ var layers []string
+ for j, l := range img.layers {
+ id := fmt.Sprintf("%064x", i*1000+j+1)
+ entries = append(entries, tarFile{id + "/", nil, true}, tarFile{id + "/layer.tar", l, false})
+ layers = append(layers, id+"/layer.tar")
+ }
+ manifest = append(manifest, map[string]any{
+ "Config": cfgName,
+ "RepoTags": []string{fmt.Sprintf("shop:%d", i)},
+ "Layers": layers,
+ })
+ }
+ mj, err := json.Marshal(manifest)
+ if err != nil {
+ t.Fatal(err)
+ }
+ entries = append(entries, tarFile{"manifest.json", mj, false})
+ writeTarFile(t, path, entries)
+}
+
+type tarFile struct {
+ name string
+ body []byte
+ dir bool
+}
+
+func writeTarFile(t *testing.T, path string, files []tarFile) {
+ t.Helper()
+ var buf bytes.Buffer
+ tw := tar.NewWriter(&buf)
+ for _, f := range files {
+ h := &tar.Header{Name: f.name, Mode: 0o644, ModTime: time.Unix(defaultMtime, 0), Typeflag: tar.TypeReg, Size: int64(len(f.body))}
+ if f.dir {
+ h.Typeflag, h.Mode, h.Size = tar.TypeDir, 0o755, 0
+ }
+ if err := tw.WriteHeader(h); err != nil {
+ t.Fatal(err)
+ }
+ if !f.dir {
+ if _, err := tw.Write(f.body); err != nil {
+ t.Fatal(err)
+ }
+ }
+ }
+ if err := tw.Close(); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(path, buf.Bytes(), 0o644); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// apply applies one layer and fails the test on an error.
+func apply(t *testing.T, tr *Tree, layer []byte) {
+ t.Helper()
+ if err := tr.ApplyLayer(bytes.NewReader(layer)); err != nil {
+ t.Fatalf("ApplyLayer: %v", err)
+ }
+}
+
+// readAll is a regular file's bytes.
+func readAll(t *testing.T, tr *Tree, e *Entry) string {
+ t.Helper()
+ rc, err := tr.Open(e)
+ if err != nil {
+ t.Fatalf("Open %s: %v", e.Path, err)
+ }
+ defer rc.Close()
+ b, err := io.ReadAll(rc)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return string(b)
+}
+
+// mustLookup is the entry at p, failing the test when there is none.
+func mustLookup(t *testing.T, tr *Tree, p string) *Entry {
+ t.Helper()
+ e, ok := tr.Lookup(p)
+ if !ok {
+ t.Fatalf("%s missing", p)
+ }
+ return e
+}
diff --git a/internal/rootfsimage/tree.go b/internal/rootfsimage/tree.go
new file mode 100644
index 000000000..528be1adf
--- /dev/null
+++ b/internal/rootfsimage/tree.go
@@ -0,0 +1,329 @@
+// Package rootfsimage turns an OCI image, a docker save or podman save
+// archive, or a plain directory into the flattened tree a rootfs release
+// ships (rootfs apps Plan 1, spec §5.1).
+//
+// Every source is untrusted: it runs on a developer's machine over images
+// pulled from anywhere. Nothing from a source is ever written to the disk by
+// its own name. A layer's file bytes go to the spool under a counter, and
+// its names only ever become keys of an in-memory tree, so a path that
+// leaves the root, a symlink that points out of it or a hardlink to a file
+// outside it can be refused but never written through.
+package rootfsimage
+
+import (
+ "fmt"
+ "io"
+ "io/fs"
+ "os"
+ "path"
+ "path/filepath"
+ "slices"
+ "strconv"
+ "strings"
+ "time"
+)
+
+// EntryType is what an entry is. Device files, sockets and pipes never
+// enter a tree (Tree.Skipped counts them).
+type EntryType uint8
+
+const (
+ TypeDir EntryType = iota + 1
+ TypeFile
+ TypeSymlink
+)
+
+// Entry is one file, directory or symlink of a tree.
+type Entry struct {
+ Path string // clean, slash-separated, relative to the root, never "" or "." (Tree.Root's Path is ".")
+ Type EntryType //
+ Mode fs.FileMode // permission bits plus fs.ModeSetuid, ModeSetgid, ModeSticky
+ UID, GID uint32 //
+ ModTime time.Time // whole seconds, UTC
+ Size int64 // a regular file's bytes; 0 otherwise
+ LinkTarget string // symlinks, as the image gives it (resolved only by Resolve)
+ src string // the file holding a regular file's bytes; entries sharing src are hardlinks
+}
+
+// ImageConfig is what the image's OCI config says that a rootfs release
+// keeps (contract deviation 2).
+type ImageConfig struct {
+ Architecture, OS, Variant string
+ Env []string // KEY=VALUE, the config's order
+ WorkingDir string
+}
+
+// The tree's bounds. A source past any of them is refused (errTooLarge)
+// before it can fill the developer's disk or memory.
+const (
+ // defaultMaxBytes bounds the file bytes written to the spool, counted
+ // cumulatively over every layer (a file a later layer replaces still
+ // used its space). A release's squashfs is at most 2 GiB compressed
+ // (release.MaxRootfsBytes); four times that leaves room for any image
+ // that could fit.
+ defaultMaxBytes int64 = 8 << 30
+ // defaultMaxHeaders bounds the tar headers read over every layer, which
+ // bounds the entries, the spool's file count and the tree's memory.
+ defaultMaxHeaders = 1 << 20
+ // maxPathBytes bounds an entry's name and a link's target (PATH_MAX).
+ maxPathBytes = 4096
+ // maxLinks is how many symlinks one resolution follows, as Linux does.
+ maxLinks = 40
+ // headerOverhead is the most tar bytes one entry costs beyond its data
+ // (a header, a PAX header holding two maxPathBytes names, padding),
+ // for the bound on a layer's decoded stream.
+ headerOverhead = 16 << 10
+)
+
+// Tree is a flattened root filesystem: every entry by path, the bytes of
+// its regular files in a spool directory, and the image's config.
+type Tree struct {
+ Root Entry // the root directory's owner, mode and time ("./" in a layer, else 0:0 0755)
+ Config ImageConfig // zero for a directory source
+ Skipped int // device files, sockets and pipes left out
+
+ spool string
+ entries map[string]*Entry
+ kids map[string]map[string]struct{} // a directory's path ("" for the root) to its children's paths
+ blobs int // the spool's counter
+
+ // The bounds, and what has been used of them.
+ maxBytes int64
+ maxHeaders int
+ usedBytes int64
+ headers int
+}
+
+// NewTree is an empty tree whose file bytes go to spool, an existing
+// directory the tree owns: the caller removes it when done with the tree.
+// Its root is 0:0 0755 at the Unix epoch until a layer's "./" entry says
+// otherwise.
+func NewTree(spool string) *Tree {
+ return &Tree{
+ Root: Entry{Path: ".", Type: TypeDir, Mode: 0o755, ModTime: time.Unix(0, 0).UTC()},
+ spool: spool,
+ entries: map[string]*Entry{},
+ kids: map[string]map[string]struct{}{},
+ maxBytes: defaultMaxBytes,
+ maxHeaders: defaultMaxHeaders,
+ }
+}
+
+// Open opens a source: a directory holding oci-layout is an OCI layout
+// (OpenLayout), a regular file a save archive (OpenArchive). Anything else
+// is NotAnImageError, a directory without oci-layout included for now.
+func Open(src, spool string) (*Tree, error) {
+ fi, err := os.Stat(src)
+ if err != nil {
+ return nil, err
+ }
+ switch {
+ case fi.IsDir():
+ if lfi, err := os.Stat(filepath.Join(src, "oci-layout")); err == nil && lfi.Mode().IsRegular() {
+ return OpenLayout(src, spool)
+ }
+ case fi.Mode().IsRegular():
+ return OpenArchive(src, spool)
+ }
+ return nil, NotAnImageError{Source: src}
+}
+
+// Entries is every entry but the root, sorted by Path.
+func (t *Tree) Entries() []*Entry {
+ out := make([]*Entry, 0, len(t.entries))
+ for _, e := range t.entries {
+ out = append(out, e)
+ }
+ slices.SortFunc(out, func(a, b *Entry) int { return strings.Compare(a.Path, b.Path) })
+ return out
+}
+
+// Lookup is the entry at exactly p, following no symlink. p is cleaned
+// first, so "/etc/passwd" finds "etc/passwd"; "." (or "" or "/") is the root.
+func (t *Tree) Lookup(p string) (*Entry, bool) {
+ p = cleanRel(p)
+ if p == "" {
+ return &t.Root, true
+ }
+ e, ok := t.entries[p]
+ return e, ok
+}
+
+// Resolve follows symlinks in every component of p, the last included, the
+// way the kernel would inside a chroot at the root: an absolute target
+// starts again from the root, a relative one from the link's directory, and
+// ".." stops at the root, so no resolution ever leaves it. It answers the
+// final path ("." for the root) and whether an entry exists there. A path
+// whose resolution needs more than 40 symlinks (a loop) answers false.
+func (t *Tree) Resolve(p string) (string, bool) {
+ r, ok, err := t.resolve(p, true)
+ if err != nil {
+ return dotRoot(r), false
+ }
+ return dotRoot(r), ok
+}
+
+// Open reads a regular file's bytes.
+func (t *Tree) Open(e *Entry) (io.ReadCloser, error) {
+ if e.Type != TypeFile {
+ return nil, fmt.Errorf("rootfsimage: %s is not a regular file", e.Path)
+ }
+ return os.Open(e.src)
+}
+
+// Newest is the latest ModTime in the tree, the root's included.
+func (t *Tree) Newest() time.Time {
+ n := t.Root.ModTime
+ for _, e := range t.entries {
+ if e.ModTime.After(n) {
+ n = e.ModTime
+ }
+ }
+ return n
+}
+
+// Files is the number of regular files.
+func (t *Tree) Files() int {
+ n := 0
+ for _, e := range t.entries {
+ if e.Type == TypeFile {
+ n++
+ }
+ }
+ return n
+}
+
+// resolve is Resolve with the error for a resolution past maxLinks
+// (LinkError naming the symlink it stopped at) and the choice of following
+// a symlink in the last component. The path it answers is "" for the root.
+// A component that is missing, or that is a file with more components after
+// it, ends the walk's knowledge: the rest is joined lexically (still clamped
+// at the root) and the answer is false.
+func (t *Tree) resolve(p string, followLast bool) (string, bool, error) {
+ comps := splitPath(cleanRel(p))
+ cur := ""
+ exists := true
+ links := 0
+ for len(comps) > 0 {
+ c := comps[0]
+ comps = comps[1:]
+ if c == ".." {
+ cur = parentOf(cur)
+ continue
+ }
+ next := joinRel(cur, c)
+ if !exists {
+ cur = next
+ continue
+ }
+ e, ok := t.entries[next]
+ switch {
+ case !ok:
+ exists = false
+ cur = next
+ case e.Type == TypeSymlink && (len(comps) > 0 || followLast):
+ links++
+ if links > maxLinks {
+ return next, false, LinkError{Path: next, Target: e.LinkTarget}
+ }
+ if strings.HasPrefix(e.LinkTarget, "/") {
+ cur = ""
+ }
+ comps = append(splitPath(path.Clean(e.LinkTarget)), comps...)
+ case e.Type == TypeFile && len(comps) > 0:
+ exists = false
+ cur = next
+ default:
+ cur = next
+ }
+ }
+ return cur, exists, nil
+}
+
+// add puts e in the tree, replacing whatever was at its path (a directory's
+// subtree with it). Its parent must already be a directory.
+func (t *Tree) add(e *Entry) {
+ if old, ok := t.entries[e.Path]; ok && old.Type == TypeDir && e.Type != TypeDir {
+ t.remove(e.Path)
+ }
+ t.entries[e.Path] = e
+ parent := parentOf(e.Path)
+ if t.kids[parent] == nil {
+ t.kids[parent] = map[string]struct{}{}
+ }
+ t.kids[parent][e.Path] = struct{}{}
+}
+
+// remove deletes the entry at p and everything under it.
+func (t *Tree) remove(p string) {
+ if _, ok := t.entries[p]; !ok {
+ return
+ }
+ for k := range t.kids[p] {
+ t.remove(k)
+ }
+ delete(t.kids, p)
+ delete(t.entries, p)
+ delete(t.kids[parentOf(p)], p)
+}
+
+// writeBlob copies exactly n bytes of r to a new spool file, named by the
+// counter and never by anything the image says, and answers its path.
+func (t *Tree) writeBlob(r io.Reader, n int64) (string, error) {
+ if n < 0 || n > t.maxBytes-t.usedBytes {
+ return "", fmt.Errorf("%w: its files pass %d bytes", errTooLarge, t.maxBytes)
+ }
+ t.usedBytes += n
+ t.blobs++
+ name := filepath.Join(t.spool, strconv.Itoa(t.blobs))
+ f, err := os.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
+ if err != nil {
+ return "", fmt.Errorf("rootfsimage: spool: %w", err)
+ }
+ if _, err := io.CopyN(f, r, n); err != nil {
+ f.Close()
+ return "", fmt.Errorf("rootfsimage: reading a file of the layer: %w", err)
+ }
+ if err := f.Close(); err != nil {
+ return "", fmt.Errorf("rootfsimage: spool: %w", err)
+ }
+ return name, nil
+}
+
+// cleanRel is p as a tree path: slash-separated, relative, clean, "" for
+// the root, with any ".." that would climb out of the root dropped. It is
+// for lookups; a layer's names go through cleanName, which refuses that.
+func cleanRel(p string) string {
+ p = path.Clean("/" + p)
+ return strings.TrimPrefix(p, "/")
+}
+
+func splitPath(p string) []string {
+ p = strings.TrimPrefix(p, "/")
+ if p == "" || p == "." {
+ return nil
+ }
+ return strings.Split(p, "/")
+}
+
+func parentOf(p string) string {
+ i := strings.LastIndexByte(p, '/')
+ if i < 0 {
+ return ""
+ }
+ return p[:i]
+}
+
+func joinRel(dir, base string) string {
+ if dir == "" {
+ return base
+ }
+ return dir + "/" + base
+}
+
+func dotRoot(p string) string {
+ if p == "" {
+ return "."
+ }
+ return p
+}