amadan

carlos / platform Public

moves/archive: the deterministic tar+zstd files archive, sealed by the config keypair (§8)

Build, PlainDigest, Seal, Open, Verify and Extract. The data key is
sealed with appconfig.SealBound (always v2) and Open refuses a v1 data
key (F4). The cap counts content only; entry-count, path-length and
decoded-size limits are shared by Build, Verify and Extract, so a bomb is
bounded on the target and every archive Build writes lands (F25). Build
reads through an os.Root and checks each opened file's identity, so a
file swapped after the walk cannot carry a host file into the archive.
Extract stages beside the destination and renames into place only once
the whole archive is accepted, so a refused archive writes nothing.
Manifest.SealedDataKey per F29. Copy rows MV-N77 to MV-N81 (pending).

Adds github.com/klauspost/compress (zstd) as a direct dependency at
v1.18.0, already in the module graph through litestream.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude 91d0326f6a1e48aa51870d34a9280e55a24f881a parent a81caec
10 files changed, +2316 −0
  • docs/superpowers/plans/2026-10-06-moves-phase1-copy.md +5 −0
  • go.mod +1 −0
  • internal/moves/archive/archive.go +963 −0
  • internal/moves/archive/archive_test.go +1017 −0
  • internal/moves/archive/archive_unix_test.go +66 −0
  • internal/moves/archive/open_other.go +6 −0
  • internal/moves/archive/open_unix.go +9 −0
  • internal/moves/archive/seal.go +188 −0
  • internal/moves/archive/sync_linux.go +38 −0
  • internal/moves/archive/sync_other.go +23 −0
diff --git a/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md b/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md
index 3a0296fe3..1920a54c0 100644
--- a/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md
+++ b/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md
@@ -116,6 +116,11 @@ Columns: id, file, text, status.
| MV-N74 | internal/host/movesreport.go | the host claim has not touched <region> yet | pending |
| MV-N75 | internal/host/movesreport.go | waiting for the root stage pass | pending |
| MV-N76 | internal/host/movesreport.go | the source has not reported its release yet | pending |
+| MV-N77 | internal/moves/archive/archive.go | archive: only regular files and directories move: <path> | pending |
+| MV-N78 | internal/moves/archive/archive.go | archive: the files directory is over 256 MiB | pending |
+| MV-N79 | internal/moves/archive/archive.go | archive: the files directory holds more than 131072 entries | pending |
+| MV-N80 | internal/moves/archive/archive.go | archive: a file changed while it was being archived: <path> | pending |
+| MV-N81 | internal/moves/archive/archive.go | archive: refused path: <path> is longer than 4096 bytes | pending |
## Plan 2
diff --git a/go.mod b/go.mod
index ae7c93327..8498846ed 100644
--- a/go.mod
+++ b/go.mod
@@ -82,6 +82,7 @@ require (
require (
amadan.net/carlos/platform.git/launchctx v0.0.0
github.com/aws/aws-sdk-go-v2/service/sns v1.43.0
+ github.com/klauspost/compress v1.18.0
github.com/oschwald/maxminddb-golang v1.13.1
)
diff --git a/internal/moves/archive/archive.go b/internal/moves/archive/archive.go
new file mode 100644
index 000000000..9ccc0754e
--- /dev/null
+++ b/internal/moves/archive/archive.go
@@ -0,0 +1,963 @@
+// Package archive is the files archive of the move-any-app design §8: a
+// deterministic tar of an instance's files directory and its <host>.env,
+// compressed with zstd, then sealed under the deployment's config keypair.
+//
+// The source box builds and seals it; the target box opens, verifies and
+// extracts it. The target treats every archive as hostile. The config
+// public key is widely held, so anyone who can write the staging bucket can
+// seal an archive that opens: authentication proves who it was sealed for,
+// not who sealed it. Every limit Build obeys, Verify and Extract enforce
+// again, and Build obeys every limit Verify and Extract enforce, so an
+// archive Build wrote always lands.
+package archive
+
+import (
+ "archive/tar"
+ "crypto/ecdh"
+ "crypto/sha256"
+ "encoding/hex"
+ "errors"
+ "fmt"
+ "hash"
+ "io"
+ "io/fs"
+ "os"
+ "path"
+ "path/filepath"
+ "runtime"
+ "sort"
+ "strings"
+
+ "github.com/klauspost/compress/zstd"
+
+ "github.com/carlosframework/platform/internal/appconfig"
+)
+
+// Cap is the archive's content cap (M5): the sum of the regular files'
+// sizes, <host>.env included. Tar headers and padding are not content and
+// do not count against it (F25); they are bounded by MaxEntries and
+// MaxPathLen instead.
+const Cap int64 = 256 << 20
+
+// MaxEntries bounds the number of tar entries (files, directories and env)
+// in one archive, and MaxPathLen the length of a path inside the files
+// directory. With Cap they bound the decoded stream (MaxDecoded), so a zstd
+// bomb is bounded on the target even after it passed Verify.
+const (
+ MaxEntries = 1 << 17
+ MaxPathLen = 4096
+)
+
+// entryOverhead is the most tar bytes one entry costs beyond its content:
+// a 512-byte header, a PAX header (512 bytes plus the path record, at most
+// len("files/")+MaxPathLen+1 bytes and a length prefix, rounded up to 4608)
+// when the path does not fit the USTAR fields, and under 512 bytes of
+// padding. That is at most 6143; 8 KiB leaves room. Owners, times and
+// sizes under Cap always fit USTAR, so no other PAX record is written.
+// TestTheLongestPathFitsTheEntryOverhead checks the arithmetic.
+const entryOverhead = 8 << 10
+
+// MaxDecoded is the most bytes a decoded archive (the tar stream) may hold:
+// content, per-entry overhead and the two-block end marker.
+const MaxDecoded int64 = Cap + MaxEntries*entryOverhead + 2*512
+
+// ManifestVersion is the Manifest.V this package writes and reads.
+const ManifestVersion = 1
+
+// MarkerName is the protocol marker inside the files directory, never
+// archived or digested (§5.1, §6.1 F5), and refused if an archive carries
+// it: an extracted marker would claim an installation that never happened.
+const MarkerName = ".files-gen"
+
+// EnvEntry is <host>.env's path inside the archive; FilesPrefix is the
+// files directory's namespace. The two never overlap.
+const (
+ EnvEntry = "env"
+ FilesPrefix = "files/"
+)
+
+// zstdWindow is the compression window, and the largest window a decoder
+// accepts: a hostile frame cannot make the target allocate more.
+const zstdWindow = 8 << 20
+
+// Errors from Build (ErrSpecialFile, ErrOverCap, ErrTooManyEntries,
+// ErrChanged and the long-path ErrBadPath) reach the operator through the
+// source report's S1; their texts are rows MV-N77 to MV-N81 of the copy
+// record. The rest are the target's refusals.
+var (
+ ErrSpecialFile = errors.New("archive: only regular files and directories move")
+ ErrOverCap = errors.New("archive: the files directory is over 256 MiB")
+ ErrTooManyEntries = fmt.Errorf("archive: the files directory holds more than %d entries", MaxEntries)
+ ErrChanged = errors.New("archive: a file changed while it was being archived")
+ ErrBadPath = errors.New("archive: refused path")
+ ErrTooLarge = errors.New("archive: the decoded archive is over its size limit")
+ ErrNotEmpty = errors.New("archive: the destination files directory is not empty")
+)
+
+// errVerifyAborted is what Verify closes its pipe with when it stops
+// reading before the end, so Open's next write fails and its goroutine
+// exits instead of blocking on the pipe for the rest of the archive.
+var errVerifyAborted = errors.New("archive: verify stopped reading")
+
+// The limits as enforced. Variables only so tests can lower them; nothing
+// else writes them.
+var (
+ entryLimit = MaxEntries
+ decodedLimit = MaxDecoded
+)
+
+// Entry is one regular file in the manifest: its archive path ("env" or
+// "files/<path>"), size and sha256.
+type Entry struct {
+ Path string `json:"path"`
+ Size int64 `json:"size"`
+ SHA256 string `json:"sha256"`
+}
+
+// Manifest travels beside the sealed object (§8.2). SealedDataKey is the
+// archive's data key sealed to the deployment's config keypair (a v2
+// envelope bound to the app); it is not an object key (F29).
+type Manifest struct {
+ V int `json:"v"`
+ Op string `json:"op"`
+ Incarnation string `json:"incarnation"`
+ Plain string `json:"plain"`
+ Sealed string `json:"sealed"`
+ Bytes int64 `json:"bytes"`
+ Entries []Entry `json:"entries"`
+ SealedDataKey string `json:"sealed_data_key"`
+ Chunk int `json:"chunk"`
+}
+
+// item is one listed entry: its archive name, where to open it, and its
+// Lstat at listing time, which the open must match.
+type item struct {
+ name string // "env", "files/<rel>"
+ rel string // path inside the files root; "" for env
+ root *os.Root // the files root; nil for env
+ path string // envPath, for env
+ info fs.FileInfo // as listed
+}
+
+// walk lists the files directory and envPath in archive order, refusing
+// anything that is not a regular file or a directory and excluding the
+// marker. The files directory is read through an os.Root, so nothing a
+// tenant does inside it (a directory swapped for a symlink mid-walk) can
+// make the walk or a later open leave it.
+func walk(root *os.Root, envPath string) ([]item, error) {
+ var items []item
+ add := func(it item) error {
+ if len(items) >= entryLimit {
+ return ErrTooManyEntries
+ }
+ items = append(items, it)
+ return nil
+ }
+ if root != nil {
+ err := fs.WalkDir(root.FS(), ".", func(p string, d fs.DirEntry, err error) error {
+ if err != nil {
+ return err
+ }
+ if p == "." || p == MarkerName {
+ return nil
+ }
+ info, err := root.Lstat(p)
+ if err != nil {
+ return err
+ }
+ if !info.Mode().IsRegular() && !info.IsDir() {
+ return fmt.Errorf("%w: %s", ErrSpecialFile, p)
+ }
+ if len(p) > MaxPathLen {
+ return fmt.Errorf("%w: %s is longer than %d bytes", ErrBadPath, p, MaxPathLen)
+ }
+ return add(item{name: FilesPrefix + p, rel: p, root: root, info: info})
+ })
+ if err != nil {
+ return nil, err
+ }
+ }
+ if envPath != "" {
+ info, err := os.Lstat(envPath)
+ switch {
+ case errors.Is(err, fs.ErrNotExist):
+ case err != nil:
+ return nil, err
+ case !info.Mode().IsRegular():
+ return nil, fmt.Errorf("%w: %s", ErrSpecialFile, filepath.Base(envPath))
+ default:
+ if err := add(item{name: EnvEntry, path: envPath, info: info}); err != nil {
+ return nil, err
+ }
+ }
+ }
+ sort.Slice(items, func(i, j int) bool { return items[i].name < items[j].name })
+ return items, nil
+}
+
+// openItem opens a listed regular file. A variable so a test can stand in
+// a file that grows after it was listed.
+var openItem = openItemChecked
+
+// openItemChecked opens it.rel inside the files root (or the env path) and
+// requires the opened file to be the one listed: a regular file with the
+// listed identity. A file swapped after the walk (for a FIFO, another file,
+// or a symlink the root would refuse to leave by) fails here. The open is
+// non-blocking where the platform has the flag, so a swapped-in FIFO
+// returns at once instead of waiting for a writer.
+func openItemChecked(it item) (io.ReadCloser, error) {
+ var f *os.File
+ var err error
+ if it.root != nil {
+ f, err = it.root.OpenFile(it.rel, os.O_RDONLY|openNonblock, 0)
+ } else {
+ f, err = os.OpenFile(it.path, os.O_RDONLY|openNonblock, 0)
+ }
+ if err != nil {
+ return nil, err
+ }
+ got, err := f.Stat()
+ if err != nil {
+ f.Close()
+ return nil, err
+ }
+ if !got.Mode().IsRegular() || !os.SameFile(it.info, got) {
+ f.Close()
+ return nil, fmt.Errorf("%w: %s", ErrChanged, displayName(it))
+ }
+ return f, nil
+}
+
+func displayName(it item) string {
+ if it.root == nil {
+ return filepath.Base(it.path)
+ }
+ return it.rel
+}
+
+// hashWriter hashes and counts the tar stream on its way to the
+// compressor. It enforces no limit: content is capped where sizes are
+// admitted, and the stream's length follows from the entry limits.
+type hashWriter struct {
+ w io.Writer
+ h hash.Hash
+ n int64
+}
+
+func (c *hashWriter) Write(p []byte) (int, error) {
+ c.h.Write(p)
+ c.n += int64(len(p))
+ return c.w.Write(p)
+}
+
+func digestString(h hash.Hash) string {
+ return "sha256:" + hex.EncodeToString(h.Sum(nil))
+}
+
+// built is one pass over the tree.
+type built struct {
+ plain string
+ entries []Entry
+ content int64
+ tarBytes int64
+ count int // tar entries: files, directories and env
+}
+
+// openFilesRoot opens filesDir as an os.Root, requiring it to be a real
+// directory (not a symlink to one) at both the Lstat and the open. A
+// missing directory is an empty one: the archive then carries env only.
+func openFilesRoot(filesDir string) (*os.Root, error) {
+ if filesDir == "" {
+ return nil, errors.New("archive: no files directory named")
+ }
+ info, err := os.Lstat(filesDir)
+ if errors.Is(err, fs.ErrNotExist) {
+ return nil, nil
+ }
+ if err != nil {
+ return nil, err
+ }
+ if !info.IsDir() {
+ return nil, fmt.Errorf("%w: %s", ErrSpecialFile, filesDir)
+ }
+ root, err := os.OpenRoot(filesDir)
+ if err != nil {
+ return nil, err
+ }
+ got, err := root.Stat(".")
+ if err != nil {
+ root.Close()
+ return nil, err
+ }
+ if !os.SameFile(info, got) {
+ root.Close()
+ return nil, fmt.Errorf("%w: %s", ErrChanged, filesDir)
+ }
+ return root, nil
+}
+
+// build writes the tar of filesDir and envPath to out and returns what
+// Build and PlainDigest report.
+func build(filesDir, envPath string, out io.Writer) (built, error) {
+ root, err := openFilesRoot(filesDir)
+ if err != nil {
+ return built{}, err
+ }
+ if root != nil {
+ defer root.Close()
+ }
+ items, err := walk(root, envPath)
+ if err != nil {
+ return built{}, err
+ }
+ var b built
+ hw := &hashWriter{w: out, h: sha256.New()}
+ tw := tar.NewWriter(hw)
+ for _, it := range items {
+ hdr := &tar.Header{Name: it.name, Mode: int64(it.info.Mode().Perm()), Format: tar.FormatPAX}
+ if it.info.IsDir() {
+ hdr.Typeflag = tar.TypeDir
+ hdr.Name += "/"
+ } else {
+ hdr.Typeflag = tar.TypeReg
+ hdr.Size = it.info.Size()
+ // Admitted against the cap before the header commits the size.
+ if hdr.Size > Cap-b.content {
+ return built{}, ErrOverCap
+ }
+ b.content += hdr.Size
+ }
+ if err := tw.WriteHeader(hdr); err != nil {
+ return built{}, err
+ }
+ b.count++
+ if hdr.Typeflag != tar.TypeReg {
+ continue
+ }
+ e, err := copyItem(tw, it, hdr.Size)
+ if err != nil {
+ return built{}, err
+ }
+ b.entries = append(b.entries, e)
+ }
+ if err := tw.Close(); err != nil {
+ return built{}, err
+ }
+ b.plain, b.tarBytes = digestString(hw.h), hw.n
+ return b, nil
+}
+
+// copyItem streams exactly the admitted size of one file into the tar.
+// This is the cap while streaming: a file is never read past the size its
+// header committed (and the cap admitted), so a file that grows after it
+// was listed cannot carry the archive past Cap. It is refused as changed,
+// as is one that shrank, rather than archived torn.
+func copyItem(tw io.Writer, it item, size int64) (Entry, error) {
+ rc, err := openItem(it)
+ if err != nil {
+ return Entry{}, err
+ }
+ defer rc.Close()
+ fh := sha256.New()
+ n, err := io.CopyN(io.MultiWriter(tw, fh), rc, size)
+ if errors.Is(err, io.EOF) {
+ return Entry{}, fmt.Errorf("%w: %s", ErrChanged, displayName(it))
+ }
+ if err != nil {
+ return Entry{}, err
+ }
+ var one [1]byte
+ m, err := io.ReadFull(rc, one[:])
+ if m > 0 {
+ return Entry{}, fmt.Errorf("%w: %s", ErrChanged, displayName(it))
+ }
+ if err != io.EOF {
+ return Entry{}, err
+ }
+ return Entry{Path: it.name, Size: n, SHA256: digestString(fh)}, nil
+}
+
+// Build writes the archive (tar then zstd) to w and returns the plaintext
+// digest (of the tar stream), the regular-file entries and the content
+// byte count. Owners and times are zeroed, modes kept, so equal content
+// gives equal plaintext (§8.1). The files directory's entries live under
+// "files/" and <host>.env is "env", in separate namespaces, so an
+// application file or directory named "env" is carried as "files/env" and
+// can never collide with the overlay. A missing envPath (or "") means the
+// archive has no env entry; a missing filesDir is an empty one.
+func Build(filesDir, envPath string, w io.Writer) (plain string, entries []Entry, bytes int64, err error) {
+ zw, err := zstd.NewWriter(w, zstd.WithEncoderConcurrency(1), zstd.WithWindowSize(zstdWindow))
+ if err != nil {
+ return "", nil, 0, err
+ }
+ b, err := build(filesDir, envPath, zw)
+ if err != nil {
+ zw.Close()
+ return "", nil, 0, err
+ }
+ if err := zw.Close(); err != nil {
+ return "", nil, 0, err
+ }
+ return b.plain, b.entries, b.content, nil
+}
+
+// PlainDigest runs Build's walk and tar without compressing or writing
+// anything, and returns the same plaintext digest Build would, the content
+// byte count and the number of regular-file entries (env included). It is
+// the baseline the source records at `stopped` before any archive exists
+// (§8.3), and the comparison every recovery makes.
+func PlainDigest(filesDir, envPath string) (plain string, bytes int64, entries int, err error) {
+ b, err := build(filesDir, envPath, io.Discard)
+ if err != nil {
+ return "", 0, 0, err
+ }
+ return b.plain, b.content, len(b.entries), nil
+}
+
+// checkName validates one archive path (a directory's without its
+// trailing slash) and says which namespace it is in. It is the one rule
+// Verify, Extract and the manifest check share.
+func checkName(name string, dir bool) (rel string, isEnv bool, err error) {
+ bad := func(why string) error { return fmt.Errorf("%w: %q %s", ErrBadPath, name, why) }
+ switch {
+ case name == "":
+ return "", false, bad("is empty")
+ case len(name) > len(FilesPrefix)+MaxPathLen:
+ return "", false, bad("is too long")
+ case strings.ContainsRune(name, 0):
+ return "", false, bad("holds a NUL")
+ case path.IsAbs(name):
+ return "", false, bad("is absolute")
+ case path.Clean(name) != name:
+ return "", false, bad("is not clean")
+ }
+ for _, c := range strings.Split(name, "/") {
+ if c == ".." || c == "." || c == "" {
+ return "", false, bad("leaves its directory")
+ }
+ }
+ if name == EnvEntry {
+ if dir {
+ return "", false, bad("is the env file but not a regular file")
+ }
+ return "", true, nil
+ }
+ rel, ok := strings.CutPrefix(name, FilesPrefix)
+ if !ok || rel == "" {
+ return "", false, bad("is outside files/ and env")
+ }
+ if rel == MarkerName {
+ return "", false, bad("is the protocol marker")
+ }
+ // The platform's own rules on top (Windows reserved names and drive
+ // letters; a backslash is a separator there and a file name here).
+ if !filepath.IsLocal(filepath.FromSlash(rel)) || (filepath.Separator != '/' && strings.ContainsRune(rel, filepath.Separator)) {
+ return "", false, bad("is not a local path on this platform")
+ }
+ return rel, false, nil
+}
+
+// headerName validates a tar header's name and type: regular files and
+// directories only.
+func headerName(hdr *tar.Header) (name, rel string, isEnv bool, err error) {
+ switch hdr.Typeflag {
+ case tar.TypeReg, tar.TypeDir:
+ default:
+ return "", "", false, fmt.Errorf("%w: %q", ErrSpecialFile, hdr.Name)
+ }
+ if hdr.Size < 0 || (hdr.Typeflag == tar.TypeDir && hdr.Size != 0) {
+ return "", "", false, fmt.Errorf("%w: %q has size %d", ErrBadPath, hdr.Name, hdr.Size)
+ }
+ name = hdr.Name
+ if hdr.Typeflag == tar.TypeDir {
+ name = strings.TrimSuffix(name, "/")
+ }
+ rel, isEnv, err = checkName(name, hdr.Typeflag == tar.TypeDir)
+ return name, rel, isEnv, err
+}
+
+// boundedReader stops a decoded stream at its limit: past it, one more
+// byte is ErrTooLarge, so a stream exactly at the limit still ends cleanly.
+type boundedReader struct {
+ r io.Reader
+ left int64
+}
+
+func (b *boundedReader) Read(p []byte) (int, error) {
+ if b.left <= 0 {
+ var one [1]byte
+ n, err := b.r.Read(one[:])
+ if n > 0 {
+ return 0, ErrTooLarge
+ }
+ return 0, err
+ }
+ if int64(len(p)) > b.left {
+ p = p[:b.left]
+ }
+ n, err := b.r.Read(p)
+ b.left -= int64(n)
+ return n, err
+}
+
+func newDecoder(r io.Reader) (*zstd.Decoder, error) {
+ return zstd.NewReader(r, zstd.WithDecoderConcurrency(1), zstd.WithDecoderMaxWindow(zstdWindow))
+}
+
+// requireEOF reads the raw stream once more after the decoder finished. A
+// stream fed by Open through a pipe reports Open's terminal verdict here
+// (the final chunk's authentication); trailing bytes are refused.
+func requireEOF(r io.Reader) error {
+ var one [1]byte
+ n, err := io.ReadFull(r, one[:])
+ if n > 0 {
+ return fmt.Errorf("%w: data after the archive", ErrBadPath)
+ }
+ if err == io.EOF {
+ return nil
+ }
+ return err
+}
+
+// check validates a manifest on its own, before any byte of the archive is
+// read: the version and chunk size this package writes, the digests and
+// key present, and an entry list Build could have written (valid, unique
+// paths within the limits, sizes summing to Bytes, at most Cap).
+func (m Manifest) check() error {
+ switch {
+ case m.V != ManifestVersion:
+ return fmt.Errorf("archive: manifest version %d, want %d", m.V, ManifestVersion)
+ case m.Chunk != ChunkSize:
+ return fmt.Errorf("archive: manifest chunk %d, want %d", m.Chunk, ChunkSize)
+ case m.Plain == "" || m.Sealed == "" || m.SealedDataKey == "":
+ return errors.New("archive: manifest carries no sealed digest, plaintext digest or sealed data key")
+ case len(m.Entries) > entryLimit:
+ return ErrTooManyEntries
+ }
+ seen := make(map[string]bool, len(m.Entries))
+ var sum int64
+ for _, e := range m.Entries {
+ if _, _, err := checkName(e.Path, false); err != nil {
+ return err
+ }
+ if seen[e.Path] {
+ return fmt.Errorf("%w: %q named twice in the manifest", ErrBadPath, e.Path)
+ }
+ seen[e.Path] = true
+ if e.Size < 0 || e.Size > Cap-sum {
+ return ErrOverCap
+ }
+ sum += e.Size
+ }
+ if sum != m.Bytes {
+ return fmt.Errorf("archive: manifest entries hold %d bytes, manifest says %d", sum, m.Bytes)
+ }
+ return nil
+}
+
+// Verify is T5 (§7.3): it opens the sealed object with priv, streams the
+// tar out of zstd and checks everything the manifest promised: the
+// plaintext digest, every entry's path, size and sha256, the entry types
+// (regular files and directories only), the limits, and the sealed digest
+// of the bytes it read. It writes nothing and returns the first mismatch.
+// Open's own verdict (every chunk authenticated, the final chunk present
+// and last) wins over any mismatch found in what it produced.
+func Verify(sealed io.Reader, m Manifest, priv *ecdh.PrivateKey, b appconfig.Binding) error {
+ if err := m.check(); err != nil {
+ return err
+ }
+ sealedHash := sha256.New()
+ pr, pw := io.Pipe()
+ done := make(chan error, 1)
+ go func() {
+ err := Open(io.TeeReader(sealed, sealedHash), m.SealedDataKey, priv, b, pw)
+ pw.CloseWithError(err)
+ done <- err
+ }()
+ plain, err := verifyStream(pr, m)
+ // Break the pipe whatever happened, so a producer still writing fails
+ // and exits, then wait for it. After a clean read to EOF this is a
+ // no-op: Open has already returned.
+ pr.CloseWithError(errVerifyAborted)
+ if openErr := <-done; openErr != nil && !errors.Is(openErr, errVerifyAborted) {
+ return openErr
+ }
+ if err != nil {
+ return err
+ }
+ if plain != m.Plain {
+ return fmt.Errorf("archive: plaintext digest %s, manifest says %s", plain, m.Plain)
+ }
+ if got := digestString(sealedHash); got != m.Sealed {
+ return fmt.Errorf("archive: sealed digest %s, manifest says %s", got, m.Sealed)
+ }
+ return nil
+}
+
+// verifyStream reads the opened archive from r to its end and checks it
+// against m, returning the plaintext digest. h hashes the decoded tar,
+// end marker included, which is exactly what Build's hashWriter hashed (it
+// sits between tar.Writer and the compressor), so Plain is the same bytes
+// on both sides (round-4 finding 8).
+func verifyStream(r io.Reader, m Manifest) (string, error) {
+ zr, err := newDecoder(r)
+ if err != nil {
+ return "", err
+ }
+ defer zr.Close()
+ h := sha256.New()
+ dec := &boundedReader{r: zr, left: decodedLimit}
+ tr := tar.NewReader(io.TeeReader(dec, h))
+ want := make(map[string]Entry, len(m.Entries))
+ for _, e := range m.Entries {
+ want[e.Path] = e
+ }
+ seen := map[string]bool{}
+ var content int64
+ for count := 0; ; count++ {
+ hdr, err := tr.Next()
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ return "", err
+ }
+ if count >= entryLimit {
+ return "", ErrTooManyEntries
+ }
+ name, _, _, err := headerName(hdr)
+ if err != nil {
+ return "", err
+ }
+ if seen[name] {
+ return "", fmt.Errorf("%w: %q named twice", ErrBadPath, hdr.Name)
+ }
+ seen[name] = true
+ if hdr.Typeflag == tar.TypeDir {
+ continue
+ }
+ if hdr.Size > Cap-content {
+ return "", ErrOverCap
+ }
+ content += hdr.Size
+ e, ok := want[name]
+ if !ok {
+ return "", fmt.Errorf("archive: entry %q is not in the manifest", hdr.Name)
+ }
+ if e.Size != hdr.Size {
+ return "", fmt.Errorf("archive: entry %q size differs from the manifest", hdr.Name)
+ }
+ fh := sha256.New()
+ if _, err := io.CopyN(fh, tr, hdr.Size); err != nil {
+ return "", err
+ }
+ if digestString(fh) != e.SHA256 {
+ return "", fmt.Errorf("archive: entry %q digest differs", hdr.Name)
+ }
+ delete(want, name)
+ }
+ // tar.Reader stops at the end-of-archive marker; whatever follows in
+ // the decoded stream must still reach h, and the raw stream must then
+ // end, so Open's terminal result becomes this read's error. A Verify
+ // that returned at the tar marker would trust an archive whose tail
+ // was never authenticated.
+ if _, err := io.Copy(h, dec); err != nil {
+ return "", err
+ }
+ if err := requireEOF(r); err != nil {
+ return "", err
+ }
+ if len(want) != 0 {
+ return "", fmt.Errorf("archive: %d manifest entries missing from the archive", len(want))
+ }
+ if content != m.Bytes {
+ return "", fmt.Errorf("archive: the archive holds %d bytes, manifest says %d", content, m.Bytes)
+ }
+ return digestString(h), nil
+}
+
+// Extract writes an opened archive into filesDir and envPath, refusing
+// anything a tar could smuggle: absolute paths, "..", the protocol marker,
+// links, devices and FIFOs, two entries naming one destination, more
+// entries than MaxEntries, content over Cap and a decoded stream over
+// MaxDecoded.
+//
+// Nothing reaches the destination unless the whole archive is accepted:
+// entries are written into a private staging directory beside filesDir
+// (and env into a temporary file beside envPath), and moved into place
+// only after the stream has been read to its end. filesDir must be absent
+// or an empty directory (ErrNotEmpty otherwise; a non-empty directory
+// would keep files the archive does not name), and its parent and
+// envPath's parent must exist. envPath is replaced when the archive
+// carries env and removed when it does not, so afterwards PlainDigest of
+// the pair equals the archive's. The files move first and env second; a
+// failure between the two renames leaves the files in place and is
+// returned.
+func Extract(r io.Reader, filesDir, envPath string) error {
+ if filesDir == "" {
+ return errors.New("archive: no files directory named")
+ }
+ filesDir = filepath.Clean(filesDir)
+ if envPath != "" {
+ envPath = filepath.Clean(envPath)
+ if rel, err := filepath.Rel(filesDir, envPath); err == nil && filepath.IsLocal(rel) {
+ return fmt.Errorf("%w: the env file %s is inside the files directory", ErrBadPath, envPath)
+ }
+ }
+ rootMode, existed, err := checkDestination(filesDir)
+ if err != nil {
+ return err
+ }
+ stage, err := os.MkdirTemp(filepath.Dir(filesDir), "."+filepath.Base(filesDir)+".extract-")
+ if err != nil {
+ return err
+ }
+ x := &extraction{stage: stage, envPath: envPath}
+ defer x.cleanup()
+ if x.root, err = os.OpenRoot(stage); err != nil {
+ return err
+ }
+ if err := x.read(r); err != nil {
+ return err
+ }
+ return x.commit(filesDir, rootMode, existed)
+}
+
+// checkDestination requires filesDir to be absent or an empty directory
+// (not a symlink to one) and returns the mode the extracted root takes:
+// the existing directory's, else 0755.
+func checkDestination(filesDir string) (fs.FileMode, bool, error) {
+ info, err := os.Lstat(filesDir)
+ if errors.Is(err, fs.ErrNotExist) {
+ return 0o755, false, nil
+ }
+ if err != nil {
+ return 0, false, err
+ }
+ if !info.IsDir() {
+ return 0, false, fmt.Errorf("%w: the destination %s is not a directory", ErrBadPath, filesDir)
+ }
+ f, err := os.Open(filesDir)
+ if err != nil {
+ return 0, false, err
+ }
+ names, err := f.Readdirnames(1)
+ f.Close()
+ if len(names) > 0 {
+ return 0, false, fmt.Errorf("%w: %s", ErrNotEmpty, filesDir)
+ }
+ if err != nil && err != io.EOF {
+ return 0, false, err
+ }
+ return info.Mode().Perm(), true, nil
+}
+
+type modeFix struct {
+ rel string
+ mode fs.FileMode
+}
+
+// extraction is one Extract in progress.
+type extraction struct {
+ stage string // the staging directory, removed unless committed
+ root *os.Root // stage, for every write
+ envPath string
+ envTmp string // the staged env file, if the archive carried one
+ envMode fs.FileMode
+ dirs []modeFix
+}
+
+func (x *extraction) cleanup() {
+ if x.root != nil {
+ x.root.Close()
+ }
+ if x.stage != "" {
+ removeStage(x.stage)
+ }
+ if x.envTmp != "" {
+ os.Remove(x.envTmp)
+ }
+}
+
+// removeStage removes a staging directory whose directories may already
+// carry their final, read-only modes. It holds only what Extract created:
+// regular files and directories.
+func removeStage(stage string) {
+ filepath.WalkDir(stage, func(p string, d fs.DirEntry, err error) error {
+ if err == nil && d.IsDir() {
+ os.Chmod(p, 0o700)
+ }
+ return nil
+ })
+ os.RemoveAll(stage)
+}
+
+// read writes every entry into the staging area and reads the stream to
+// its end.
+func (x *extraction) read(r io.Reader) error {
+ zr, err := newDecoder(r)
+ if err != nil {
+ return err
+ }
+ defer zr.Close()
+ dec := &boundedReader{r: zr, left: decodedLimit}
+ tr := tar.NewReader(dec)
+ seen := map[string]bool{}
+ var content int64
+ for count := 0; ; count++ {
+ hdr, err := tr.Next()
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ return err
+ }
+ if count >= entryLimit {
+ return ErrTooManyEntries
+ }
+ name, rel, isEnv, err := headerName(hdr)
+ if err != nil {
+ return err
+ }
+ if seen[name] {
+ return fmt.Errorf("%w: %q named twice", ErrBadPath, hdr.Name)
+ }
+ seen[name] = true
+ mode := fs.FileMode(hdr.Mode).Perm()
+ if hdr.Typeflag == tar.TypeDir {
+ // Created writable so children can land; the archived mode is
+ // set once every entry is in (commit).
+ if err := x.root.MkdirAll(rel, 0o700); err != nil {
+ return err
+ }
+ x.dirs = append(x.dirs, modeFix{rel, mode})
+ continue
+ }
+ if hdr.Size > Cap-content {
+ return ErrOverCap
+ }
+ content += hdr.Size
+ if isEnv {
+ err = x.writeEnv(tr, hdr.Size, mode)
+ } else {
+ err = x.writeFile(tr, rel, hdr.Size, mode)
+ }
+ if err != nil {
+ return err
+ }
+ }
+ if _, err := io.Copy(io.Discard, dec); err != nil {
+ return err
+ }
+ return requireEOF(r)
+}
+
+func (x *extraction) writeFile(src io.Reader, rel string, size int64, mode fs.FileMode) error {
+ if dir := path.Dir(rel); dir != "." {
+ if err := x.root.MkdirAll(dir, 0o700); err != nil {
+ return err
+ }
+ }
+ // O_EXCL: a name that already exists in the fresh staging tree (on a
+ // case-insensitive filesystem, "A" after "a") is refused, not merged.
+ f, err := x.root.OpenFile(rel, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
+ if err != nil {
+ return err
+ }
+ return finishFile(f, src, size, mode)
+}
+
+func (x *extraction) writeEnv(src io.Reader, size int64, mode fs.FileMode) error {
+ if x.envPath == "" {
+ return fmt.Errorf("%w: the archive carries env and no env path was given", ErrBadPath)
+ }
+ f, err := os.CreateTemp(filepath.Dir(x.envPath), "."+filepath.Base(x.envPath)+".extract-")
+ if err != nil {
+ return err
+ }
+ x.envTmp, x.envMode = f.Name(), mode
+ return finishFile(f, src, size, mode)
+}
+
+// finishFile copies exactly size bytes, sets the archived mode (fchmod, so
+// the umask does not apply; §8.1) and closes. On Linux it does not sync:
+// commit syncs the staged filesystem once before anything moves into
+// place, one flush rather than one per file (the landing runs inside the
+// cut's offline window; per-file fsync took 6 ms a file on ext4).
+func finishFile(f *os.File, src io.Reader, size int64, mode fs.FileMode) error {
+ if _, err := io.CopyN(f, src, size); err != nil {
+ f.Close()
+ return err
+ }
+ if err := f.Chmod(mode); err != nil {
+ f.Close()
+ return err
+ }
+ if syncEachFile {
+ if err := f.Sync(); err != nil {
+ f.Close()
+ return err
+ }
+ }
+ return f.Close()
+}
+
+// commit syncs the staged tree, restores directory modes deepest first (a
+// read-only 0555 directory is reconstructed exactly without blocking its
+// own extraction), then moves the staged tree and env file into place.
+func (x *extraction) commit(filesDir string, rootMode fs.FileMode, existed bool) error {
+ if err := x.syncStaged(); err != nil {
+ return err
+ }
+ sort.Slice(x.dirs, func(i, j int) bool { return len(x.dirs[i].rel) > len(x.dirs[j].rel) })
+ for _, d := range x.dirs {
+ if err := x.root.Chmod(d.rel, d.mode); err != nil {
+ return err
+ }
+ }
+ if err := os.Chmod(x.stage, rootMode); err != nil {
+ return err
+ }
+ if existed {
+ // os.Remove refuses a directory that is no longer empty.
+ if err := os.Remove(filesDir); err != nil {
+ return err
+ }
+ }
+ if err := os.Rename(x.stage, filesDir); err != nil {
+ return err
+ }
+ x.stage = ""
+ if err := syncDir(filepath.Dir(filesDir)); err != nil {
+ return err
+ }
+ if x.envPath == "" {
+ return nil
+ }
+ if x.envTmp == "" {
+ if err := os.Remove(x.envPath); err != nil && !errors.Is(err, fs.ErrNotExist) {
+ return err
+ }
+ } else {
+ if err := os.Rename(x.envTmp, x.envPath); err != nil {
+ return err
+ }
+ x.envTmp = ""
+ }
+ return syncDir(filepath.Dir(x.envPath))
+}
+
+// syncDir makes a rename durable. Windows cannot sync a directory; the
+// platform's boxes are Linux, and this only keeps that build whole.
+func syncDir(dir string) error {
+ if runtime.GOOS == "windows" {
+ return nil
+ }
+ d, err := os.Open(dir)
+ if err != nil {
+ return err
+ }
+ err = d.Sync()
+ d.Close()
+ return err
+}
diff --git a/internal/moves/archive/archive_test.go b/internal/moves/archive/archive_test.go
new file mode 100644
index 000000000..ff4982d04
--- /dev/null
+++ b/internal/moves/archive/archive_test.go
@@ -0,0 +1,1017 @@
+package archive
+
+import (
+ "archive/tar"
+ "bytes"
+ "crypto/ecdh"
+ "crypto/rand"
+ "crypto/sha256"
+ "encoding/binary"
+ "encoding/hex"
+ "errors"
+ "io"
+ "io/fs"
+ "os"
+ "path/filepath"
+ "runtime"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/klauspost/compress/zstd"
+
+ "github.com/carlosframework/platform/internal/appconfig"
+)
+
+func filesFixture(t *testing.T) (dir, env string) {
+ t.Helper()
+ dir = filepath.Join(t.TempDir(), "h")
+ if err := os.MkdirAll(filepath.Join(dir, "sub"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ mustWrite(t, filepath.Join(dir, "a.txt"), []byte("alpha"), 0o644)
+ mustWrite(t, filepath.Join(dir, "sub", "b.bin"), []byte{1, 2, 3}, 0o600)
+ mustWrite(t, filepath.Join(dir, ".files-gen"), []byte(`{"op":"x"}`), 0o644)
+ // A legal application file named env, and a directory named env with
+ // a file in it: both must survive beside the <host>.env overlay.
+ mustWrite(t, filepath.Join(dir, "env"), []byte("app-env"), 0o644)
+ if err := os.MkdirAll(filepath.Join(dir, "sub", "env"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ mustWrite(t, filepath.Join(dir, "sub", "env", "x"), []byte("x"), 0o644)
+ env = filepath.Join(t.TempDir(), "h.env")
+ mustWrite(t, env, []byte("CARLOS_ADMIN_TOKEN=t\n"), 0o600)
+ return dir, env
+}
+
+func mustWrite(t *testing.T, p string, b []byte, mode fs.FileMode) {
+ t.Helper()
+ if err := os.WriteFile(p, b, mode); err != nil {
+ t.Fatal(err)
+ }
+ // WriteFile's mode passes through the umask; the fixtures want it exact.
+ if err := os.Chmod(p, mode); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// §8.1: equal content gives equal plaintext; the marker is excluded.
+func TestBuildIsDeterministicAndExcludesTheMarker(t *testing.T) {
+ dir, env := filesFixture(t)
+ var b1, b2 bytes.Buffer
+ p1, entries, n, err := Build(dir, env, &b1)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Chtimes(filepath.Join(dir, "a.txt"), time.Unix(1, 0), time.Unix(1, 0)); err != nil {
+ t.Fatal(err)
+ }
+ p2, _, _, err := Build(dir, env, &b2)
+ if err != nil || p1 != p2 || !bytes.Equal(b1.Bytes(), b2.Bytes()) {
+ t.Fatalf("not deterministic: %q %q %v", p1, p2, err)
+ }
+ paths := map[string]bool{}
+ for _, e := range entries {
+ paths[e.Path] = true
+ }
+ if paths["files/.files-gen"] || !paths["files/a.txt"] || !paths["files/sub/b.bin"] || !paths["env"] || !paths["files/env"] || !paths["files/sub/env/x"] || n != 5+3+7+1+int64(len("CARLOS_ADMIN_TOKEN=t\n")) {
+ t.Fatalf("entries %v bytes %d", entries, n)
+ }
+ // An application file and directory named "env" are legal and carried
+ // under files/, never confused with the overlay (round-tripped in
+ // TestSealOpenExtractRoundTrip through the extra fixture entries).
+ plain, bytesN, count, err := PlainDigest(dir, env)
+ if err != nil || plain != p1 || bytesN != n || count != len(entries) {
+ t.Fatalf("PlainDigest %q %d %d %v, want %q %d %d", plain, bytesN, count, err, p1, n, len(entries))
+ }
+ // Owners and times are zeroed and modes kept, in the tar itself.
+ tr := tar.NewReader(mustDecode(t, b1.Bytes()))
+ for {
+ hdr, err := tr.Next()
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ t.Fatal(err)
+ }
+ if hdr.Uid != 0 || hdr.Gid != 0 || hdr.Uname != "" || hdr.Gname != "" || !hdr.ModTime.Equal(time.Unix(0, 0)) {
+ t.Fatalf("%s carries an owner or a time: %+v", hdr.Name, hdr)
+ }
+ if hdr.Name == "files/sub/b.bin" && hdr.Mode != 0o600 {
+ t.Fatalf("mode %o", hdr.Mode)
+ }
+ }
+}
+
+func mustDecode(t *testing.T, b []byte) io.Reader {
+ t.Helper()
+ zr, err := zstd.NewReader(bytes.NewReader(b))
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(zr.Close)
+ return zr
+}
+
+// Review Focus 3: a symlink refuses with its path, and so does a files
+// directory that is itself a symlink (WalkDir would otherwise archive
+// nothing beneath it and the move would carry an empty tree).
+func TestArchiveRefusesSpecialFiles(t *testing.T) {
+ dir, env := filesFixture(t)
+ if err := os.Symlink("a.txt", filepath.Join(dir, "link")); err != nil {
+ t.Fatal(err)
+ }
+ _, _, _, err := Build(dir, env, &bytes.Buffer{})
+ if !errors.Is(err, ErrSpecialFile) || !strings.Contains(err.Error(), "link") {
+ t.Fatalf("symlink: %v", err)
+ }
+ dir2, env2 := filesFixture(t)
+ linked := filepath.Join(t.TempDir(), "linked")
+ if err := os.Symlink(dir2, linked); err != nil {
+ t.Fatal(err)
+ }
+ if _, _, _, err := Build(linked, env2, &bytes.Buffer{}); !errors.Is(err, ErrSpecialFile) {
+ t.Fatalf("symlinked files directory: %v", err)
+ }
+ envLink := filepath.Join(t.TempDir(), "h.env")
+ if err := os.Symlink(env2, envLink); err != nil {
+ t.Fatal(err)
+ }
+ if _, _, _, err := Build(dir2, envLink, &bytes.Buffer{}); !errors.Is(err, ErrSpecialFile) || !strings.Contains(err.Error(), "h.env") {
+ t.Fatalf("symlinked env: %v", err)
+ }
+}
+
+// tarZst writes a hand-built archive: what a hostile sealer could produce.
+type rawEntry struct {
+ hdr tar.Header
+ body []byte
+}
+
+func tarZst(t *testing.T, entries []rawEntry, trailer []byte) []byte {
+ t.Helper()
+ var buf bytes.Buffer
+ zw, err := zstd.NewWriter(&buf)
+ if err != nil {
+ t.Fatal(err)
+ }
+ tw := tar.NewWriter(zw)
+ for _, e := range entries {
+ hdr := e.hdr
+ if hdr.Mode == 0 {
+ hdr.Mode = 0o600
+ }
+ if err := tw.WriteHeader(&hdr); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := tw.Write(e.body); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if err := tw.Close(); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := zw.Write(trailer); err != nil {
+ t.Fatal(err)
+ }
+ if err := zw.Close(); err != nil {
+ t.Fatal(err)
+ }
+ return buf.Bytes()
+}
+
+func reg(name, body string) rawEntry {
+ return rawEntry{hdr: tar.Header{Name: name, Typeflag: tar.TypeReg, Size: int64(len(body))}, body: []byte(body)}
+}
+
+func dirEntry(name string) rawEntry {
+ return rawEntry{hdr: tar.Header{Name: name, Typeflag: tar.TypeDir, Mode: 0o755}}
+}
+
+// extractTarget is a destination whose parent holds nothing else, so a
+// test can prove a refused archive left no byte behind.
+func extractTarget(t *testing.T) (parent, files, env string) {
+ t.Helper()
+ parent = t.TempDir()
+ envDir := t.TempDir()
+ return parent, filepath.Join(parent, "out"), filepath.Join(envDir, "out.env")
+}
+
+func assertUntouched(t *testing.T, parent, files, env string) {
+ t.Helper()
+ left, err := os.ReadDir(parent)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(left) != 0 {
+ t.Fatalf("a refused archive left %v in the destination's parent", left)
+ }
+ if _, err := os.Lstat(files); !errors.Is(err, fs.ErrNotExist) {
+ t.Fatalf("files directory written: %v", err)
+ }
+ envLeft, _ := os.ReadDir(filepath.Dir(env))
+ if len(envLeft) != 0 {
+ t.Fatalf("a refused archive left %v beside the env file", envLeft)
+ }
+}
+
+// A crafted archive naming one destination twice (two "env" entries, a
+// file twice, or a directory and then a file of one name) is refused
+// before any byte reaches the destination: Extract writes into a private
+// staging directory and moves it into place only once the whole archive
+// has been read and checked.
+func TestExtractRefusesDuplicateDestinations(t *testing.T) {
+ cases := map[string][]rawEntry{
+ "env twice": {reg("env", "x"), reg("env", "y")},
+ "file twice": {reg("files/a", "x"), reg("files/a", "y")},
+ "dir then file": {dirEntry("files/a/"), reg("files/a", "y")},
+ "file then env dup": {reg("files/a", "x"), reg("env", "e"), reg("files/a", "z")},
+ }
+ for name, entries := range cases {
+ t.Run(name, func(t *testing.T) {
+ parent, files, env := extractTarget(t)
+ err := Extract(bytes.NewReader(tarZst(t, entries, nil)), files, env)
+ if !errors.Is(err, ErrBadPath) {
+ t.Fatalf("duplicate: %v", err)
+ }
+ assertUntouched(t, parent, files, env)
+ })
+ }
+}
+
+// Traversal, the protocol marker, entries outside files/ and env, and any
+// type but a regular file or a directory are refused, and nothing lands.
+func TestExtractRefusesTraversalAndSpecialEntries(t *testing.T) {
+ cases := map[string]struct {
+ entries []rawEntry
+ want error
+ }{
+ "dotdot": {[]rawEntry{reg("files/../x", "x")}, ErrBadPath},
+ "leading dotdot": {[]rawEntry{reg("../x", "x")}, ErrBadPath},
+ "absolute": {[]rawEntry{reg("/etc/x", "x")}, ErrBadPath},
+ "dot segment": {[]rawEntry{reg("files/./a", "x")}, ErrBadPath},
+ "double slash": {[]rawEntry{reg("files//a", "x")}, ErrBadPath},
+ "files root": {[]rawEntry{dirEntry("files/")}, ErrBadPath},
+ "outside": {[]rawEntry{reg("other", "x")}, ErrBadPath},
+ "marker": {[]rawEntry{reg("files/.files-gen", `{"op":"forged"}`)}, ErrBadPath},
+ "env as dir": {[]rawEntry{dirEntry("env/")}, ErrBadPath},
+ "too long": {[]rawEntry{reg("files/"+strings.Repeat("a", MaxPathLen+1), "x")}, ErrBadPath},
+ "symlink": {[]rawEntry{{hdr: tar.Header{Name: "files/l", Typeflag: tar.TypeSymlink, Linkname: "/etc/passwd"}}}, ErrSpecialFile},
+ "hard link": {[]rawEntry{reg("files/a", "x"), {hdr: tar.Header{Name: "files/l", Typeflag: tar.TypeLink, Linkname: "files/a"}}}, ErrSpecialFile},
+ "char device": {[]rawEntry{{hdr: tar.Header{Name: "files/d", Typeflag: tar.TypeChar, Devmajor: 1, Devminor: 3}}}, ErrSpecialFile},
+ "fifo": {[]rawEntry{{hdr: tar.Header{Name: "files/f", Typeflag: tar.TypeFifo}}}, ErrSpecialFile},
+ "late bad entry": {[]rawEntry{reg("files/a", "fine"), reg("env", "e"), reg("files/../escape", "x")}, ErrBadPath},
+ "over cap header": {[]rawEntry{{hdr: tar.Header{Name: "files/big", Typeflag: tar.TypeReg, Size: Cap + 1}}}, ErrOverCap},
+ }
+ for name, c := range cases {
+ t.Run(name, func(t *testing.T) {
+ var raw []byte
+ if c.want == ErrOverCap {
+ raw = overCapArchive(t)
+ } else {
+ raw = tarZst(t, c.entries, nil)
+ }
+ parent, files, env := extractTarget(t)
+ if err := Extract(bytes.NewReader(raw), files, env); !errors.Is(err, c.want) {
+ t.Fatalf("%s: %v, want %v", name, err, c.want)
+ }
+ assertUntouched(t, parent, files, env)
+ })
+ }
+}
+
+// overCapArchive is a tar.zst whose one header claims Cap+1 bytes. Only
+// the header and a few bytes of content are present: every reader must
+// refuse on the header, before reading the content.
+func overCapArchive(t *testing.T) []byte {
+ t.Helper()
+ var tarBuf bytes.Buffer
+ tw := tar.NewWriter(&tarBuf)
+ if err := tw.WriteHeader(&tar.Header{Name: "files/big", Typeflag: tar.TypeReg, Size: Cap + 1, Mode: 0o600}); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := tw.Write([]byte("abc")); err != nil {
+ t.Fatal(err)
+ }
+ // No Close: the content is short on purpose.
+ var buf bytes.Buffer
+ zw, _ := zstd.NewWriter(&buf)
+ zw.Write(tarBuf.Bytes())
+ zw.Close()
+ return buf.Bytes()
+}
+
+// The destination must be absent or an empty directory, and the env file
+// may not live inside the files directory (Build would carry it twice).
+func TestExtractRefusesAnOccupiedDestination(t *testing.T) {
+ dir, env := filesFixture(t)
+ var tarz bytes.Buffer
+ if _, _, _, err := Build(dir, env, &tarz); err != nil {
+ t.Fatal(err)
+ }
+ occupied := t.TempDir()
+ mustWrite(t, filepath.Join(occupied, "stale"), []byte("s"), 0o644)
+ if err := Extract(bytes.NewReader(tarz.Bytes()), occupied, filepath.Join(t.TempDir(), "e.env")); !errors.Is(err, ErrNotEmpty) {
+ t.Fatalf("occupied: %v", err)
+ }
+ inside := t.TempDir()
+ if err := Extract(bytes.NewReader(tarz.Bytes()), inside, filepath.Join(inside, "h.env")); !errors.Is(err, ErrBadPath) {
+ t.Fatalf("env inside files: %v", err)
+ }
+ // An empty existing directory is fine, and keeps its mode.
+ empty := filepath.Join(t.TempDir(), "empty")
+ if err := os.Mkdir(empty, 0o750); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Chmod(empty, 0o750); err != nil {
+ t.Fatal(err)
+ }
+ if err := Extract(bytes.NewReader(tarz.Bytes()), empty, filepath.Join(t.TempDir(), "e.env")); err != nil {
+ t.Fatal(err)
+ }
+ if info, _ := os.Stat(empty); info.Mode().Perm() != 0o750 {
+ t.Fatalf("root mode %o", info.Mode().Perm())
+ }
+}
+
+// The stat-time check: a file already over the cap when it is listed is
+// refused before any content is read (a sparse file, so nothing is read).
+func TestArchiveCapIsEnforcedAtStat(t *testing.T) {
+ dir, env := filesFixture(t)
+ f, err := os.Create(filepath.Join(dir, "big"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := f.Truncate(Cap + 1); err != nil {
+ t.Fatal(err)
+ }
+ f.Close()
+ if _, _, _, err := Build(dir, env, &bytes.Buffer{}); !errors.Is(err, ErrOverCap) {
+ t.Fatalf("over cap: %v", err)
+ }
+}
+
+// lyingFile is a file that keeps growing after it was listed: it yields
+// bytes forever and counts how many it was asked for.
+type lyingFile struct{ read int64 }
+
+func (l *lyingFile) Read(p []byte) (int, error) {
+ for i := range p {
+ p[i] = 'g'
+ }
+ l.read += int64(len(p))
+ return len(p), nil
+}
+func (l *lyingFile) Close() error { return nil }
+
+// The streaming half of the cap: the tar header commits each file's size
+// as listed, and every file is read only up to that size. A file that grows
+// after it was listed (here without end) cannot push the archive past what
+// was admitted against the cap; it is refused as changed, naming its path,
+// having been read for at most one byte past its listed size.
+func TestArchiveCapHoldsWhenAFileGrowsWhileStreaming(t *testing.T) {
+ dir, env := filesFixture(t)
+ liar := &lyingFile{}
+ orig := openItem
+ t.Cleanup(func() { openItem = orig })
+ openItem = func(it item) (io.ReadCloser, error) {
+ if it.name == "files/a.txt" {
+ return liar, nil
+ }
+ return orig(it)
+ }
+ var out countingWriter
+ _, _, _, err := Build(dir, env, &out)
+ if !errors.Is(err, ErrChanged) || !strings.Contains(err.Error(), "a.txt") {
+ t.Fatalf("growing file: %v", err)
+ }
+ if liar.read > int64(len("alpha"))+1 {
+ t.Fatalf("read %d bytes of a file listed at 5", liar.read)
+ }
+ if _, _, _, err := PlainDigest(dir, env); !errors.Is(err, ErrChanged) {
+ t.Fatalf("PlainDigest of a growing file: %v", err)
+ }
+}
+
+type countingWriter struct{ n int64 }
+
+func (c *countingWriter) Write(p []byte) (int, error) { c.n += int64(len(p)); return len(p), nil }
+
+// F25: the cap counts content, not tar headers and padding. Content just
+// under the cap plus a few hundred small files (whose headers alone run
+// past the old 64 KiB allowance) archives; so do thousands of small files.
+func TestManySmallFilesUnderTheCapArchive(t *testing.T) {
+ if testing.Short() {
+ t.Skip("reads 256 MiB of a sparse file")
+ }
+ dir, env := filesFixture(t)
+ // Fixture content is 37 bytes; leave room for 400 one-byte files.
+ f, err := os.Create(filepath.Join(dir, "near-cap"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := f.Truncate(Cap - 37 - 400 - 100); err != nil {
+ t.Fatal(err)
+ }
+ f.Close()
+ many := filepath.Join(dir, "many")
+ if err := os.Mkdir(many, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ for i := 0; i < 400; i++ {
+ mustWrite(t, filepath.Join(many, strings.Repeat("f", 40)+string(rune('a'+i%26))+hex.EncodeToString([]byte{byte(i >> 8), byte(i)})), []byte("z"), 0o644)
+ }
+ b, err := build(dir, env, io.Discard)
+ if err != nil {
+ t.Fatalf("content under the cap refused: %v", err)
+ }
+ if b.content > Cap || b.tarBytes <= Cap+64<<10 {
+ t.Fatalf("fixture does not exercise header overhead: content %d tar %d", b.content, b.tarBytes)
+ }
+}
+
+func TestThousandsOfSmallFilesArchive(t *testing.T) {
+ dir, env := filesFixture(t)
+ for i := 0; i < 3000; i++ {
+ mustWrite(t, filepath.Join(dir, "s"+hex.EncodeToString([]byte{byte(i >> 8), byte(i)})), []byte{byte(i)}, 0o644)
+ }
+ var tarz bytes.Buffer
+ plain, entries, _, err := Build(dir, env, &tarz)
+ if err != nil || len(entries) != 3005 {
+ t.Fatalf("%d entries, %v", len(entries), err)
+ }
+ _, files, outEnv := extractTarget(t)
+ if err := Extract(bytes.NewReader(tarz.Bytes()), files, outEnv); err != nil {
+ t.Fatal(err)
+ }
+ if p2, _, _, _ := PlainDigest(files, outEnv); p2 != plain {
+ t.Fatal("round trip changed the plain digest")
+ }
+}
+
+// The per-entry overhead bound behind MaxDecoded holds for the longest
+// path Build admits, so every archive Build writes is one Extract accepts.
+func TestTheLongestPathFitsTheEntryOverhead(t *testing.T) {
+ dir := filepath.Join(t.TempDir(), "h")
+ // Components of 200 bytes nested to a relative path of MaxPathLen-ish.
+ rel := ""
+ for len(rel)+201 <= MaxPathLen-10 {
+ if rel != "" {
+ rel += "/"
+ }
+ rel += strings.Repeat("d", 200)
+ }
+ if err := os.Mkdir(dir, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ leaf := rel + "/" + strings.Repeat("f", MaxPathLen-len(rel)-1)
+ if len(leaf) != MaxPathLen {
+ t.Fatalf("leaf %d", len(leaf))
+ }
+ // Created through a Root (openat), as Build reads it: the absolute
+ // path is past PATH_MAX, the relative one is not.
+ root, err := os.OpenRoot(dir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer root.Close()
+ if err := root.MkdirAll(rel, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := root.WriteFile(leaf, []byte("x"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ b, err := build(dir, "", io.Discard)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if per := (b.tarBytes - 2*512 - b.content) / int64(b.count); per > entryOverhead {
+ t.Fatalf("per-entry overhead %d over the bound %d", per, entryOverhead)
+ }
+ // One byte longer is refused by Build, naming the path.
+ if err := root.Rename(leaf, leaf+"x"); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := build(dir, "", io.Discard); !errors.Is(err, ErrBadPath) {
+ t.Fatalf("over-long path: %v", err)
+ }
+}
+
+// Build, Verify and Extract share one entry-count limit, and Extract and
+// Verify bound the decoded stream, so a zstd bomb that passed the sealing
+// is still bounded on the target.
+func TestEntryAndDecodedLimitsAreShared(t *testing.T) {
+ dir, env := filesFixture(t)
+ var tarz bytes.Buffer
+ plain, entries, n, err := Build(dir, env, &tarz)
+ if err != nil {
+ t.Fatal(err)
+ }
+ pub, priv, b := keys(t)
+ var sealed bytes.Buffer
+ key, sealedSHA, err := Seal(bytes.NewReader(tarz.Bytes()), pub, b, &sealed)
+ if err != nil {
+ t.Fatal(err)
+ }
+ m := Manifest{V: ManifestVersion, Op: "mv_x", Incarnation: "i", Plain: plain, Sealed: sealedSHA, Bytes: n, Entries: entries, SealedDataKey: key, Chunk: ChunkSize}
+ if err := Verify(bytes.NewReader(sealed.Bytes()), m, priv, b); err != nil {
+ t.Fatal(err)
+ }
+
+ origEntries := entryLimit
+ entryLimit = 4
+ if _, _, _, err := Build(dir, env, io.Discard); !errors.Is(err, ErrTooManyEntries) {
+ t.Fatalf("Build over the entry limit: %v", err)
+ }
+ if err := Verify(bytes.NewReader(sealed.Bytes()), m, priv, b); !errors.Is(err, ErrTooManyEntries) {
+ t.Fatalf("Verify over the entry limit: %v", err)
+ }
+ parent, files, outEnv := extractTarget(t)
+ if err := Extract(bytes.NewReader(tarz.Bytes()), files, outEnv); !errors.Is(err, ErrTooManyEntries) {
+ t.Fatalf("Extract over the entry limit: %v", err)
+ }
+ assertUntouched(t, parent, files, outEnv)
+ entryLimit = origEntries
+
+ // A bomb: a valid tar followed by 8 MiB of zeros inside the zstd
+ // stream (a few hundred bytes compressed).
+ bomb := tarZst(t, []rawEntry{reg("files/a", "x")}, make([]byte, 8<<20))
+ origDecoded := decodedLimit
+ t.Cleanup(func() { decodedLimit = origDecoded; entryLimit = origEntries })
+ decodedLimit = 1 << 20
+ parent, files, outEnv = extractTarget(t)
+ if err := Extract(bytes.NewReader(bomb), files, outEnv); !errors.Is(err, ErrTooLarge) {
+ t.Fatalf("bomb: %v", err)
+ }
+ assertUntouched(t, parent, files, outEnv)
+ var bombSealed bytes.Buffer
+ bombKey, bombSHA, err := Seal(bytes.NewReader(bomb), pub, b, &bombSealed)
+ if err != nil {
+ t.Fatal(err)
+ }
+ bm := Manifest{V: ManifestVersion, Plain: "sha256:x", Sealed: bombSHA, Bytes: 1, Entries: []Entry{{Path: "files/a", Size: 1, SHA256: digestBytes([]byte("x"))}}, SealedDataKey: bombKey, Chunk: ChunkSize}
+ if err := Verify(bytes.NewReader(bombSealed.Bytes()), bm, priv, b); !errors.Is(err, ErrTooLarge) {
+ t.Fatalf("Verify of a bomb: %v", err)
+ }
+}
+
+func digestBytes(b []byte) string {
+ sum := sha256.Sum256(b)
+ return "sha256:" + hex.EncodeToString(sum[:])
+}
+
+func keys(t *testing.T) (*ecdh.PublicKey, *ecdh.PrivateKey, appconfig.Binding) {
+ t.Helper()
+ privB64, pubB64, err := appconfig.GenerateKey()
+ if err != nil {
+ t.Fatal(err)
+ }
+ pk, err := appconfig.ParsePrivateKey(privB64, "test")
+ if err != nil {
+ t.Fatal(err)
+ }
+ pu, err := appconfig.ParsePublicKey(pubB64, "test")
+ if err != nil {
+ t.Fatal(err)
+ }
+ return pu, pk, appconfig.Binding{Account: "bab", App: "app"}
+}
+
+func TestSealOpenExtractRoundTrip(t *testing.T) {
+ dir, env := filesFixture(t)
+ var tarz bytes.Buffer
+ plain, _, _, err := Build(dir, env, &tarz)
+ if err != nil {
+ t.Fatal(err)
+ }
+ pub, priv, b := keys(t)
+ var sealed bytes.Buffer
+ key, sealedSHA, err := Seal(bytes.NewReader(tarz.Bytes()), pub, b, &sealed)
+ if err != nil {
+ t.Fatal(err)
+ }
+ sum := sha256.Sum256(sealed.Bytes())
+ if sealedSHA != "sha256:"+hex.EncodeToString(sum[:]) {
+ t.Fatal("sealed digest")
+ }
+ var opened bytes.Buffer
+ if err := Open(bytes.NewReader(sealed.Bytes()), key, priv, b, &opened); err != nil {
+ t.Fatal(err)
+ }
+ if !bytes.Equal(opened.Bytes(), tarz.Bytes()) {
+ t.Fatal("open != build")
+ }
+ // A flipped byte fails: GCM, not a checksum.
+ bad := append([]byte{}, sealed.Bytes()...)
+ bad[len(bad)/2] ^= 1
+ if err := Open(bytes.NewReader(bad), key, priv, b, &bytes.Buffer{}); !errors.Is(err, ErrSealed) {
+ t.Fatalf("tampered archive opened: %v", err)
+ }
+ // Another binding cannot open the data key (F4: SealBound, always v2).
+ if err := Open(bytes.NewReader(sealed.Bytes()), key, priv, appconfig.Binding{Account: "bab", App: "other"}, &bytes.Buffer{}); !errors.Is(err, ErrSealed) {
+ t.Fatalf("wrong binding opened: %v", err)
+ }
+ out := filepath.Join(t.TempDir(), "out")
+ outEnv := filepath.Join(t.TempDir(), "out.env")
+ if err := Extract(bytes.NewReader(opened.Bytes()), out, outEnv); err != nil {
+ t.Fatal(err)
+ }
+ got, _ := os.ReadFile(filepath.Join(out, "sub", "b.bin"))
+ envGot, _ := os.ReadFile(outEnv)
+ appEnv, _ := os.ReadFile(filepath.Join(out, "env"))
+ nested, _ := os.ReadFile(filepath.Join(out, "sub", "env", "x"))
+ if string(got) != "\x01\x02\x03" || string(envGot) != "CARLOS_ADMIN_TOKEN=t\n" || string(appEnv) != "app-env" || string(nested) != "x" {
+ t.Fatal("extract content")
+ }
+ if _, err := os.Lstat(filepath.Join(out, MarkerName)); !errors.Is(err, fs.ErrNotExist) {
+ t.Fatalf("marker carried: %v", err)
+ }
+ if info, _ := os.Stat(outEnv); info.Mode().Perm() != 0o600 {
+ t.Fatalf("env mode %o", info.Mode().Perm())
+ }
+ p2, _, _, _ := PlainDigest(out, outEnv)
+ if p2 != plain {
+ t.Fatal("extracted tree has another plain digest")
+ }
+ // Encryption is randomized (§8.2).
+ var sealed2 bytes.Buffer
+ if _, _, err := Seal(bytes.NewReader(tarz.Bytes()), pub, b, &sealed2); err != nil {
+ t.Fatal(err)
+ }
+ if bytes.Equal(sealed.Bytes(), sealed2.Bytes()) {
+ t.Fatal("deterministic sealing")
+ }
+}
+
+// F4: a data key sealed as a v1 (unbound) envelope opens under any app's
+// binding, so a manifest downgraded to one would unbind the archive. Open
+// refuses it before trying the key.
+func TestOpenRefusesAnUnboundDataKey(t *testing.T) {
+ dir, env := filesFixture(t)
+ var tarz bytes.Buffer
+ if _, _, _, err := Build(dir, env, &tarz); err != nil {
+ t.Fatal(err)
+ }
+ pub, priv, b := keys(t)
+ var sealed bytes.Buffer
+ key, _, err := Seal(bytes.NewReader(tarz.Bytes()), pub, b, &sealed)
+ if err != nil {
+ t.Fatal(err)
+ }
+ raw, err := appconfig.Open(priv, b, key)
+ if err != nil {
+ t.Fatal(err)
+ }
+ v1, err := appconfig.SealUnbound(pub, raw)
+ if err != nil {
+ t.Fatal(err)
+ }
+ // The v1 key is the right key: appconfig opens it under any binding.
+ if again, err := appconfig.Open(priv, appconfig.Binding{Account: "bab", App: "other"}, v1); err != nil || !bytes.Equal(again, raw) {
+ t.Fatalf("fixture: %v", err)
+ }
+ if err := Open(bytes.NewReader(sealed.Bytes()), v1, priv, b, &bytes.Buffer{}); !errors.Is(err, ErrSealed) {
+ t.Fatalf("v1 data key opened: %v", err)
+ }
+}
+
+// A 0555 directory holding a 0400 file round-trips to exactly those modes,
+// and the extracted tree digests as the original did.
+func TestExtractPreservesModes(t *testing.T) {
+ dir, env := filesFixture(t)
+ ro := filepath.Join(dir, "ro")
+ if err := os.Mkdir(ro, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ mustWrite(t, filepath.Join(ro, "f"), []byte("secret"), 0o400)
+ if err := os.Chmod(ro, 0o555); err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { os.Chmod(ro, 0o755) })
+ var tarz bytes.Buffer
+ plain, _, _, err := Build(dir, env, &tarz)
+ if err != nil {
+ t.Fatal(err)
+ }
+ out := filepath.Join(t.TempDir(), "out")
+ outEnv := filepath.Join(t.TempDir(), "out.env")
+ t.Cleanup(func() { os.Chmod(filepath.Join(out, "ro"), 0o755) })
+ if err := Extract(bytes.NewReader(tarz.Bytes()), out, outEnv); err != nil {
+ t.Fatal(err)
+ }
+ di, _ := os.Stat(filepath.Join(out, "ro"))
+ fi, _ := os.Stat(filepath.Join(out, "ro", "f"))
+ if di.Mode().Perm() != 0o555 || fi.Mode().Perm() != 0o400 {
+ t.Fatalf("modes %o %o", di.Mode().Perm(), fi.Mode().Perm())
+ }
+ if p2, _, _, err := PlainDigest(out, outEnv); err != nil || p2 != plain {
+ t.Fatalf("plain after extract %q %v", p2, err)
+ }
+}
+
+// An archive with no env entry removes a stale env file, so the extracted
+// pair always digests as the archive does.
+func TestExtractWithoutEnvLeavesNoEnv(t *testing.T) {
+ dir, _ := filesFixture(t)
+ var tarz bytes.Buffer
+ plain, _, _, err := Build(dir, "", &tarz)
+ if err != nil {
+ t.Fatal(err)
+ }
+ out := filepath.Join(t.TempDir(), "out")
+ stale := filepath.Join(t.TempDir(), "out.env")
+ mustWrite(t, stale, []byte("OLD=1\n"), 0o600)
+ if err := Extract(bytes.NewReader(tarz.Bytes()), out, stale); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := os.Lstat(stale); !errors.Is(err, fs.ErrNotExist) {
+ t.Fatalf("stale env kept: %v", err)
+ }
+ if p2, _, _, _ := PlainDigest(out, stale); p2 != plain {
+ t.Fatal("plain differs")
+ }
+}
+
+// sealedArchive is a good archive with its manifest, over a tree large
+// enough to seal into three chunks.
+type sealedArchive struct {
+ tarz, sealed []byte
+ m Manifest
+ pub *ecdh.PublicKey
+ priv *ecdh.PrivateKey
+ b appconfig.Binding
+}
+
+func threeChunkArchive(t *testing.T) sealedArchive {
+ t.Helper()
+ dir, env := filesFixture(t)
+ big := make([]byte, 2*ChunkSize+ChunkSize/2)
+ if _, err := rand.Read(big); err != nil {
+ t.Fatal(err)
+ }
+ mustWrite(t, filepath.Join(dir, "random.bin"), big, 0o644)
+ var tarz bytes.Buffer
+ plain, entries, n, err := Build(dir, env, &tarz)
+ if err != nil {
+ t.Fatal(err)
+ }
+ pub, priv, b := keys(t)
+ var sealed bytes.Buffer
+ key, sealedSHA, err := Seal(bytes.NewReader(tarz.Bytes()), pub, b, &sealed)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got := len(chunkOffsets(t, sealed.Bytes())); got != 3 {
+ t.Fatalf("fixture seals into %d chunks", got)
+ }
+ return sealedArchive{
+ tarz: tarz.Bytes(), sealed: sealed.Bytes(), pub: pub, priv: priv, b: b,
+ m: Manifest{V: ManifestVersion, Op: "mv_x", Incarnation: "i", Plain: plain, Sealed: sealedSHA, Bytes: n, Entries: entries, SealedDataKey: key, Chunk: ChunkSize},
+ }
+}
+
+// chunkOffsets returns where each sealed chunk's length prefix starts.
+func chunkOffsets(t *testing.T, sealed []byte) []int {
+ t.Helper()
+ var offs []int
+ for off := 0; off < len(sealed); {
+ if off+4 > len(sealed) {
+ t.Fatal("ragged sealed fixture")
+ }
+ offs = append(offs, off)
+ off += 4 + int(binary.BigEndian.Uint32(sealed[off:]))
+ }
+ return offs
+}
+
+// sealRaw seals a hand-built archive, with a manifest naming its regular
+// entries as given.
+func (a sealedArchive) sealRaw(t *testing.T, raw []byte, entries []Entry) ([]byte, Manifest) {
+ t.Helper()
+ var sealed bytes.Buffer
+ key, sha, err := Seal(bytes.NewReader(raw), a.pub, a.b, &sealed)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var n int64
+ for _, e := range entries {
+ n += e.Size
+ }
+ return sealed.Bytes(), Manifest{V: ManifestVersion, Plain: "sha256:unchecked", Sealed: sha, Bytes: n, Entries: entries, SealedDataKey: key, Chunk: ChunkSize}
+}
+
+func TestVerifyRefusesWhatTheManifestDoesNotPromise(t *testing.T) {
+ a := threeChunkArchive(t)
+ if err := Verify(bytes.NewReader(a.sealed), a.m, a.priv, a.b); err != nil {
+ t.Fatalf("good archive: %v", err)
+ }
+ withManifest := func(edit func(m *Manifest)) Manifest {
+ m := a.m
+ m.Entries = append([]Entry(nil), a.m.Entries...)
+ edit(&m)
+ return m
+ }
+ idx := func(m *Manifest, p string) int {
+ for i, e := range m.Entries {
+ if e.Path == p {
+ return i
+ }
+ }
+ t.Fatalf("no %s", p)
+ return -1
+ }
+ manifestCases := map[string]struct {
+ m Manifest
+ want string
+ }{
+ "plain": {withManifest(func(m *Manifest) { m.Plain = digestBytes([]byte("other")) }), "plaintext digest"},
+ "sealed": {withManifest(func(m *Manifest) { m.Sealed = digestBytes([]byte("other")) }), "sealed digest"},
+ "missing entry": {withManifest(func(m *Manifest) {
+ i := idx(m, "files/a.txt")
+ m.Bytes -= m.Entries[i].Size
+ m.Entries = append(m.Entries[:i], m.Entries[i+1:]...)
+ }), "not in the manifest"},
+ "entry digest": {withManifest(func(m *Manifest) { m.Entries[idx(m, "files/a.txt")].SHA256 = digestBytes([]byte("beta!")) }), "digest differs"},
+ "bytes": {withManifest(func(m *Manifest) { m.Bytes++ }), "bytes"},
+ "version": {withManifest(func(m *Manifest) { m.V = 2 }), "version"},
+ "chunk": {withManifest(func(m *Manifest) { m.Chunk = 4096 }), "chunk"},
+ "no key": {withManifest(func(m *Manifest) { m.SealedDataKey = "" }), "no sealed"},
+ "duplicate manifest entry": {withManifest(func(m *Manifest) {
+ e := m.Entries[idx(m, "files/a.txt")]
+ m.Entries = append(m.Entries, e)
+ m.Bytes += e.Size
+ }), "named twice"},
+ "traversal in manifest": {withManifest(func(m *Manifest) { m.Entries[idx(m, "files/a.txt")].Path = "files/../a.txt" }), "refused path"},
+ }
+ for name, c := range manifestCases {
+ t.Run(name, func(t *testing.T) {
+ err := Verify(bytes.NewReader(a.sealed), c.m, a.priv, a.b)
+ if err == nil || !strings.Contains(err.Error(), c.want) {
+ t.Fatalf("%v, want %q", err, c.want)
+ }
+ })
+ }
+
+ // Hand-built sealed archives.
+ symlink := tarZst(t, []rawEntry{{hdr: tar.Header{Name: "files/l", Typeflag: tar.TypeSymlink, Linkname: "/etc/shadow"}}}, nil)
+ dup := tarZst(t, []rawEntry{reg("files/a", "x"), reg("files/a", "x")}, nil)
+ marker := tarZst(t, []rawEntry{reg("files/.files-gen", "x")}, nil)
+ xEntry := []Entry{{Path: "files/a", Size: 1, SHA256: digestBytes([]byte("x"))}}
+ rawCases := map[string]struct {
+ raw []byte
+ entries []Entry
+ want error
+ }{
+ "symlink entry": {symlink, nil, ErrSpecialFile},
+ "over cap": {overCapArchive(t), nil, ErrOverCap},
+ "duplicate in tar": {dup, xEntry, ErrBadPath},
+ "marker in tar": {marker, nil, ErrBadPath},
+ }
+ for name, c := range rawCases {
+ t.Run(name, func(t *testing.T) {
+ sealed, m := a.sealRaw(t, c.raw, c.entries)
+ if err := Verify(bytes.NewReader(sealed), m, a.priv, a.b); !errors.Is(err, c.want) {
+ t.Fatalf("%v, want %v", err, c.want)
+ }
+ })
+ }
+
+ // Damage to the sealed object itself: each is Open's verdict.
+ offs := chunkOffsets(t, a.sealed)
+ flipped := append([]byte(nil), a.sealed...)
+ flipped[len(flipped)-1] ^= 1
+ cut := append([]byte(nil), a.sealed[:offs[len(offs)-1]]...)
+ appended := append(append([]byte(nil), a.sealed...), 0)
+ twice := append(append([]byte(nil), a.sealed...), a.sealed...)
+ damage := map[string][]byte{
+ "last byte flipped": flipped,
+ "final chunk cut off": cut,
+ "byte appended": appended,
+ "archive concatenated": twice,
+ }
+ for name, sealed := range damage {
+ t.Run(name, func(t *testing.T) {
+ // The manifest's sealed digest is made to match the damaged
+ // object, so only Open's own checks stand between it and nil.
+ m := a.m
+ m.Sealed = digestBytes(sealed)
+ err := Verify(bytes.NewReader(sealed), m, a.priv, a.b)
+ if !errors.Is(err, ErrSealed) {
+ t.Fatalf("%v, want ErrSealed", err)
+ }
+ if (name == "byte appended" || name == "archive concatenated") && !strings.Contains(err.Error(), "after the final chunk") {
+ t.Fatalf("%v", err)
+ }
+ })
+ }
+
+ // A downgraded data key in the manifest is refused.
+ raw, err := appconfig.Open(a.priv, a.b, a.m.SealedDataKey)
+ if err != nil {
+ t.Fatal(err)
+ }
+ v1, _ := appconfig.SealUnbound(a.pub, raw)
+ m := a.m
+ m.SealedDataKey = v1
+ if err := Verify(bytes.NewReader(a.sealed), m, a.priv, a.b); !errors.Is(err, ErrSealed) {
+ t.Fatalf("v1 key in the manifest: %v", err)
+ }
+}
+
+// An early mismatch must not strand Open's goroutine on the pipe for the
+// rest of the archive (round-5 finding 12).
+func TestVerifyReleasesTheProducerOnEarlyMismatch(t *testing.T) {
+ a := threeChunkArchive(t)
+ m := a.m
+ m.Entries = append([]Entry(nil), a.m.Entries...)
+ // "env" is the first regular entry in tar order; its size is off by
+ // one, with Bytes kept consistent so the manifest itself passes.
+ for i := range m.Entries {
+ if m.Entries[i].Path == EnvEntry {
+ m.Entries[i].Size++
+ m.Bytes++
+ }
+ }
+ runtime.GC()
+ baseline := runtime.NumGoroutine()
+ err := Verify(bytes.NewReader(a.sealed), m, a.priv, a.b)
+ if err == nil || !strings.Contains(err.Error(), "size differs") {
+ t.Fatalf("early mismatch: %v", err)
+ }
+ deadline := time.Now().Add(time.Second)
+ for runtime.NumGoroutine() > baseline {
+ if time.Now().After(deadline) {
+ t.Fatalf("goroutines %d, baseline %d", runtime.NumGoroutine(), baseline)
+ }
+ time.Sleep(10 * time.Millisecond)
+ }
+}
+
+// Seal and Open agree on the chunk framing at the edges: empty input, and
+// input of exactly one chunk (whose final chunk is then empty).
+func TestSealChunkEdges(t *testing.T) {
+ pub, priv, b := keys(t)
+ for _, size := range []int{0, 1, ChunkSize - 1, ChunkSize, ChunkSize + 1, 2 * ChunkSize} {
+ plain := make([]byte, size)
+ rand.Read(plain)
+ var sealed, opened bytes.Buffer
+ key, _, err := Seal(bytes.NewReader(plain), pub, b, &sealed)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := Open(bytes.NewReader(sealed.Bytes()), key, priv, b, &opened); err != nil || !bytes.Equal(opened.Bytes(), plain) {
+ t.Fatalf("size %d: %v", size, err)
+ }
+ // Every prefix that ends on a chunk boundary is refused.
+ offs := chunkOffsets(t, sealed.Bytes())
+ for _, off := range offs {
+ if err := Open(bytes.NewReader(sealed.Bytes()[:off]), key, priv, b, io.Discard); !errors.Is(err, ErrSealed) {
+ t.Fatalf("size %d prefix %d opened: %v", size, off, err)
+ }
+ }
+ }
+}
+
+// The final flag is authenticated, not inferred: a holder of the data key
+// who frames a short chunk without the flag, or a full chunk with it, is
+// refused, so the length rule and the flag must agree (§8.2).
+func TestOpenRequiresTheFinalFlag(t *testing.T) {
+ pub, priv, b := keys(t)
+ var sealed bytes.Buffer
+ key, _, err := Seal(bytes.NewReader([]byte("x")), pub, b, &sealed)
+ if err != nil {
+ t.Fatal(err)
+ }
+ raw, err := appconfig.Open(priv, b, key)
+ if err != nil {
+ t.Fatal(err)
+ }
+ gcm, err := newGCM(raw)
+ if err != nil {
+ t.Fatal(err)
+ }
+ frame := func(index uint64, last bool, pt []byte) []byte {
+ nonce, aad := nonceFor(index, last)
+ ct := gcm.Seal(nil, nonce, pt, aad)
+ return append(binary.BigEndian.AppendUint32(nil, uint32(len(ct))), ct...)
+ }
+ full := make([]byte, ChunkSize)
+ forged := map[string][]byte{
+ "short chunk unflagged": frame(0, false, []byte("x")),
+ "full chunk flagged": frame(0, true, full),
+ "full flagged, short after": append(frame(0, true, full), frame(1, true, []byte("x"))...),
+ "well formed (control case)": append(frame(0, false, full), frame(1, true, []byte("x"))...),
+ }
+ for name, stream := range forged {
+ err := Open(bytes.NewReader(stream), key, priv, b, io.Discard)
+ if name == "well formed (control case)" {
+ if err != nil {
+ t.Fatalf("%s: %v", name, err)
+ }
+ continue
+ }
+ if !errors.Is(err, ErrSealed) {
+ t.Fatalf("%s opened: %v", name, err)
+ }
+ }
+}
diff --git a/internal/moves/archive/archive_unix_test.go b/internal/moves/archive/archive_unix_test.go
new file mode 100644
index 000000000..b9f34d68b
--- /dev/null
+++ b/internal/moves/archive/archive_unix_test.go
@@ -0,0 +1,66 @@
+//go:build unix
+
+package archive
+
+import (
+ "errors"
+ "os"
+ "path/filepath"
+ "syscall"
+ "testing"
+ "time"
+)
+
+// Between the walk and the open, a tenant can swap a listed file for a
+// FIFO (an open would block forever), another file, or a symlink out of
+// the files directory (the open would read a host file into the archive).
+// Each is refused.
+func TestOpenRefusesASwappedEntry(t *testing.T) {
+ dir := t.TempDir()
+ mustWrite(t, filepath.Join(dir, "a"), []byte("a"), 0o644)
+ mustWrite(t, filepath.Join(dir, "b"), []byte("b"), 0o644)
+ outside := filepath.Join(t.TempDir(), "host-secret")
+ mustWrite(t, outside, []byte("secret"), 0o600)
+ if err := os.Symlink(outside, filepath.Join(dir, "esc")); err != nil {
+ t.Fatal(err)
+ }
+ if err := syscall.Mkfifo(filepath.Join(dir, "fifo"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ root, err := os.OpenRoot(dir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer root.Close()
+ listed, err := root.Lstat("a")
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, swapped := range []string{"fifo", "b", "esc"} {
+ done := make(chan error, 1)
+ go func() {
+ rc, err := openItemChecked(item{name: "files/" + swapped, rel: swapped, root: root, info: listed})
+ if rc != nil {
+ rc.Close()
+ }
+ done <- err
+ }()
+ select {
+ case err := <-done:
+ if err == nil {
+ t.Fatalf("%s opened as the listed file", swapped)
+ }
+ if swapped != "esc" && !errors.Is(err, ErrChanged) {
+ t.Fatalf("%s: %v", swapped, err)
+ }
+ case <-time.After(5 * time.Second):
+ t.Fatalf("%s: open blocked", swapped)
+ }
+ }
+ // The listed file itself opens.
+ rc, err := openItemChecked(item{name: "files/a", rel: "a", root: root, info: listed})
+ if err != nil {
+ t.Fatal(err)
+ }
+ rc.Close()
+}
diff --git a/internal/moves/archive/open_other.go b/internal/moves/archive/open_other.go
new file mode 100644
index 000000000..e43f1a616
--- /dev/null
+++ b/internal/moves/archive/open_other.go
@@ -0,0 +1,6 @@
+//go:build !unix
+
+package archive
+
+// openNonblock: no FIFO opens block here.
+const openNonblock = 0
diff --git a/internal/moves/archive/open_unix.go b/internal/moves/archive/open_unix.go
new file mode 100644
index 000000000..2644dc602
--- /dev/null
+++ b/internal/moves/archive/open_unix.go
@@ -0,0 +1,9 @@
+//go:build unix
+
+package archive
+
+import "syscall"
+
+// openNonblock keeps an open of a FIFO swapped in after the walk from
+// waiting for a writer; a regular file ignores it.
+const openNonblock = syscall.O_NONBLOCK
diff --git a/internal/moves/archive/seal.go b/internal/moves/archive/seal.go
new file mode 100644
index 000000000..a4a2c5c92
--- /dev/null
+++ b/internal/moves/archive/seal.go
@@ -0,0 +1,188 @@
+package archive
+
+import (
+ "crypto/aes"
+ "crypto/cipher"
+ "crypto/ecdh"
+ "crypto/rand"
+ "crypto/sha256"
+ "encoding/base64"
+ "encoding/binary"
+ "errors"
+ "fmt"
+ "io"
+
+ "github.com/carlosframework/platform/internal/appconfig"
+)
+
+// ChunkSize is the AES-GCM chunk (§8.2). Each chunk is a 4-byte big-endian
+// length and the ciphertext; its nonce is its index, and its associated
+// data flags whether it is the last. Every chunk but the last holds exactly
+// ChunkSize bytes of plaintext and the last holds fewer (possibly none), so
+// the framing of a plaintext under a key is unique, Open knows from a
+// chunk's length which flag it must carry, and a stream cut at a chunk
+// boundary never ends on a chunk flagged last.
+const ChunkSize = 1 << 20
+
+// maxChunks bounds a sealed object: the largest decoded archive, plus
+// zstd's worst-case expansion of incompressible input (under 1/256, and
+// a margin for frame headers), in chunks, plus the final chunk.
+const maxChunks = uint64((MaxDecoded+MaxDecoded/256+64<<10)/ChunkSize + 2)
+
+// ErrSealed is every refusal to open: a data key that is unbound or sealed
+// for another app, a chunk that does not authenticate, a stream cut short
+// or carrying data after its final chunk.
+var ErrSealed = errors.New("archive: sealed object does not open")
+
+func nonceFor(index uint64, last bool) ([]byte, []byte) {
+ nonce := make([]byte, 12)
+ binary.BigEndian.PutUint64(nonce[4:], index)
+ aad := []byte{0}
+ if last {
+ aad[0] = 1
+ }
+ return nonce, aad
+}
+
+// Seal encrypts plain into w under a fresh 256-bit data key and returns
+// the data key sealed with appconfig.SealBound (a v2 envelope bound to b,
+// with its own fresh ephemeral key and nonce) and the sha256 of what it
+// wrote. Never deterministic. SealBound, not appconfig.Seal: Seal writes
+// unbound v1 envelopes unless the process turned bound sealing on, and the
+// host agent never does (F4).
+func Seal(plain io.Reader, pub *ecdh.PublicKey, b appconfig.Binding, w io.Writer) (sealedDataKey, sealedSHA string, err error) {
+ key := make([]byte, 32)
+ if _, err := rand.Read(key); err != nil {
+ return "", "", err
+ }
+ defer clear(key)
+ gcm, err := newGCM(key)
+ if err != nil {
+ return "", "", err
+ }
+ h := sha256.New()
+ out := io.MultiWriter(w, h)
+ buf := make([]byte, ChunkSize)
+ for index := uint64(0); ; index++ {
+ if index >= maxChunks {
+ return "", "", fmt.Errorf("%w: more than %d chunks", ErrTooLarge, maxChunks)
+ }
+ n, rerr := io.ReadFull(plain, buf)
+ last := rerr == io.EOF || rerr == io.ErrUnexpectedEOF
+ if rerr != nil && !last {
+ return "", "", rerr
+ }
+ nonce, aad := nonceFor(index, last)
+ ct := gcm.Seal(nil, nonce, buf[:n], aad)
+ var hdr [4]byte
+ binary.BigEndian.PutUint32(hdr[:], uint32(len(ct)))
+ if _, err := out.Write(hdr[:]); err != nil {
+ return "", "", err
+ }
+ if _, err := out.Write(ct); err != nil {
+ return "", "", err
+ }
+ if last {
+ break
+ }
+ }
+ sealedDataKey, err = appconfig.SealBound(pub, b, key)
+ if err != nil {
+ return "", "", err
+ }
+ return sealedDataKey, digestString(h), nil
+}
+
+func newGCM(key []byte) (cipher.AEAD, error) {
+ block, err := aes.NewCipher(key)
+ if err != nil {
+ return nil, err
+ }
+ return cipher.NewGCM(block)
+}
+
+// boundEnvelope reports whether an appconfig envelope is v2 (bound to an
+// account and app). Twin of appconfig's unexported boundPrefix and
+// splitVersion (crypto.go): appconfig.Open opens a v1 envelope under any
+// binding, so Open checks the version itself.
+func boundEnvelope(envelope string) bool {
+ raw, err := base64.StdEncoding.DecodeString(envelope)
+ return err == nil && len(raw) > 4 && string(raw[:4]) == "CSB2"
+}
+
+// Open reverses Seal into w. An unbound (v1) data key, a data key sealed
+// for another binding, any tampered chunk, a stream cut short and data
+// after the final chunk all return ErrSealed. Each chunk is written to w
+// only after it authenticates, but the stream as a whole is authentic only
+// when Open returns nil: a reader must treat what it got as provisional
+// until then.
+func Open(sealed io.Reader, sealedDataKey string, priv *ecdh.PrivateKey, b appconfig.Binding, w io.Writer) error {
+ if !boundEnvelope(sealedDataKey) {
+ return fmt.Errorf("%w: the data key is not bound to an app", ErrSealed)
+ }
+ key, err := appconfig.Open(priv, b, sealedDataKey)
+ if err != nil {
+ return fmt.Errorf("%w: data key: %v", ErrSealed, err)
+ }
+ defer clear(key)
+ if len(key) != 32 {
+ return fmt.Errorf("%w: data key of %d bytes", ErrSealed, len(key))
+ }
+ gcm, err := newGCM(key)
+ if err != nil {
+ return err
+ }
+ full := uint32(ChunkSize + gcm.Overhead())
+ ct := make([]byte, full)
+ pt := make([]byte, 0, ChunkSize)
+ for index := uint64(0); ; index++ {
+ if index >= maxChunks {
+ return fmt.Errorf("%w: more than %d chunks", ErrSealed, maxChunks)
+ }
+ var hdr [4]byte
+ if _, err := io.ReadFull(sealed, hdr[:]); err != nil {
+ return truncated(err, index)
+ }
+ n := binary.BigEndian.Uint32(hdr[:])
+ if n < uint32(gcm.Overhead()) || n > full {
+ return fmt.Errorf("%w: chunk %d has length %d", ErrSealed, index, n)
+ }
+ if _, err := io.ReadFull(sealed, ct[:n]); err != nil {
+ return truncated(err, index)
+ }
+ last := n < full
+ nonce, aad := nonceFor(index, last)
+ // Into pt, never ct[:0]: a failed Open may overwrite its dst.
+ out, err := gcm.Open(pt[:0], nonce, ct[:n], aad)
+ if err != nil {
+ return fmt.Errorf("%w: chunk %d does not authenticate", ErrSealed, index)
+ }
+ if _, err := w.Write(out); err != nil {
+ return err
+ }
+ if !last {
+ continue
+ }
+ // The final chunk must end the stream: trailing ciphertext would
+ // otherwise pass Verify against a manifest whose sealed digest
+ // covers only this authenticated prefix (round-6 finding 6).
+ var one [1]byte
+ m, err := io.ReadFull(sealed, one[:])
+ if m > 0 {
+ return fmt.Errorf("%w: data after the final chunk", ErrSealed)
+ }
+ if err != io.EOF {
+ return err
+ }
+ return nil
+ }
+}
+
+// truncated classifies a short read: the stream ending is ErrSealed, any
+// other read error is the reader's own.
+func truncated(err error, index uint64) error {
+ if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
+ return fmt.Errorf("%w: truncated at chunk %d", ErrSealed, index)
+ }
+ return err
+}
diff --git a/internal/moves/archive/sync_linux.go b/internal/moves/archive/sync_linux.go
new file mode 100644
index 000000000..5061dd7ad
--- /dev/null
+++ b/internal/moves/archive/sync_linux.go
@@ -0,0 +1,38 @@
+//go:build linux
+
+package archive
+
+import (
+ "os"
+ "path/filepath"
+
+ "golang.org/x/sys/unix"
+)
+
+// syncEachFile is false on Linux: syncStaged flushes the whole staged
+// tree with syncfs(2) instead.
+const syncEachFile = false
+
+// syncStaged makes the staged tree and env file durable before they move
+// into place: one syncfs(2) on each filesystem involved. Through the
+// directories, which are still writable by their owner at this point,
+// never through the files, whose archived mode may refuse an open.
+func (x *extraction) syncStaged() error {
+ if err := syncfs(x.stage); err != nil {
+ return err
+ }
+ if x.envTmp != "" {
+ return syncfs(filepath.Dir(x.envTmp))
+ }
+ return nil
+}
+
+func syncfs(dir string) error {
+ d, err := os.Open(dir)
+ if err != nil {
+ return err
+ }
+ err = unix.Syncfs(int(d.Fd()))
+ d.Close()
+ return err
+}
diff --git a/internal/moves/archive/sync_other.go b/internal/moves/archive/sync_other.go
new file mode 100644
index 000000000..0ed34041b
--- /dev/null
+++ b/internal/moves/archive/sync_other.go
@@ -0,0 +1,23 @@
+//go:build !linux
+
+package archive
+
+import (
+ "io/fs"
+ "path/filepath"
+)
+
+// syncEachFile is true where there is no syncfs(2): finishFile syncs each
+// file while it is still open (its archived mode may refuse a later open).
+const syncEachFile = true
+
+// syncStaged syncs the staged directories, the files having been synced
+// as they were written.
+func (x *extraction) syncStaged() error {
+ return filepath.WalkDir(x.stage, func(p string, d fs.DirEntry, err error) error {
+ if err != nil || !d.IsDir() {
+ return err
+ }
+ return syncDir(p)
+ })
+}