amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package archive
2
3import (
4 "archive/tar"
5 "bytes"
6 "crypto/ecdh"
7 "crypto/rand"
8 "crypto/sha256"
9 "encoding/binary"
10 "encoding/hex"
11 "errors"
12 "io"
13 "io/fs"
14 "os"
15 "path/filepath"
16 "runtime"
17 "strings"
18 "sync"
19 "testing"
20 "time"
21
22 "github.com/klauspost/compress/zstd"
23
24 "github.com/carlosframework/platform/internal/appconfig"
25)
26
27func filesFixture(t *testing.T) (dir, env string) {
28 t.Helper()
29 dir = filepath.Join(t.TempDir(), "h")
30 if err := os.MkdirAll(filepath.Join(dir, "sub"), 0o755); err != nil {
31 t.Fatal(err)
32 }
33 mustWrite(t, filepath.Join(dir, "a.txt"), []byte("alpha"), 0o644)
34 mustWrite(t, filepath.Join(dir, "sub", "b.bin"), []byte{1, 2, 3}, 0o600)
35 mustWrite(t, filepath.Join(dir, ".files-gen"), []byte(`{"op":"x"}`), 0o644)
36 // A legal application file named env, and a directory named env with
37 // a file in it: both must survive beside the <host>.env overlay.
38 mustWrite(t, filepath.Join(dir, "env"), []byte("app-env"), 0o644)
39 if err := os.MkdirAll(filepath.Join(dir, "sub", "env"), 0o755); err != nil {
40 t.Fatal(err)
41 }
42 mustWrite(t, filepath.Join(dir, "sub", "env", "x"), []byte("x"), 0o644)
43 env = filepath.Join(t.TempDir(), "h.env")
44 mustWrite(t, env, []byte("CARLOS_ADMIN_TOKEN=t\n"), 0o600)
45 return dir, env
46}
47
48func mustWrite(t *testing.T, p string, b []byte, mode fs.FileMode) {
49 t.Helper()
50 if err := os.WriteFile(p, b, mode); err != nil {
51 t.Fatal(err)
52 }
53 // WriteFile's mode passes through the umask; the fixtures want it exact.
54 if err := os.Chmod(p, mode); err != nil {
55 t.Fatal(err)
56 }
57}
58
59// §8.1: equal content gives equal plaintext; the marker is excluded.
60func TestBuildIsDeterministicAndExcludesTheMarker(t *testing.T) {
61 dir, env := filesFixture(t)
62 var b1, b2 bytes.Buffer
63 p1, entries, n, err := Build(dir, env, &b1)
64 if err != nil {
65 t.Fatal(err)
66 }
67 if err := os.Chtimes(filepath.Join(dir, "a.txt"), time.Unix(1, 0), time.Unix(1, 0)); err != nil {
68 t.Fatal(err)
69 }
70 p2, _, _, err := Build(dir, env, &b2)
71 if err != nil || p1 != p2 || !bytes.Equal(b1.Bytes(), b2.Bytes()) {
72 t.Fatalf("not deterministic: %q %q %v", p1, p2, err)
73 }
74 paths := map[string]bool{}
75 for _, e := range entries {
76 paths[e.Path] = true
77 }
78 if paths["files/.files-gen"] || !paths["files/a.txt"] || !paths["files/sub/b.bin"] || !paths["env"] || !paths["files/env"] || !paths["files/sub/env/x"] || n != 5+3+7+1+int64(len("CARLOS_ADMIN_TOKEN=t\n")) {
79 t.Fatalf("entries %v bytes %d", entries, n)
80 }
81 // An application file and directory named "env" are legal and carried
82 // under files/, never confused with the overlay (round-tripped in
83 // TestSealOpenExtractRoundTrip through the extra fixture entries).
84 plain, bytesN, count, err := PlainDigest(dir, env)
85 if err != nil || plain != p1 || bytesN != n || count != len(entries) {
86 t.Fatalf("PlainDigest %q %d %d %v, want %q %d %d", plain, bytesN, count, err, p1, n, len(entries))
87 }
88 // Owners and times are zeroed and modes kept, in the tar itself.
89 tr := tar.NewReader(mustDecode(t, b1.Bytes()))
90 for {
91 hdr, err := tr.Next()
92 if err == io.EOF {
93 break
94 }
95 if err != nil {
96 t.Fatal(err)
97 }
98 if hdr.Uid != 0 || hdr.Gid != 0 || hdr.Uname != "" || hdr.Gname != "" || !hdr.ModTime.Equal(time.Unix(0, 0)) {
99 t.Fatalf("%s carries an owner or a time: %+v", hdr.Name, hdr)
100 }
101 if hdr.Name == "files/sub/b.bin" && hdr.Mode != 0o600 {
102 t.Fatalf("mode %o", hdr.Mode)
103 }
104 }
105}
106
107// BuildFiltered over a nested-shaped directory leaves out exactly what
108// skip accepts (a top-level file, a top-level directory with everything
109// under it, and a symlink, which unskipped would refuse the walk), never
110// asks about env, and writes byte for byte the archive Build writes over a
111// flat directory holding only the app's files; that archive extracts flat
112// and its extraction digests to the same plaintext. Build is BuildFiltered
113// with a nil skip.
114func TestBuildFilteredLeavesOutWhatSkipAccepts(t *testing.T) {
115 flat, env := filesFixture(t)
116 nested := filepath.Join(t.TempDir(), "nested")
117 if err := os.CopyFS(nested, os.DirFS(flat)); err != nil {
118 t.Fatal(err)
119 }
120 for _, rel := range []string{"a.txt", "sub", "sub/b.bin", "env", "sub/env", "sub/env/x", ".files-gen"} {
121 info, err := os.Lstat(filepath.Join(flat, rel))
122 if err != nil {
123 t.Fatal(err)
124 }
125 if err := os.Chmod(filepath.Join(nested, rel), info.Mode().Perm()); err != nil {
126 t.Fatal(err)
127 }
128 }
129 mustWrite(t, filepath.Join(nested, "h.db"), []byte("database"), 0o600)
130 if err := os.MkdirAll(filepath.Join(nested, ".activator-h.db-litestream", "ltx", "0"), 0o700); err != nil {
131 t.Fatal(err)
132 }
133 mustWrite(t, filepath.Join(nested, ".activator-h.db-litestream", "ltx", "0", "f.ltx"), []byte("ltx"), 0o600)
134 if err := os.Symlink("/etc/passwd", filepath.Join(nested, "h.config.env")); err != nil {
135 t.Fatal(err)
136 }
137 // A nested file that shares a skipped name below the top level is kept.
138 mustWrite(t, filepath.Join(nested, "sub", "h.db"), []byte("an app file"), 0o644)
139 mustWrite(t, filepath.Join(flat, "sub", "h.db"), []byte("an app file"), 0o644)
140 platform := map[string]bool{"h.db": true, ".activator-h.db-litestream": true, "h.config.env": true}
141 var asked []string
142 skip := func(rel string) bool {
143 asked = append(asked, rel)
144 return !strings.Contains(rel, "/") && platform[rel]
145 }
146 if _, _, _, err := Build(nested, env, io.Discard); !errors.Is(err, ErrSpecialFile) {
147 t.Fatalf("unfiltered, the symlink should refuse the walk: %v", err)
148 }
149 var got, want bytes.Buffer
150 plain, entries, n, err := BuildFiltered(nested, env, skip, &got)
151 if err != nil {
152 t.Fatal(err)
153 }
154 wplain, wentries, wn, err := Build(flat, env, &want)
155 if err != nil {
156 t.Fatal(err)
157 }
158 if plain != wplain || n != wn || len(entries) != len(wentries) || !bytes.Equal(got.Bytes(), want.Bytes()) {
159 t.Fatalf("filtered %s %d %v, flat %s %d %v", plain, n, entries, wplain, wn, wentries)
160 }
161 for _, rel := range asked {
162 if strings.HasPrefix(rel, ".activator-h.db-litestream/") {
163 t.Fatalf("asked about %q inside a skipped directory", rel)
164 }
165 }
166 if !slicesContains(asked, "sub/h.db") || !slicesContains(asked, "h.db") {
167 t.Fatalf("asked %v", asked)
168 }
169 // Extract flat, then digest the extraction as Build would.
170 parent := t.TempDir()
171 files, outEnv := filepath.Join(parent, "h"), filepath.Join(parent, "h.env")
172 if err := Extract(bytes.NewReader(got.Bytes()), files, outEnv); err != nil {
173 t.Fatal(err)
174 }
175 if p, _, _, err := PlainDigest(files, outEnv); err != nil || p != plain {
176 t.Fatalf("extraction digests to %s (%v), want %s", p, err, plain)
177 }
178 for _, name := range []string{"h.db", ".activator-h.db-litestream", "h.config.env"} {
179 if _, err := os.Lstat(filepath.Join(files, name)); !errors.Is(err, fs.ErrNotExist) {
180 t.Fatalf("%s extracted: %v", name, err)
181 }
182 }
183 // A nil skip is Build.
184 var nilSkip bytes.Buffer
185 if p, _, _, err := BuildFiltered(flat, env, nil, &nilSkip); err != nil || p != wplain || !bytes.Equal(nilSkip.Bytes(), want.Bytes()) {
186 t.Fatalf("nil skip: %s %v", p, err)
187 }
188}
189
190func slicesContains(s []string, v string) bool {
191 for _, x := range s {
192 if x == v {
193 return true
194 }
195 }
196 return false
197}
198
199func mustDecode(t *testing.T, b []byte) io.Reader {
200 t.Helper()
201 zr, err := zstd.NewReader(bytes.NewReader(b))
202 if err != nil {
203 t.Fatal(err)
204 }
205 t.Cleanup(zr.Close)
206 return zr
207}
208
209// Review Focus 3: a symlink refuses with its path, and so does a files
210// directory that is itself a symlink (WalkDir would otherwise archive
211// nothing beneath it and the move would carry an empty tree).
212func TestArchiveRefusesSpecialFiles(t *testing.T) {
213 dir, env := filesFixture(t)
214 if err := os.Symlink("a.txt", filepath.Join(dir, "link")); err != nil {
215 t.Fatal(err)
216 }
217 _, _, _, err := Build(dir, env, &bytes.Buffer{})
218 if !errors.Is(err, ErrSpecialFile) || !strings.Contains(err.Error(), "link") {
219 t.Fatalf("symlink: %v", err)
220 }
221 dir2, env2 := filesFixture(t)
222 linked := filepath.Join(t.TempDir(), "linked")
223 if err := os.Symlink(dir2, linked); err != nil {
224 t.Fatal(err)
225 }
226 if _, _, _, err := Build(linked, env2, &bytes.Buffer{}); !errors.Is(err, ErrSpecialFile) {
227 t.Fatalf("symlinked files directory: %v", err)
228 }
229 envLink := filepath.Join(t.TempDir(), "h.env")
230 if err := os.Symlink(env2, envLink); err != nil {
231 t.Fatal(err)
232 }
233 if _, _, _, err := Build(dir2, envLink, &bytes.Buffer{}); !errors.Is(err, ErrSpecialFile) || !strings.Contains(err.Error(), "h.env") {
234 t.Fatalf("symlinked env: %v", err)
235 }
236}
237
238// tarZst writes a hand-built archive: what a hostile sealer could produce.
239type rawEntry struct {
240 hdr tar.Header
241 body []byte
242}
243
244func tarZst(t *testing.T, entries []rawEntry, trailer []byte) []byte {
245 t.Helper()
246 var buf bytes.Buffer
247 zw, err := zstd.NewWriter(&buf)
248 if err != nil {
249 t.Fatal(err)
250 }
251 tw := tar.NewWriter(zw)
252 for _, e := range entries {
253 hdr := e.hdr
254 if err := tw.WriteHeader(&hdr); err != nil {
255 t.Fatal(err)
256 }
257 if _, err := tw.Write(e.body); err != nil {
258 t.Fatal(err)
259 }
260 }
261 if err := tw.Close(); err != nil {
262 t.Fatal(err)
263 }
264 if _, err := zw.Write(trailer); err != nil {
265 t.Fatal(err)
266 }
267 if err := zw.Close(); err != nil {
268 t.Fatal(err)
269 }
270 return buf.Bytes()
271}
272
273func reg(name, body string) rawEntry {
274 return rawEntry{hdr: tar.Header{Name: name, Typeflag: tar.TypeReg, Size: int64(len(body)), Mode: 0o600}, body: []byte(body)}
275}
276
277func dirEntry(name string) rawEntry {
278 return rawEntry{hdr: tar.Header{Name: name, Typeflag: tar.TypeDir, Mode: 0o755}}
279}
280
281// extractTarget is a destination whose parent holds nothing else, so a
282// test can prove a refused archive left no byte behind.
283func extractTarget(t *testing.T) (parent, files, env string) {
284 t.Helper()
285 parent = t.TempDir()
286 envDir := t.TempDir()
287 return parent, filepath.Join(parent, "out"), filepath.Join(envDir, "out.env")
288}
289
290func assertUntouched(t *testing.T, parent, files, env string) {
291 t.Helper()
292 left, err := os.ReadDir(parent)
293 if err != nil {
294 t.Fatal(err)
295 }
296 if len(left) != 0 {
297 t.Fatalf("a refused archive left %v in the destination's parent", left)
298 }
299 if _, err := os.Lstat(files); !errors.Is(err, fs.ErrNotExist) {
300 t.Fatalf("files directory written: %v", err)
301 }
302 envLeft, _ := os.ReadDir(filepath.Dir(env))
303 if len(envLeft) != 0 {
304 t.Fatalf("a refused archive left %v beside the env file", envLeft)
305 }
306}
307
308// duplicateStreams name one destination twice.
309func duplicateStreams() map[string][]rawEntry {
310 return map[string][]rawEntry{
311 "env twice": {reg("env", "x"), reg("env", "y")},
312 "file twice": {reg("files/a", "x"), reg("files/a", "y")},
313 "dir then file": {dirEntry("files/a/"), reg("files/a", "y")},
314 "file then env dup": {reg("files/a", "x"), reg("env", "e"), reg("files/a", "z")},
315 }
316}
317
318// A crafted archive naming one destination twice (two "env" entries, a
319// file twice, or a directory and then a file of one name) is refused
320// before any byte reaches the destination: Extract writes into a private
321// staging directory and moves it into place only once the whole archive
322// has been read and checked.
323func TestExtractRefusesDuplicateDestinations(t *testing.T) {
324 for name, entries := range duplicateStreams() {
325 t.Run(name, func(t *testing.T) {
326 parent, files, env := extractTarget(t)
327 err := Extract(bytes.NewReader(tarZst(t, entries, nil)), files, env)
328 if !errors.Is(err, ErrBadPath) {
329 t.Fatalf("duplicate: %v", err)
330 }
331 assertUntouched(t, parent, files, env)
332 })
333 }
334}
335
336// Traversal, the protocol marker, entries outside files/ and env, and any
337// type but a regular file or a directory are refused, and nothing lands.
338// refusedStreamCase is one hostile stream and the refusal every reader owes it.
339type refusedStreamCase struct {
340 entries []rawEntry
341 want error
342}
343
344// refusedStreams are the hostile streams Extract and Verify both refuse.
345func refusedStreams() map[string]refusedStreamCase {
346 return map[string]refusedStreamCase{
347 "dotdot": {[]rawEntry{reg("files/../x", "x")}, ErrBadPath},
348 "leading dotdot": {[]rawEntry{reg("../x", "x")}, ErrBadPath},
349 "absolute": {[]rawEntry{reg("/etc/x", "x")}, ErrBadPath},
350 "dot segment": {[]rawEntry{reg("files/./a", "x")}, ErrBadPath},
351 "double slash": {[]rawEntry{reg("files//a", "x")}, ErrBadPath},
352 "files root": {[]rawEntry{dirEntry("files/")}, ErrBadPath},
353 "outside": {[]rawEntry{reg("other", "x")}, ErrBadPath},
354 "marker": {[]rawEntry{reg("files/.files-gen", `{"op":"forged"}`)}, ErrBadPath},
355 "marker dir": {[]rawEntry{dirEntry("files/.files-gen/")}, ErrBadPath},
356 "marker child": {[]rawEntry{reg("files/.files-gen/child", "x")}, ErrBadPath},
357 "before its dir": {[]rawEntry{reg("files/a/b", "x")}, ErrBadPath},
358 "late dir": {[]rawEntry{reg("files/a/b", "x"), dirEntry("files/a/")}, ErrBadPath},
359 "nested late dir": {[]rawEntry{dirEntry("files/a/"), reg("files/a/b/c", "x"), dirEntry("files/a/b/")}, ErrBadPath},
360 "env as dir": {[]rawEntry{dirEntry("env/")}, ErrBadPath},
361 "too long": {[]rawEntry{reg("files/"+strings.Repeat("a", MaxPathLen+1), "x")}, ErrBadPath},
362 "past tree cap": {[]rawEntry{reg("files/"+strings.Repeat("a", TreePathLen+1), "x")}, ErrBadPath},
363 "symlink": {[]rawEntry{{hdr: tar.Header{Name: "files/l", Typeflag: tar.TypeSymlink, Linkname: "/etc/passwd"}}}, ErrSpecialFile},
364 "hard link": {[]rawEntry{reg("files/a", "x"), {hdr: tar.Header{Name: "files/l", Typeflag: tar.TypeLink, Linkname: "files/a"}}}, ErrSpecialFile},
365 "char device": {[]rawEntry{{hdr: tar.Header{Name: "files/d", Typeflag: tar.TypeChar, Devmajor: 1, Devminor: 3}}}, ErrSpecialFile},
366 "fifo": {[]rawEntry{{hdr: tar.Header{Name: "files/f", Typeflag: tar.TypeFifo}}}, ErrSpecialFile},
367 "late bad entry": {[]rawEntry{reg("files/a", "fine"), reg("env", "e"), reg("files/../escape", "x")}, ErrBadPath},
368 "over cap header": {[]rawEntry{{hdr: tar.Header{Name: "files/big", Typeflag: tar.TypeReg, Size: Cap + 1}}}, ErrOverCap},
369 }
370}
371
372func TestExtractRefusesTraversalAndSpecialEntries(t *testing.T) {
373 cases := refusedStreams()
374 for name, c := range cases {
375 t.Run(name, func(t *testing.T) {
376 var raw []byte
377 if c.want == ErrOverCap {
378 raw = overCapArchive(t)
379 } else {
380 raw = tarZst(t, c.entries, nil)
381 }
382 parent, files, env := extractTarget(t)
383 if err := Extract(bytes.NewReader(raw), files, env); !errors.Is(err, c.want) {
384 t.Fatalf("%s: %v, want %v", name, err, c.want)
385 }
386 assertUntouched(t, parent, files, env)
387 })
388 }
389}
390
391// overCapArchive is a tar.zst whose one header claims Cap+1 bytes. Only
392// the header and a few bytes of content are present: every reader must
393// refuse on the header, before reading the content.
394func overCapArchive(t *testing.T) []byte {
395 t.Helper()
396 var tarBuf bytes.Buffer
397 tw := tar.NewWriter(&tarBuf)
398 if err := tw.WriteHeader(&tar.Header{Name: "files/big", Typeflag: tar.TypeReg, Size: Cap + 1, Mode: 0o600}); err != nil {
399 t.Fatal(err)
400 }
401 if _, err := tw.Write([]byte("abc")); err != nil {
402 t.Fatal(err)
403 }
404 // No Close: the content is short on purpose.
405 var buf bytes.Buffer
406 zw, _ := zstd.NewWriter(&buf)
407 zw.Write(tarBuf.Bytes())
408 zw.Close()
409 return buf.Bytes()
410}
411
412// The destination must be absent or an empty directory, and the env file
413// may not live inside the files directory (Build would carry it twice).
414func TestExtractRefusesAnOccupiedDestination(t *testing.T) {
415 dir, env := filesFixture(t)
416 var tarz bytes.Buffer
417 if _, _, _, err := Build(dir, env, &tarz); err != nil {
418 t.Fatal(err)
419 }
420 occupied := t.TempDir()
421 mustWrite(t, filepath.Join(occupied, "stale"), []byte("s"), 0o644)
422 if err := Extract(bytes.NewReader(tarz.Bytes()), occupied, filepath.Join(t.TempDir(), "e.env")); !errors.Is(err, ErrNotEmpty) {
423 t.Fatalf("occupied: %v", err)
424 }
425 inside := t.TempDir()
426 if err := Extract(bytes.NewReader(tarz.Bytes()), inside, filepath.Join(inside, "h.env")); !errors.Is(err, ErrBadPath) {
427 t.Fatalf("env inside files: %v", err)
428 }
429 // An empty existing directory is fine, and keeps its mode.
430 empty := filepath.Join(t.TempDir(), "empty")
431 if err := os.Mkdir(empty, 0o750); err != nil {
432 t.Fatal(err)
433 }
434 if err := os.Chmod(empty, 0o750); err != nil {
435 t.Fatal(err)
436 }
437 if err := Extract(bytes.NewReader(tarz.Bytes()), empty, filepath.Join(t.TempDir(), "e.env")); err != nil {
438 t.Fatal(err)
439 }
440 if info, _ := os.Stat(empty); info.Mode().Perm() != 0o750 {
441 t.Fatalf("root mode %o", info.Mode().Perm())
442 }
443}
444
445// The stat-time check: a file already over the cap when it is listed is
446// refused before any content is read (a sparse file, so nothing is read).
447func TestArchiveCapIsEnforcedAtStat(t *testing.T) {
448 dir, env := filesFixture(t)
449 f, err := os.Create(filepath.Join(dir, "big"))
450 if err != nil {
451 t.Fatal(err)
452 }
453 if err := f.Truncate(Cap + 1); err != nil {
454 t.Fatal(err)
455 }
456 f.Close()
457 if _, _, _, err := Build(dir, env, &bytes.Buffer{}); !errors.Is(err, ErrOverCap) {
458 t.Fatalf("over cap: %v", err)
459 }
460}
461
462// lyingFile is a file that keeps growing after it was listed: it yields
463// bytes forever and counts how many it was asked for.
464type lyingFile struct{ read int64 }
465
466func (l *lyingFile) Read(p []byte) (int, error) {
467 for i := range p {
468 p[i] = 'g'
469 }
470 l.read += int64(len(p))
471 return len(p), nil
472}
473func (l *lyingFile) Close() error { return nil }
474
475// The streaming half of the cap: the tar header commits each file's size
476// as listed, and every file is read only up to that size. A file that grows
477// after it was listed (here without end) cannot push the archive past what
478// was admitted against the cap; it is refused as changed, naming its path,
479// having been read for at most one byte past its listed size.
480func TestArchiveCapHoldsWhenAFileGrowsWhileStreaming(t *testing.T) {
481 dir, env := filesFixture(t)
482 liar := &lyingFile{}
483 orig := openItem
484 t.Cleanup(func() { openItem = orig })
485 openItem = func(it item) (io.ReadCloser, error) {
486 if it.name == "files/a.txt" {
487 return liar, nil
488 }
489 return orig(it)
490 }
491 var out countingWriter
492 _, _, _, err := Build(dir, env, &out)
493 if !errors.Is(err, ErrChanged) || !strings.Contains(err.Error(), "a.txt") {
494 t.Fatalf("growing file: %v", err)
495 }
496 if liar.read > int64(len("alpha"))+1 {
497 t.Fatalf("read %d bytes of a file listed at 5", liar.read)
498 }
499 if _, _, _, err := PlainDigest(dir, env); !errors.Is(err, ErrChanged) {
500 t.Fatalf("PlainDigest of a growing file: %v", err)
501 }
502}
503
504type countingWriter struct{ n int64 }
505
506func (c *countingWriter) Write(p []byte) (int, error) { c.n += int64(len(p)); return len(p), nil }
507
508// F25: the cap counts content, not tar headers and padding. Content just
509// under the cap plus a few hundred small files (whose headers alone run
510// past the old 64 KiB allowance) archives; so do thousands of small files.
511func TestManySmallFilesUnderTheCapArchive(t *testing.T) {
512 if testing.Short() {
513 t.Skip("reads 256 MiB of a sparse file")
514 }
515 dir, env := filesFixture(t)
516 // Fixture content is 37 bytes; leave room for 400 one-byte files.
517 f, err := os.Create(filepath.Join(dir, "near-cap"))
518 if err != nil {
519 t.Fatal(err)
520 }
521 if err := f.Truncate(Cap - 37 - 400 - 100); err != nil {
522 t.Fatal(err)
523 }
524 f.Close()
525 many := filepath.Join(dir, "many")
526 if err := os.Mkdir(many, 0o755); err != nil {
527 t.Fatal(err)
528 }
529 for i := 0; i < 400; i++ {
530 mustWrite(t, filepath.Join(many, strings.Repeat("f", 40)+string(rune('a'+i%26))+hex.EncodeToString([]byte{byte(i >> 8), byte(i)})), []byte("z"), 0o644)
531 }
532 b, err := build(dir, env, io.Discard)
533 if err != nil {
534 t.Fatalf("content under the cap refused: %v", err)
535 }
536 if b.content > Cap || b.tarBytes <= Cap+64<<10 {
537 t.Fatalf("fixture does not exercise header overhead: content %d tar %d", b.content, b.tarBytes)
538 }
539}
540
541func TestThousandsOfSmallFilesArchive(t *testing.T) {
542 dir, env := filesFixture(t)
543 for i := 0; i < 3000; i++ {
544 mustWrite(t, filepath.Join(dir, "s"+hex.EncodeToString([]byte{byte(i >> 8), byte(i)})), []byte{byte(i)}, 0o644)
545 }
546 var tarz bytes.Buffer
547 plain, entries, _, err := Build(dir, env, &tarz)
548 if err != nil || len(entries) != 3005 {
549 t.Fatalf("%d entries, %v", len(entries), err)
550 }
551 _, files, outEnv := extractTarget(t)
552 if err := Extract(bytes.NewReader(tarz.Bytes()), files, outEnv); err != nil {
553 t.Fatal(err)
554 }
555 if p2, _, _, _ := PlainDigest(files, outEnv); p2 != plain {
556 t.Fatal("round trip changed the plain digest")
557 }
558}
559
560// The per-entry overhead bound behind MaxDecoded holds for the longest
561// path Build admits, so every archive Build writes is one Extract accepts.
562func TestTheLongestPathFitsTheEntryOverhead(t *testing.T) {
563 dir := filepath.Join(t.TempDir(), "h")
564 // Components of 200 bytes nested to a relative path of TreePathLen-ish:
565 // the longest Build admits is the files tree's cap (#236 fix round 1).
566 rel := ""
567 for len(rel)+201 <= TreePathLen-10 {
568 if rel != "" {
569 rel += "/"
570 }
571 rel += strings.Repeat("d", 200)
572 }
573 if err := os.Mkdir(dir, 0o755); err != nil {
574 t.Fatal(err)
575 }
576 leaf := rel + "/" + strings.Repeat("f", TreePathLen-len(rel)-1)
577 if len(leaf) != TreePathLen {
578 t.Fatalf("leaf %d", len(leaf))
579 }
580 // Created through a Root (openat), as Build reads it: the absolute
581 // path is past PATH_MAX, the relative one is not.
582 root, err := os.OpenRoot(dir)
583 if err != nil {
584 t.Fatal(err)
585 }
586 defer root.Close()
587 if err := root.MkdirAll(rel, 0o755); err != nil {
588 t.Fatal(err)
589 }
590 if err := root.WriteFile(leaf, []byte("x"), 0o644); err != nil {
591 t.Fatal(err)
592 }
593 b, err := build(dir, "", io.Discard)
594 if err != nil {
595 t.Fatal(err)
596 }
597 if per := (b.tarBytes - 2*512 - b.content) / int64(b.count); per > entryOverhead {
598 t.Fatalf("per-entry overhead %d over the bound %d", per, entryOverhead)
599 }
600 // One byte longer (MaxPathLen) is past the tree's cap: refused by Build
601 // with ErrPastTreeCap, naming the path.
602 if err := root.Rename(leaf, leaf+"x"); err != nil {
603 t.Fatal(err)
604 }
605 var pe *PathError
606 if _, err := build(dir, "", io.Discard); !errors.Is(err, ErrPastTreeCap) || errors.Is(err, ErrBadPath) || !errors.As(err, &pe) || pe.Path != leaf+"x" {
607 t.Fatalf("a path of MaxPathLen: %v", err)
608 }
609 // Past MaxPathLen it is the format's own refusal (MV-N81), as before.
610 if err := root.Rename(leaf+"x", leaf+"xy"); err != nil {
611 t.Fatal(err)
612 }
613 if _, err := build(dir, "", io.Discard); !errors.Is(err, ErrBadPath) || !errors.As(err, &pe) || pe.Path != leaf+"xy" {
614 t.Fatalf("over-long path: %v", err)
615 }
616}
617
618// #236 fix round 1: a path of exactly the files tree's cap (TreePathLen,
619// hostfiles.MaxPathBytes) goes through Build, Seal, Verify and Extract, into
620// a destination whose absolute path is past PATH_MAX once the path is
621// added; one byte longer never leaves the source, and a stream that carries
622// it (an older source's) is refused by Verify and Extract (the "past tree
623// cap" hostile stream).
624func TestATreeCapPathRoundTrips(t *testing.T) {
625 // TreePathLen is hostfiles.MaxPathBytes by definition; MaxPathLen is
626 // one byte more, which the next line fails to compile without.
627 const _ = uint(MaxPathLen-1-TreePathLen) + uint(TreePathLen-(MaxPathLen-1))
628 dir := filepath.Join(t.TempDir(), "h")
629 if err := os.Mkdir(dir, 0o755); err != nil {
630 t.Fatal(err)
631 }
632 leaf := treeCapPath(TreePathLen)
633 root, err := os.OpenRoot(dir)
634 if err != nil {
635 t.Fatal(err)
636 }
637 defer root.Close()
638 if err := root.MkdirAll(filepath.Dir(leaf), 0o755); err != nil {
639 t.Fatal(err)
640 }
641 if err := root.WriteFile(leaf, []byte("deep"), 0o644); err != nil {
642 t.Fatal(err)
643 }
644 var tarz bytes.Buffer
645 plain, entries, n, err := Build(dir, "", &tarz)
646 if err != nil {
647 t.Fatal(err)
648 }
649 pub, priv, b := keys(t)
650 var sealed bytes.Buffer
651 key, sealedSHA, err := Seal(bytes.NewReader(tarz.Bytes()), pub, b, &sealed)
652 if err != nil {
653 t.Fatal(err)
654 }
655 m := Manifest{V: ManifestVersion, Op: "mv_x", Incarnation: "i", Plain: plain, Sealed: sealedSHA, Bytes: n, Entries: entries, SealedDataKey: key, Chunk: ChunkSize}
656 if err := Verify(bytes.NewReader(sealed.Bytes()), m, priv, b); err != nil {
657 t.Fatalf("Verify of a tree-cap path: %v", err)
658 }
659 _, files, env := extractTarget(t)
660 if len(files)+1+len(leaf) <= 4096 {
661 t.Fatalf("the destination %s does not put the path past PATH_MAX", files)
662 }
663 if err := Extract(bytes.NewReader(tarz.Bytes()), files, env); err != nil {
664 t.Fatalf("Extract of a tree-cap path: %v", err)
665 }
666 out, err := os.OpenRoot(files)
667 if err != nil {
668 t.Fatal(err)
669 }
670 defer out.Close()
671 if got, err := out.ReadFile(leaf); err != nil || string(got) != "deep" {
672 t.Fatalf("extracted %q, %v", got, err)
673 }
674}
675
676// treeCapPath is a relative path of exactly n bytes, 200-byte directory
677// components and a leaf.
678func treeCapPath(n int) string {
679 var b strings.Builder
680 for b.Len()+201 <= n-10 {
681 if b.Len() > 0 {
682 b.WriteByte('/')
683 }
684 b.WriteString(strings.Repeat("d", 200))
685 }
686 b.WriteByte('/')
687 b.WriteString(strings.Repeat("f", n-b.Len()))
688 return b.String()
689}
690
691// Build, Verify and Extract share one entry-count limit, and Extract and
692// Verify bound the decoded stream, so a zstd bomb that passed the sealing
693// is still bounded on the target.
694func TestEntryAndDecodedLimitsAreShared(t *testing.T) {
695 dir, env := filesFixture(t)
696 var tarz bytes.Buffer
697 plain, entries, n, err := Build(dir, env, &tarz)
698 if err != nil {
699 t.Fatal(err)
700 }
701 pub, priv, b := keys(t)
702 var sealed bytes.Buffer
703 key, sealedSHA, err := Seal(bytes.NewReader(tarz.Bytes()), pub, b, &sealed)
704 if err != nil {
705 t.Fatal(err)
706 }
707 m := Manifest{V: ManifestVersion, Op: "mv_x", Incarnation: "i", Plain: plain, Sealed: sealedSHA, Bytes: n, Entries: entries, SealedDataKey: key, Chunk: ChunkSize}
708 if err := Verify(bytes.NewReader(sealed.Bytes()), m, priv, b); err != nil {
709 t.Fatal(err)
710 }
711
712 origEntries := entryLimit
713 entryLimit = 4
714 if _, _, _, err := Build(dir, env, io.Discard); !errors.Is(err, ErrTooManyEntries) {
715 t.Fatalf("Build over the entry limit: %v", err)
716 }
717 if err := Verify(bytes.NewReader(sealed.Bytes()), m, priv, b); !errors.Is(err, ErrTooManyEntries) {
718 t.Fatalf("Verify over the entry limit: %v", err)
719 }
720 parent, files, outEnv := extractTarget(t)
721 if err := Extract(bytes.NewReader(tarz.Bytes()), files, outEnv); !errors.Is(err, ErrTooManyEntries) {
722 t.Fatalf("Extract over the entry limit: %v", err)
723 }
724 assertUntouched(t, parent, files, outEnv)
725 entryLimit = origEntries
726
727 // A bomb: a valid tar followed by 8 MiB of zeros inside the zstd
728 // stream (a few hundred bytes compressed).
729 bomb := tarZst(t, []rawEntry{reg("files/a", "x")}, make([]byte, 8<<20))
730 origDecoded := decodedLimit
731 t.Cleanup(func() { decodedLimit = origDecoded; entryLimit = origEntries })
732 decodedLimit = 1 << 20
733 parent, files, outEnv = extractTarget(t)
734 if err := Extract(bytes.NewReader(bomb), files, outEnv); !errors.Is(err, ErrTooLarge) {
735 t.Fatalf("bomb: %v", err)
736 }
737 assertUntouched(t, parent, files, outEnv)
738 var bombSealed bytes.Buffer
739 bombKey, bombSHA, err := Seal(bytes.NewReader(bomb), pub, b, &bombSealed)
740 if err != nil {
741 t.Fatal(err)
742 }
743 bm := Manifest{V: ManifestVersion, Plain: "sha256:x", Sealed: bombSHA, Bytes: 1, Entries: []Entry{{Path: "files/a", Size: 1, SHA256: digestBytes([]byte("x"))}}, SealedDataKey: bombKey, Chunk: ChunkSize}
744 if err := Verify(bytes.NewReader(bombSealed.Bytes()), bm, priv, b); !errors.Is(err, ErrTooLarge) {
745 t.Fatalf("Verify of a bomb: %v", err)
746 }
747}
748
749func digestBytes(b []byte) string {
750 sum := sha256.Sum256(b)
751 return "sha256:" + hex.EncodeToString(sum[:])
752}
753
754func keys(t *testing.T) (*ecdh.PublicKey, *ecdh.PrivateKey, appconfig.Binding) {
755 t.Helper()
756 privB64, pubB64, err := appconfig.GenerateKey()
757 if err != nil {
758 t.Fatal(err)
759 }
760 pk, err := appconfig.ParsePrivateKey(privB64, "test")
761 if err != nil {
762 t.Fatal(err)
763 }
764 pu, err := appconfig.ParsePublicKey(pubB64, "test")
765 if err != nil {
766 t.Fatal(err)
767 }
768 return pu, pk, appconfig.Binding{Account: "bab", App: "app"}
769}
770
771func TestSealOpenExtractRoundTrip(t *testing.T) {
772 dir, env := filesFixture(t)
773 var tarz bytes.Buffer
774 plain, _, _, err := Build(dir, env, &tarz)
775 if err != nil {
776 t.Fatal(err)
777 }
778 pub, priv, b := keys(t)
779 var sealed bytes.Buffer
780 key, sealedSHA, err := Seal(bytes.NewReader(tarz.Bytes()), pub, b, &sealed)
781 if err != nil {
782 t.Fatal(err)
783 }
784 sum := sha256.Sum256(sealed.Bytes())
785 if sealedSHA != "sha256:"+hex.EncodeToString(sum[:]) {
786 t.Fatal("sealed digest")
787 }
788 var opened bytes.Buffer
789 if err := Open(bytes.NewReader(sealed.Bytes()), key, priv, b, &opened); err != nil {
790 t.Fatal(err)
791 }
792 if !bytes.Equal(opened.Bytes(), tarz.Bytes()) {
793 t.Fatal("open != build")
794 }
795 // A flipped byte fails: GCM, not a checksum.
796 bad := append([]byte{}, sealed.Bytes()...)
797 bad[len(bad)/2] ^= 1
798 if err := Open(bytes.NewReader(bad), key, priv, b, &bytes.Buffer{}); !errors.Is(err, ErrSealed) {
799 t.Fatalf("tampered archive opened: %v", err)
800 }
801 // Another binding cannot open the data key (F4: SealBound, always v2).
802 if err := Open(bytes.NewReader(sealed.Bytes()), key, priv, appconfig.Binding{Account: "bab", App: "other"}, &bytes.Buffer{}); !errors.Is(err, ErrSealed) {
803 t.Fatalf("wrong binding opened: %v", err)
804 }
805 out := filepath.Join(t.TempDir(), "out")
806 outEnv := filepath.Join(t.TempDir(), "out.env")
807 if err := Extract(bytes.NewReader(opened.Bytes()), out, outEnv); err != nil {
808 t.Fatal(err)
809 }
810 got, _ := os.ReadFile(filepath.Join(out, "sub", "b.bin"))
811 envGot, _ := os.ReadFile(outEnv)
812 appEnv, _ := os.ReadFile(filepath.Join(out, "env"))
813 nested, _ := os.ReadFile(filepath.Join(out, "sub", "env", "x"))
814 if string(got) != "\x01\x02\x03" || string(envGot) != "CARLOS_ADMIN_TOKEN=t\n" || string(appEnv) != "app-env" || string(nested) != "x" {
815 t.Fatal("extract content")
816 }
817 if _, err := os.Lstat(filepath.Join(out, MarkerName)); !errors.Is(err, fs.ErrNotExist) {
818 t.Fatalf("marker carried: %v", err)
819 }
820 if info, _ := os.Stat(outEnv); info.Mode().Perm() != 0o600 {
821 t.Fatalf("env mode %o", info.Mode().Perm())
822 }
823 p2, _, _, _ := PlainDigest(out, outEnv)
824 if p2 != plain {
825 t.Fatal("extracted tree has another plain digest")
826 }
827 // Encryption is randomized (§8.2).
828 var sealed2 bytes.Buffer
829 if _, _, err := Seal(bytes.NewReader(tarz.Bytes()), pub, b, &sealed2); err != nil {
830 t.Fatal(err)
831 }
832 if bytes.Equal(sealed.Bytes(), sealed2.Bytes()) {
833 t.Fatal("deterministic sealing")
834 }
835}
836
837// F4: a data key sealed as a v1 (unbound) envelope opens under any app's
838// binding, so a manifest downgraded to one would unbind the archive. Open
839// refuses it before trying the key.
840func TestOpenRefusesAnUnboundDataKey(t *testing.T) {
841 dir, env := filesFixture(t)
842 var tarz bytes.Buffer
843 if _, _, _, err := Build(dir, env, &tarz); err != nil {
844 t.Fatal(err)
845 }
846 pub, priv, b := keys(t)
847 var sealed bytes.Buffer
848 key, _, err := Seal(bytes.NewReader(tarz.Bytes()), pub, b, &sealed)
849 if err != nil {
850 t.Fatal(err)
851 }
852 raw, err := appconfig.Open(priv, b, key)
853 if err != nil {
854 t.Fatal(err)
855 }
856 v1, err := appconfig.SealUnbound(pub, raw)
857 if err != nil {
858 t.Fatal(err)
859 }
860 // The v1 key is the right key: appconfig opens it under any binding.
861 if again, err := appconfig.Open(priv, appconfig.Binding{Account: "bab", App: "other"}, v1); err != nil || !bytes.Equal(again, raw) {
862 t.Fatalf("fixture: %v", err)
863 }
864 if err := Open(bytes.NewReader(sealed.Bytes()), v1, priv, b, &bytes.Buffer{}); !errors.Is(err, ErrSealed) {
865 t.Fatalf("v1 data key opened: %v", err)
866 }
867}
868
869// A 0555 directory holding a 0400 file round-trips to exactly those modes,
870// and the extracted tree digests as the original did.
871func TestExtractPreservesModes(t *testing.T) {
872 dir, env := filesFixture(t)
873 ro := filepath.Join(dir, "ro")
874 if err := os.Mkdir(ro, 0o755); err != nil {
875 t.Fatal(err)
876 }
877 mustWrite(t, filepath.Join(ro, "f"), []byte("secret"), 0o400)
878 if err := os.Chmod(ro, 0o555); err != nil {
879 t.Fatal(err)
880 }
881 t.Cleanup(func() { os.Chmod(ro, 0o755) })
882 var tarz bytes.Buffer
883 plain, _, _, err := Build(dir, env, &tarz)
884 if err != nil {
885 t.Fatal(err)
886 }
887 out := filepath.Join(t.TempDir(), "out")
888 outEnv := filepath.Join(t.TempDir(), "out.env")
889 t.Cleanup(func() { os.Chmod(filepath.Join(out, "ro"), 0o755) })
890 if err := Extract(bytes.NewReader(tarz.Bytes()), out, outEnv); err != nil {
891 t.Fatal(err)
892 }
893 di, _ := os.Stat(filepath.Join(out, "ro"))
894 fi, _ := os.Stat(filepath.Join(out, "ro", "f"))
895 if di.Mode().Perm() != 0o555 || fi.Mode().Perm() != 0o400 {
896 t.Fatalf("modes %o %o", di.Mode().Perm(), fi.Mode().Perm())
897 }
898 if p2, _, _, err := PlainDigest(out, outEnv); err != nil || p2 != plain {
899 t.Fatalf("plain after extract %q %v", p2, err)
900 }
901}
902
903// An archive with no env entry removes a stale env file, so the extracted
904// pair always digests as the archive does.
905func TestExtractWithoutEnvLeavesNoEnv(t *testing.T) {
906 dir, _ := filesFixture(t)
907 var tarz bytes.Buffer
908 plain, _, _, err := Build(dir, "", &tarz)
909 if err != nil {
910 t.Fatal(err)
911 }
912 out := filepath.Join(t.TempDir(), "out")
913 stale := filepath.Join(t.TempDir(), "out.env")
914 mustWrite(t, stale, []byte("OLD=1\n"), 0o600)
915 if err := Extract(bytes.NewReader(tarz.Bytes()), out, stale); err != nil {
916 t.Fatal(err)
917 }
918 if _, err := os.Lstat(stale); !errors.Is(err, fs.ErrNotExist) {
919 t.Fatalf("stale env kept: %v", err)
920 }
921 if p2, _, _, _ := PlainDigest(out, stale); p2 != plain {
922 t.Fatal("plain differs")
923 }
924}
925
926// A small archive handed over in a *bytes.Buffer (under the decoder's
927// 128 KiB shortcut for a reader with Bytes() and Len()) is read as a
928// stream like any other: Extract, the decode Verify runs, and Verify itself
929// accept it, rather than finding the whole archive still unread after it.
930func TestASmallArchiveInABytesBufferExtractsAndVerifies(t *testing.T) {
931 dir, env := filesFixture(t)
932 var tarz bytes.Buffer
933 plain, entries, n, err := Build(dir, env, &tarz)
934 if err != nil {
935 t.Fatal(err)
936 }
937 if tarz.Len() >= 128<<10 {
938 t.Fatalf("fixture: %d bytes is not under 128 KiB", tarz.Len())
939 }
940 raw := append([]byte(nil), tarz.Bytes()...)
941 parent := t.TempDir()
942 files, envOut := filepath.Join(parent, "h"), filepath.Join(parent, "h.env")
943 if err := Extract(bytes.NewBuffer(append([]byte(nil), raw...)), files, envOut); err != nil {
944 t.Fatalf("Extract from a *bytes.Buffer: %v", err)
945 }
946 if got, _, _, err := PlainDigest(files, envOut); err != nil || got != plain {
947 t.Fatalf("the extraction digests to %s (%v), want %s", got, err, plain)
948 }
949 m := Manifest{V: ManifestVersion, Op: "mv_x", Plain: plain, Bytes: n, Entries: entries, Chunk: ChunkSize}
950 if got, err := verifyStream(bytes.NewBuffer(append([]byte(nil), raw...)), m); err != nil || got != plain {
951 t.Fatalf("verifyStream from a *bytes.Buffer: %s, %v", got, err)
952 }
953 pub, priv, b := keys(t)
954 var sealed bytes.Buffer
955 key, sealedSHA, err := Seal(bytes.NewReader(raw), pub, b, &sealed)
956 if err != nil {
957 t.Fatal(err)
958 }
959 m.Sealed, m.SealedDataKey = sealedSHA, key
960 if err := Verify(&sealed, m, priv, b); err != nil {
961 t.Fatalf("Verify from a *bytes.Buffer: %v", err)
962 }
963}
964
965// sealedArchive is a good archive with its manifest, over a tree large
966// enough to seal into three chunks.
967type sealedArchive struct {
968 tarz, sealed []byte
969 m Manifest
970 pub *ecdh.PublicKey
971 priv *ecdh.PrivateKey
972 b appconfig.Binding
973}
974
975func threeChunkArchive(t *testing.T) sealedArchive {
976 t.Helper()
977 dir, env := filesFixture(t)
978 big := make([]byte, 2*ChunkSize+ChunkSize/2)
979 if _, err := rand.Read(big); err != nil {
980 t.Fatal(err)
981 }
982 mustWrite(t, filepath.Join(dir, "random.bin"), big, 0o644)
983 var tarz bytes.Buffer
984 plain, entries, n, err := Build(dir, env, &tarz)
985 if err != nil {
986 t.Fatal(err)
987 }
988 pub, priv, b := keys(t)
989 var sealed bytes.Buffer
990 key, sealedSHA, err := Seal(bytes.NewReader(tarz.Bytes()), pub, b, &sealed)
991 if err != nil {
992 t.Fatal(err)
993 }
994 if got := len(chunkOffsets(t, sealed.Bytes())); got != 3 {
995 t.Fatalf("fixture seals into %d chunks", got)
996 }
997 return sealedArchive{
998 tarz: tarz.Bytes(), sealed: sealed.Bytes(), pub: pub, priv: priv, b: b,
999 m: Manifest{V: ManifestVersion, Op: "mv_x", Incarnation: "i", Plain: plain, Sealed: sealedSHA, Bytes: n, Entries: entries, SealedDataKey: key, Chunk: ChunkSize},
1000 }
1001}
1002
1003// chunkOffsets returns where each sealed chunk's length prefix starts.
1004func chunkOffsets(t *testing.T, sealed []byte) []int {
1005 t.Helper()
1006 var offs []int
1007 for off := 0; off < len(sealed); {
1008 if off+4 > len(sealed) {
1009 t.Fatal("ragged sealed fixture")
1010 }
1011 offs = append(offs, off)
1012 off += 4 + int(binary.BigEndian.Uint32(sealed[off:]))
1013 }
1014 return offs
1015}
1016
1017// sealRaw seals a hand-built archive, with a manifest naming its regular
1018// entries as given.
1019func (a sealedArchive) sealRaw(t *testing.T, raw []byte, entries []Entry) ([]byte, Manifest) {
1020 t.Helper()
1021 var sealed bytes.Buffer
1022 key, sha, err := Seal(bytes.NewReader(raw), a.pub, a.b, &sealed)
1023 if err != nil {
1024 t.Fatal(err)
1025 }
1026 var n int64
1027 for _, e := range entries {
1028 n += e.Size
1029 }
1030 return sealed.Bytes(), Manifest{V: ManifestVersion, Plain: "sha256:unchecked", Sealed: sha, Bytes: n, Entries: entries, SealedDataKey: key, Chunk: ChunkSize}
1031}
1032
1033func TestVerifyRefusesWhatTheManifestDoesNotPromise(t *testing.T) {
1034 a := threeChunkArchive(t)
1035 if err := Verify(bytes.NewReader(a.sealed), a.m, a.priv, a.b); err != nil {
1036 t.Fatalf("good archive: %v", err)
1037 }
1038 withManifest := func(edit func(m *Manifest)) Manifest {
1039 m := a.m
1040 m.Entries = append([]Entry(nil), a.m.Entries...)
1041 edit(&m)
1042 return m
1043 }
1044 idx := func(m *Manifest, p string) int {
1045 for i, e := range m.Entries {
1046 if e.Path == p {
1047 return i
1048 }
1049 }
1050 t.Fatalf("no %s", p)
1051 return -1
1052 }
1053 manifestCases := map[string]struct {
1054 m Manifest
1055 want string
1056 }{
1057 "plain": {withManifest(func(m *Manifest) { m.Plain = digestBytes([]byte("other")) }), "plaintext digest"},
1058 "sealed": {withManifest(func(m *Manifest) { m.Sealed = digestBytes([]byte("other")) }), "sealed digest"},
1059 "missing entry": {withManifest(func(m *Manifest) {
1060 i := idx(m, "files/a.txt")
1061 m.Bytes -= m.Entries[i].Size
1062 m.Entries = append(m.Entries[:i], m.Entries[i+1:]...)
1063 }), "not in the manifest"},
1064 "entry digest": {withManifest(func(m *Manifest) { m.Entries[idx(m, "files/a.txt")].SHA256 = digestBytes([]byte("beta!")) }), "digest differs"},
1065 "bytes": {withManifest(func(m *Manifest) { m.Bytes++ }), "bytes"},
1066 "version": {withManifest(func(m *Manifest) { m.V = 2 }), "version"},
1067 "chunk": {withManifest(func(m *Manifest) { m.Chunk = 4096 }), "chunk"},
1068 "no key": {withManifest(func(m *Manifest) { m.SealedDataKey = "" }), "no sealed"},
1069 "duplicate manifest entry": {withManifest(func(m *Manifest) {
1070 e := m.Entries[idx(m, "files/a.txt")]
1071 m.Entries = append(m.Entries, e)
1072 m.Bytes += e.Size
1073 }), "named twice"},
1074 "traversal in manifest": {withManifest(func(m *Manifest) { m.Entries[idx(m, "files/a.txt")].Path = "files/../a.txt" }), "refused path"},
1075 }
1076 for name, c := range manifestCases {
1077 t.Run(name, func(t *testing.T) {
1078 err := Verify(bytes.NewReader(a.sealed), c.m, a.priv, a.b)
1079 if err == nil || !strings.Contains(err.Error(), c.want) {
1080 t.Fatalf("%v, want %q", err, c.want)
1081 }
1082 })
1083 }
1084
1085 // Hand-built sealed archives.
1086 symlink := tarZst(t, []rawEntry{{hdr: tar.Header{Name: "files/l", Typeflag: tar.TypeSymlink, Linkname: "/etc/shadow"}}}, nil)
1087 dup := tarZst(t, []rawEntry{reg("files/a", "x"), reg("files/a", "x")}, nil)
1088 marker := tarZst(t, []rawEntry{reg("files/.files-gen", "x")}, nil)
1089 xEntry := []Entry{{Path: "files/a", Size: 1, SHA256: digestBytes([]byte("x"))}}
1090 rawCases := map[string]struct {
1091 raw []byte
1092 entries []Entry
1093 want error
1094 }{
1095 "symlink entry": {symlink, nil, ErrSpecialFile},
1096 "over cap": {overCapArchive(t), nil, ErrOverCap},
1097 "duplicate in tar": {dup, xEntry, ErrBadPath},
1098 "marker in tar": {marker, nil, ErrBadPath},
1099 "marker child": {tarZst(t, []rawEntry{reg("files/.files-gen/child", "x")}, nil), nil, ErrBadPath},
1100 }
1101 for name, c := range rawCases {
1102 t.Run(name, func(t *testing.T) {
1103 sealed, m := a.sealRaw(t, c.raw, c.entries)
1104 if err := Verify(bytes.NewReader(sealed), m, a.priv, a.b); !errors.Is(err, c.want) {
1105 t.Fatalf("%v, want %v", err, c.want)
1106 }
1107 })
1108 }
1109
1110 // Damage to the sealed object itself: each is Open's verdict.
1111 offs := chunkOffsets(t, a.sealed)
1112 flipped := append([]byte(nil), a.sealed...)
1113 flipped[len(flipped)-1] ^= 1
1114 cut := append([]byte(nil), a.sealed[:offs[len(offs)-1]]...)
1115 appended := append(append([]byte(nil), a.sealed...), 0)
1116 twice := append(append([]byte(nil), a.sealed...), a.sealed...)
1117 damage := map[string][]byte{
1118 "last byte flipped": flipped,
1119 "final chunk cut off": cut,
1120 "byte appended": appended,
1121 "archive concatenated": twice,
1122 }
1123 for name, sealed := range damage {
1124 t.Run(name, func(t *testing.T) {
1125 // The manifest's sealed digest is made to match the damaged
1126 // object, so only Open's own checks stand between it and nil.
1127 m := a.m
1128 m.Sealed = digestBytes(sealed)
1129 err := Verify(bytes.NewReader(sealed), m, a.priv, a.b)
1130 if !errors.Is(err, ErrSealed) {
1131 t.Fatalf("%v, want ErrSealed", err)
1132 }
1133 if (name == "byte appended" || name == "archive concatenated") && !strings.Contains(err.Error(), "after the final chunk") {
1134 t.Fatalf("%v", err)
1135 }
1136 })
1137 }
1138
1139 // A downgraded data key in the manifest is refused.
1140 raw, err := appconfig.Open(a.priv, a.b, a.m.SealedDataKey)
1141 if err != nil {
1142 t.Fatal(err)
1143 }
1144 v1, _ := appconfig.SealUnbound(a.pub, raw)
1145 m := a.m
1146 m.SealedDataKey = v1
1147 if err := Verify(bytes.NewReader(a.sealed), m, a.priv, a.b); !errors.Is(err, ErrSealed) {
1148 t.Fatalf("v1 key in the manifest: %v", err)
1149 }
1150}
1151
1152// An early mismatch must not strand Open's goroutine on the pipe for the
1153// rest of the archive (round-5 finding 12).
1154func TestVerifyReleasesTheProducerOnEarlyMismatch(t *testing.T) {
1155 a := threeChunkArchive(t)
1156 m := a.m
1157 m.Entries = append([]Entry(nil), a.m.Entries...)
1158 // "env" is the first regular entry in tar order; its size is off by
1159 // one, with Bytes kept consistent so the manifest itself passes.
1160 for i := range m.Entries {
1161 if m.Entries[i].Path == EnvEntry {
1162 m.Entries[i].Size++
1163 m.Bytes++
1164 }
1165 }
1166 runtime.GC()
1167 baseline := runtime.NumGoroutine()
1168 err := Verify(bytes.NewReader(a.sealed), m, a.priv, a.b)
1169 if err == nil || !strings.Contains(err.Error(), "size differs") {
1170 t.Fatalf("early mismatch: %v", err)
1171 }
1172 deadline := time.Now().Add(time.Second)
1173 for runtime.NumGoroutine() > baseline {
1174 if time.Now().After(deadline) {
1175 t.Fatalf("goroutines %d, baseline %d", runtime.NumGoroutine(), baseline)
1176 }
1177 time.Sleep(10 * time.Millisecond)
1178 }
1179}
1180
1181// Seal and Open agree on the chunk framing at the edges: empty input, and
1182// input of exactly one chunk (whose final chunk is then empty).
1183func TestSealChunkEdges(t *testing.T) {
1184 pub, priv, b := keys(t)
1185 for _, size := range []int{0, 1, ChunkSize - 1, ChunkSize, ChunkSize + 1, 2 * ChunkSize} {
1186 plain := make([]byte, size)
1187 rand.Read(plain)
1188 var sealed, opened bytes.Buffer
1189 key, _, err := Seal(bytes.NewReader(plain), pub, b, &sealed)
1190 if err != nil {
1191 t.Fatal(err)
1192 }
1193 if err := Open(bytes.NewReader(sealed.Bytes()), key, priv, b, &opened); err != nil || !bytes.Equal(opened.Bytes(), plain) {
1194 t.Fatalf("size %d: %v", size, err)
1195 }
1196 // Every prefix that ends on a chunk boundary is refused.
1197 offs := chunkOffsets(t, sealed.Bytes())
1198 for _, off := range offs {
1199 if err := Open(bytes.NewReader(sealed.Bytes()[:off]), key, priv, b, io.Discard); !errors.Is(err, ErrSealed) {
1200 t.Fatalf("size %d prefix %d opened: %v", size, off, err)
1201 }
1202 }
1203 }
1204}
1205
1206// The final flag is authenticated, not inferred: a holder of the data key
1207// who frames a short chunk without the flag, or a full chunk with it, is
1208// refused, so the length rule and the flag must agree (§8.2).
1209func TestOpenRequiresTheFinalFlag(t *testing.T) {
1210 pub, priv, b := keys(t)
1211 var sealed bytes.Buffer
1212 key, _, err := Seal(bytes.NewReader([]byte("x")), pub, b, &sealed)
1213 if err != nil {
1214 t.Fatal(err)
1215 }
1216 raw, err := appconfig.Open(priv, b, key)
1217 if err != nil {
1218 t.Fatal(err)
1219 }
1220 gcm, err := newGCM(raw)
1221 if err != nil {
1222 t.Fatal(err)
1223 }
1224 frame := func(index uint64, last bool, pt []byte) []byte {
1225 nonce, aad := nonceFor(index, last)
1226 ct := gcm.Seal(nil, nonce, pt, aad)
1227 return append(binary.BigEndian.AppendUint32(nil, uint32(len(ct))), ct...)
1228 }
1229 full := make([]byte, ChunkSize)
1230 forged := map[string][]byte{
1231 "short chunk unflagged": frame(0, false, []byte("x")),
1232 "full chunk flagged": frame(0, true, full),
1233 "full flagged, short after": append(frame(0, true, full), frame(1, true, []byte("x"))...),
1234 "well formed (control case)": append(frame(0, false, full), frame(1, true, []byte("x"))...),
1235 }
1236 for name, stream := range forged {
1237 err := Open(bytes.NewReader(stream), key, priv, b, io.Discard)
1238 if name == "well formed (control case)" {
1239 if err != nil {
1240 t.Fatalf("%s: %v", name, err)
1241 }
1242 continue
1243 }
1244 if !errors.Is(err, ErrSealed) {
1245 t.Fatalf("%s opened: %v", name, err)
1246 }
1247 }
1248}
1249
1250// Fix round 1, ruling 2: a directory named .files-gen is skipped whole,
1251// children included, so it neither reaches the archive nor blocks the
1252// marker file the landing writes; and a manifest cannot name a path under
1253// it.
1254func TestBuildSkipsAMarkerDirectory(t *testing.T) {
1255 dir, env := filesFixture(t)
1256 if err := os.Remove(filepath.Join(dir, MarkerName)); err != nil {
1257 t.Fatal(err)
1258 }
1259 before, _, _, err := PlainDigest(dir, env)
1260 if err != nil {
1261 t.Fatal(err)
1262 }
1263 if err := os.MkdirAll(filepath.Join(dir, MarkerName, "deeper"), 0o755); err != nil {
1264 t.Fatal(err)
1265 }
1266 mustWrite(t, filepath.Join(dir, MarkerName, "child"), []byte("c"), 0o644)
1267 mustWrite(t, filepath.Join(dir, MarkerName, "deeper", "x"), []byte("x"), 0o644)
1268 plain, entries, _, err := Build(dir, env, io.Discard)
1269 if err != nil {
1270 t.Fatal(err)
1271 }
1272 for _, e := range entries {
1273 if strings.Contains(e.Path, MarkerName) {
1274 t.Fatalf("marker subtree archived: %s", e.Path)
1275 }
1276 }
1277 if plain != before {
1278 t.Fatal("a marker directory changed the plain digest")
1279 }
1280 m := Manifest{V: ManifestVersion, Plain: "p", Sealed: "s", SealedDataKey: "k", Chunk: ChunkSize, Bytes: 1, Entries: []Entry{{Path: "files/.files-gen/child", Size: 1}}}
1281 if err := m.check(); !errors.Is(err, ErrBadPath) {
1282 t.Fatalf("manifest naming a marker child: %v", err)
1283 }
1284}
1285
1286// Fix round 1, rulings 3 and 4: every extracted file and directory is
1287// fsynced, each directory already carrying its archived mode when it is,
1288// then the destination's parent (and env's) after the renames. Restrictive
1289// modes (a directory without read permission, a file without any) do not
1290// stop the sync, and are in place once Extract returns.
1291func TestExtractSyncsEverythingAfterRestoringModes(t *testing.T) {
1292 raw := tarZst(t, []rawEntry{
1293 {hdr: tar.Header{Name: "files/locked/", Typeflag: tar.TypeDir, Mode: 0o300}},
1294 {hdr: tar.Header{Name: "files/locked/f", Typeflag: tar.TypeReg, Mode: 0o000, Size: 1}, body: []byte("f")},
1295 {hdr: tar.Header{Name: "files/ro/", Typeflag: tar.TypeDir, Mode: 0o555}},
1296 {hdr: tar.Header{Name: "files/ro/inner/", Typeflag: tar.TypeDir, Mode: 0o500}},
1297 {hdr: tar.Header{Name: "files/ro/inner/g", Typeflag: tar.TypeReg, Mode: 0o400, Size: 1}, body: []byte("g")},
1298 {hdr: tar.Header{Name: "env", Typeflag: tar.TypeReg, Mode: 0o600, Size: 2}, body: []byte("E\n")},
1299 }, nil)
1300 parent, files, env := extractTarget(t)
1301 t.Cleanup(func() {
1302 for _, d := range []string{"locked", "ro/inner", "ro"} {
1303 os.Chmod(filepath.Join(files, d), 0o755)
1304 }
1305 })
1306 type synced struct {
1307 mode fs.FileMode
1308 dir bool
1309 }
1310 var got = map[string]synced{}
1311 var order []string
1312 orig := fsyncFile
1313 t.Cleanup(func() { fsyncFile = orig })
1314 var mu sync.Mutex
1315 fsyncFile = func(f *os.File) error {
1316 info, err := f.Stat()
1317 if err != nil {
1318 return err
1319 }
1320 mu.Lock()
1321 got[f.Name()] = synced{info.Mode().Perm(), info.IsDir()}
1322 order = append(order, f.Name())
1323 mu.Unlock()
1324 return orig(f)
1325 }
1326 if err := Extract(bytes.NewReader(raw), files, env); err != nil {
1327 t.Fatal(err)
1328 }
1329 find := func(suffix string) synced {
1330 t.Helper()
1331 for name, s := range got {
1332 if strings.HasSuffix(filepath.ToSlash(name), suffix) {
1333 return s
1334 }
1335 }
1336 t.Fatalf("%s never synced; synced %v", suffix, order)
1337 return synced{}
1338 }
1339 want := map[string]fs.FileMode{"/locked": 0o300, "/ro": 0o555, "/ro/inner": 0o500, "/locked/f": 0o000, "/ro/inner/g": 0o400}
1340 for suffix, mode := range want {
1341 if s := find(suffix); s.mode != mode {
1342 t.Fatalf("%s synced with mode %o, archived %o", suffix, s.mode, mode)
1343 }
1344 }
1345 // The staged root (its mode is the destination's, 0755 when absent),
1346 // the env file, and both parents after the renames.
1347 stageRoot := false
1348 for name, s := range got {
1349 if s.dir && strings.Contains(filepath.Base(name), ".out.extract-") && s.mode == 0o755 {
1350 stageRoot = true
1351 }
1352 }
1353 if !stageRoot {
1354 t.Fatalf("staged root not synced with its final mode: %v", order)
1355 }
1356 envStaged := false
1357 for name, s := range got {
1358 if !s.dir && strings.HasPrefix(filepath.Base(name), ".out.env.extract-") && s.mode == 0o600 {
1359 envStaged = true
1360 }
1361 }
1362 if !envStaged {
1363 t.Fatalf("staged env not synced: %v", order)
1364 }
1365 if s, ok := got[parent]; !ok || !s.dir {
1366 t.Fatalf("destination parent not synced: %v", order)
1367 }
1368 if s, ok := got[filepath.Dir(env)]; !ok || !s.dir {
1369 t.Fatalf("env parent not synced: %v", order)
1370 }
1371 if order[len(order)-1] != filepath.Dir(env) {
1372 t.Fatalf("env parent is not the last sync: %v", order)
1373 }
1374 // Modes in place on return.
1375 for suffix, mode := range want {
1376 info, err := os.Lstat(filepath.Join(files, filepath.FromSlash(strings.TrimPrefix(suffix, "/"))))
1377 if err != nil || info.Mode().Perm() != mode {
1378 t.Fatalf("%s: %v %v", suffix, info, err)
1379 }
1380 }
1381}
1382
1383// Ruling 4's measurement: serial against parallel fsync on this machine's
1384// disk, for many small files and for one large one. Run by hand:
1385// CARLOS_ARCHIVE_FSYNC_TIMING=1 go test -run TestExtractFsyncTiming -v.
1386func TestExtractFsyncTiming(t *testing.T) {
1387 if os.Getenv("CARLOS_ARCHIVE_FSYNC_TIMING") == "" {
1388 t.Skip("set CARLOS_ARCHIVE_FSYNC_TIMING=1 to measure")
1389 }
1390 small := make([]rawEntry, 0, 3000)
1391 for i := 0; i < 3000; i++ {
1392 small = append(small, reg("files/s"+hex.EncodeToString([]byte{byte(i >> 8), byte(i)}), "z"))
1393 }
1394 big := make([]byte, 200<<20)
1395 rand.Read(big)
1396 fixtures := map[string][]byte{
1397 "3000 small files": tarZst(t, small, nil),
1398 "one 200 MiB file": tarZst(t, []rawEntry{{hdr: tar.Header{Name: "files/big", Typeflag: tar.TypeReg, Size: int64(len(big))}, body: big}}, nil),
1399 }
1400 orig := syncWorkers
1401 t.Cleanup(func() { syncWorkers = orig })
1402 for _, name := range []string{"3000 small files", "one 200 MiB file"} {
1403 for _, workers := range []int{1, 16} {
1404 syncWorkers = workers
1405 _, files, env := extractTarget(t)
1406 start := time.Now()
1407 if err := Extract(bytes.NewReader(fixtures[name]), files, env); err != nil {
1408 t.Fatal(err)
1409 }
1410 t.Logf("%s, %2d workers: %v", name, workers, time.Since(start).Round(time.Millisecond))
1411 }
1412 }
1413}
1414
1415// Fix round 2: Verify and Extract share one set of stream rules, so a
1416// stream Extract would refuse never passes Verify only to fail at landing.
1417// Every refused stream above is sealed with a manifest that names its
1418// valid regular entries correctly, so only the stream rules can refuse it,
1419// and Verify must give the same refusal Extract does.
1420func TestVerifyRefusesEveryStreamExtractRefuses(t *testing.T) {
1421 a := threeChunkArchive(t)
1422 all := refusedStreams()
1423 for name, entries := range duplicateStreams() {
1424 all["duplicate: "+name] = refusedStreamCase{entries, ErrBadPath}
1425 }
1426 for name, c := range all {
1427 t.Run(name, func(t *testing.T) {
1428 raw := overCapArchive(t)
1429 if c.want != ErrOverCap {
1430 raw = tarZst(t, c.entries, nil)
1431 }
1432 var entries []Entry
1433 named := map[string]bool{}
1434 for _, e := range c.entries {
1435 if e.hdr.Typeflag != tar.TypeReg || named[e.hdr.Name] {
1436 continue
1437 }
1438 if _, _, err := checkName(e.hdr.Name, false); err != nil {
1439 continue
1440 }
1441 named[e.hdr.Name] = true
1442 entries = append(entries, Entry{Path: e.hdr.Name, Size: int64(len(e.body)), SHA256: digestBytes(e.body)})
1443 }
1444 sealed, m := a.sealRaw(t, raw, entries)
1445 if err := m.check(); err != nil {
1446 t.Fatalf("fixture manifest refused on its own: %v", err)
1447 }
1448 if err := Verify(bytes.NewReader(sealed), m, a.priv, a.b); !errors.Is(err, c.want) {
1449 t.Fatalf("Verify: %v, want %v", err, c.want)
1450 }
1451 })
1452 }
1453 // The two directory-order shapes, named as the review asked.
1454 for name, entries := range map[string][]rawEntry{
1455 "missing parent": {reg("files/a/b", "x")},
1456 "late parent": {reg("files/a/b", "x"), dirEntry("files/a/")},
1457 } {
1458 sealed, m := a.sealRaw(t, tarZst(t, entries, nil), []Entry{{Path: "files/a/b", Size: 1, SHA256: digestBytes([]byte("x"))}})
1459 if err := Verify(bytes.NewReader(sealed), m, a.priv, a.b); !errors.Is(err, ErrBadPath) || !strings.Contains(err.Error(), "before its directory") {
1460 t.Fatalf("%s: %v", name, err)
1461 }
1462 }
1463}
1464