amadan
rootfsimage: write squashfs in pure Go, round-tripped through its reader (rootfs Plan 1 Task 6)
WriteSquashfs writes a Tree with github.com/KarpelesLab/squashfs v1.2.0 (zlib, 128 KiB blocks), after Tree.Normalise: every entry 0:0, others read every file and enter every directory, run any file with an x bit, setuid and setgid cleared, sticky kept. Hardlinks share one copy of the bytes (CloneInode). Owner and time are set on every path and the root after AddFS, so epoch-0 times stay 0. The image is padded to 4 KiB for loop devices, and refused past release.MaxRootfsBytes while it is written (RFS-1-C86) or for a single file over it (RFS-1-C87), both new rows for the end-of-plan copy pass. The kernel-mount proof moves to Plan 2 Task 10 (controller ruling).
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
12 files changed,
+1020
−6
approved_rootfs_copy_test.go+4 −2docs/superpowers/plans/2026-10-24-rootfs-apps-1-ship.md+3 −1docs/superpowers/plans/2026-10-24-rootfs-apps-copy.md+3 −1go.mod+3 −0go.sum+10 −0internal/rootfsimage/errors.go+19 −0internal/rootfsimage/normalise.go+58 −0internal/rootfsimage/normalise_test.go+79 −0internal/rootfsimage/squashfs.go+308 −0internal/rootfsimage/squashfs_test.go+457 −0internal/rootfsimage/squashfs_unix_test.go+72 −0internal/rootfsimage/testimage_test.go+4 −2
CI cancelled — superseded by a newer push — run details
CI log
=== gate === .amadan/test-gate.sh go: downloading github.com/KarpelesLab/squashfs v1.2.0 amadan: job cancelled — a newer push superseded it
diff --git a/approved_rootfs_copy_test.go b/approved_rootfs_copy_test.go| index 52de3f963..b6ff162af 100644 |
| --- a/approved_rootfs_copy_test.go |
| +++ b/approved_rootfs_copy_test.go |
| @@ -25,7 +25,7 @@ const rootfsCopyRecordPath = "docs/superpowers/plans/2026-10-24-rootfs-apps-copy |
| var rootfsCopyIDPattern = regexp.MustCompile(`^RFS-[1-6]-C(0[1-9]|[1-9][0-9]{1,2})$`) |
| // rootfsCopyIDs is the deck's inventory, pinned as a literal list in the |
| -// deck's order: RFS-1-C01 to RFS-1-C85, then each later plan's rows. A |
| +// deck's order: RFS-1-C01 to RFS-1-C87, then each later plan's rows. A |
| // struck row stays in it: the deck keeps the row for history. |
| var rootfsCopyIDs = []string{ |
| // Plan 1 |
| @@ -50,7 +50,7 @@ var rootfsCopyIDs = []string{ |
| "RFS-1-C73", "RFS-1-C74", "RFS-1-C75", "RFS-1-C76", |
| "RFS-1-C77", "RFS-1-C78", "RFS-1-C79", "RFS-1-C80", |
| "RFS-1-C81", "RFS-1-C82", "RFS-1-C83", "RFS-1-C84", |
| - "RFS-1-C85", |
| + "RFS-1-C85", "RFS-1-C86", "RFS-1-C87", |
| // Plan 2 |
| "RFS-2-C01", "RFS-2-C02", "RFS-2-C03", "RFS-2-C04", |
| "RFS-2-C05", "RFS-2-C06", "RFS-2-C07", "RFS-2-C08", |
| @@ -154,6 +154,8 @@ var rootfsCopyBuilt = []string{ |
| "RFS-1-C45", "RFS-1-C46", "RFS-1-C50", "RFS-1-C51", "RFS-1-C53", "RFS-1-C55", "RFS-1-C78", |
| // Task 5: directory sources, mount points, program architecture (internal/rootfsimage/errors.go, dir_other.go) |
| "RFS-1-C47", "RFS-1-C52", "RFS-1-C56", |
| + // Task 6: the squashfs writer's size refusals (internal/rootfsimage/errors.go) |
| + "RFS-1-C86", "RFS-1-C87", |
| } |
| func mustParseRootfsRecord(t *testing.T) []copyRow { |
diff --git a/docs/superpowers/plans/2026-10-24-rootfs-apps-1-ship.md b/docs/superpowers/plans/2026-10-24-rootfs-apps-1-ship.md| index 9f951d140..0616aa6aa 100644 |
| --- a/docs/superpowers/plans/2026-10-24-rootfs-apps-1-ship.md |
| +++ b/docs/superpowers/plans/2026-10-24-rootfs-apps-1-ship.md |
| @@ -212,8 +212,10 @@ Placeholders: `<app>` an app name; `<name>` a process name; `<var>` an environme |
| | RFS-1-C83 | internal/release/rootfs_ship.go | <app>'s first ship, <version>, made its rootfs releases hibernate, so this release must too. ship it with --hibernate, or use a new app | approved | |
| | RFS-1-C84 | cli.go | let this rootfs release's processes run a JIT compiler, as Node, Bun, Deno, the JVM and Ruby's YJIT need. without it a box never lets memory be writable and executable at once | approved | |
| | RFS-1-C85 | ship_check.go | the <name> process runs <program>, which needs a JIT, and this release does not declare one. it passes here but stops at start on a box. ship it with --jit | approved | |
| +| RFS-1-C86 | internal/rootfsimage/errors.go | the rootfs image is over 2 GiB, the most one release can carry. make the image smaller and ship again | approved | |
| +| RFS-1-C87 | internal/rootfsimage/errors.go | <path> is over 2 GiB, the largest file a rootfs image can hold. make it smaller and ship again | approved | |
| -RFS-1-C78's `<digest>` is a `sha256:<hex>` digest. RFS-1-C80 to C83 are the first-ship forms of C18 to C21: the app has no other release, and its kind marker (deviation 10) names the ship that fixed its kind, which may not have published, so they never say "is one". RFS-1-C85 is a warning: it does not fail the check. Kept out of the table on purpose (diagnostics, not human text): the probe's one-line JSON on stdout, `validateManifest`'s existing terse messages, store errors wrapped for logs. |
| +RFS-1-C78's `<digest>` is a `sha256:<hex>` digest. RFS-1-C80 to C83 are the first-ship forms of C18 to C21: the app has no other release, and its kind marker (deviation 10) names the ship that fixed its kind, which may not have published, so they never say "is one". RFS-1-C85 is a warning: it does not fail the check. Kept out of the table on purpose (diagnostics, not human text): the probe's one-line JSON on stdout, `validateManifest`'s existing terse messages, store errors wrapped for logs. RFS-1-C86 and C87 were added in the build (Task 6, the controller ruling on strings found in the build), for the Luna pass at the end of Plan 1: C86 is the writer's refusal when the image passes 2 GiB while it is written (the text of C26), C87 its refusal of a single file over 2 GiB before anything is written. |
| --- |
diff --git a/docs/superpowers/plans/2026-10-24-rootfs-apps-copy.md b/docs/superpowers/plans/2026-10-24-rootfs-apps-copy.md| index d3141e57b..8c051e7b2 100644 |
| --- a/docs/superpowers/plans/2026-10-24-rootfs-apps-copy.md |
| +++ b/docs/superpowers/plans/2026-10-24-rootfs-apps-copy.md |
| @@ -124,8 +124,10 @@ From the plan: Placeholders: `<app>` an app name; `<name>` a process name; `<var |
| | RFS-1-C83 | internal/release/rootfs_ship.go | <app>'s first ship, <version>, made its rootfs releases hibernate, so this release must too. ship it with --hibernate, or use a new app | approved | |
| | RFS-1-C84 | cli.go | let this rootfs release's processes run a JIT compiler, as Node, Bun, Deno, the JVM and Ruby's YJIT need. without it a box never lets memory be writable and executable at once | approved | |
| | RFS-1-C85 | ship_check.go | the <name> process runs <program>, which needs a JIT, and this release does not declare one. it passes here but stops at start on a box. ship it with --jit | approved | |
| +| RFS-1-C86 | internal/rootfsimage/errors.go | the rootfs image is over 2 GiB, the most one release can carry. make the image smaller and ship again | approved | |
| +| RFS-1-C87 | internal/rootfsimage/errors.go | <path> is over 2 GiB, the largest file a rootfs image can hold. make it smaller and ship again | approved | |
| -From the plan: RFS-1-C78's `<digest>` is a `sha256:<hex>` digest. RFS-1-C80 to C83 are the first-ship forms of C18 to C21: the app has no other release, and its kind marker (deviation 10) names the ship that fixed its kind, which may not have published, so they never say "is one". RFS-1-C85 is a warning: it does not fail the check. Kept out of the table on purpose (diagnostics, not human text): the probe's one-line JSON on stdout, `validateManifest`'s existing terse messages, store errors wrapped for logs. |
| +From the plan: RFS-1-C78's `<digest>` is a `sha256:<hex>` digest. RFS-1-C80 to C83 are the first-ship forms of C18 to C21: the app has no other release, and its kind marker (deviation 10) names the ship that fixed its kind, which may not have published, so they never say "is one". RFS-1-C85 is a warning: it does not fail the check. Kept out of the table on purpose (diagnostics, not human text): the probe's one-line JSON on stdout, `validateManifest`'s existing terse messages, store errors wrapped for logs. RFS-1-C86 and C87 were added in the build (Task 6, the controller ruling on strings found in the build), for the Luna pass at the end of Plan 1: C86 is the writer's refusal when the image passes 2 GiB while it is written (the text of C26), C87 its refusal of a single file over 2 GiB before anything is written. |
| ## Plan 2: box isolation (`2026-10-24-rootfs-apps-2-box-isolation.md`) |
diff --git a/go.mod b/go.mod| index c320d08df..d1432938e 100644 |
| --- a/go.mod |
| +++ b/go.mod |
| @@ -51,6 +51,7 @@ require ( |
| github.com/google/uuid v1.6.0 // indirect |
| github.com/hablullah/go-hijri v1.0.2 // indirect |
| github.com/hablullah/go-juliandays v1.0.0 // indirect |
| + github.com/hanwen/go-fuse/v2 v2.1.0 // indirect |
| github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect |
| github.com/jalaali/go-jalaali v0.0.0-20210801064154-80525e88d958 // indirect |
| github.com/lmittmann/tint v1.1.3 // indirect |
| @@ -70,6 +71,7 @@ require ( |
| github.com/psanford/sqlite3vfs v0.0.0-20251127171934-4e34e03a991a // indirect |
| github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect |
| github.com/tetratelabs/wazero v1.2.1 // indirect |
| + github.com/ulikunitz/xz v0.5.10 // indirect |
| github.com/wasilibs/go-re2 v1.3.0 // indirect |
| github.com/wlynxg/anet v0.0.5 // indirect |
| golang.org/x/mod v0.41.0 // indirect |
| @@ -83,6 +85,7 @@ require ( |
| require ( |
| amadan.net/carlos/platform.git/launchctx v0.0.0 |
| + github.com/KarpelesLab/squashfs v1.2.0 |
| github.com/aws/aws-sdk-go-v2/service/sns v1.43.0 |
| github.com/klauspost/compress v1.18.0 |
| github.com/oschwald/maxminddb-golang v1.13.1 |
diff --git a/go.sum b/go.sum| index a7dd6e987..68e971383 100644 |
| --- a/go.sum |
| +++ b/go.sum |
| @@ -16,6 +16,8 @@ github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2 h1:FwladfywkNirM+FZY |
| github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2/go.mod h1:vv5Ad0RrIoT1lJFdWBZwt4mB1+j+V8DUroixmKDTCdk= |
| github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs= |
| github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= |
| +github.com/KarpelesLab/squashfs v1.2.0 h1:YXM4M7Va+rgshoY5P1VV3VZQdwXM2VDLSx/CDM1fzSw= |
| +github.com/KarpelesLab/squashfs v1.2.0/go.mod h1:izTsPqb0O5ReI3jT24oyGtZorvsUl+uCxx36zwjfXTg= |
| github.com/aliyun/alibabacloud-oss-go-sdk-v2 v1.3.0 h1:wQlqotpyjYPjJz+Noh5bRu7Snmydk8SKC5Z6u1CR20Y= |
| github.com/aliyun/alibabacloud-oss-go-sdk-v2 v1.3.0/go.mod h1:FTzydeQVmR24FI0D6XWUOMKckjXehM/jgMn1xC+DA9M= |
| github.com/aws/aws-sdk-go-v2 v1.44.0 h1:4IbaHhtzy+4h37z4JQyO9a2QsiCml3CNYHtq5hIHigo= |
| @@ -119,6 +121,9 @@ github.com/hablullah/go-hijri v1.0.2 h1:drT/MZpSZJQXo7jftf5fthArShcaMtsal0Zf/dnm |
| github.com/hablullah/go-hijri v1.0.2/go.mod h1:OS5qyYLDjORXzK4O1adFw9Q5WfhOcMdAKglDkcTxgWQ= |
| github.com/hablullah/go-juliandays v1.0.0 h1:A8YM7wIj16SzlKT0SRJc9CD29iiaUzpBLzh5hr0/5p0= |
| github.com/hablullah/go-juliandays v1.0.0/go.mod h1:0JOYq4oFOuDja+oospuc61YoX+uNEn7Z6uHYTbBzdGc= |
| +github.com/hanwen/go-fuse v1.0.0/go.mod h1:unqXarDXqzAk0rt98O2tVndEPIpUgLD9+rwFisZH3Ok= |
| +github.com/hanwen/go-fuse/v2 v2.1.0 h1:+32ffteETaLYClUj0a3aHjZ1hOPxxaNEHiZiujuDaek= |
| +github.com/hanwen/go-fuse/v2 v2.1.0/go.mod h1:oRyA5eK+pvJyv5otpO/DgccS8y/RvYMaO00GgRLGryc= |
| github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= |
| github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= |
| github.com/jalaali/go-jalaali v0.0.0-20210801064154-80525e88d958 h1:qxLoi6CAcXVzjfvu+KXIXJOAsQB62LXjsfbOaErsVzE= |
| @@ -127,6 +132,7 @@ github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zt |
| github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= |
| github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= |
| github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= |
| +github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= |
| github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= |
| github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= |
| github.com/lmittmann/tint v1.1.3 h1:Hv4EaHWXQr+GTFnOU4VKf8UvAtZgn0VuKT+G0wFlO3I= |
| @@ -192,6 +198,8 @@ github.com/superfly/ltx v0.5.1 h1:vGUeBhKvBKZ2s2TsTrOSahz+m0PswfqOWDAD+ICiiYY= |
| github.com/superfly/ltx v0.5.1/go.mod h1:Nf50QAIXU/ET4ua3AuQ2fh31MbgNQZA7r/DYx6Os77s= |
| github.com/tetratelabs/wazero v1.2.1 h1:J4X2hrGzJvt+wqltuvcSjHQ7ujQxA9gb6PeMs4qlUWs= |
| github.com/tetratelabs/wazero v1.2.1/go.mod h1:wYx2gNRg8/WihJfSDxA1TIL8H+GkfLYm+bIfbblu9VQ= |
| +github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= |
| +github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= |
| github.com/wasilibs/go-re2 v1.3.0 h1:LFhBNzoStM3wMie6rN2slD1cuYH2CGiHpvNL3UtcsMw= |
| github.com/wasilibs/go-re2 v1.3.0/go.mod h1:AafrCXVvGRJJOImMajgJ2M7rVmWyisVK7sFshbxnVrg= |
| github.com/wasilibs/nottinygc v0.4.0 h1:h1TJMihMC4neN6Zq+WKpLxgd9xCFMw7O9ETLwY2exJQ= |
| @@ -220,8 +228,10 @@ golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= |
| golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= |
| golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= |
| golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= |
| +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= |
| golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= |
| golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= |
| +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= |
| golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= |
| golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= |
| golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= |
diff --git a/internal/rootfsimage/errors.go b/internal/rootfsimage/errors.go| index 966300e54..f3dfe6288 100644 |
| --- a/internal/rootfsimage/errors.go |
| +++ b/internal/rootfsimage/errors.go |
| @@ -88,6 +88,25 @@ func (e DigestError) Error() string { |
| return fmt.Sprintf("the image's layer %s does not match its digest, so it was not shipped", shown(e.Digest)) |
| } |
| +// ImageTooLargeError refuses an image that grows past the bytes one |
| +// release can carry (release.MaxRootfsBytes) while it is written |
| +// (RFS-1-C86). Writing stops there, so a directory source read in place |
| +// cannot fill the disk. |
| +type ImageTooLargeError struct{} |
| + |
| +func (ImageTooLargeError) Error() string { |
| + return "the rootfs image is over 2 GiB, the most one release can carry. make the image smaller and ship again" |
| +} |
| + |
| +// FileTooLargeError refuses a regular file larger than an image can be |
| +// (RFS-1-C87), before anything is written: the writer holds one file's |
| +// bytes in memory at a time. |
| +type FileTooLargeError struct{ Path string } |
| + |
| +func (e FileTooLargeError) Error() string { |
| + return fmt.Sprintf("%s is over 2 GiB, the largest file a rootfs image can hold. make it smaller and ship again", shown(e.Path)) |
| +} |
| + |
| // errTooLarge is wrapped by the refusal of a tree past its bounds (maxBytes, |
| // maxHeaders) and of a JSON document past maxJSONBytes. carlos ship's own |
| // size refusal (RFS-1-C43) speaks for the built image; this one stops a |
diff --git a/internal/rootfsimage/normalise.go b/internal/rootfsimage/normalise.go| new file mode 100644 |
| index 000000000..04ef1ab64 |
| --- /dev/null |
| +++ b/internal/rootfsimage/normalise.go |
| @@ -0,0 +1,58 @@ |
| +package rootfsimage |
| + |
| +import ( |
| + "io/fs" |
| + "math" |
| + "time" |
| +) |
| + |
| +// Normalise makes every entry what a box can use, and WriteSquashfs calls |
| +// it before writing (controller ruling, index "Ownership and modes are |
| +// normalised at ship"). A box runs every process of an image as one |
| +// unprivileged user, not as the image's USER, so a file at 0600 or a |
| +// directory at 0700 would be closed to the app itself. The image is |
| +// read-only and mounted only in its own instance, so opening it to that |
| +// user exposes nothing to anyone else. |
| +// |
| +// - Every entry, the root included, is owned by 0:0. |
| +// - Others may read every file and enter every directory, and may run |
| +// every file that has any execute bit. No other bit is cleared. |
| +// - Setuid and setgid are cleared (they cannot work under the box's |
| +// NoNewPrivileges and empty capability set); sticky stays. |
| +// - A symlink's mode is 0777, as Linux gives every symlink. |
| +// - The root is 0755: the squashfs writer gives the root that mode |
| +// whatever the tree says. |
| +// - A time before the epoch becomes the epoch, and one past what |
| +// squashfs's unsigned 32-bit seconds hold becomes the last it holds. |
| +// |
| +// Calling it again changes nothing. |
| +func (t *Tree) Normalise() { |
| + normalise(&t.Root) |
| + t.Root.Mode = 0o755 |
| + for _, e := range t.entries { |
| + normalise(e) |
| + } |
| +} |
| + |
| +func normalise(e *Entry) { |
| + e.UID, e.GID = 0, 0 |
| + m := e.Mode &^ (fs.ModeSetuid | fs.ModeSetgid) |
| + switch e.Type { |
| + case TypeDir: |
| + m |= 0o005 |
| + case TypeFile: |
| + m |= 0o004 |
| + if m&0o111 != 0 { |
| + m |= 0o001 |
| + } |
| + case TypeSymlink: |
| + m = 0o777 |
| + } |
| + e.Mode = m |
| + switch s := e.ModTime.Unix(); { |
| + case s < 0: |
| + e.ModTime = time.Unix(0, 0).UTC() |
| + case s > math.MaxUint32: |
| + e.ModTime = time.Unix(math.MaxUint32, 0).UTC() |
| + } |
| +} |
diff --git a/internal/rootfsimage/normalise_test.go b/internal/rootfsimage/normalise_test.go| new file mode 100644 |
| index 000000000..5d65ff785 |
| --- /dev/null |
| +++ b/internal/rootfsimage/normalise_test.go |
| @@ -0,0 +1,79 @@ |
| +package rootfsimage |
| + |
| +import ( |
| + "archive/tar" |
| + "io/fs" |
| + "math" |
| + "testing" |
| + "time" |
| +) |
| + |
| +// The ship's normalisation (controller ruling, index "Ownership and modes |
| +// are normalised at ship"): a box runs every image process as one |
| +// unprivileged user, so every entry becomes 0:0, others may read every file |
| +// and enter every directory, and setuid and setgid go. |
| +func TestNormalise(t *testing.T) { |
| + tr := NewTree(t.TempDir()) |
| + apply(t, tr, layerTar(t, |
| + tEntry{name: "./", typ: tar.TypeDir, mode: 0o700, uid: 5, gid: 5}, |
| + tEntry{name: "config/", typ: tar.TypeDir, mode: 0o755}, |
| + tEntry{name: "config/master.key", typ: tar.TypeReg, mode: 0o600, uid: 1000, gid: 1000, body: "k"}, |
| + tEntry{name: "private/", typ: tar.TypeDir, mode: 0o700, uid: 1000}, |
| + tEntry{name: "private/run", typ: tar.TypeReg, mode: 0o750, body: "#!/bin/sh\n"}, |
| + tEntry{name: "usr/bin/su", typ: tar.TypeReg, mode: 0o4755, body: "su"}, |
| + tEntry{name: "usr/bin/wall", typ: tar.TypeReg, mode: 0o2711, body: "wall"}, |
| + tEntry{name: "srv/", typ: tar.TypeDir, mode: 0o2770}, |
| + tEntry{name: "srv/user-only", typ: tar.TypeReg, mode: 0o100, body: "x"}, |
| + tEntry{name: "tmp/", typ: tar.TypeDir, mode: 0o1777}, |
| + tEntry{name: "var/", typ: tar.TypeDir, mode: 0o777}, |
| + tEntry{name: "var/open", typ: tar.TypeReg, mode: 0o666, body: "o"}, |
| + tEntry{name: "bin", typ: tar.TypeSymlink, mode: 0o755, link: "usr/bin"}, |
| + tEntry{name: "old", typ: tar.TypeReg, mode: 0o644, body: "o", mtime: -5}, |
| + tEntry{name: "late", typ: tar.TypeReg, mode: 0o644, body: "l", mtime: 1 << 33}, |
| + )) |
| + tr.Normalise() |
| + want := map[string]fs.FileMode{ |
| + "config": 0o755, |
| + "config/master.key": 0o604, // 0600: others read |
| + "private": 0o705, // 0700: others enter |
| + "private/run": 0o755, // 0750: others read, and run what has an x bit |
| + "usr/bin/su": 0o755, // 04755: setuid cleared |
| + "usr/bin/wall": 0o715, // 02711: setgid cleared |
| + "srv": 0o775, // 02770 directory: setgid cleared |
| + "srv/user-only": 0o105, // any x bit makes others' x |
| + "tmp": 0o777 | fs.ModeSticky, |
| + "var": 0o777, // never narrowed |
| + "var/open": 0o666, |
| + "bin": 0o777, // a symlink's mode means nothing; links get the usual one |
| + "old": 0o644, |
| + "late": 0o644, |
| + } |
| + for p, m := range want { |
| + e := mustLookup(t, tr, p) |
| + if e.Mode != m || e.UID != 0 || e.GID != 0 { |
| + t.Errorf("%s = %d:%d %v, want 0:0 %v", p, e.UID, e.GID, e.Mode, m) |
| + } |
| + } |
| + if r := tr.Root; r.Mode != 0o755 || r.UID != 0 || r.GID != 0 { |
| + t.Errorf("root = %d:%d %v, want 0:0 0755", r.UID, r.GID, r.Mode) |
| + } |
| + // Times squashfs cannot record are brought to the nearest it can. |
| + if got := mustLookup(t, tr, "old").ModTime; !got.Equal(time.Unix(0, 0)) { |
| + t.Errorf("old mtime = %v, want the epoch", got) |
| + } |
| + if got := mustLookup(t, tr, "late").ModTime; got.Unix() != math.MaxUint32 { |
| + t.Errorf("late mtime = %d, want %d", got.Unix(), uint32(math.MaxUint32)) |
| + } |
| + |
| + // A second pass changes nothing. |
| + before := map[string]Entry{} |
| + for _, e := range tr.Entries() { |
| + before[e.Path] = *e |
| + } |
| + tr.Normalise() |
| + for _, e := range tr.Entries() { |
| + if *e != before[e.Path] { |
| + t.Errorf("%s changed on a second pass: %+v, was %+v", e.Path, *e, before[e.Path]) |
| + } |
| + } |
| +} |
diff --git a/internal/rootfsimage/squashfs.go b/internal/rootfsimage/squashfs.go| new file mode 100644 |
| index 000000000..59d9bbc6d |
| --- /dev/null |
| +++ b/internal/rootfsimage/squashfs.go |
| @@ -0,0 +1,308 @@ |
| +package rootfsimage |
| + |
| +import ( |
| + "errors" |
| + "fmt" |
| + "io" |
| + "io/fs" |
| + "os" |
| + "path" |
| + "slices" |
| + "strings" |
| + "time" |
| + |
| + "github.com/KarpelesLab/squashfs" |
| + |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// Stats is what WriteSquashfs wrote: the image file's bytes and its number |
| +// of regular files (hardlinks counted once per name). |
| +type Stats struct { |
| + Bytes int64 |
| + Files int |
| +} |
| + |
| +// imageAlign is what the image is padded to. A loop device (systemd's |
| +// RootImage= mounts through one) is a whole number of 512-byte sectors and |
| +// drops a partial last one, and the kernel refuses a squashfs whose tables |
| +// end past the device; mksquashfs pads to 4 KiB for the same reason. |
| +const imageAlign = 4096 |
| + |
| +// WriteSquashfs writes t to out as the squashfs image of a rootfs release: |
| +// zlib, 128 KiB blocks, the tree's own owners, modes and times after |
| +// Normalise (which it calls, changing t), and no extended attributes (a |
| +// file capability means nothing under a box's empty capability set). |
| +// Entries that share a file's bytes (hardlinks) share one copy of them. |
| +// Two writes of the same tree are the same bytes. |
| +// |
| +// It reads one file at a time, holding that file's bytes, and writes |
| +// through out at offsets, so out must be a file opened for writing and not |
| +// for appending. An image that would pass release.MaxRootfsBytes is |
| +// ImageTooLargeError, and a single file that large FileTooLargeError; out |
| +// then holds a partial image the caller removes. |
| +func WriteSquashfs(t *Tree, out *os.File) (Stats, error) { |
| + return writeSquashfs(t, out, release.MaxRootfsBytes) |
| +} |
| + |
| +func writeSquashfs(t *Tree, out *os.File, limit int64) (Stats, error) { |
| + t.Normalise() |
| + entries := t.Entries() |
| + |
| + // The first name, in path order, of each file's bytes. The writer is |
| + // given only that one; each later name is a clone of it. |
| + first := map[string]string{} |
| + var clones []*Entry |
| + for _, e := range entries { |
| + if e.Type != TypeFile { |
| + continue |
| + } |
| + if e.Size > limit { |
| + return Stats{}, FileTooLargeError{Path: e.Path} |
| + } |
| + if _, ok := first[e.src]; ok { |
| + clones = append(clones, e) |
| + } else { |
| + first[e.src] = e.Path |
| + } |
| + } |
| + |
| + dst := &boundedFile{f: out, limit: limit} |
| + w, err := squashfs.NewWriter(dst, |
| + squashfs.WithModTime(t.Newest()), |
| + squashfs.WithCompression(squashfs.GZip), |
| + squashfs.WithBlockSize(128<<10)) |
| + if err != nil { |
| + return Stats{}, fmt.Errorf("rootfsimage: squashfs: %w", err) |
| + } |
| + if err := w.AddFS(treeFS{t: t, first: first}); err != nil { |
| + return Stats{}, writeErr(err) |
| + } |
| + for _, e := range clones { |
| + if err := w.CloneInode(e.Path, first[e.src]); err != nil { |
| + return Stats{}, writeErr(err) |
| + } |
| + } |
| + // Owner and time again for every path, clones included, and for the |
| + // root, which the writer otherwise gives the time of the write. Its |
| + // CloneInode and Add* methods replace a zero time with the time of the |
| + // write too; SetModTime stores what it is given, so an epoch-0 entry |
| + // stays 0 (preflight 3.4). |
| + for _, e := range append([]*Entry{&t.Root}, entries...) { |
| + if err := w.SetOwner(e.Path, e.UID, e.GID); err != nil { |
| + return Stats{}, writeErr(err) |
| + } |
| + if err := w.SetModTime(e.Path, e.ModTime); err != nil { |
| + return Stats{}, writeErr(err) |
| + } |
| + } |
| + if err := w.Finalize(); err != nil { |
| + return Stats{}, writeErr(err) |
| + } |
| + |
| + size := (dst.end + imageAlign - 1) / imageAlign * imageAlign |
| + if pad := size - dst.end; pad > 0 { |
| + if _, err := dst.WriteAt(make([]byte, pad), dst.end); err != nil { |
| + return Stats{}, writeErr(err) |
| + } |
| + } |
| + if err := out.Truncate(size); err != nil { |
| + return Stats{}, writeErr(err) |
| + } |
| + fi, err := out.Stat() |
| + if err != nil { |
| + return Stats{}, writeErr(err) |
| + } |
| + return Stats{Bytes: fi.Size(), Files: t.Files()}, nil |
| +} |
| + |
| +// writeErr is err as WriteSquashfs answers it: a refusal as it is, anything |
| +// else named as the image's. |
| +func writeErr(err error) error { |
| + if errors.As(err, new(ImageTooLargeError)) || errors.As(err, new(FileTooLargeError)) { |
| + return err |
| + } |
| + return fmt.Errorf("rootfsimage: writing the squashfs image: %w", err) |
| +} |
| + |
| +// boundedFile is the writer's output: a file written at offsets that refuses |
| +// any byte past limit, and remembers the end of what it wrote. |
| +type boundedFile struct { |
| + f *os.File |
| + limit, end int64 |
| +} |
| + |
| +func (b *boundedFile) WriteAt(p []byte, off int64) (int, error) { |
| + if off < 0 || int64(len(p)) > b.limit-off { |
| + return 0, ImageTooLargeError{} |
| + } |
| + n, err := b.f.WriteAt(p, off) |
| + b.end = max(b.end, off+int64(n)) |
| + return n, err |
| +} |
| + |
| +// Write is never called: the squashfs writer uses WriteAt when its output |
| +// has it, and buffers the whole image in memory when it does not. |
| +func (b *boundedFile) Write([]byte) (int, error) { |
| + return 0, errors.New("rootfsimage: the squashfs writer wrote without an offset") |
| +} |
| + |
| +// treeFS is a Tree as the squashfs writer reads it: fs.FS, fs.ReadDirFS, |
| +// fs.StatFS, fs.ReadFileFS and fs.ReadLinkFS. A file's later names (first |
| +// says which name of its bytes is the first) are left out of its |
| +// directory's listing, for the writer to add as clones. |
| +type treeFS struct { |
| + t *Tree |
| + first map[string]string |
| +} |
| + |
| +func (s treeFS) entry(op, name string) (*Entry, error) { |
| + if !fs.ValidPath(name) { |
| + return nil, &fs.PathError{Op: op, Path: name, Err: fs.ErrInvalid} |
| + } |
| + e, ok := s.t.Lookup(name) |
| + if !ok { |
| + return nil, &fs.PathError{Op: op, Path: name, Err: fs.ErrNotExist} |
| + } |
| + return e, nil |
| +} |
| + |
| +func (s treeFS) Stat(name string) (fs.FileInfo, error) { |
| + e, err := s.entry("stat", name) |
| + if err != nil { |
| + return nil, err |
| + } |
| + return entryInfo{e}, nil |
| +} |
| + |
| +// Lstat is Stat: the tree never follows a symlink. |
| +func (s treeFS) Lstat(name string) (fs.FileInfo, error) { return s.Stat(name) } |
| + |
| +func (s treeFS) ReadDir(name string) ([]fs.DirEntry, error) { |
| + e, err := s.entry("readdir", name) |
| + if err != nil { |
| + return nil, err |
| + } |
| + if e.Type != TypeDir { |
| + return nil, &fs.PathError{Op: "readdir", Path: name, Err: errors.New("not a directory")} |
| + } |
| + dir := cleanRel(name) |
| + var out []fs.DirEntry |
| + for p := range s.t.kids[dir] { |
| + k := s.t.entries[p] |
| + if k.Type == TypeFile && s.first[k.src] != p { |
| + continue |
| + } |
| + out = append(out, fs.FileInfoToDirEntry(entryInfo{k})) |
| + } |
| + slices.SortFunc(out, func(a, b fs.DirEntry) int { return strings.Compare(a.Name(), b.Name()) }) |
| + return out, nil |
| +} |
| + |
| +func (s treeFS) ReadLink(name string) (string, error) { |
| + e, err := s.entry("readlink", name) |
| + if err != nil { |
| + return "", err |
| + } |
| + if e.Type != TypeSymlink { |
| + return "", &fs.PathError{Op: "readlink", Path: name, Err: fs.ErrInvalid} |
| + } |
| + return e.LinkTarget, nil |
| +} |
| + |
| +// ReadFile is a regular file's bytes, exactly as many as the tree says it |
| +// has: a directory source is read in place, and a file that changed size |
| +// since the tree was read would otherwise make an image whose inode |
| +// disagrees with its data. |
| +func (s treeFS) ReadFile(name string) ([]byte, error) { |
| + e, err := s.entry("read", name) |
| + if err != nil { |
| + return nil, err |
| + } |
| + rc, err := s.t.Open(e) |
| + if err != nil { |
| + return nil, err |
| + } |
| + defer rc.Close() |
| + b := make([]byte, e.Size) |
| + if _, err := io.ReadFull(rc, b); err != nil { |
| + if errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, io.EOF) { |
| + return nil, changedErr(e) |
| + } |
| + return nil, fmt.Errorf("reading %s: %w", shown(e.Path), err) |
| + } |
| + if n, _ := rc.Read(make([]byte, 1)); n > 0 { |
| + return nil, changedErr(e) |
| + } |
| + return b, nil |
| +} |
| + |
| +func changedErr(e *Entry) error { |
| + return fmt.Errorf("reading %s: it changed size while the image was written", shown(e.Path)) |
| +} |
| + |
| +// Open is for fs.FS: the writer reads through ReadFile, ReadDir and |
| +// ReadLink, so a file opened here only reads and stats. |
| +func (s treeFS) Open(name string) (fs.File, error) { |
| + e, err := s.entry("open", name) |
| + if err != nil { |
| + return nil, err |
| + } |
| + f := &treeFile{info: entryInfo{e}} |
| + if e.Type == TypeFile { |
| + if f.data, err = s.ReadFile(name); err != nil { |
| + return nil, err |
| + } |
| + } |
| + return f, nil |
| +} |
| + |
| +type treeFile struct { |
| + info entryInfo |
| + data []byte |
| + off int |
| +} |
| + |
| +func (f *treeFile) Stat() (fs.FileInfo, error) { return f.info, nil } |
| +func (f *treeFile) Close() error { return nil } |
| +func (f *treeFile) Read(p []byte) (int, error) { |
| + if f.info.e.Type != TypeFile { |
| + return 0, &fs.PathError{Op: "read", Path: f.info.e.Path, Err: fs.ErrInvalid} |
| + } |
| + if f.off >= len(f.data) { |
| + return 0, io.EOF |
| + } |
| + n := copy(p, f.data[f.off:]) |
| + f.off += n |
| + return n, nil |
| +} |
| + |
| +// entryInfo is an entry as fs.FileInfo. Its Sys has the Uid, Gid and Rdev |
| +// methods the squashfs writer reads owners through. |
| +type entryInfo struct{ e *Entry } |
| + |
| +func (i entryInfo) Name() string { |
| + if i.e.Path == "." { |
| + return "." |
| + } |
| + return path.Base(i.e.Path) |
| +} |
| +func (i entryInfo) Size() int64 { return i.e.Size } |
| +func (i entryInfo) ModTime() time.Time { return i.e.ModTime } |
| +func (i entryInfo) IsDir() bool { return i.e.Type == TypeDir } |
| +func (i entryInfo) Sys() any { return owner{i.e.UID, i.e.GID} } |
| +func (i entryInfo) Mode() fs.FileMode { |
| + switch i.e.Type { |
| + case TypeDir: |
| + return fs.ModeDir | i.e.Mode |
| + case TypeSymlink: |
| + return fs.ModeSymlink | i.e.Mode |
| + } |
| + return i.e.Mode |
| +} |
| + |
| +type owner struct{ uid, gid uint32 } |
| + |
| +func (o owner) Uid() uint32 { return o.uid } |
| +func (o owner) Gid() uint32 { return o.gid } |
| +func (o owner) Rdev() uint32 { return 0 } |
diff --git a/internal/rootfsimage/squashfs_test.go b/internal/rootfsimage/squashfs_test.go| new file mode 100644 |
| index 000000000..59c5e17c6 |
| --- /dev/null |
| +++ b/internal/rootfsimage/squashfs_test.go |
| @@ -0,0 +1,457 @@ |
| +package rootfsimage |
| + |
| +import ( |
| + "archive/tar" |
| + "bytes" |
| + "crypto/sha256" |
| + "errors" |
| + "io" |
| + "io/fs" |
| + "os" |
| + "os/exec" |
| + "path/filepath" |
| + "slices" |
| + "strconv" |
| + "strings" |
| + "testing" |
| + |
| + "github.com/KarpelesLab/squashfs" |
| +) |
| + |
| +// bigBody is 300 KiB that zlib cannot shrink, so the image shows whether it |
| +// holds the bytes once or twice: two full 128 KiB blocks and a 44 KiB |
| +// fragment. |
| +func bigBody() string { |
| + var b bytes.Buffer |
| + sum := sha256.Sum256([]byte("shop")) |
| + for b.Len() < 300<<10 { |
| + b.Write(sum[:]) |
| + sum = sha256.Sum256(sum[:]) |
| + } |
| + return b.String()[:300<<10] |
| +} |
| + |
| +// roundTripLayout writes the round trip's OCI layout and answers its |
| +// directory. Its lower layer holds every kind of entry an image can, with |
| +// owners and modes the ship normalises; its upper layer whites out a file. |
| +func roundTripLayout(t *testing.T) string { |
| + t.Helper() |
| + lower := layerTar(t, |
| + tEntry{name: "srv/", typ: tar.TypeDir, mode: 0o750, uid: 1000, gid: 1000}, |
| + tEntry{name: "srv/big.bin", typ: tar.TypeReg, mode: 0o644, body: bigBody()}, |
| + tEntry{name: "srv/big.link", typ: tar.TypeLink, link: "srv/big.bin"}, |
| + tEntry{name: "srv/empty", typ: tar.TypeReg, mode: 0o644}, |
| + tEntry{name: "usr/bin/su", typ: tar.TypeReg, mode: 0o4755, body: "#!/bin/sh\n"}, |
| + tEntry{name: "usr/bin/abs", typ: tar.TypeSymlink, link: "/usr/bin/su"}, |
| + tEntry{name: "usr/bin/rel", typ: tar.TypeSymlink, link: "su"}, |
| + tEntry{name: "dev/null", typ: tar.TypeChar, mode: 0o666}, |
| + tEntry{name: "etc/old.conf", typ: tar.TypeReg, mode: 0o644, body: "old"}, |
| + tEntry{name: "etc/master.key", typ: tar.TypeReg, mode: 0o600, uid: 1000, body: "secret"}, |
| + tEntry{name: "etc/epoch", typ: tar.TypeReg, mode: 0o644, body: "built at 0", epoch: true}, |
| + tEntry{name: "opt/private/", typ: tar.TypeDir, mode: 0o700}, |
| + tEntry{name: "opt/private/run", typ: tar.TypeReg, mode: 0o750, body: "#!/bin/sh\n"}, |
| + tEntry{name: "tmp/", typ: tar.TypeDir, mode: 0o1777}, |
| + ) |
| + upper := layerTar(t, tEntry{name: "etc/.wh.old.conf", typ: tar.TypeReg}) |
| + dir := t.TempDir() |
| + writeLayout(t, dir, false, tImage{platform: "linux/arm64", layers: [][]byte{lower, upper}}) |
| + return dir |
| +} |
| + |
| +// openForShip opens a source as carlos ship does before writing: the tree, |
| +// then the mount points. |
| +func openForShip(t *testing.T, src string) *Tree { |
| + t.Helper() |
| + tr, err := Open(src, t.TempDir()) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := AddMountPoints(tr); err != nil { |
| + t.Fatal(err) |
| + } |
| + return tr |
| +} |
| + |
| +func roundTripTree(t *testing.T) *Tree { |
| + t.Helper() |
| + return openForShip(t, roundTripLayout(t)) |
| +} |
| + |
| +// writeImage writes tr with WriteSquashfs to a new file and answers its path. |
| +func writeImage(t *testing.T, tr *Tree) (string, Stats) { |
| + t.Helper() |
| + img := filepath.Join(t.TempDir(), "rootfs.squashfs") |
| + f, err := os.Create(img) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + defer f.Close() |
| + st, err := WriteSquashfs(tr, f) |
| + if err != nil { |
| + t.Fatalf("WriteSquashfs: %v", err) |
| + } |
| + return img, st |
| +} |
| + |
| +// openSquashfs reads an image back with the writer's own library. |
| +func openSquashfs(t *testing.T, img string) *squashfs.Superblock { |
| + t.Helper() |
| + f, err := os.Open(img) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + t.Cleanup(func() { f.Close() }) |
| + sb, err := squashfs.New(f) |
| + if err != nil { |
| + t.Fatalf("squashfs.New: %v", err) |
| + } |
| + return sb |
| +} |
| + |
| +// wantMode is the fs.FileMode the image should give e. |
| +func wantMode(e *Entry) fs.FileMode { |
| + switch e.Type { |
| + case TypeDir: |
| + return fs.ModeDir | e.Mode |
| + case TypeSymlink: |
| + return fs.ModeSymlink | e.Mode |
| + } |
| + return e.Mode |
| +} |
| + |
| +func inodeOf(t *testing.T, fi fs.FileInfo) *squashfs.Inode { |
| + t.Helper() |
| + in, ok := fi.Sys().(*squashfs.Inode) |
| + if !ok { |
| + t.Fatalf("%s: Sys() is %T", fi.Name(), fi.Sys()) |
| + } |
| + return in |
| +} |
| + |
| +// imagePaths is every path of the image but the root, sorted. |
| +func imagePaths(t *testing.T, sb *squashfs.Superblock) []string { |
| + t.Helper() |
| + var out []string |
| + err := fs.WalkDir(sb, ".", func(p string, d fs.DirEntry, err error) error { |
| + if err != nil { |
| + return err |
| + } |
| + if p != "." { |
| + out = append(out, p) |
| + } |
| + return nil |
| + }) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + slices.Sort(out) |
| + return out |
| +} |
| + |
| +func TestSquashfsRoundTrip(t *testing.T) { |
| + tr := roundTripTree(t) |
| + if tr.Skipped != 1 { |
| + t.Fatalf("Skipped = %d, want the char device", tr.Skipped) |
| + } |
| + img, st := writeImage(t, tr) |
| + fi, err := os.Stat(img) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if fi.Size() != st.Bytes { |
| + t.Fatalf("Stats.Bytes = %d, file is %d", st.Bytes, fi.Size()) |
| + } |
| + if st.Bytes%4096 != 0 { |
| + t.Errorf("the image is %d bytes, not padded to 4 KiB for a loop device", st.Bytes) |
| + } |
| + if st.Files != tr.Files() { |
| + t.Errorf("Stats.Files = %d, want %d", st.Files, tr.Files()) |
| + } |
| + sb := openSquashfs(t, img) |
| + if sb.Comp != squashfs.GZip || sb.BlockSize != 128<<10 { |
| + t.Errorf("compression %v, block size %d; want gzip, 128 KiB", sb.Comp, sb.BlockSize) |
| + } |
| + if int64(sb.ModTime) != tr.Newest().Unix() { |
| + t.Errorf("superblock time %d, want the tree's newest %d", sb.ModTime, tr.Newest().Unix()) |
| + } |
| + |
| + // Every entry of the tree, as the tree has it after the write. |
| + var want []string |
| + for _, e := range tr.Entries() { |
| + want = append(want, e.Path) |
| + fi, err := sb.Lstat(e.Path) |
| + if err != nil { |
| + t.Errorf("%s missing: %v", e.Path, err) |
| + continue |
| + } |
| + in := inodeOf(t, fi) |
| + if in.GetUid() != e.UID || in.GetGid() != e.GID { |
| + t.Errorf("%s owner %d:%d, want %d:%d", e.Path, in.GetUid(), in.GetGid(), e.UID, e.GID) |
| + } |
| + if fi.Mode() != wantMode(e) { |
| + t.Errorf("%s mode %v, want %v", e.Path, fi.Mode(), wantMode(e)) |
| + } |
| + if fi.ModTime().Unix() != e.ModTime.Unix() { |
| + t.Errorf("%s mtime %v, want %v", e.Path, fi.ModTime(), e.ModTime) |
| + } |
| + switch e.Type { |
| + case TypeSymlink: |
| + target, err := in.Readlink() |
| + if err != nil || string(target) != e.LinkTarget { |
| + t.Errorf("%s -> %q (%v), want %q", e.Path, target, err, e.LinkTarget) |
| + } |
| + case TypeFile: |
| + if fi.Size() != e.Size { |
| + t.Errorf("%s size %d, want %d", e.Path, fi.Size(), e.Size) |
| + } |
| + f, err := sb.Open(e.Path) |
| + if err != nil { |
| + t.Errorf("open %s: %v", e.Path, err) |
| + continue |
| + } |
| + got, err := io.ReadAll(f) |
| + f.Close() |
| + if err != nil || string(got) != readAll(t, tr, e) { |
| + t.Errorf("%s bytes differ from the tree's (%v)", e.Path, err) |
| + } |
| + } |
| + } |
| + // Nothing else: no device inode, and the whited-out file is gone. |
| + if got := imagePaths(t, sb); !slices.Equal(got, want) { |
| + t.Errorf("image paths %q\nwant the tree's %q", got, want) |
| + } |
| + |
| + // The root is the tree's, owned by root. |
| + root, err := sb.Lstat(".") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if in := inodeOf(t, root); in.GetUid() != 0 || in.GetGid() != 0 || root.Mode() != fs.ModeDir|0o755 || root.ModTime().Unix() != tr.Root.ModTime.Unix() { |
| + t.Errorf("root %d:%d %v %v, want 0:0 0755 at %v", in.GetUid(), in.GetGid(), root.Mode(), root.ModTime(), tr.Root.ModTime) |
| + } |
| + for _, p := range MountPoints { |
| + if fi, err := sb.Lstat(p); err != nil || !fi.IsDir() { |
| + t.Errorf("mount point %s: %v, %v", p, fi, err) |
| + } |
| + } |
| + |
| + // The ship's normalisation, as the image carries it. |
| + for p, m := range map[string]fs.FileMode{ |
| + "srv": fs.ModeDir | 0o755, // 0750, owned 1000:1000 |
| + "etc/master.key": 0o604, // 0600, owned 1000 |
| + "opt/private": fs.ModeDir | 0o705, // 0700 |
| + "opt/private/run": 0o755, // 0750 |
| + "usr/bin/su": 0o755, // 04755 |
| + "tmp": fs.ModeDir | fs.ModeSticky | 0o777, // sticky kept |
| + } { |
| + fi, err := sb.Lstat(p) |
| + if err != nil { |
| + t.Errorf("%s: %v", p, err) |
| + continue |
| + } |
| + if in := inodeOf(t, fi); fi.Mode() != m || in.GetUid() != 0 || in.GetGid() != 0 { |
| + t.Errorf("%s = %d:%d %v, want 0:0 %v", p, in.GetUid(), in.GetGid(), fi.Mode(), m) |
| + } |
| + } |
| + |
| + // A SOURCE_DATE_EPOCH=0 time stays 0, not the time of the write. |
| + if fi, err := sb.Lstat("etc/epoch"); err != nil || fi.ModTime().Unix() != 0 { |
| + t.Errorf("etc/epoch mtime %v (%v), want the epoch", fi.ModTime(), err) |
| + } |
| + |
| + // The hardlinked pair is one copy of the bytes. |
| + a, err := sb.Lstat("srv/big.bin") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + b, err := sb.Lstat("srv/big.link") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + ia, ib := inodeOf(t, a), inodeOf(t, b) |
| + if ia.StartBlock != ib.StartBlock || !slices.Equal(ia.Blocks, ib.Blocks) || ia.FragBlock != ib.FragBlock || ia.FragOfft != ib.FragOfft { |
| + t.Errorf("the hardlinks have separate data: %+v and %+v", ia, ib) |
| + } |
| + if ia.NLink != 2 || ib.NLink != 2 { |
| + t.Errorf("nlink %d and %d, want 2", ia.NLink, ib.NLink) |
| + } |
| + if two := int64(2 * len(bigBody())); st.Bytes >= two { |
| + t.Errorf("the image is %d bytes, as large as two copies of the big file (%d)", st.Bytes, two) |
| + } |
| +} |
| + |
| +// The same source written twice is the same image, byte for byte (Review |
| +// Focus 4: a re-ship of the same build is the same release). |
| +func TestWriteSquashfsIsDeterministic(t *testing.T) { |
| + assertSameImage(t, roundTripLayout(t)) |
| +} |
| + |
| +// assertSameImage opens src twice, into two spools, and writes each tree. |
| +func assertSameImage(t *testing.T, src string) { |
| + t.Helper() |
| + one, _ := writeImage(t, openForShip(t, src)) |
| + two, _ := writeImage(t, openForShip(t, src)) |
| + a, err := os.ReadFile(one) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + b, err := os.ReadFile(two) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if !bytes.Equal(a, b) { |
| + t.Errorf("two writes of %s differ (%d and %d bytes)", src, len(a), len(b)) |
| + } |
| +} |
| + |
| +// lsMode is m as ls -l and unsquashfs -lln print it. |
| +func lsMode(m fs.FileMode) string { |
| + b := []byte("----------") |
| + switch { |
| + case m.IsDir(): |
| + b[0] = 'd' |
| + case m&fs.ModeSymlink != 0: |
| + b[0] = 'l' |
| + } |
| + const rwx = "rwxrwxrwx" |
| + for i := range 9 { |
| + if m&(1<<uint(8-i)) != 0 { |
| + b[i+1] = rwx[i] |
| + } |
| + } |
| + special := func(i int, set bool, on, off byte) { |
| + if set { |
| + if b[i] == 'x' { |
| + b[i] = on |
| + } else { |
| + b[i] = off |
| + } |
| + } |
| + } |
| + special(3, m&fs.ModeSetuid != 0, 's', 'S') |
| + special(6, m&fs.ModeSetgid != 0, 's', 'S') |
| + special(9, m&fs.ModeSticky != 0, 't', 'T') |
| + return string(b) |
| +} |
| + |
| +// The C reference reader agrees with the Go one. |
| +func TestSquashfsMatchesUnsquashfs(t *testing.T) { |
| + bin, err := exec.LookPath("unsquashfs") |
| + if err != nil { |
| + t.Skip("unsquashfs is not installed") |
| + } |
| + tr := roundTripTree(t) |
| + img, _ := writeImage(t, tr) |
| + out, err := exec.Command(bin, "-lln", img).CombinedOutput() |
| + if err != nil { |
| + t.Fatalf("unsquashfs -lln: %v\n%s", err, out) |
| + } |
| + type listed struct{ mode, owner string } |
| + got := map[string]listed{} |
| + for _, line := range strings.Split(string(out), "\n") { |
| + f := strings.Fields(line) |
| + if len(f) < 6 || !strings.HasPrefix(f[5], "squashfs-root") { |
| + continue |
| + } |
| + p := strings.TrimPrefix(strings.TrimPrefix(f[5], "squashfs-root"), "/") |
| + if p == "" { |
| + p = "." |
| + } |
| + got[p] = listed{f[0], f[1]} |
| + } |
| + want := map[string]listed{".": {lsMode(fs.ModeDir | tr.Root.Mode), "0/0"}} |
| + for _, e := range tr.Entries() { |
| + want[e.Path] = listed{lsMode(wantMode(e)), strconv.Itoa(int(e.UID)) + "/" + strconv.Itoa(int(e.GID))} |
| + } |
| + for p, w := range want { |
| + if g, ok := got[p]; !ok || g != w { |
| + t.Errorf("%s: unsquashfs lists %+v, want %+v", p, g, w) |
| + } |
| + } |
| + if len(got) != len(want) { |
| + t.Errorf("unsquashfs lists %d entries, the tree has %d", len(got), len(want)) |
| + } |
| +} |
| + |
| +// writeLimited is writeSquashfs to a new file with limit, answering the |
| +// error and the size the file was left at. |
| +func writeLimited(t *testing.T, tr *Tree, limit int64) (Stats, int64, error) { |
| + t.Helper() |
| + f, err := os.Create(filepath.Join(t.TempDir(), "rootfs.squashfs")) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + defer f.Close() |
| + st, werr := writeSquashfs(tr, f, limit) |
| + fi, err := f.Stat() |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + return st, fi.Size(), werr |
| +} |
| + |
| +func bigTree(t *testing.T) *Tree { |
| + t.Helper() |
| + tr := NewTree(t.TempDir()) |
| + apply(t, tr, layerTar(t, tEntry{name: "srv/big.bin", typ: tar.TypeReg, mode: 0o644, body: bigBody()})) |
| + return tr |
| +} |
| + |
| +func TestWriteSquashfsRefusesAnImageOverTheLimit(t *testing.T) { |
| + // Two files each under the limit, together over it. |
| + tr := NewTree(t.TempDir()) |
| + big := bigBody() |
| + apply(t, tr, layerTar(t, |
| + tEntry{name: "srv/a.bin", typ: tar.TypeReg, mode: 0o644, body: big[:150<<10]}, |
| + tEntry{name: "srv/b.bin", typ: tar.TypeReg, mode: 0o644, body: big[150<<10:]}, |
| + )) |
| + _, size, err := writeLimited(t, tr, 200<<10) |
| + if !errors.As(err, new(ImageTooLargeError)) { |
| + t.Fatalf("err = %v, want ImageTooLargeError", err) |
| + } |
| + if size > 200<<10 { |
| + t.Errorf("the refused image left %d bytes, past the %d limit", size, 200<<10) |
| + } |
| + |
| + // At the limit exactly is an image; a byte under it is not. |
| + st, _, err := writeLimited(t, bigTree(t), 1<<30) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, _, err := writeLimited(t, bigTree(t), st.Bytes); err != nil { |
| + t.Errorf("an image of exactly the limit (%d) was refused: %v", st.Bytes, err) |
| + } |
| + if _, _, err := writeLimited(t, bigTree(t), st.Bytes-1); !errors.As(err, new(ImageTooLargeError)) { |
| + t.Errorf("an image a byte over the limit: err = %v, want ImageTooLargeError", err) |
| + } |
| +} |
| + |
| +// A file larger than the limit is refused before anything is written: the |
| +// writer holds one file's bytes in memory, and squashfs cannot record a |
| +// file of 4 GiB or more in the inodes this writer uses. |
| +func TestWriteSquashfsRefusesAFileOverTheLimit(t *testing.T) { |
| + tr := NewTree(t.TempDir()) |
| + apply(t, tr, layerTar(t, tEntry{name: "srv/zeros", typ: tar.TypeReg, mode: 0o644, body: strings.Repeat("\x00", 64<<10)})) |
| + _, size, err := writeLimited(t, tr, 32<<10) |
| + var fe FileTooLargeError |
| + if !errors.As(err, &fe) || fe.Path != "srv/zeros" { |
| + t.Fatalf("err = %v, want FileTooLargeError for srv/zeros", err) |
| + } |
| + if size != 0 { |
| + t.Errorf("the refusal left %d bytes written", size) |
| + } |
| +} |
| + |
| +func TestSquashfsRefusalsSpeakTheDeck(t *testing.T) { |
| + for _, c := range []struct { |
| + err error |
| + want string |
| + }{ |
| + {ImageTooLargeError{}, "the rootfs image is over 2 GiB, the most one release can carry. make the image smaller and ship again"}, |
| + {FileTooLargeError{Path: "srv/model.bin"}, "srv/model.bin is over 2 GiB, the largest file a rootfs image can hold. make it smaller and ship again"}, |
| + } { |
| + if got := c.err.Error(); got != c.want { |
| + t.Errorf("%q\nwant %q", got, c.want) |
| + } |
| + } |
| +} |
diff --git a/internal/rootfsimage/squashfs_unix_test.go b/internal/rootfsimage/squashfs_unix_test.go| new file mode 100644 |
| index 000000000..6e2eba970 |
| --- /dev/null |
| +++ b/internal/rootfsimage/squashfs_unix_test.go |
| @@ -0,0 +1,72 @@ |
| +//go:build unix |
| + |
| +package rootfsimage |
| + |
| +import ( |
| + "os" |
| + "path/filepath" |
| + "strings" |
| + "testing" |
| + "time" |
| +) |
| + |
| +// A directory source written twice is the same image, its hardlinks are one |
| +// copy of the bytes, and an epoch-0 time stays 0. |
| +func TestWriteSquashfsIsDeterministicForADirectory(t *testing.T) { |
| + dir := t.TempDir() |
| + writeSrc(t, filepath.Join(dir, "srv", "big.bin"), bigBody(), 0o644) |
| + if err := os.Link(filepath.Join(dir, "srv", "big.bin"), filepath.Join(dir, "srv", "big.link")); err != nil { |
| + t.Fatal(err) |
| + } |
| + writeSrc(t, filepath.Join(dir, "config", "master.key"), "secret", 0o600) |
| + writeSrc(t, filepath.Join(dir, "bin", "run"), "#!/bin/sh\n", 0o750) |
| + writeSrc(t, filepath.Join(dir, "epoch"), "built at 0", 0o644) |
| + if err := os.Symlink("bin/run", filepath.Join(dir, "start")); err != nil { |
| + t.Fatal(err) |
| + } |
| + at := time.Unix(1_700_000_000, 0) |
| + for _, p := range []string{"srv/big.bin", "config/master.key", "bin/run", "srv", "config", "bin", "."} { |
| + if err := os.Chtimes(filepath.Join(dir, p), at, at); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + if err := os.Chtimes(filepath.Join(dir, "epoch"), time.Unix(0, 0), time.Unix(0, 0)); err != nil { |
| + t.Fatal(err) |
| + } |
| + assertSameImage(t, dir) |
| + |
| + img, _ := writeImage(t, openForShip(t, dir)) |
| + sb := openSquashfs(t, img) |
| + a, err := sb.Lstat("srv/big.bin") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + b, err := sb.Lstat("srv/big.link") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if inodeOf(t, a).StartBlock != inodeOf(t, b).StartBlock { |
| + t.Error("the directory's hardlinks have separate data") |
| + } |
| + if e, err := sb.Lstat("epoch"); err != nil || e.ModTime().Unix() != 0 { |
| + t.Errorf("epoch: %v, want the epoch as its time", err) |
| + } |
| + if k, err := sb.Lstat("config/master.key"); err != nil || k.Mode() != 0o604 { |
| + t.Errorf("config/master.key: %v, want mode 0604", err) |
| + } |
| +} |
| + |
| +// A file that changes size after the tree was read (a directory source is |
| +// read in place) fails the write instead of making a corrupt image. |
| +func TestWriteSquashfsRefusesAFileThatChanged(t *testing.T) { |
| + dir := t.TempDir() |
| + writeSrc(t, filepath.Join(dir, "log"), "short", 0o644) |
| + tr, err := OpenDir(dir, t.TempDir()) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + writeSrc(t, filepath.Join(dir, "log"), "much longer now", 0o644) |
| + if _, _, err := writeLimited(t, tr, 1<<30); err == nil || !strings.Contains(err.Error(), "changed size") { |
| + t.Errorf("err = %v, want the file's change named", err) |
| + } |
| +} |
diff --git a/internal/rootfsimage/testimage_test.go b/internal/rootfsimage/testimage_test.go| index 6daa07d78..ca5e60dc2 100644 |
| --- a/internal/rootfsimage/testimage_test.go |
| +++ b/internal/rootfsimage/testimage_test.go |
| @@ -21,7 +21,8 @@ import ( |
| // Builders for the tests: layer tars, OCI layouts and the two docker save |
| // archive forms, written with archive/tar and encoding/json only. |
| -// defaultMtime is the time an entry gets when its tEntry leaves mtime 0. |
| +// defaultMtime is the time an entry gets when its tEntry leaves mtime 0 |
| +// (and epoch false). |
| const defaultMtime int64 = 1_700_000_000 |
| type tEntry struct { |
| @@ -32,6 +33,7 @@ type tEntry struct { |
| body string |
| link string |
| mtime int64 |
| + epoch bool // mtime is the Unix epoch itself, as SOURCE_DATE_EPOCH=0 builds write |
| } |
| // layerTar is one uncompressed layer holding es, in order. |
| @@ -41,7 +43,7 @@ func layerTar(t *testing.T, es ...tEntry) []byte { |
| tw := tar.NewWriter(&buf) |
| for _, e := range es { |
| mt := e.mtime |
| - if mt == 0 { |
| + if mt == 0 && !e.epoch { |
| mt = defaultMtime |
| } |
| h := &tar.Header{ |