carlos / platform Public

release: stage a rootfs image at the version path, read-only, by its own artifact key (rootfs apps plan 2, task 3)

Manifest.BoxArtifact picks the one artifact a box stages: Artifacts[platform]
for a binary, Artifacts["rootfs-"+platform] for a rootfs release. stageFrom,
the adopt pass's sha readers, RetainedStaged and the proof gate's manifestSHA
read it (preflight 1.7); signedhead stays binary-only. An image is staged
0644 at VersionPath, streamed and sha256-checked; a binary keeps 0755.

A row stages only the kind its runtime runs (Astra R2 finding 6):
stageCheck.runtime, runtimeMatches/errRuntimeMismatch first in stageFrom
(before any read), and adoptHost re-checks the manifest it holds, which
covers stageAuthorized's retained-copy recovery. For a rootfs release a
staged-manifest sibling that cannot be written fails the stage (preflight
1.6). BoxPlatform, FleetOwnerKey and FleetOwner for later tasks. The sweep
test is a pin (preflight 2.8).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

claude 2 hours ago 931bdf2887c175780d021146bad0051168471f9a parent ab302a1

8 files changed, +710 −33

internal/host/versionsweep_test.go +32 −0
internal/release/adopt.go +97 −16
internal/release/adopt_rootfs_test.go +514 −0
internal/release/proofgate.go +5 −3
internal/release/release.go +43 −0
internal/release/stage_stream_test.go +5 −5
internal/release/staged.go +9 −7
internal/release/versions.go +5 −2

diff --git a/internal/host/versionsweep_test.go b/internal/host/versionsweep_test.go

index 5221bba79..df387844f 100644

--- a/internal/host/versionsweep_test.go

+++ b/internal/host/versionsweep_test.go

@@ -394,3 +394,35 @@ func TestSweepStagedVersionsDeletesNothingWhenSettingsPinsAreUnknown(t *testing.

})

}

+// Spec §5.1: a superseded image is swept after the grace, a referenced one

+// never. The image sits at the version path (contract deviation 3), so the

+// pin set that keeps a binary keeps it; this test is the proof that the

+// path choice holds, not a second mechanism.

+func TestSweepStagedVersionsTreatsRootfsImagesAsVersions(t *testing.T) {

+ t.Parallel()

+ optDir := t.TempDir()

+ stage := func(ver string, age time.Duration) {

+ p := release.VersionPath(optDir, "acct1", ver, "shop")

+ if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {

+ t.Fatal(err)

+ }

+ if err := os.WriteFile(p, []byte("squashfs"), 0o644); err != nil {

+ t.Fatal(err)

+ }

+ when := time.Now().Add(-age)

+ if err := os.Chtimes(p, when, when); err != nil {

+ t.Fatal(err)

+ }

+ stage("img-old", 48*time.Hour)

+ stage("img-live", 48*time.Hour)

+ reg := fakeLister{routes: []registry.Route{{Host: "shop.example.com", Kind: "instance", Version: "img-live", Runtime: registry.RuntimeRootfsV1}}}

+ sweepStagedVersions(reg, optDir, noGenerations(t), release.DefaultSweepGrace, time.Now())

+ if staged(t, optDir, "img-old") {

+ t.Error("the superseded image survived the sweep")

+ }

+ if !staged(t, optDir, "img-live") {

+ t.Error("the image a route runs was swept")

+ }

diff --git a/internal/release/adopt.go b/internal/release/adopt.go

index d3b5c56b3..049ff3561 100644

--- a/internal/release/adopt.go

+++ b/internal/release/adopt.go

@@ -30,6 +30,11 @@ import (

// tests can exercise resolution without faking runtime.GOOS/GOARCH.

var adoptPlatform = runtime.GOOS + "-" + runtime.GOARCH

+// BoxPlatform is the platform this box stages builds for (adoptPlatform):

+// a binary release's artifact key, and the suffix of a rootfs release's

+// (Manifest.BoxArtifact). For the control service.

+func BoxPlatform() string { return adoptPlatform }

// PassOption configures one call to Pass or PassFilter beyond their required

// arguments — NagExpiredCanaries and WithCapabilities today. The type exists

// so a call site opts in explicitly and a future option composes the same

@@ -1608,6 +1613,10 @@ func adoptHost(ctx context.Context, work hostwrite.Work, reg *registry.Registry,

// has no pointer digest, and staging it unbound on a digested channel is

// the #253 replay (invariant 3).

var m *Manifest

+ // The row's runtime rides with the stage check (a copy: verified stays

+ // the pass's shared map), so this row stages only the kind it runs.

+ chk := cfg.stage

+ chk.runtime = rt.Runtime

switch {

case !settings || dec.Evidence == EvidencePointer:

if ptr == nil || ver != ptr.Version {

@@ -1615,9 +1624,9 @@ func adoptHost(ctx context.Context, work hostwrite.Work, reg *registry.Registry,

cfg.proof.clear(rt.Host)

return

}

- m, err = ensureStagedDigest(ctx, s, optDir, RouteAccount(rt), rt.App, ver, rt.Host, ptr.DigestOf(ver), cfg.stage)

+ m, err = ensureStagedDigest(ctx, s, optDir, RouteAccount(rt), rt.App, ver, rt.Host, ptr.DigestOf(ver), chk)

case dec.Evidence == EvidenceRecord:

- m, err = stageAuthorized(ctx, s, cfg.pub, optDir, RouteAccount(rt), rt.App, ver, rt.Host, cfg.stage)

+ m, err = stageAuthorized(ctx, s, cfg.pub, optDir, RouteAccount(rt), rt.App, ver, rt.Host, chk)

default:

fmt.Printf(" %s: %s: no evidence this pass can check; not staged\n", rt.Host, ver)

cfg.proof.clear(rt.Host)

@@ -1637,6 +1646,13 @@ func adoptHost(ctx context.Context, work hostwrite.Work, reg *registry.Registry,

}

return

}

+ // Again on the manifest this pass ends up holding: stageAuthorized's

+ // retained-copy recovery (RetainedStaged) answers one that never passed

+ // stageFrom's check (Astra R2 finding 6).

+ if err := runtimeMatches(m, rt.Runtime); err != nil {

+ fmt.Printf(" %s: %s: %v; not staged for this row\n", rt.Host, ver, err)

+ return

+ }

id, err := releaseIdentity(m)

if err != nil {

fmt.Printf(" %s: release identity: %v\n", rt.Host, err)

@@ -1693,7 +1709,8 @@ func adoptHost(ctx context.Context, work hostwrite.Work, reg *registry.Registry,

if (len(hist) > 0 || cfg.requireRootHistory && !fresh) && !slices.Contains(hist, newProg) && !programChange {

fmt.Printf(" %s: %s is %s, but this route has only run %s. not pointed live (ship a new version with carlos ship --program-change and promote it, if the change is intended)\n",

rt.Host, ver, newProg, strings.Join(hist, ", "))

- sha := m.Artifacts[adoptPlatform].SHA256

+ art, _ := m.BoxArtifact(adoptPlatform)

+ sha := art.SHA256

// Nothing was restarted, so the box is still on its old build.

if err := cfg.recordRefusal(ctx, rt, Refusal{Host: rt.Host, App: rt.App, Version: ver, SHA256: sha,

Platform: adoptPlatform, PointerPromotedAt: sel.PromotedAt, RefusedAt: now.UTC(),

@@ -1713,7 +1730,11 @@ func adoptHost(ctx context.Context, work hostwrite.Work, reg *registry.Registry,

}

var historyAttempt string

if cfg.requireRootHistory {

- if m == nil || m.Artifacts[adoptPlatform].SHA256 == "" {

+ var art Artifact

+ if m != nil {

+ art, _ = m.BoxArtifact(adoptPlatform)

+ }

+ if art.SHA256 == "" {

cfg.historyError(os.ErrPermission)

return

}

@@ -1721,7 +1742,7 @@ func adoptHost(ctx context.Context, work hostwrite.Work, reg *registry.Registry,

if !cfg.hostWrites.Enabled {

historyIdentity = hostwrite.Identity{}

}

- historyAttempt, err = cfg.rootHistory.begin(ctx, rt, cfg.historySelected[rt.Host], sel, strings.ToLower(m.Artifacts[adoptPlatform].SHA256), now, historyIdentity)

+ historyAttempt, err = cfg.rootHistory.begin(ctx, rt, cfg.historySelected[rt.Host], sel, strings.ToLower(art.SHA256), now, historyIdentity)

if err != nil {

cfg.historyError(err)

return

@@ -2104,7 +2125,8 @@ func adoptHost(ctx context.Context, work hostwrite.Work, reg *registry.Registry,

}

sha := ""

if m != nil {

- sha = m.Artifacts[adoptPlatform].SHA256

+ art, _ := m.BoxArtifact(adoptPlatform)

+ sha = art.SHA256

}

writeGroupRefusals(cfg, group, rt.App, sel, ver, sha, adoptPlatform, notUp, rec, now)

}

@@ -2217,6 +2239,32 @@ type stageCheck struct {

// only for the sha its own checked manifest names (stageFrom). nil

// remembers nothing.

verified map[string]string

+ // runtime is the Route.Runtime of the row this staging is for: "" a

+ // binary row, registry.RuntimeRootfsV1 a rootfs row. A row stages only

+ // the kind it runs (runtimeMatches), so the zero value, a binary row,

+ // never stages a rootfs image (Astra R2 finding 6).

+ runtime string

+// errRuntimeMismatch is a release of one kind meeting a row that runs the

+// other: a rootfs image for a binary row, or a binary for a rootfs row.

+var errRuntimeMismatch = errors.New("release: the release's kind does not match the row's runtime")

+// runtimeMatches is nil when m is a release the row whose Route.Runtime is

+// runtime can run: a rootfs manifest for a rootfs row, a binary or static

+// one for a binary row. Anything else, a runtime this binary does not know

+// included, is errRuntimeMismatch. A nil m (a staged copy

+// whose manifest neither disk nor bucket answered, ensureStagedChecked) is

+// a binary row's today's answer and a refusal for a rootfs row, which never

+// runs without its manifest.

+func runtimeMatches(m *Manifest, runtime string) error {

+ switch {

+ case runtime == "" && (m == nil || !m.IsRootfs()):

+ return nil

+ case runtime == registry.RuntimeRootfsV1 && m != nil && m.IsRootfs():

+ return nil

+ }

+ return errRuntimeMismatch

}

// ensureStaged makes ver's build of app available in the version store,

@@ -2319,16 +2367,42 @@ func ensureStagedChecked(ctx context.Context, s Store, optDir, acct, app, ver, h

// and the sibling is made exactly raw (syncSibling) rather than repaired only

// when it does not read. Unbound, it is ensureStagedChecked as zdd-4 wrote it.

func stageFrom(ctx context.Context, s Store, optDir, acct, app, ver, host string, chk stageCheck, staged bool, m *Manifest, raw []byte, bound bool) (*Manifest, error) {

+ // Before the staged-copy check and before any download: a row stages

+ // only the kind it runs (Astra R2 finding 6).

+ if err := runtimeMatches(m, chk.runtime); err != nil {

+ return nil, fmt.Errorf("%s/%s: %w", app, ver, err)

+ }

path := VersionPath(optDir, acct, ver, app)

sibling := repairSibling

if bound {

sibling = syncSibling

}

- art, hasBuild := m.Artifacts[adoptPlatform]

+ // A rootfs start is validated against the staged sibling (root refuses

+ // one without it), so for a rootfs release a sibling that is not there

+ // fails the stage and nothing is pointed live; a binary's stays

+ // best-effort (preflight 1.6).

+ reused := func() (*Manifest, error) {

+ sibling(optDir, acct, app, ver, host, raw, chk)

+ if !m.IsRootfs() {

+ return m, nil

+ }

+ sib, err := ReadStagedManifest(optDir, acct, ver, app)

+ switch {

+ case err != nil:

+ case sib == nil:

+ err = os.ErrNotExist

+ case !sib.IsRootfs():

+ err = errRuntimeMismatch

+ }

+ if err != nil {

+ return nil, fmt.Errorf("%s/%s: stage manifest sibling: %w", app, ver, err)

+ }

+ return m, nil

+ }

+ art, hasBuild := m.BoxArtifact(adoptPlatform)

if staged {

if v := chk.verified[path]; v != "" && (!bound || hasBuild && strings.EqualFold(v, art.SHA256)) {

- sibling(optDir, acct, app, ver, host, raw, chk)

- return m, nil

+ return reused()

}

sum, herr := FileSHA256(path)

if herr != nil {

@@ -2338,8 +2412,7 @@ func stageFrom(ctx context.Context, s Store, optDir, acct, app, ver, host string

if chk.verified != nil {

chk.verified[path] = art.SHA256

}

- sibling(optDir, acct, app, ver, host, raw, chk)

- return m, nil

+ return reused()

}

// Different bytes under the same name. Overwrite only a copy nothing

// refers to: overwriting a referenced one would change the bytes under

@@ -2372,7 +2445,7 @@ func stageFrom(ctx context.Context, s Store, optDir, acct, app, ver, host string

// A box facing a release with no build for its platform: skip, never

// install the wrong arch. The next release ships it and the box

// catches up.

- return nil, fmt.Errorf("release %s has no %s build — skipping", ver, adoptPlatform)

+ return nil, fmt.Errorf("release %s has no %s build — skipping", ver, boxArtifactKey(m, adoptPlatform))

}

// Streamed to disk and hashed on the way (#269): a build read whole

// outgrew the stage pass's memory cap.

@@ -2381,7 +2454,11 @@ func stageFrom(ctx context.Context, s Store, optDir, acct, app, ver, host string

return nil, err

}

- err := stageArtifactFrom(optDir, acct, ver, app, art.SHA256, func() (io.ReadCloser, error) {

+ mode := os.FileMode(0o755)

+ if m.IsRootfs() {

+ mode = 0o644 // an image is mounted, never executed

+ }

+ err := stageArtifactFrom(optDir, acct, ver, app, art.SHA256, mode, func() (io.ReadCloser, error) {

rc, _, err := OpenQualified(ctx, s, ArtifactKey(acct, app, ver, art.File))

return rc, err

})

@@ -2392,10 +2469,14 @@ func stageFrom(ctx context.Context, s Store, optDir, acct, app, ver, host string

chk.verified[path] = art.SHA256

}

// The sibling is the CANONICAL bytes (spec §4.2), and a restage replaces

- // whatever sibling the stale copy had. Best-effort: a failure here is not

- // fatal to staging the binary, and backfillManifest gets another chance

- // at it on every later pass.

+ // whatever sibling the stale copy had. Best-effort for a binary: a

+ // failure here is not fatal to staging it, and backfillManifest gets

+ // another chance at it on every later pass. Fatal for a rootfs image,

+ // which root never starts without it (preflight 1.6).

if err := StageManifest(optDir, acct, ver, app, raw); err != nil {

+ if m.IsRootfs() {

+ return nil, fmt.Errorf("%s/%s: stage manifest sibling: %w", app, ver, err)

+ }

fmt.Printf(" %s: stage manifest sibling for %s: %v\n", host, ver, err)

}

return m, nil

diff --git a/internal/release/adopt_rootfs_test.go b/internal/release/adopt_rootfs_test.go

new file mode 100644

index 000000000..3cd12807e

--- /dev/null

+++ b/internal/release/adopt_rootfs_test.go

@@ -0,0 +1,514 @@

+package release

+import (

+ "bytes"

+ "context"

+ "crypto/ed25519"

+ "crypto/sha256"

+ "encoding/hex"

+ "encoding/json"

+ "errors"

+ "io"

+ "os"

+ "path/filepath"

+ "strings"

+ "sync/atomic"

+ "testing"

+ "github.com/carlosframework/platform/internal/registry"

+// stagedRootfsManifest is a valid rootfs manifest as Plan 1's ship writes one:

+// kind rootfs, the artifact under RootfsArtifactKey, and never under the

+// binary key. (Named apart from Plan 1's rootfsManifest() fixture in the

+// same package.)

+func stagedRootfsManifest(t *testing.T, image []byte) *Manifest {

+ t.Helper()

+ sum := sha256.Sum256(image)

+ return &Manifest{Schema: 1, App: "shop", Version: "v1", Kind: KindRootfs,

+ Artifacts: map[string]Artifact{RootfsArtifactKey: {File: "rootfs.squashfs", SHA256: hex.EncodeToString(sum[:]), Size: int64(len(image))}},

+ Processes: map[string]Process{ProcessWeb: {Command: []string{"/app/serve"}}}}

+// onArm64 makes this box a linux-arm64 one for the test, the one platform a

+// rootfs release names (RootfsArtifactKey). Not parallel: adoptPlatform is a

+// package var.

+func onArm64(t *testing.T) {

+ t.Helper()

+ old := adoptPlatform

+ adoptPlatform = "linux-arm64"

+ t.Cleanup(func() { adoptPlatform = old })

+// putArtifact puts body at the release's artifact key, as ship leaves it.

+func putArtifact(t *testing.T, s Store, acct, app, ver, file string, body []byte) {

+ t.Helper()

+ if _, err := s.Put(context.Background(), ArtifactKey(acct, app, ver, file), body, ""); err != nil {

+ t.Fatalf("put artifact: %v", err)

+ }

+// artifactReadCounter wraps a store and counts every Get or Open of a

+// release's artifact (any key under a release that is not its manifest).

+type artifactReadCounter struct {

+ Store

+ n atomic.Int64

+func (c *artifactReadCounter) count(key string) {

+ if strings.Contains(key, "/releases/") && !strings.HasSuffix(key, "/manifest.json") {

+ c.n.Add(1)

+ }

+func (c *artifactReadCounter) Get(ctx context.Context, key string) ([]byte, string, error) {

+ c.count(key)

+ return c.Store.Get(ctx, key)

+func (c *artifactReadCounter) Open(ctx context.Context, key string) (io.ReadCloser, string, error) {

+ c.count(key)

+ return OpenObject(ctx, c.Store, key)

+func countArtifactReads(s Store) (Store, func() int64) {

+ c := &artifactReadCounter{Store: s}

+ return c, c.n.Load

+// C2: a box from before this feature looks up Artifacts[adoptPlatform]. On a

+// rootfs manifest that key must not exist, so such a box stages nothing

+// (stageFrom's "no build for this platform, skipping" branch), whatever

+// else it understood. Pinned on the manifest shape, not on behaviour, so a

+// later change that adds a "linux-arm64" entry to a rootfs manifest fails

+// here before it reaches a box.

+func TestRootfsManifestNeverCarriesTheBinaryArtifactKey(t *testing.T) {

+ m := stagedRootfsManifest(t, []byte("squashfs"))

+ if _, ok := m.Artifacts["linux-arm64"]; ok {

+ t.Fatal("a rootfs manifest carries the binary key; an old box would stage the image as a binary")

+ }

+ if _, ok := m.Artifacts[adoptPlatform]; ok {

+ t.Fatal("a rootfs manifest carries this box's binary key")

+ }

+ if validateManifest(m) != nil {

+ t.Fatalf("fixture is not a valid rootfs manifest: %v", validateManifest(m))

+ }

+func TestBoxArtifactPicksTheKindsOwnKey(t *testing.T) {

+ m := stagedRootfsManifest(t, []byte("squashfs"))

+ if a, ok := m.BoxArtifact("linux-arm64"); !ok || a.File != "rootfs.squashfs" {

+ t.Errorf("BoxArtifact(rootfs) = %+v %v", a, ok)

+ }

+ b := &Manifest{Kind: "binary", Artifacts: map[string]Artifact{"linux-arm64": {File: "shop"}, RootfsArtifactKey: {File: "rootfs.squashfs"}}}

+ if a, ok := b.BoxArtifact("linux-arm64"); !ok || a.File != "shop" {

+ t.Errorf("BoxArtifact(binary) = %+v %v", a, ok)

+ }

+ if _, ok := b.BoxArtifact("linux-amd64"); ok {

+ t.Error("BoxArtifact found a build for a platform the manifest lacks")

+ }

+ if _, ok := m.BoxArtifact("linux-amd64"); ok {

+ t.Error("BoxArtifact found an image for a platform the manifest lacks")

+ }

+ if rootfsArtifactKeyFor("linux-arm64") != RootfsArtifactKey {

+ t.Errorf("rootfsArtifactKeyFor(linux-arm64) = %q, want %q", rootfsArtifactKeyFor("linux-arm64"), RootfsArtifactKey)

+ }

+ if !m.IsRootfs() || b.IsRootfs() {

+ t.Error("IsRootfs does not follow Kind")

+ }

+// The image is staged where a binary is (contract deviation 3), hashed on

+// the way in, and never executable.

+func TestStageFromStagesTheImageAtTheVersionPathReadOnly(t *testing.T) {

+ onArm64(t)

+ image := []byte("not really a squashfs, but the bytes are the test")

+ m := stagedRootfsManifest(t, image)

+ optDir := t.TempDir()

+ s := NewDirStore(t.TempDir())

+ putArtifact(t, s, "bab", "shop", "v1", "rootfs.squashfs", image)

+ raw, _ := json.Marshal(m)

+ got, err := stageFrom(context.Background(), s, optDir, "bab", "shop", "v1", "shop.example.com", stageCheck{runtime: registry.RuntimeRootfsV1}, false, m, raw, false)

+ if err != nil || got == nil {

+ t.Fatalf("stageFrom: %v", err)

+ }

+ fi, err := os.Stat(VersionPath(optDir, "bab", "v1", "shop"))

+ if err != nil {

+ t.Fatalf("image not at the version path: %v", err)

+ }

+ if fi.Mode().Perm() != 0o644 {

+ t.Errorf("image mode %o, want 0644 (an image is mounted, never executed)", fi.Mode().Perm())

+ }

+ if sum, _ := FileSHA256(VersionPath(optDir, "bab", "v1", "shop")); !strings.EqualFold(sum, m.Artifacts[RootfsArtifactKey].SHA256) {

+ t.Error("staged bytes do not hash to the manifest's digest")

+ }

+ // Corrupt bytes are refused and nothing is left under the name.

+ putArtifact(t, s, "bab", "shop", "v2", "rootfs.squashfs", []byte("tampered"))

+ m2 := stagedRootfsManifest(t, image)

+ m2.Version = "v2"

+ raw2, _ := json.Marshal(m2)

+ if _, err := stageFrom(context.Background(), s, optDir, "bab", "shop", "v2", "shop.example.com", stageCheck{runtime: registry.RuntimeRootfsV1}, false, m2, raw2, false); !errors.Is(err, errArtifactMismatch) {

+ t.Fatalf("tampered image: err = %v, want errArtifactMismatch", err)

+ }

+ if _, err := os.Stat(VersionPath(optDir, "bab", "v2", "shop")); err == nil {

+ t.Error("a tampered image was left staged")

+ }

+// A binary keeps its executable mode: the mode follows the kind.

+func TestStageFromStagesABinaryExecutable(t *testing.T) {

+ onArm64(t)

+ body := []byte("a binary")

+ sum := sha256.Sum256(body)

+ m := &Manifest{Schema: 1, App: "shop", Version: "v1", Kind: "binary",

+ Artifacts: map[string]Artifact{"linux-arm64": {File: "linux-arm64", SHA256: hex.EncodeToString(sum[:]), Size: int64(len(body))}}}

+ optDir := t.TempDir()

+ s := NewDirStore(t.TempDir())

+ putArtifact(t, s, "bab", "shop", "v1", "linux-arm64", body)

+ raw, _ := json.Marshal(m)

+ if _, err := stageFrom(context.Background(), s, optDir, "bab", "shop", "v1", "shop.example.com", stageCheck{}, false, m, raw, false); err != nil {

+ t.Fatalf("stageFrom: %v", err)

+ }

+ fi, err := os.Stat(VersionPath(optDir, "bab", "v1", "shop"))

+ if err != nil {

+ t.Fatal(err)

+ }

+ if fi.Mode().Perm() != 0o755 {

+ t.Errorf("binary mode %o, want 0755", fi.Mode().Perm())

+ }

+// Astra R2 finding 6, the stage half: a row stages only its own kind, and

+// the refusal comes before a byte is downloaded.

+func TestStageFromRefusesAReleaseOfTheOtherRuntime(t *testing.T) {

+ onArm64(t)

+ image := []byte("squashfs")

+ rfs := stagedRootfsManifest(t, image)

+ bin := &Manifest{Schema: 1, App: "shop", Version: "v1", Kind: "binary",

+ Artifacts: map[string]Artifact{"linux-arm64": {File: "linux-arm64", SHA256: rfs.Artifacts[RootfsArtifactKey].SHA256, Size: int64(len(image))}}}

+ for _, c := range []struct {

+ name, file, runtime string

+ m *Manifest

+ }{

+ {"rootfs release, binary row", "rootfs.squashfs", "", rfs},

+ {"binary release, rootfs row", "linux-arm64", registry.RuntimeRootfsV1, bin},

+ {"rootfs release, unknown runtime", "rootfs.squashfs", "rootfs-v2", rfs},

+ } {

+ optDir := t.TempDir()

+ s, reads := countArtifactReads(NewDirStore(t.TempDir()))

+ putArtifact(t, s, "bab", "shop", "v1", c.file, image)

+ raw, _ := json.Marshal(c.m)

+ before := reads()

+ if _, err := stageFrom(context.Background(), s, optDir, "bab", "shop", "v1", "shop.example.com", stageCheck{runtime: c.runtime}, false, c.m, raw, false); !errors.Is(err, errRuntimeMismatch) {

+ t.Errorf("%s: err = %v, want errRuntimeMismatch", c.name, err)

+ }

+ if reads() != before {

+ t.Errorf("%s: the artifact was read before the refusal", c.name)

+ }

+ if _, err := os.Lstat(VersionPath(optDir, "bab", "v1", "shop")); err == nil {

+ t.Errorf("%s: something was staged", c.name)

+ }

+// binaryRowOnV1 is the adopt pass's starting point for a binary row: v1, a

+// binary, staged and live on host; v2 a rootfs release promoted onto the

+// channel. It returns the image's bytes.

+func binaryRowOnV1(t *testing.T, s Store, priv ed25519.PrivateKey, opt string, reg *registry.Registry, host string, settings bool) []byte {

+ t.Helper()

+ ctx := context.Background()

+ shipRelease(t, s, "bab", "shop", "v1", adoptPlatform, []byte("good"))

+ if err := Promote(ctx, s, priv, "bab", "shop", "canary/x", "v1"); err != nil {

+ t.Fatalf("Promote v1: %v", err)

+ }

+ if err := StageArtifact(opt, "bab", "v1", "shop", []byte("good")); err != nil {

+ t.Fatal(err)

+ }

+ if settings {

+ settingsRoute(t, reg, host, "shop", "canary/x", UpdateSettings{Updates: UpdatesAuto, Channel: "canary/x", Pin: "v2"}, BuildStateCopy{Known: true})

+ } else {

+ addRoute(t, reg, host, "shop", "canary/x", false)

+ }

+ if err := PointLive(opt, host, "bab", "v1", "shop"); err != nil {

+ t.Fatal(err)

+ }

+ image := []byte("a rootfs image")

+ putRootfsRelease(t, s, "bab", "shop", "v2", image, false)

+ if err := Promote(ctx, s, priv, "bab", "shop", "canary/x", "v2"); err != nil {

+ t.Fatalf("Promote v2: %v", err)

+ }

+ return image

+// adoptPassOut runs one adopt pass, recording restarts, and returns what it

+// printed.

+func adoptPassOut(t *testing.T, reg *registry.Registry, s Store, pub ed25519.PublicKey, opt string, restarted *[]string, opts ...PassOption) string {

+ t.Helper()

+ var out bytes.Buffer

+ withCapturedStdout(t, &out, func() {

+ if err := Pass(context.Background(), reg, s, pub, opt, func(h string) error {

+ *restarted = append(*restarted, h)

+ return nil

+ }, opts...); err != nil {

+ t.Fatalf("Pass: %v", err)

+ }

+ })

+ return out.String()

+// refusalLine is the printed line that names host, ver and the runtime

+// refusal, or "".

+func refusalLine(out, host, ver string) string {

+ for _, l := range strings.Split(out, "\n") {

+ if strings.Contains(l, host) && strings.Contains(l, ver) && strings.Contains(l, "runtime") {

+ return l

+ }

+ return ""

+// Astra R2 finding 6, at the layer that receives the row: the adopt pass.

+// A binary row whose channel moves to a rootfs release keeps serving what

+// it has, whether the image would arrive by a fresh stage or from a copy

+// already staged and verified on the box (stageAuthorized's retained-copy

+// recovery, which never passes stageFrom's check).

+func TestAdoptPassRefusesARootfsReleaseForABinaryRow(t *testing.T) {

+ onArm64(t)

+ const host = "shop.example.com"

+ t.Run("fresh", func(t *testing.T) {

+ s := NewDirStore(t.TempDir())

+ priv, pub := testKeys(t)

+ opt, reg := t.TempDir(), testRegistry(t)

+ binaryRowOnV1(t, s, priv, opt, reg, host, false)

+ var restarted []string

+ out := adoptPassOut(t, reg, s, pub, opt, &restarted)

+ if v, _ := LiveVersion(opt, host); v != "v1" {

+ t.Errorf("live = %q, want v1", v)

+ }

+ if _, err := os.Lstat(VersionPath(opt, "bab", "v2", "shop")); err == nil {

+ t.Error("the rootfs image was staged for a binary row")

+ }

+ if len(restarted) != 0 {

+ t.Errorf("restarted = %v, want none", restarted)

+ }

+ if refusalLine(out, host, "v2") == "" {

+ t.Errorf("no line names %s, v2 and the runtime:\n%s", host, out)

+ }

+ })

+ t.Run("retained copy", func(t *testing.T) {

+ ctx := context.Background()

+ s := NewDirStore(t.TempDir())

+ priv, pub := testKeys(t)

+ opt, reg := t.TempDir(), testRegistry(t)

+ image := binaryRowOnV1(t, s, priv, opt, reg, host, true)

+ // The box already has v2's image and its sibling, verified; the record

+ // is gone (promoted "before the feature"), so the pin's evidence is

+ // the retained copy alone.

+ raw, _, err := s.Get(ctx, ManifestKey("bab", "shop", "v2"))

+ if err != nil {

+ t.Fatal(err)

+ }

+ if err := StageArtifact(opt, "bab", "v2", "shop", image); err != nil {

+ t.Fatal(err)

+ }

+ if err := StageManifest(opt, "bab", "v2", "shop", raw); err != nil {

+ t.Fatal(err)

+ }

+ if err := s.Delete(ctx, PromotionRecordKey("bab", "shop", "v2")); err != nil {

+ t.Fatal(err)

+ }

+ if id, _ := RetainedStaged(opt, "bab", "shop", "v2"); id == nil {

+ t.Fatal("fixture: the retained copy does not verify, so this case would not reach the recovery path")

+ }

+ var restarted []string

+ out := adoptPassOut(t, reg, s, pub, opt, &restarted, WithUpdateSettings(nil, nil, true))

+ if !strings.Contains(out, "recovered from this box's verified staged copy") {

+ t.Fatalf("the pass did not take the retained-copy path:\n%s", out)

+ }

+ if v, _ := LiveVersion(opt, host); v != "v1" {

+ t.Errorf("live = %q, want v1", v)

+ }

+ if _, err := os.Lstat(VersionPath(opt, "bab", "v2", "shop")); err != nil {

+ t.Errorf("the retained copy went away: %v", err)

+ }

+ if len(restarted) != 0 {

+ t.Errorf("restarted = %v, want none", restarted)

+ }

+ if refusalLine(out, host, "v2") == "" {

+ t.Errorf("no line names %s, v2 and the runtime:\n%s", host, out)

+ }

+ })

+// The other direction: a rootfs row moving to a rootfs v2 stages it at the

+// version path, read-only, with its sibling, and points it live.

+func TestAdoptPassStagesARootfsReleaseForARootfsRow(t *testing.T) {

+ onArm64(t)

+ const host = "shop.example.com"

+ ctx := context.Background()

+ s := NewDirStore(t.TempDir())

+ priv, pub := testKeys(t)

+ opt, reg := t.TempDir(), testRegistry(t)

+ if err := reg.Upsert(registry.Route{Host: host, Kind: "instance", App: "shop", Channel: "canary/x", Runtime: registry.RuntimeRootfsV1}); err != nil {

+ t.Fatal(err)

+ }

+ var restarted []string

+ putRootfsRelease(t, s, "bab", "shop", "v1", []byte("image one"), false)

+ if err := Promote(ctx, s, priv, "bab", "shop", "canary/x", "v1"); err != nil {

+ t.Fatal(err)

+ }

+ adoptPassOut(t, reg, s, pub, opt, &restarted)

+ if v, _ := LiveVersion(opt, host); v != "v1" {

+ t.Fatalf("live = %q after the first pass, want v1", v)

+ }

+ image := []byte("image two")

+ putRootfsRelease(t, s, "bab", "shop", "v2", image, false)

+ if err := Promote(ctx, s, priv, "bab", "shop", "canary/x", "v2"); err != nil {

+ t.Fatal(err)

+ }

+ out := adoptPassOut(t, reg, s, pub, opt, &restarted)

+ p := VersionPath(opt, "bab", "v2", "shop")

+ fi, err := os.Stat(p)

+ if err != nil {

+ t.Fatalf("v2 not staged: %v\n%s", err, out)

+ }

+ if fi.Mode().Perm() != 0o644 {

+ t.Errorf("image mode %o, want 0644", fi.Mode().Perm())

+ }

+ if sum, _ := FileSHA256(p); sum != etagOf(image) {

+ t.Error("staged bytes do not hash to the manifest's rootfs artifact")

+ }

+ sib, err := ReadStagedManifest(opt, "bab", "v2", "shop")

+ if err != nil || sib == nil || sib.Kind != KindRootfs {

+ t.Errorf("staged sibling = %+v, %v; want a rootfs manifest", sib, err)

+ }

+ if v, _ := LiveVersion(opt, host); v != "v2" {

+ t.Errorf("live = %q, want v2\n%s", v, out)

+ }

+// Preflight 1.6: the sibling is root's source of truth for a rootfs start,

+// so a rootfs stage whose sibling cannot be written fails; a binary stage

+// with the same fault still succeeds (best-effort, as today).

+func TestStageFromFailsARootfsStageWhoseSiblingCannotBeWritten(t *testing.T) {

+ onArm64(t)

+ body := []byte("bytes")

+ sum := sha256.Sum256(body)

+ rfs := stagedRootfsManifest(t, body)

+ bin := &Manifest{Schema: 1, App: "shop", Version: "v1", Kind: "binary",

+ Artifacts: map[string]Artifact{"linux-arm64": {File: "linux-arm64", SHA256: hex.EncodeToString(sum[:]), Size: int64(len(body))}}}

+ for _, c := range []struct {

+ name, file, runtime string

+ m *Manifest

+ staged, wantErr bool

+ }{

+ {"rootfs", "rootfs.squashfs", registry.RuntimeRootfsV1, rfs, false, true},

+ {"binary", "linux-arm64", "", bin, false, false},

+ // The staged-copy reuse path: the bytes are already there and verify.

+ {"rootfs, staged copy", "rootfs.squashfs", registry.RuntimeRootfsV1, rfs, true, true},

+ {"binary, staged copy", "linux-arm64", "", bin, true, false},

+ } {

+ optDir := t.TempDir()

+ s := NewDirStore(t.TempDir())

+ putArtifact(t, s, "bab", "shop", "v1", c.file, body)

+ raw, _ := json.Marshal(c.m)

+ if c.staged {

+ if err := StageArtifact(optDir, "bab", "v1", "shop", body); err != nil {

+ t.Fatal(err)

+ }

+ // A directory where the sibling goes: the write's rename fails.

+ sibDir := StagedManifestPath(optDir, "bab", "v1", "shop")

+ if err := os.MkdirAll(sibDir, 0o755); err != nil {

+ t.Fatal(err)

+ }

+ if err := os.WriteFile(filepath.Join(sibDir, "occupied"), nil, 0o644); err != nil {

+ t.Fatal(err)

+ }

+ var err error

+ var out bytes.Buffer

+ withCapturedStdout(t, &out, func() {

+ _, err = stageFrom(context.Background(), s, optDir, "bab", "shop", "v1", "shop.example.com", stageCheck{runtime: c.runtime}, c.staged, c.m, raw, false)

+ })

+ switch {

+ case c.wantErr && (err == nil || !strings.Contains(err.Error(), "sibling")):

+ t.Errorf("%s: err = %v, want one naming the sibling", c.name, err)

+ case !c.wantErr && err != nil:

+ t.Errorf("%s: err = %v, want nil (the binary sibling is best-effort)", c.name, err)

+ }

+// Preflight 1.7: retained-copy recovery finds a staged image by its own key,

+// and only while its bytes still hash to the sibling's digest.

+func TestRetainedStagedFindsAStagedRootfsImage(t *testing.T) {

+ onArm64(t)

+ opt := t.TempDir()

+ image := []byte("a staged image")

+ m := stagedRootfsManifest(t, image)

+ raw, _ := json.Marshal(m)

+ if err := StageArtifact(opt, "bab", "v1", "shop", image); err != nil {

+ t.Fatal(err)

+ }

+ if err := StageManifest(opt, "bab", "v1", "shop", raw); err != nil {

+ t.Fatal(err)

+ }

+ id, got := RetainedStaged(opt, "bab", "shop", "v1")

+ if id == nil || got == nil {

+ t.Fatal("RetainedStaged found no staged image")

+ }

+ if id.Digest != etagOf(image) {

+ t.Errorf("digest = %q, want the image's sha256", id.Digest)

+ }

+ if !got.IsRootfs() {

+ t.Error("the recovered manifest is not a rootfs one")

+ }

+ if err := os.WriteFile(VersionPath(opt, "bab", "v1", "shop"), []byte("changed"), 0o644); err != nil {

+ t.Fatal(err)

+ }

+ if id, _ := RetainedStaged(opt, "bab", "shop", "v1"); id != nil {

+ t.Error("RetainedStaged vouched for bytes that no longer match")

+ }

+// Preflight 1.7: the proof gate reads a rootfs release's digest, so an

+// unproven image is held like an unproven binary.

+func TestProofGateHoldsAnUnprovenRootfsRelease(t *testing.T) {

+ onArm64(t)

+ e := newProofEnv(t)

+ image := []byte("bad image")

+ // "new" becomes a rootfs release: its binary manifest goes, the image's

+ // comes.

+ if err := e.store.Delete(e.ctx, ManifestKey("bab", "console", "new")); err != nil {

+ t.Fatal(err)

+ }

+ putRootfsRelease(t, e.store, "bab", "console", "new", image, false)

+ if err := Promote(e.ctx, e.store, e.priv, "bab", "console", "canary/x", "new"); err != nil {

+ t.Fatalf("Promote new: %v", err)

+ }

+ if err := e.reg.Delete(proofHost); err != nil {

+ t.Fatal(err)

+ }

+ if err := e.reg.Upsert(registry.Route{Host: proofHost, Kind: "instance", App: "console", Channel: "canary/x", Runtime: registry.RuntimeRootfsV1}); err != nil {

+ t.Fatal(err)

+ }

+ e.requireDev(t)

+ out := e.pass(t, e.store)

+ if !strings.Contains(out, "is not proven on dev; staying on old") {

+ t.Fatalf("printed %q, want the not-proven line", out)

+ }

+ if e.live() != "old" {

+ t.Errorf("live = %q, want old", e.live())

+ }

+ if w := e.gate.Waits(); len(w) != 1 || w[0].SHA256 != etagOf(image) {

+ t.Errorf("waits = %+v, want one naming the image's sha256", w)

+ }

+func TestFleetOwnerKey(t *testing.T) {

+ if got := FleetOwnerKey("acct1", "pi"); got != "accounts/acct1/fleets/pi.json" {

+ t.Errorf("FleetOwnerKey = %q", got)

+ }

diff --git a/internal/release/proofgate.go b/internal/release/proofgate.go

index 2f83a657a..318179aa8 100644

--- a/internal/release/proofgate.go

+++ b/internal/release/proofgate.go

@@ -306,8 +306,9 @@ func (g *ProofGate) admit(ctx context.Context, s Store, optDir string, rt regist

return v, false

}

-// manifestSHA is the manifest's sha256 for this platform, from the staged

-// sibling when there is one and the bucket otherwise, cached per promotion.

+// manifestSHA is the manifest's sha256 for this platform (its binary, or its

+// rootfs image: Manifest.BoxArtifact), from the staged sibling when there is

+// one and the bucket otherwise, cached per promotion.

func (g *ProofGate) manifestSHA(ctx context.Context, s Store, optDir, acct, app string, ptr *ChannelPointer) (string, error) {

k := promotionKey{acct, app, ptr.Version, ptr.PromotedAt.Unix()}

g.mu.Lock()

@@ -323,7 +324,8 @@ func (g *ProofGate) manifestSHA(ctx context.Context, s Store, optDir, acct, app

return "", err

}

- sha = m.Artifacts[adoptPlatform].SHA256

+ art, _ := m.BoxArtifact(adoptPlatform)

+ sha = art.SHA256

g.mu.Lock()

g.shas[k] = sha

g.mu.Unlock()

diff --git a/internal/release/release.go b/internal/release/release.go

index da8c02546..37703b24a 100644

--- a/internal/release/release.go

+++ b/internal/release/release.go

@@ -126,6 +126,49 @@ type Artifact struct {

Size int64 `json:"size"`

}

+// rootfsArtifactKeyFor is the MANIFEST artifact key a rootfs release's image

+// sits under for a box of platform: "rootfs-" + platform. On linux-arm64 it

+// is Plan 1's RootfsArtifactKey; on any other platform it names a key no

+// rootfs manifest Plan 1 accepts carries, so such a box finds no image.

+// Distinct from IsRootfsArtifactKey (rootfs_ship.go), which tests a BUCKET

+// key.

+func rootfsArtifactKeyFor(platform string) string { return "rootfs-" + platform }

+// IsRootfs reports whether m is a rootfs release.

+func (m *Manifest) IsRootfs() bool { return m.Kind == KindRootfs }

+// boxArtifactKey is the artifact key a box of platform stages from m: the

+// platform itself for a binary or static release, rootfsArtifactKeyFor for

+// a rootfs one. Never the other kind's key.

+func boxArtifactKey(m *Manifest, platform string) string {

+ if m.IsRootfs() {

+ return rootfsArtifactKeyFor(platform)

+ }

+ return platform

+// BoxArtifact is the one artifact a box of platform stages from m:

+// Artifacts[platform] for a binary, Artifacts[rootfsArtifactKeyFor(platform)]

+// for a rootfs release. Never the other key, so a rootfs release that also

+// carried a binary entry (Plan 1 refuses one) would still stage its image.

+func (m *Manifest) BoxArtifact(platform string) (Artifact, bool) {

+ a, ok := m.Artifacts[boxArtifactKey(m, platform)]

+ return a, ok

+// FleetOwnerKey is the bucket key of the record naming the account that owns

+// fleet (contract deviation 2; Plan 4 writes it).

+func FleetOwnerKey(acct, fleet string) string {

+ return "accounts/" + acct + "/fleets/" + fleet + ".json"

+// FleetOwner is the record at FleetOwnerKey.

+type FleetOwner struct {

+ Fleet string `json:"fleet"`

+ Account string `json:"account"`

+ CreatedAt int64 `json:"created_at,omitempty"`

// ChannelPointer says which version a channel currently serves. This is the

// document the fleet trusts, so it ships wrapped in signedWrapper and only

// moves via CAS.

diff --git a/internal/release/stage_stream_test.go b/internal/release/stage_stream_test.go

index 60fffa519..d78a3c799 100644

--- a/internal/release/stage_stream_test.go

+++ b/internal/release/stage_stream_test.go

@@ -84,7 +84,7 @@ func TestStageArtifactFromKeepsMemoryFlat(t *testing.T) {

runtime.GC()

var before, after runtime.MemStats

runtime.ReadMemStats(&before)

- err := stageArtifactFrom(opt, "bab", "v1", "big", want, func() (io.ReadCloser, error) {

+ err := stageArtifactFrom(opt, "bab", "v1", "big", want, 0o755, func() (io.ReadCloser, error) {

return io.NopCloser(&patternReader{n: size}), nil

})

runtime.ReadMemStats(&after)

@@ -131,7 +131,7 @@ func TestStageArtifactFromLeavesTheStagedCopyOnFailure(t *testing.T) {

if err := StageArtifact(opt, "bab", "v1", "km", []byte("old build")); err != nil {

t.Fatal(err)

}

- err := stageArtifactFrom(opt, "bab", "v1", "km", tc.want, func() (io.ReadCloser, error) {

+ err := stageArtifactFrom(opt, "bab", "v1", "km", tc.want, 0o755, func() (io.ReadCloser, error) {

return io.NopCloser(tc.r()), nil

})

if err == nil || !strings.Contains(err.Error(), "download v1") {

@@ -159,7 +159,7 @@ func TestStageArtifactFromChecksTheStoreBeforeOpening(t *testing.T) {

if err := os.WriteFile(opt, nil, 0o644); err != nil {

t.Fatal(err)

}

- err := stageArtifactFrom(opt, "bab", "v1", "km", "", func() (io.ReadCloser, error) {

+ err := stageArtifactFrom(opt, "bab", "v1", "km", "", 0o755, func() (io.ReadCloser, error) {

t.Error("the build was opened though the version store cannot be written")

return io.NopCloser(strings.NewReader("")), nil

})

@@ -182,13 +182,13 @@ func TestStageArtifactFromOverlappingWriters(t *testing.T) {

bWrote, release, bDone := make(chan struct{}), make(chan struct{}), make(chan error, 1)

a := &overlapReader{body: good, atEOF: func() {

go func() {

- bDone <- stageArtifactFrom(opt, "bab", "v1", "km", want, func() (io.ReadCloser, error) {

+ bDone <- stageArtifactFrom(opt, "bab", "v1", "km", want, 0o755, func() (io.ReadCloser, error) {

return io.NopCloser(&stallingReader{wrote: bWrote, release: release}), nil

})

}()

<-bWrote

}}

- if err := stageArtifactFrom(opt, "bab", "v1", "km", want, func() (io.ReadCloser, error) {

+ if err := stageArtifactFrom(opt, "bab", "v1", "km", want, 0o755, func() (io.ReadCloser, error) {

return io.NopCloser(a), nil

}); err != nil {

t.Fatalf("writer A: %v", err)

diff --git a/internal/release/staged.go b/internal/release/staged.go

index df1e43e28..cf0ddc256 100644

--- a/internal/release/staged.go

+++ b/internal/release/staged.go

@@ -140,19 +140,21 @@ func ResolveLiveLink(optDir, host string) (StagedIdentity, error) {

return StagedIdentity{Version: versionOfTarget(optDir, raw), StagedPath: hops[len(hops)-1]}, nil

}

-// RetainedStaged is ver's staged executable of app on this box, when its

-// bytes still hash to this platform's artifact digest in its staged manifest

-// sibling: the manifest the copy was verified against when it was staged

-// (settings spec §4.1's recovery check; root's stageAuthorized and the held

-// wake check both use it). nil when the sibling is absent or unreadable,

-// names no artifact for this platform, or the bytes differ. Demand is left

+// RetainedStaged is ver's staged build of app on this box (a binary, or a

+// rootfs image), when its bytes still hash to this platform's artifact

+// digest in its staged manifest sibling (Manifest.BoxArtifact): the manifest

+// the copy was verified against when it was staged (settings spec §4.1's

+// recovery check; root's stageAuthorized and the held wake check both use

+// it). nil when the sibling is absent or unreadable, names no artifact for

+// this platform, or the bytes differ. The manifest's kind is the caller's to

+// check against the row (adoptHost's runtimeMatches). Demand is left

// empty, as ResolveLive leaves it: only CanonicalDemand can vouch for it.

func RetainedStaged(optDir, acct, app, ver string) (*StagedIdentity, *Manifest) {

m, err := ReadStagedManifest(optDir, acct, ver, app)

if err != nil || m == nil {

return nil, nil

}

- a, ok := m.Artifacts[adoptPlatform]

+ a, ok := m.BoxArtifact(adoptPlatform)

if !ok {

return nil, nil

}

diff --git a/internal/release/versions.go b/internal/release/versions.go

index 25b7886f1..8ae314c7d 100644

--- a/internal/release/versions.go

+++ b/internal/release/versions.go

@@ -115,10 +115,13 @@ var errArtifactMismatch = errors.New("sha256 mismatch")

// leaves whatever was staged under the name untouched, exactly as verifying

// in memory first did.

+// mode is the staged file's mode: 0o755 for a binary, 0o644 for a rootfs

+// image, which is mounted and never executed.

+//

// open is called only once the version dir is writable, so a sandboxed

// agent fails on the mkdir (stageBlocked relays that to root) without

// opening the build at all.

-func stageArtifactFrom(optDir, acct, ver, app, wantSHA string, open func() (io.ReadCloser, error)) error {

+func stageArtifactFrom(optDir, acct, ver, app, wantSHA string, mode os.FileMode, open func() (io.ReadCloser, error)) error {

dst, err := stagePath(optDir, acct, ver, app)

if err != nil {

return err

@@ -128,7 +131,7 @@ func stageArtifactFrom(optDir, acct, ver, app, wantSHA string, open func() (io.R

return fmt.Errorf("download %s: %w", ver, err)

}

defer in.Close()

- return safefile.Write(dst, 0o755, func(out io.Writer) error {

+ return safefile.Write(dst, mode, func(out io.Writer) error {

h := sha256.New()

if _, err := io.Copy(io.MultiWriter(out, h), in); err != nil {

return fmt.Errorf("download %s: %w", ver, err)