amadan

carlos / platform Public

ship: carlos ship --rootfs builds the image locally and ships it (rootfs Plan 1 Task 9)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude 7a92396be528fca0617bd8c53f66a102dbb355fb parent bfa83eb
6 files changed, +975 −44
  • apimode_rootfs.go +84 −0
  • apimode_rootfs_test.go +151 −0
  • approved_rootfs_copy_test.go +2 −0
  • cli.go +91 −44
  • ship_rootfs.go +300 −0
  • ship_rootfs_test.go +347 −0
diff --git a/apimode_rootfs.go b/apimode_rootfs.go
new file mode 100644
index 000000000..0d5007db2
--- /dev/null
+++ b/apimode_rootfs.go
@@ -0,0 +1,84 @@
+package main
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "mime/multipart"
+ "net/http"
+ "time"
+
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// shipRootfsTimeout bounds one rootfs upload: up to 2 GiB, against
+// shipTimeout's 15 minutes for a binary of tens of MiB.
+const shipRootfsTimeout = 75 * time.Minute
+
+// shipOldConsoleKind is the refusal a console from before rootfs releases
+// gives kind "rootfs" (internal/console/apicli.go before rootfs apps Plan 1).
+const shipOldConsoleKind = `invalid kind (want "binary" or "static")`
+
+// errShipRootfsOldConsole is RFS-1-C44.
+var errShipRootfsOldConsole = errors.New("this console does not take rootfs releases yet. update the console, then ship again")
+
+// shipRootfs is ship for a rootfs release: the meta part first, carrying
+// the processes and the image's size so the console can refuse before
+// reading the image, then the image as the one file part
+// rootfs-linux-arm64, streamed from disk. hibernate and jit are sent only
+// when true, so the meta of a release without them names neither.
+func (s *apiSession) shipRootfs(ctx context.Context, a apiAcct, app, version string, b *builtRootfs, hibernate bool, label, notes string, messages ...string) (apiShipResult, error) {
+ message := firstMutationMessage(messages)
+ ctx, cancel := context.WithTimeout(ctx, shipRootfsTimeout)
+ defer cancel()
+
+ status, body, err := s.c.doMultipart(ctx, apiPath(a, app, "/releases"), func(mw *multipart.Writer) error {
+ metaJSON, err := json.Marshal(struct {
+ Version string `json:"version"`
+ Kind string `json:"kind"`
+ Label string `json:"label,omitempty"`
+ Notes string `json:"notes,omitempty"`
+ Message string `json:"message,omitempty"`
+ Swap string `json:"swap,omitempty"`
+ Processes map[string]release.Process `json:"processes"`
+ Hibernate bool `json:"hibernate,omitempty"`
+ JIT bool `json:"jit,omitempty"`
+ RootfsSize int64 `json:"rootfs_size"`
+ }{version, release.KindRootfs, label, notes, message, s.shipSwap, b.procs, hibernate, b.jit, b.size})
+ if err != nil {
+ return err
+ }
+ metaPart, err := mw.CreateFormField("meta")
+ if err != nil {
+ return err
+ }
+ if _, err := metaPart.Write(metaJSON); err != nil {
+ return err
+ }
+ return writeFilePart(mw, release.RootfsArtifactKey, b.image)
+ })
+ if err != nil {
+ return apiShipResult{}, err
+ }
+ switch status {
+ case http.StatusCreated, http.StatusOK:
+ if err := checkSwapEcho(s.shipSwap, body); err != nil {
+ return apiShipResult{}, err
+ }
+ var out struct {
+ Version string `json:"version"`
+ BuildProbe string `json:"build_probe"`
+ }
+ if err := json.Unmarshal(body, &out); err != nil {
+ return apiShipResult{}, fmt.Errorf("ship: malformed response: %w", err)
+ }
+ return apiShipResult{NoOp: status == http.StatusOK, Version: out.Version, BuildProbe: out.BuildProbe}, nil
+ default:
+ err := apiErrorFromBody(status, body)
+ if status == http.StatusBadRequest && err.Error() == shipOldConsoleKind {
+ return apiShipResult{}, errShipRootfsOldConsole
+ }
+ return apiShipResult{}, err
+ }
+}
diff --git a/apimode_rootfs_test.go b/apimode_rootfs_test.go
new file mode 100644
index 000000000..e11892bdb
--- /dev/null
+++ b/apimode_rootfs_test.go
@@ -0,0 +1,151 @@
+package main
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "io"
+ "maps"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "slices"
+ "testing"
+
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// rootfsTestBuilt is a builtRootfs whose image is a file of image bytes, as
+// buildRootfs leaves it for shipRootfs.
+func rootfsTestBuilt(t *testing.T, image []byte, jit bool) *builtRootfs {
+ t.Helper()
+ p := filepath.Join(t.TempDir(), "rootfs.squashfs")
+ if err := os.WriteFile(p, image, 0o600); err != nil {
+ t.Fatal(err)
+ }
+ return &builtRootfs{
+ procs: map[string]release.Process{"web": {Command: []string{"/usr/bin/shop", "serve"}}},
+ image: p, size: int64(len(image)), jit: jit,
+ }
+}
+
+type rootfsTestPart struct {
+ name, file string
+ body []byte
+}
+
+// rootfsTestConsole answers every ship with status and body, recording each
+// part it read.
+func rootfsTestConsole(t *testing.T, status int, body any) (*httptest.Server, *[]rootfsTestPart) {
+ t.Helper()
+ var parts []rootfsTestPart
+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ mr, err := r.MultipartReader()
+ if err != nil {
+ t.Errorf("request was not multipart: %v", err)
+ w.WriteHeader(http.StatusInternalServerError)
+ return
+ }
+ for {
+ part, err := mr.NextPart()
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ t.Errorf("reading part: %v", err)
+ w.WriteHeader(http.StatusInternalServerError)
+ return
+ }
+ b, err := io.ReadAll(part)
+ if err != nil {
+ t.Errorf("reading part body: %v", err)
+ w.WriteHeader(http.StatusInternalServerError)
+ return
+ }
+ parts = append(parts, rootfsTestPart{part.FormName(), part.FileName(), b})
+ }
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(status)
+ json.NewEncoder(w).Encode(body)
+ }))
+ t.Cleanup(ts.Close)
+ return ts, &parts
+}
+
+func TestAPISessionShipRootfsSendsMetaThenTheImage(t *testing.T) {
+ image := bytes.Repeat([]byte("squashfs image bytes "), 1000)
+ for _, c := range []struct {
+ name string
+ hibernate, jit bool
+ want []string
+ }{
+ {"plain", false, false, []string{"kind", "processes", "rootfs_size", "version"}},
+ {"hibernate", true, false, []string{"hibernate", "kind", "processes", "rootfs_size", "version"}},
+ {"jit", false, true, []string{"jit", "kind", "processes", "rootfs_size", "version"}},
+ } {
+ t.Run(c.name, func(t *testing.T) {
+ ts, parts := rootfsTestConsole(t, http.StatusCreated, map[string]any{
+ "result": "shipped", "version": "0.1.0-1", "artifacts": map[string]any{},
+ })
+ s := newTestSession(ts.URL)
+ b := rootfsTestBuilt(t, image, c.jit)
+ res, err := s.shipRootfs(context.Background(), apiAcct{Seg: "acct"}, "shop", "0.1.0-1", b, c.hibernate, "", "")
+ if err != nil {
+ t.Fatalf("shipRootfs: %v", err)
+ }
+ if res.NoOp || res.Version != "0.1.0-1" {
+ t.Errorf("result %+v", res)
+ }
+ if len(*parts) != 2 {
+ t.Fatalf("%d parts, want meta and the image", len(*parts))
+ }
+ meta, img := (*parts)[0], (*parts)[1]
+ if meta.name != "meta" {
+ t.Fatalf("first part %q, want meta", meta.name)
+ }
+ var got map[string]json.RawMessage
+ if err := json.Unmarshal(meta.body, &got); err != nil {
+ t.Fatalf("meta %s: %v", meta.body, err)
+ }
+ if keys := slices.Sorted(maps.Keys(got)); !slices.Equal(keys, c.want) {
+ t.Errorf("meta keys %v, want %v (%s)", keys, c.want, meta.body)
+ }
+ var m struct {
+ Version string `json:"version"`
+ Kind string `json:"kind"`
+ Processes map[string]release.Process `json:"processes"`
+ Hibernate bool `json:"hibernate"`
+ JIT bool `json:"jit"`
+ RootfsSize int64 `json:"rootfs_size"`
+ }
+ if err := json.Unmarshal(meta.body, &m); err != nil {
+ t.Fatal(err)
+ }
+ if m.Version != "0.1.0-1" || m.Kind != "rootfs" || m.RootfsSize != int64(len(image)) ||
+ m.Hibernate != c.hibernate || m.JIT != c.jit || !release.ProcessesEqual(m.Processes, b.procs) {
+ t.Errorf("meta %s", meta.body)
+ }
+ if img.name != "rootfs-linux-arm64" || img.file == "" || !bytes.Equal(img.body, image) {
+ t.Errorf("image part %q (file %q), %d bytes", img.name, img.file, len(img.body))
+ }
+ })
+ }
+}
+
+func TestAPISessionShipRootfsNamesAnOldConsole(t *testing.T) {
+ ts, _ := rootfsTestConsole(t, http.StatusBadRequest, map[string]string{"error": `invalid kind (want "binary" or "static")`})
+ s := newTestSession(ts.URL)
+ _, err := s.shipRootfs(context.Background(), apiAcct{Seg: "acct"}, "shop", "0.1.0-1", rootfsTestBuilt(t, []byte("image"), false), false, "", "")
+ if want := "this console does not take rootfs releases yet. update the console, then ship again"; err == nil || err.Error() != want {
+ t.Fatalf("shipRootfs: %v, want %q", err, want)
+ }
+
+ // Any other refusal is the console's own sentence.
+ ts, _ = rootfsTestConsole(t, http.StatusConflict, map[string]string{"error": "shop is a binary app"})
+ s = newTestSession(ts.URL)
+ _, err = s.shipRootfs(context.Background(), apiAcct{Seg: "acct"}, "shop", "0.1.0-1", rootfsTestBuilt(t, []byte("image"), false), false, "", "")
+ if err == nil || err.Error() != "shop is a binary app" {
+ t.Fatalf("shipRootfs: %v, want the console's refusal", err)
+ }
+}
diff --git a/approved_rootfs_copy_test.go b/approved_rootfs_copy_test.go
index c2ee4b454..6ebacb071 100644
--- a/approved_rootfs_copy_test.go
+++ b/approved_rootfs_copy_test.go
@@ -160,6 +160,8 @@ var rootfsCopyBuilt = []string{
"RFS-1-C48", "RFS-1-C54",
// Task 8: the console's ship door takes rootfs (internal/console/apicli.go, apiship_rootfs.go)
"RFS-1-C22", "RFS-1-C23", "RFS-1-C24", "RFS-1-C25", "RFS-1-C26", "RFS-1-C27", "RFS-1-C28", "RFS-1-C29", "RFS-1-C79",
+ // Task 9: carlos ship --rootfs (cli.go, ship_rootfs.go, apimode_rootfs.go)
+ "RFS-1-C30", "RFS-1-C31", "RFS-1-C32", "RFS-1-C33", "RFS-1-C34", "RFS-1-C35", "RFS-1-C36", "RFS-1-C37", "RFS-1-C38", "RFS-1-C39", "RFS-1-C40", "RFS-1-C41", "RFS-1-C42", "RFS-1-C43", "RFS-1-C44", "RFS-1-C84",
}
func mustParseRootfsRecord(t *testing.T) []copyRow {
diff --git a/cli.go b/cli.go
index 135fda274..d959c5d11 100644
--- a/cli.go
+++ b/cli.go
@@ -2170,8 +2170,30 @@ func runShip(args []string) {
notesFile := fs.String("notes-file", "", "file holding the changelog fragment (mutually exclusive with --notes)")
swap := fs.String("swap", "", shipSwapFlagHelp)
programChange := fs.Bool("program-change", false, "accept that this release's Go main package differs from the app's previous release (recorded in the manifest)")
+ rootfs := fs.String("rootfs", "", "an OCI image (a layout directory, a docker save archive or a local image name) or a directory, shipped as a rootfs release")
+ processes := fs.String("processes", "", "JSON file holding the rootfs release's processes: web, any workers, release and console")
+ hibernate := fs.Bool("hibernate", false, "let this rootfs app hibernate when idle. Rootfs apps are always on unless their first release opts in")
+ jit := fs.Bool("jit", false, "let this rootfs release's processes run a JIT compiler, as Node, Bun, Deno, the JVM and Ruby's YJIT need. without it a box never lets memory be writable and executable at once")
+ check := fs.Bool("check", false, "run the rootfs image here with Docker or Podman, the way a box would, and ship nothing")
fs.Usage = flagUsage(fs, "")
fs.Parse(args)
+ // A rootfs release is its own body (ship_rootfs.go): its flags are
+ // refused anywhere else, and the binary and static path below never
+ // sees one.
+ if *rootfs == "" {
+ if err := shipRootfsOnlyRefusal(fs); err != nil {
+ log.Fatal(err)
+ }
+ } else {
+ runShipRootfs(fs, rootfsShipOpts{
+ src: *rootfs, processesFile: *processes,
+ hibernate: *hibernate, jit: *jit, check: *check,
+ app: *app, ver: *ver, label: *label, notes: *notes, notesFile: *notesFile, swap: *swap,
+ console: *consoleFlag, acctRaw: acctRaw, acctIDRaw: acctIDRaw, acctOf: acctOf,
+ messageFlags: messageFlags,
+ })
+ return
+ }
// --version is no longer required: with a version target set, the next
// iteration under it is inferable, and inferring it is the whole point of
// the target (release versioning spec, 2026-08-08). Everything else about
@@ -2182,18 +2204,9 @@ func runShip(args []string) {
log.Fatal(usage)
}
- // --notes and --notes-file are two spellings of ONE manifest field, so both
- // together is a refusal rather than a precedence rule nobody remembers.
- if *notes != "" && *notesFile != "" {
- log.Fatal("pass --notes or --notes-file, not both")
- }
- noteText := *notes
- if *notesFile != "" {
- b, err := os.ReadFile(*notesFile)
- if err != nil {
- log.Fatal(err)
- }
- noteText = string(b)
+ noteText, err := shipNoteText(*notes, *notesFile)
+ if err != nil {
+ log.Fatal(err)
}
// Bounded HERE, before the mode split, for the same reason the capability
// block is: both doors must refuse the same bytes with the same message
@@ -2221,7 +2234,6 @@ func runShip(args []string) {
// bytes with the same message; an omitted --app just moves the moment.
var capsSet *caps.Set
if *app != "" {
- var err error
capsSet, err = readCapabilities(*capsFile, *app)
if err != nil {
log.Fatal(err)
@@ -2310,28 +2322,7 @@ func runShip(args []string) {
// drift from them), and release.Ship writes through the same handle.
st := openStore()
acct := acctOf()
- shipVersion := *ver
- if shipVersion == "" {
- vt, _, err := release.GetTarget(context.Background(), st, acct, *app)
- if err != nil {
- log.Fatal(err)
- }
- if vt == nil {
- log.Fatal("no --version and no version target set — set one with `carlos version target --app " + *app + " <semver>`, or pass --version")
- }
- t, ok := release.ParseSemVer(vt.Target)
- if !ok {
- // GetTarget already refuses a stored target that isn't a bare
- // semver, so this is unreachable through that door — kept because
- // the alternative to a named refusal here is a silent zero SemVer
- // minting "0.0.0-1" against an app nobody meant to touch.
- log.Fatalf("stored version target %q is not a semver — set it again", vt.Target)
- }
- shipVersion, err = release.NextIteration(context.Background(), st, acct, *app, t)
- if err != nil {
- log.Fatal(err)
- }
- }
+ shipVersion := mintDirectVersion(st, acct, *app, *ver)
m := release.Manifest{
Schema: 1, App: *app, Version: shipVersion, Kind: *kind,
BuiltBy: "carlos " + version.Build, BuiltAt: shipNow().UTC().Format(time.RFC3339),
@@ -2371,6 +2362,54 @@ func runShip(args []string) {
}
}
+// shipNoteText is the changelog fragment --notes or --notes-file gives.
+// --notes and --notes-file are two spellings of ONE manifest field, so both
+// together is a refusal rather than a precedence rule nobody remembers.
+func shipNoteText(notes, notesFile string) (string, error) {
+ if notes != "" && notesFile != "" {
+ return "", errors.New("pass --notes or --notes-file, not both")
+ }
+ if notesFile == "" {
+ return notes, nil
+ }
+ b, err := os.ReadFile(notesFile)
+ if err != nil {
+ return "", err
+ }
+ return string(b), nil
+}
+
+// mintDirectVersion is the version a direct-mode ship writes: ver when the
+// operator gave one, else the next iteration under the app's version
+// target. The release dirs are the iteration counter (there is no side
+// counter to drift from them), so it reads st. It exits on a refusal, so a
+// caller holding anything to clean up calls it first.
+func mintDirectVersion(st release.Store, acct, app, ver string) string {
+ if ver != "" {
+ return ver
+ }
+ vt, _, err := release.GetTarget(context.Background(), st, acct, app)
+ if err != nil {
+ log.Fatal(err)
+ }
+ if vt == nil {
+ log.Fatal("no --version and no version target set — set one with `carlos version target --app " + app + " <semver>`, or pass --version")
+ }
+ t, ok := release.ParseSemVer(vt.Target)
+ if !ok {
+ // GetTarget already refuses a stored target that isn't a bare
+ // semver, so this is unreachable through that door — kept because
+ // the alternative to a named refusal here is a silent zero SemVer
+ // minting "0.0.0-1" against an app nobody meant to touch.
+ log.Fatalf("stored version target %q is not a semver — set it again", vt.Target)
+ }
+ next, err := release.NextIteration(context.Background(), st, acct, app, t)
+ if err != nil {
+ log.Fatal(err)
+ }
+ return next
+}
+
// readCapabilities is the ship-time declaration surface: a file holding
// EXACTLY the manifest's "capabilities" object (not a whole manifest, not a
// wrapper), decoded through caps.Set's own strict door and then through the
@@ -2504,10 +2543,23 @@ func runShipAPI(sess *apiSession, acct apiAcct, fs *flag.FlagSet, app, ver, kind
if err != nil {
return apiShipResult{}, err
}
- // res.Version, never ver: when the caller passed no --version the CONSOLE
- // minted one from the app's target, and the response is the only place
- // that name exists. It echoes the sent version verbatim otherwise, so
- // this is the right thing to print on both paths rather than two.
+ printAPIShipped(sess, app, res)
+ // res carries more than the print above needs: BuildProbe is the app's
+ // build-report path, which `carlos deploy` turns into its watch's exit
+ // criterion. Returned rather than acted on here — ship itself has no
+ // watch.
+ return res, nil
+}
+
+// printAPIShipped is the one account of what an API-mode ship did, for
+// every kind (runShipAPI, and ship_rootfs.go's rootfs body).
+//
+// res.Version, never the version sent: when the caller passed no --version
+// the CONSOLE minted one from the app's target, and the response is the
+// only place that name exists. It echoes the sent version verbatim
+// otherwise, so this is the right thing to print on both paths rather than
+// two.
+func printAPIShipped(sess *apiSession, app string, res apiShipResult) {
if res.NoOp {
mutedf(stdout, "already shipped %s %s — identical content, nothing to do\n", app, res.Version)
} else {
@@ -2516,11 +2568,6 @@ func runShipAPI(sess *apiSession, acct apiAcct, fs *flag.FlagSet, app, ver, kind
if sess.shipSwap == release.DemandHold {
mutedf(stdout, shipHoldNote)
}
- // res carries more than the print above needs: BuildProbe is the app's
- // build-report path, which `carlos deploy` turns into its watch's exit
- // criterion. Returned rather than acted on here — ship itself has no
- // watch.
- return res, nil
}
// runPromote points a channel at a shipped version, with the ladder enforced:
diff --git a/ship_rootfs.go b/ship_rootfs.go
new file mode 100644
index 000000000..77470e32d
--- /dev/null
+++ b/ship_rootfs.go
@@ -0,0 +1,300 @@
+package main
+
+import (
+ "context"
+ "errors"
+ "flag"
+ "fmt"
+ "log"
+ "os"
+ "path/filepath"
+ "strings"
+ "time"
+
+ "github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/rootfsimage"
+ "github.com/carlosframework/platform/internal/version"
+)
+
+// carlos ship --rootfs: a rootfs release (rootfs apps Plan 1), built here
+// from an OCI image or a directory into one squashfs image, then shipped
+// through the console (API mode) or straight to the bucket (direct mode).
+
+// shipRootfsUsage is the rootfs form of ship's usage (RFS-1-C34).
+const shipRootfsUsage = "usage: carlos ship --app <a> --rootfs <image-or-dir> --processes <file.json> [--hibernate] [--jit] [--version <sha>] [--label <one line>] [--notes <text> | --notes-file <file>] [--swap hold] [--account <sqid-or-name>] [--account-id <sqid>] [--console <origin>]\n carlos ship --rootfs <image-or-dir> --processes <file.json> [--hibernate] [--jit] --check"
+
+// shipRootfsOnlyFlags are the flags that mean something only with --rootfs,
+// in the order their refusal names them (RFS-1-C38).
+var shipRootfsOnlyFlags = []string{"check", "hibernate", "jit", "processes"}
+
+// shipRootfsExcludedFlags are ship's binary and static flags, which a
+// rootfs release has no use for, in the order their refusal names them
+// (RFS-1-C35).
+var shipRootfsExcludedFlags = []string{"kind", "target", "capabilities", "program-change"}
+
+// shipRootfsMaxBytes is the largest image a ship sends (RFS-1-C43). It is
+// release.MaxRootfsBytes, the bound WriteSquashfs already writes under, so
+// with this value the check never fires for an image this binary built;
+// it stays so the bound is stated where the ship decides to send, and a
+// test lowers it to prove nothing is sent past it.
+var shipRootfsMaxBytes = release.MaxRootfsBytes
+
+// shipRootfsOnlyRefusal refuses a rootfs-only flag given without --rootfs
+// (RFS-1-C38). runShip calls it on its binary and static path.
+func shipRootfsOnlyRefusal(fs *flag.FlagSet) error {
+ for _, name := range shipRootfsOnlyFlags {
+ if flagWasSet(fs, name) {
+ return fmt.Errorf("--%s works only with --rootfs", name)
+ }
+ }
+ return nil
+}
+
+// rootfsShipOpts is one carlos ship --rootfs: the flags runShip parsed,
+// then what runShipRootfs resolves from them before anything is built.
+type rootfsShipOpts struct {
+ src, processesFile string
+ hibernate, jit, check bool
+ app, ver, label, notes, swap string
+ notesFile, console string
+ acctRaw, acctIDRaw *string
+ acctOf func() string
+ messageFlags mutationMessageFlags
+
+ // Resolved by runShipRootfs.
+ procs map[string]release.Process
+ sess *apiSession // API mode
+ acct apiAcct // API mode
+ message string // API mode
+ st release.Store // direct mode
+ acctName string // direct mode
+}
+
+// builtRootfs is a rootfs image built on this machine: the tree it was
+// built from, the processes with the image's config merged in and console
+// filled, the squashfs file (empty for a --check build, which writes none)
+// and its size, and --jit, carried for the manifest and --check's warning.
+type builtRootfs struct {
+ tree *rootfsimage.Tree
+ procs map[string]release.Process
+ image string
+ size int64
+ jit bool
+}
+
+// errRootfsCheckPending is what a --check build answers until the check
+// itself lands (Plan 1 Task 11): runShipRootfs exits 2 on it, saying
+// nothing more.
+var errRootfsCheckPending = errors.New("ship: --check is not built yet")
+
+// runShipRootfs is runShip's rootfs body. Everything that can exit (the
+// flag rules, mode and account resolution, the version mint) runs here,
+// before the scratch exists; the build and the ship run in shipRootfsRun,
+// which returns its errors so the scratch is removed on every path.
+func runShipRootfs(fs *flag.FlagSet, o rootfsShipOpts) {
+ for _, name := range shipRootfsExcludedFlags {
+ if flagWasSet(fs, name) {
+ log.Fatalf("--rootfs cannot be combined with --%s", name)
+ }
+ }
+ if fs.NArg() > 0 {
+ log.Fatal("--rootfs takes the image or directory as its value, so leave the other arguments off")
+ }
+ if o.processesFile == "" {
+ log.Fatal("--rootfs needs --processes, a JSON file holding the release's processes")
+ }
+
+ // --check runs the image here and ships nothing: it needs no app, no
+ // login and no console.
+ var sess *apiSession
+ if !o.check {
+ var err error
+ if sess, err = resolveMode(o.console); err != nil {
+ log.Fatal(err)
+ }
+ if sess == nil && o.app == "" {
+ log.Fatal(shipRootfsUsage)
+ }
+ }
+
+ procs, err := readProcessesFile(o.processesFile)
+ if err != nil {
+ log.Fatal(err)
+ }
+ o.procs = procs
+ if !o.check {
+ if o.notes, err = shipNoteText(o.notes, o.notesFile); err != nil {
+ log.Fatal(err)
+ }
+ if err := release.ValidateLabelNotes(o.label, o.notes); err != nil {
+ log.Fatal(err)
+ }
+ if err := validSwapFlag(o.swap); err != nil {
+ log.Fatal(err)
+ }
+ if sess != nil {
+ if o.message, err = resolveMutationMessage(fs, o.messageFlags); err != nil {
+ log.Fatal(err)
+ }
+ // The banner before the account resolves, as runShip prints it.
+ printUsing(stdout, sess)
+ o.acct = resolvedAccount(sess, o.acctRaw, o.acctIDRaw)
+ o.app = resolvedApp(sess, o.acct, o.app, shipRootfsUsage)
+ sess.shipSwap = o.swap
+ o.sess = sess
+ } else {
+ // Minted now, not after the build: a mint refusal exits.
+ o.st = openStore()
+ o.acctName = o.acctOf()
+ o.ver = mintDirectVersion(o.st, o.acctName, o.app, o.ver)
+ }
+ }
+
+ err = shipRootfsRun(context.Background(), o)
+ if errors.Is(err, errRootfsCheckPending) {
+ os.Exit(2)
+ }
+ if err != nil {
+ log.Fatal(err)
+ }
+}
+
+// shipRootfsRun builds the image in a scratch directory it removes on
+// every path, then ships it: through o.sess when set, else to o.st. A
+// --check build writes no image and answers errRootfsCheckPending.
+func shipRootfsRun(ctx context.Context, o rootfsShipOpts) error {
+ scratch, err := os.MkdirTemp("", "carlos-rootfs-*")
+ if err != nil {
+ return err
+ }
+ defer os.RemoveAll(scratch)
+
+ b, err := buildRootfs(ctx, o.src, o.procs, scratch, !o.check)
+ if err != nil {
+ return err
+ }
+ b.jit = o.jit
+ if o.check {
+ return errRootfsCheckPending
+ }
+ if b.size > shipRootfsMaxBytes {
+ return fmt.Errorf("the rootfs image is %s, over the 2 GiB one release can carry. make the image smaller and ship again", humanBytes(&b.size))
+ }
+
+ if o.sess != nil {
+ res, err := o.sess.shipRootfs(ctx, o.acct, o.app, o.ver, b, o.hibernate, o.label, o.notes, o.message)
+ if err != nil {
+ return err
+ }
+ printAPIShipped(o.sess, o.app, res)
+ return nil
+ }
+
+ m := release.Manifest{
+ Schema: 1, App: o.app, Version: o.ver, Kind: release.KindRootfs,
+ BuiltBy: "carlos " + version.Build, BuiltAt: shipNow().UTC().Format(time.RFC3339),
+ Artifacts: map[string]release.Artifact{release.RootfsArtifactKey: {File: release.RootfsArtifactFile}},
+ Processes: b.procs,
+ Hibernate: o.hibernate,
+ JIT: b.jit,
+ Label: o.label,
+ Notes: o.notes,
+ Swap: o.swap,
+ }
+ // Refused before a byte is written; ShipSpooled checks again at
+ // publication, which is the check that counts.
+ if err := release.CheckShipKind(ctx, o.st, o.acctName, &m); err != nil {
+ return err
+ }
+ if err := release.ShipSpooled(ctx, o.st, o.acctName, m, map[string]string{release.RootfsArtifactKey: b.image}); err != nil {
+ return err
+ }
+ fmt.Fprintf(stdout, "%s %s %s (%d artifacts)\n", colorFor(stdout).ok("shipped"), o.app, o.ver, len(m.Artifacts))
+ if o.swap == release.DemandHold {
+ mutedf(stdout, shipHoldNote)
+ }
+ return nil
+}
+
+// readProcessesFile reads --processes: exactly the manifest's processes
+// object, decoded strictly (release.DecodeProcesses). Its refusals name the
+// file (RFS-1-C39).
+func readProcessesFile(path string) (map[string]release.Process, error) {
+ raw, err := os.ReadFile(path)
+ if err == nil {
+ var procs map[string]release.Process
+ if procs, err = release.DecodeProcesses(raw); err == nil {
+ return procs, nil
+ }
+ }
+ return nil, fmt.Errorf("--processes %s: %w", path, err)
+}
+
+// buildRootfs opens src and makes it a rootfs release's image: the mount
+// points a box needs added, a directory's programs checked for arm64, the
+// image's config merged into procs, console filled in for a framework it
+// knows (RFS-1-C40), the processes validated, then their commands resolved
+// in the image. With write it writes the squashfs image to
+// <scratch>/rootfs.squashfs (RFS-1-C42). The tree's spool is under scratch,
+// which the caller removes.
+//
+// The order matters: console is filled after the config is merged, so it
+// is looked up on the image's PATH, and the processes are validated before
+// anything reads a command's first word.
+func buildRootfs(ctx context.Context, src string, procs map[string]release.Process, scratch string, write bool) (*builtRootfs, error) {
+ spool := filepath.Join(scratch, "spool")
+ if err := os.Mkdir(spool, 0o700); err != nil {
+ return nil, err
+ }
+ t, err := rootfsimage.Open(src, spool)
+ if err != nil {
+ return nil, err
+ }
+ if t.Skipped > 0 {
+ fmt.Fprintln(stdout, fmt.Sprintf("left out %d device files, sockets and pipes: a rootfs image cannot carry them", t.Skipped))
+ }
+ if err := rootfsimage.AddMountPoints(t); err != nil {
+ return nil, err
+ }
+ // Only a directory source has no image config: an OCI image always
+ // names its architecture (OpenLayout picks arm64 by it).
+ if t.Config.Architecture == "" {
+ if err := rootfsimage.CheckProgramsArch(t); err != nil {
+ return nil, err
+ }
+ }
+ if procs, err = rootfsimage.ApplyImageConfig(procs, t.Config); err != nil {
+ return nil, err
+ }
+ procs, fw := rootfsimage.FillConsole(t, procs)
+ if fw != "" {
+ fmt.Fprintln(stdout, fmt.Sprintf("filled in console: %s, because this looks like a %s app. declare console in --processes to choose another",
+ strings.Join(procs[release.ProcessConsole].Command, " "), fw))
+ }
+ if err := release.ValidateProcesses(procs); err != nil {
+ return nil, err
+ }
+ if err := rootfsimage.ResolveCommands(t, procs); err != nil {
+ return nil, err
+ }
+ b := &builtRootfs{tree: t, procs: procs}
+ if !write {
+ return b, nil
+ }
+
+ image := filepath.Join(scratch, release.RootfsArtifactFile)
+ f, err := os.OpenFile(image, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0o600)
+ if err != nil {
+ return nil, err
+ }
+ st, err := rootfsimage.WriteSquashfs(t, f)
+ if cerr := f.Close(); err == nil {
+ err = cerr
+ }
+ if err != nil {
+ return nil, err
+ }
+ fmt.Fprintln(stdout, fmt.Sprintf("built the rootfs image: %s, %d files.", humanBytes(&st.Bytes), st.Files))
+ b.image, b.size = image, st.Bytes
+ return b, nil
+}
diff --git a/ship_rootfs_test.go b/ship_rootfs_test.go
new file mode 100644
index 000000000..038fc4884
--- /dev/null
+++ b/ship_rootfs_test.go
@@ -0,0 +1,347 @@
+package main
+
+import (
+ "bytes"
+ "context"
+ "debug/elf"
+ "encoding/binary"
+ "errors"
+ "io/fs"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/KarpelesLab/squashfs"
+
+ "github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/rootfsimage"
+)
+
+// rootfsTestELF is a 64-byte little-endian ELF64 header for machine: enough
+// for rootfsimage.CheckProgramsArch, which reads only the header.
+func rootfsTestELF(machine elf.Machine) []byte {
+ h := make([]byte, 64)
+ copy(h, "\x7fELF")
+ h[4], h[5], h[6] = 2, 1, 1 // 64-bit, little-endian, version 1
+ binary.LittleEndian.PutUint16(h[16:], 2)
+ binary.LittleEndian.PutUint16(h[18:], uint16(machine))
+ return h
+}
+
+// rootfsRailsDir is a directory source shaped like a Rails app on arm64:
+// rails/bin/rails and usr/bin/ruby are aarch64 programs, and
+// rails/config/application.rb makes FillConsole see Rails. It returns the
+// directory and a processes file whose web runs bin/rails from /rails.
+func rootfsRailsDir(t *testing.T) (dir, procs string) {
+ t.Helper()
+ dir = t.TempDir()
+ files := []struct {
+ path string
+ body []byte
+ mode os.FileMode
+ }{
+ {"rails/bin/rails", rootfsTestELF(elf.EM_AARCH64), 0o755},
+ {"rails/config/application.rb", []byte("module Shop; end\n"), 0o644},
+ {"usr/bin/ruby", rootfsTestELF(elf.EM_AARCH64), 0o755},
+ }
+ for _, f := range files {
+ p := filepath.Join(dir, f.path)
+ if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(p, f.body, f.mode); err != nil {
+ t.Fatal(err)
+ }
+ }
+ procs = filepath.Join(t.TempDir(), "processes.json")
+ if err := os.WriteFile(procs, []byte(`{"web": {"command": ["bin/rails", "server"], "working_dir": "/rails"}}`), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ return dir, procs
+}
+
+// rootfsStoredFiles is every regular file in a DirStore bucket.
+func rootfsStoredFiles(t *testing.T, bucket string) []string {
+ t.Helper()
+ var out []string
+ err := filepath.WalkDir(bucket, func(p string, d fs.DirEntry, err error) error {
+ if err != nil {
+ return err
+ }
+ if d.Type().IsRegular() {
+ out = append(out, p)
+ }
+ return nil
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ return out
+}
+
+// rootfsScratchLeft is every entry of dir named carlos-rootfs-*.
+func rootfsScratchLeft(t *testing.T, dir string) []string {
+ t.Helper()
+ ents, err := os.ReadDir(dir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var out []string
+ for _, e := range ents {
+ if strings.HasPrefix(e.Name(), "carlos-rootfs-") {
+ out = append(out, e.Name())
+ }
+ }
+ return out
+}
+
+func TestShipRootfsFlagRefusals(t *testing.T) {
+ if testing.Short() {
+ t.Skip("runs the carlos binary")
+ }
+ run, bucket := versioningCLI(t)
+ dir, procs := rootfsRailsDir(t)
+ caps := filepath.Join(t.TempDir(), "caps.json")
+ if err := os.WriteFile(caps, []byte(`{}`), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ cases := []struct {
+ name string
+ args []string
+ want string
+ }{
+ {"check without rootfs", []string{"-app", "shop", "--check"}, "--check works only with --rootfs"},
+ {"hibernate without rootfs", []string{"-app", "shop", "--hibernate", "x"}, "--hibernate works only with --rootfs"},
+ {"jit without rootfs", []string{"-app", "shop", "--jit", "x"}, "--jit works only with --rootfs"},
+ {"processes without rootfs", []string{"-app", "shop", "--processes", procs, "x"}, "--processes works only with --rootfs"},
+ {"rootfs with kind", []string{"-app", "shop", "--rootfs", dir, "--processes", procs, "--kind", "static"}, "--rootfs cannot be combined with --kind"},
+ {"rootfs with target", []string{"-app", "shop", "--rootfs", dir, "--processes", procs, "--target", "linux-arm64"}, "--rootfs cannot be combined with --target"},
+ {"rootfs with capabilities", []string{"-app", "shop", "--rootfs", dir, "--processes", procs, "--capabilities", caps}, "--rootfs cannot be combined with --capabilities"},
+ {"rootfs with program-change", []string{"-app", "shop", "--rootfs", dir, "--processes", procs, "--program-change"}, "--rootfs cannot be combined with --program-change"},
+ {"rootfs with positional arguments", []string{"-app", "shop", "--rootfs", dir, "--processes", procs, "extra"}, "--rootfs takes the image or directory as its value, so leave the other arguments off"},
+ {"rootfs without processes", []string{"-app", "shop", "--rootfs", dir}, "--rootfs needs --processes, a JSON file holding the release's processes"},
+ {"rootfs without an app", []string{"--rootfs", dir, "--processes", procs}, "usage: carlos ship --app <a> --rootfs <image-or-dir> --processes <file.json> [--hibernate] [--jit]"},
+ }
+ for _, c := range cases {
+ t.Run(c.name, func(t *testing.T) {
+ out, err := run(append([]string{"ship"}, c.args...)...)
+ if err == nil {
+ t.Fatalf("ship %v succeeded:\n%s", c.args, out)
+ }
+ if !strings.Contains(out, c.want) {
+ t.Fatalf("ship %v does not say %q:\n%s", c.args, c.want, out)
+ }
+ })
+ }
+ if files := rootfsStoredFiles(t, bucket); len(files) != 0 {
+ t.Errorf("refused ships stored %v", files)
+ }
+
+ out, _ := run("ship", "-h")
+ if want := "let this rootfs release's processes run a JIT compiler, as Node, Bun, Deno, the JVM and Ruby's YJIT need. without it a box never lets memory be writable and executable at once"; !strings.Contains(out, want) {
+ t.Errorf("ship -h does not give --jit's help:\n%s", out)
+ }
+}
+
+func TestReadProcessesFileIsStrict(t *testing.T) {
+ dir := t.TempDir()
+ write := func(name, body string) string {
+ p := filepath.Join(dir, name)
+ if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ return p
+ }
+
+ unknown := write("unknown.json", `{"web": {"command": ["serve"], "cwd": "/app"}}`)
+ _, err := readProcessesFile(unknown)
+ if want := "--processes " + unknown + ": malformed processes block. want the processes object: web, any workers, release and console"; err == nil || err.Error() != want {
+ t.Errorf("unknown key: %v, want %q", err, want)
+ }
+
+ null := write("null.json", `null`)
+ _, err = readProcessesFile(null)
+ if want := "--processes " + null + ": processes: null. a rootfs release needs its processes, starting with web"; err == nil || err.Error() != want {
+ t.Errorf("null: %v, want %q", err, want)
+ }
+
+ _, err = readProcessesFile(filepath.Join(dir, "missing.json"))
+ if err == nil || !strings.HasPrefix(err.Error(), "--processes "+filepath.Join(dir, "missing.json")+": ") || !errors.Is(err, fs.ErrNotExist) {
+ t.Errorf("missing file: %v", err)
+ }
+
+ valid := write("valid.json", `{"web": {"command": ["bin/rails", "server"], "working_dir": "/rails"}, "jobs": {"command": ["bin/jobs"]}}`)
+ procs, err := readProcessesFile(valid)
+ if err != nil {
+ t.Fatalf("valid: %v", err)
+ }
+ if len(procs) != 2 || procs["web"].WorkingDir != "/rails" || procs["jobs"].Command[0] != "bin/jobs" {
+ t.Errorf("valid decoded as %+v", procs)
+ }
+}
+
+func TestShipRootfsDirectFromADirectory(t *testing.T) {
+ if testing.Short() {
+ t.Skip("runs the carlos binary")
+ }
+ run, bucket := versioningCLI(t)
+ dir, procs := rootfsRailsDir(t)
+
+ out, err := run("ship", "-app", "shop", "-version", "0.1.0-1", "--rootfs", dir, "--processes", procs)
+ if err != nil {
+ t.Fatalf("ship --rootfs: %v\n%s", err, out)
+ }
+ if want := "filled in console: bin/rails console, because this looks like a Rails app. declare console in --processes to choose another"; !strings.Contains(out, want) {
+ t.Errorf("ship does not say %q:\n%s", want, out)
+ }
+ if !strings.Contains(out, "built the rootfs image: ") || !strings.Contains(out, "shipped shop 0.1.0-1") {
+ t.Errorf("ship output:\n%s", out)
+ }
+
+ st := release.NewDirStore(bucket)
+ ctx := context.Background()
+ m, err := release.GetManifest(ctx, st, release.OpsAccount, "shop", "0.1.0-1")
+ if err != nil {
+ t.Fatalf("manifest 0.1.0-1: %v", err)
+ }
+ if m.Kind != release.KindRootfs || m.JIT || m.Hibernate {
+ t.Errorf("manifest kind %q, jit %v, hibernate %v", m.Kind, m.JIT, m.Hibernate)
+ }
+ art, ok := m.Artifacts[release.RootfsArtifactKey]
+ if len(m.Artifacts) != 1 || !ok || art.File != release.RootfsArtifactFile {
+ t.Fatalf("artifacts %+v", m.Artifacts)
+ }
+ console, ok := m.Processes[release.ProcessConsole]
+ if !ok || strings.Join(console.Command, " ") != "bin/rails console" || console.WorkingDir != "/rails" {
+ t.Errorf("console %+v (filled: %v)", console, ok)
+ }
+ if !strings.HasPrefix(m.BuiltBy, "carlos ") {
+ t.Errorf("built by %q", m.BuiltBy)
+ }
+
+ out, err = run("ship", "-app", "shop", "-version", "0.1.0-2", "--rootfs", dir, "--processes", procs, "--jit")
+ if err != nil {
+ t.Fatalf("ship --rootfs --jit: %v\n%s", err, out)
+ }
+ m2, err := release.GetManifest(ctx, st, release.OpsAccount, "shop", "0.1.0-2")
+ if err != nil {
+ t.Fatalf("manifest 0.1.0-2: %v", err)
+ }
+ if !m2.JIT {
+ t.Error("--jit did not reach the manifest")
+ }
+
+ img := filepath.Join(bucket, release.ArtifactKey(release.OpsAccount, "shop", "0.1.0-1", release.RootfsArtifactFile))
+ f, err := os.Open(img)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer f.Close()
+ if fi, err := f.Stat(); err != nil || fi.Size() != art.Size {
+ t.Fatalf("stored image: %v, size %d, manifest %d", err, fi.Size(), art.Size)
+ }
+ sb, err := squashfs.New(f)
+ if err != nil {
+ t.Fatalf("squashfs.New: %v", err)
+ }
+ want := append([]string{"rails/bin/rails", "rails/config/application.rb", "usr/bin/ruby"}, rootfsimage.MountPoints...)
+ want = append(want, rootfsimage.MountFiles...)
+ for _, p := range want {
+ if _, err := fs.Stat(sb, p); err != nil {
+ t.Errorf("image lacks %s: %v", p, err)
+ }
+ }
+ body, err := fs.ReadFile(sb, "rails/bin/rails")
+ if err != nil || !bytes.Equal(body, rootfsTestELF(elf.EM_AARCH64)) {
+ t.Errorf("rails/bin/rails in the image: %v, %q", err, body)
+ }
+}
+
+func TestShipRootfsRefusesAnAmd64Directory(t *testing.T) {
+ if testing.Short() {
+ t.Skip("runs the carlos binary")
+ }
+ run, bucket := versioningCLI(t)
+ dir, procs := rootfsRailsDir(t)
+ if err := os.WriteFile(filepath.Join(dir, "usr/bin/ruby"), rootfsTestELF(elf.EM_X86_64), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ out, err := run("ship", "-app", "shop", "-version", "0.1.0-1", "--rootfs", dir, "--processes", procs)
+ if err == nil {
+ t.Fatalf("ship of an amd64 directory succeeded:\n%s", out)
+ }
+ if want := "usr/bin/ruby is a program built for linux/amd64, and boxes run linux/arm64. build the directory's programs for linux/arm64"; !strings.Contains(out, want) {
+ t.Errorf("ship does not say %q:\n%s", want, out)
+ }
+ if files := rootfsStoredFiles(t, bucket); len(files) != 0 {
+ t.Errorf("a refused ship stored %v", files)
+ }
+}
+
+func TestShipRootfsRefusesAnOversizedImageBeforeShipping(t *testing.T) {
+ old, oldOut := shipRootfsMaxBytes, stdout
+ t.Cleanup(func() { shipRootfsMaxBytes, stdout = old, oldOut })
+ shipRootfsMaxBytes = 1
+ var out bytes.Buffer
+ stdout = &out
+
+ dir, procsFile := rootfsRailsDir(t)
+ procs, err := readProcessesFile(procsFile)
+ if err != nil {
+ t.Fatal(err)
+ }
+ bucket := t.TempDir()
+ tmp := t.TempDir()
+ t.Setenv("TMPDIR", tmp)
+ err = shipRootfsRun(context.Background(), rootfsShipOpts{
+ src: dir, procs: procs, app: "shop", ver: "0.1.0-1",
+ st: release.NewDirStore(bucket), acctName: release.OpsAccount,
+ })
+ if err == nil || !strings.HasPrefix(err.Error(), "the rootfs image is ") ||
+ !strings.HasSuffix(err.Error(), ", over the 2 GiB one release can carry. make the image smaller and ship again") {
+ t.Fatalf("shipRootfsRun: %v, want the size refusal", err)
+ }
+ if !strings.Contains(out.String(), "built the rootfs image: ") {
+ t.Errorf("the image was not built first:\n%s", out.String())
+ }
+ if _, err := release.GetManifest(context.Background(), release.NewDirStore(bucket), release.OpsAccount, "shop", "0.1.0-1"); err == nil {
+ t.Error("an oversized image left a manifest")
+ }
+ if files := rootfsStoredFiles(t, bucket); len(files) != 0 {
+ t.Errorf("an oversized image stored %v", files)
+ }
+ if left := rootfsScratchLeft(t, tmp); len(left) != 0 {
+ t.Errorf("scratch left behind: %v", left)
+ }
+}
+
+func TestShipRootfsRemovesItsScratch(t *testing.T) {
+ if testing.Short() {
+ t.Skip("runs the carlos binary")
+ }
+ run, _ := versioningCLI(t)
+ dir, procs := rootfsRailsDir(t)
+ tmp := t.TempDir()
+ t.Setenv("TMPDIR", tmp)
+
+ if out, err := run("ship", "-app", "shop", "-version", "0.1.0-1", "--rootfs", dir, "--processes", procs); err != nil {
+ t.Fatalf("ship --rootfs: %v\n%s", err, out)
+ }
+ if left := rootfsScratchLeft(t, tmp); len(left) != 0 {
+ t.Errorf("a ship left its scratch: %v", left)
+ }
+
+ // A refusal after the scratch exists: a command the image does not have.
+ bad := filepath.Join(t.TempDir(), "bad.json")
+ if err := os.WriteFile(bad, []byte(`{"web": {"command": ["bin/puma"], "working_dir": "/rails"}}`), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ out, err := run("ship", "-app", "shop", "-version", "0.1.0-2", "--rootfs", dir, "--processes", bad)
+ if err == nil || !strings.Contains(out, "the web process runs bin/puma, which the image does not have") {
+ t.Fatalf("ship with a missing command: %v\n%s", err, out)
+ }
+ if left := rootfsScratchLeft(t, tmp); len(left) != 0 {
+ t.Errorf("a refused ship left its scratch: %v", left)
+ }
+}