amadan

carlos / platform Public

ship: carlos ship --check runs the image locally the way a box would (rootfs Plan 1 Task 11)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude feff1b80459218c4fbfbb80551aef4c2c3c97ed8 parent 54dc05b
11 files changed, +2060 −33
  • approved_rootfs_copy_test.go +4 −1
  • docs/superpowers/plans/2026-10-24-rootfs-apps-1-ship.md +2 −1
  • docs/superpowers/plans/2026-10-24-rootfs-apps-copy.md +2 −1
  • main.go +5 −0
  • ship_check.go +504 −0
  • ship_check_probe.go +283 −0
  • ship_check_probe_test.go +313 −0
  • ship_check_test.go +777 −0
  • ship_rootfs.go +82 −27
  • ship_rootfs_unix_test.go +77 −0
  • ship_runtime.go +11 −3
diff --git a/approved_rootfs_copy_test.go b/approved_rootfs_copy_test.go
index f6944ff71..af0451440 100644
--- a/approved_rootfs_copy_test.go
+++ b/approved_rootfs_copy_test.go
@@ -25,7 +25,7 @@ const rootfsCopyRecordPath = "docs/superpowers/plans/2026-10-24-rootfs-apps-copy
var rootfsCopyIDPattern = regexp.MustCompile(`^RFS-[1-6]-C(0[1-9]|[1-9][0-9]{1,2})$`)
// rootfsCopyIDs is the deck's inventory, pinned as a literal list in the
-// deck's order: RFS-1-C01 to RFS-1-C88, then each later plan's rows. A
+// deck's order: RFS-1-C01 to RFS-1-C89, then each later plan's rows. A
// struck row stays in it: the deck keeps the row for history.
var rootfsCopyIDs = []string{
// Plan 1
@@ -51,6 +51,7 @@ var rootfsCopyIDs = []string{
"RFS-1-C77", "RFS-1-C78", "RFS-1-C79", "RFS-1-C80",
"RFS-1-C81", "RFS-1-C82", "RFS-1-C83", "RFS-1-C84",
"RFS-1-C85", "RFS-1-C86", "RFS-1-C87", "RFS-1-C88",
+ "RFS-1-C89",
// Plan 2
"RFS-2-C01", "RFS-2-C02", "RFS-2-C03", "RFS-2-C04",
"RFS-2-C05", "RFS-2-C06", "RFS-2-C07", "RFS-2-C08",
@@ -164,6 +165,8 @@ var rootfsCopyBuilt = []string{
"RFS-1-C30", "RFS-1-C31", "RFS-1-C32", "RFS-1-C33", "RFS-1-C34", "RFS-1-C35", "RFS-1-C36", "RFS-1-C37", "RFS-1-C38", "RFS-1-C39", "RFS-1-C40", "RFS-1-C41", "RFS-1-C42", "RFS-1-C43", "RFS-1-C44", "RFS-1-C84",
// Task 10: the local container runtime (ship_runtime.go)
"RFS-1-C49", "RFS-1-C57", "RFS-1-C58", "RFS-1-C59", "RFS-1-C88",
+ // Task 11: carlos ship --check (ship_check.go, ship_check_probe.go, ship_rootfs.go)
+ "RFS-1-C60", "RFS-1-C61", "RFS-1-C62", "RFS-1-C63", "RFS-1-C64", "RFS-1-C65", "RFS-1-C66", "RFS-1-C67", "RFS-1-C68", "RFS-1-C69", "RFS-1-C70", "RFS-1-C71", "RFS-1-C72", "RFS-1-C73", "RFS-1-C74", "RFS-1-C75", "RFS-1-C76", "RFS-1-C77", "RFS-1-C85", "RFS-1-C89",
}
func mustParseRootfsRecord(t *testing.T) []copyRow {
diff --git a/docs/superpowers/plans/2026-10-24-rootfs-apps-1-ship.md b/docs/superpowers/plans/2026-10-24-rootfs-apps-1-ship.md
index fe3d83e77..c3242a35f 100644
--- a/docs/superpowers/plans/2026-10-24-rootfs-apps-1-ship.md
+++ b/docs/superpowers/plans/2026-10-24-rootfs-apps-1-ship.md
@@ -215,8 +215,9 @@ Placeholders: `<app>` an app name; `<name>` a process name; `<var>` an environme
| RFS-1-C86 | internal/rootfsimage/errors.go | the rootfs image is over 2 GiB, the most one release can carry. make the image smaller and ship again | approved |
| RFS-1-C87 | internal/rootfsimage/errors.go | <path> is over 2 GiB, the largest file a rootfs image can hold. make it smaller and ship again | approved |
| RFS-1-C88 | ship_runtime.go | CARLOS_CONTAINER_RUNTIME must be docker or podman | approved |
+| RFS-1-C89 | ship_rootfs.go | --check ships nothing, so it cannot be combined with --<flag> | approved |
-RFS-1-C78's `<digest>` is a `sha256:<hex>` digest. RFS-1-C80 to C83 are the first-ship forms of C18 to C21: the app has no other release, and its kind marker (deviation 10) names the ship that fixed its kind, which may not have published, so they never say "is one". RFS-1-C85 is a warning: it does not fail the check. Kept out of the table on purpose (diagnostics, not human text): the probe's one-line JSON on stdout, `validateManifest`'s existing terse messages, store errors wrapped for logs. RFS-1-C86 and C87 were added in the build (Task 6, the controller ruling on strings found in the build), for the Luna pass at the end of Plan 1: C86 is the writer's refusal when the image passes 2 GiB while it is written (the text of C26), C87 its refusal of a single file over 2 GiB before anything is written. RFS-1-C88 was added in the build (Task 10, the controller ruling on strings found in the build): the refusal of a CARLOS_CONTAINER_RUNTIME that names neither runtime.
+RFS-1-C78's `<digest>` is a `sha256:<hex>` digest. RFS-1-C80 to C83 are the first-ship forms of C18 to C21: the app has no other release, and its kind marker (deviation 10) names the ship that fixed its kind, which may not have published, so they never say "is one". RFS-1-C85 is a warning: it does not fail the check. Kept out of the table on purpose (diagnostics, not human text): the probe's one-line JSON on stdout, `validateManifest`'s existing terse messages, store errors wrapped for logs. RFS-1-C86 and C87 were added in the build (Task 6, the controller ruling on strings found in the build), for the Luna pass at the end of Plan 1: C86 is the writer's refusal when the image passes 2 GiB while it is written (the text of C26), C87 its refusal of a single file over 2 GiB before anything is written. RFS-1-C88 was added in the build (Task 10, the controller ruling on strings found in the build): the refusal of a CARLOS_CONTAINER_RUNTIME that names neither runtime. RFS-1-C89 was added in the build (Task 11, the controller ruling on strings found in the build): --check refuses a flag only a ship uses (--app, --version, --label, --notes, --notes-file, --swap, --console, --destination, --account, --account-id, --registry, --message, --no-prompt) rather than ignore it.
---
diff --git a/docs/superpowers/plans/2026-10-24-rootfs-apps-copy.md b/docs/superpowers/plans/2026-10-24-rootfs-apps-copy.md
index 855587df0..bc7c2875c 100644
--- a/docs/superpowers/plans/2026-10-24-rootfs-apps-copy.md
+++ b/docs/superpowers/plans/2026-10-24-rootfs-apps-copy.md
@@ -127,8 +127,9 @@ From the plan: Placeholders: `<app>` an app name; `<name>` a process name; `<var
| RFS-1-C86 | internal/rootfsimage/errors.go | the rootfs image is over 2 GiB, the most one release can carry. make the image smaller and ship again | approved |
| RFS-1-C87 | internal/rootfsimage/errors.go | <path> is over 2 GiB, the largest file a rootfs image can hold. make it smaller and ship again | approved |
| RFS-1-C88 | ship_runtime.go | CARLOS_CONTAINER_RUNTIME must be docker or podman | approved |
+| RFS-1-C89 | ship_rootfs.go | --check ships nothing, so it cannot be combined with --<flag> | approved |
-From the plan: RFS-1-C78's `<digest>` is a `sha256:<hex>` digest. RFS-1-C80 to C83 are the first-ship forms of C18 to C21: the app has no other release, and its kind marker (deviation 10) names the ship that fixed its kind, which may not have published, so they never say "is one". RFS-1-C85 is a warning: it does not fail the check. Kept out of the table on purpose (diagnostics, not human text): the probe's one-line JSON on stdout, `validateManifest`'s existing terse messages, store errors wrapped for logs. RFS-1-C86 and C87 were added in the build (Task 6, the controller ruling on strings found in the build), for the Luna pass at the end of Plan 1: C86 is the writer's refusal when the image passes 2 GiB while it is written (the text of C26), C87 its refusal of a single file over 2 GiB before anything is written. RFS-1-C88 was added in the build (Task 10, the controller ruling on strings found in the build): the refusal of a CARLOS_CONTAINER_RUNTIME that names neither runtime.
+From the plan: RFS-1-C78's `<digest>` is a `sha256:<hex>` digest. RFS-1-C80 to C83 are the first-ship forms of C18 to C21: the app has no other release, and its kind marker (deviation 10) names the ship that fixed its kind, which may not have published, so they never say "is one". RFS-1-C85 is a warning: it does not fail the check. Kept out of the table on purpose (diagnostics, not human text): the probe's one-line JSON on stdout, `validateManifest`'s existing terse messages, store errors wrapped for logs. RFS-1-C86 and C87 were added in the build (Task 6, the controller ruling on strings found in the build), for the Luna pass at the end of Plan 1: C86 is the writer's refusal when the image passes 2 GiB while it is written (the text of C26), C87 its refusal of a single file over 2 GiB before anything is written. RFS-1-C88 was added in the build (Task 10, the controller ruling on strings found in the build): the refusal of a CARLOS_CONTAINER_RUNTIME that names neither runtime. RFS-1-C89 was added in the build (Task 11, the controller ruling on strings found in the build): --check refuses a flag only a ship uses (--app, --version, --label, --notes, --notes-file, --swap, --console, --destination, --account, --account-id, --registry, --message, --no-prompt) rather than ignore it.
## Plan 2: box isolation (`2026-10-24-rootfs-apps-2-box-isolation.md`)
diff --git a/main.go b/main.go
index 3f13a396f..6c5d4dc8d 100644
--- a/main.go
+++ b/main.go
@@ -40,6 +40,11 @@ func main() {
}
return
}
+ // carlos ship --check's health probe, run inside the web container
+ // (ship_check_probe.go): plumbing, not a command anyone types.
+ if cmd == "__rootfs-check-probe" {
+ os.Exit(runRootfsCheckProbe(args))
+ }
switch cmd {
// Asking for help is not an error: it prints on stdout and exits 0, so
// `carlos help | less` works. A bare `carlos` still fails (stderr, 2).
diff --git a/ship_check.go b/ship_check.go
new file mode 100644
index 000000000..5f3e00f75
--- /dev/null
+++ b/ship_check.go
@@ -0,0 +1,504 @@
+package main
+
+import (
+ "context"
+ "crypto/rand"
+ "encoding/hex"
+ "encoding/json"
+ "fmt"
+ "io"
+ "maps"
+ "os"
+ "path/filepath"
+ "slices"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/carlosframework/platform/internal/cgroup"
+ "github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/rootfsimage"
+)
+
+// carlos ship --check (rootfs apps Plan 1 Task 11, spec §5.8): the built
+// image run here with Docker or Podman the way a box runs it. It is a
+// convenience, not a guarantee: a box adds its egress fence, its own limits
+// and MemoryDenyWriteExecute, which no local runtime applies.
+
+const (
+ checkUID = 65534
+ checkInstanceID = "check"
+ checkHost = "shop.example.com"
+ workerSettle = 5 * time.Second
+ checkLogLines = 20
+)
+
+// The instance's paths in every check container, as a box lays them out.
+const (
+ checkDataDir = "/data/instances/" + checkInstanceID
+ checkRunDir = "/run/carlos/instances/" + checkInstanceID
+ checkSocket = checkRunDir + "/app.sock"
+ checkDB = checkDataDir + "/" + checkHost + ".db"
+)
+
+// checkNoEmulation is an arm64 program this machine cannot run (RFS-1-C62).
+const checkNoEmulation = "the arm64 image cannot run here. install emulation for arm64 (qemu and binfmt), or run the check on an arm64 machine"
+
+// checkMemory is the rootfs tier's memory (deviation 9): memory and swap
+// together, as the box's cgroup counts them.
+var checkMemory = cgroup.TierChildren.MemoryMax
+
+// checkPause waits d, or less when ctx ends first. A var so the tests do not
+// wait out workerSettle.
+var checkPause = func(ctx context.Context, d time.Duration) error {
+ t := time.NewTimer(d)
+ defer t.Stop()
+ select {
+ case <-t.C:
+ return nil
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+}
+
+// checkReport is what the check said, line by line, and whether it failed.
+type checkReport struct {
+ lines []string
+ failed bool
+}
+
+// rootfsCheck is one run of the check: the runtime, the build, and every
+// object it has asked the runtime to make, which cleanup removes.
+type rootfsCheck struct {
+ rt containerRuntime
+ b *builtRootfs
+ probe string
+ out io.Writer
+ report checkReport
+
+ id, prefix string
+ image string
+ volumes []string // data, then run
+ containers []string
+ envFiles map[string]string
+}
+
+// containerState is the part of a container's inspect State the check
+// reads. Docker and Podman both name these fields so.
+type containerState struct {
+ Running bool
+ OOMKilled bool
+ ExitCode int
+}
+
+// checkProbeResult is the probe's one JSON line, either form.
+type checkProbeResult struct {
+ Status int `json:"status"`
+ AfterMS int64 `json:"after_ms"`
+ Ports []int `json:"ports"`
+}
+
+// runRootfsCheck runs b's image with rt: release, if declared, to its end;
+// then web, probed with the binary at probe; then each worker, after a
+// healthy web. console is never started (spec §5.10). It writes each line
+// of its report to out as it goes. Every container, volume and image it
+// made is removed before it returns, on every path; when ctx ends early the
+// report is failed and says nothing more.
+func runRootfsCheck(ctx context.Context, rt containerRuntime, b *builtRootfs, probe string, out io.Writer) checkReport {
+ var raw [4]byte
+ rand.Read(raw[:]) // never fails (crypto/rand, Go 1.24 on)
+ c := &rootfsCheck{rt: rt, b: b, probe: probe, out: out, envFiles: map[string]string{}}
+ c.id = hex.EncodeToString(raw[:])
+ c.prefix = "carlos-check-" + c.id
+ defer c.cleanup(ctx)
+
+ c.check(ctx)
+ switch {
+ case ctx.Err() != nil:
+ c.report.failed = true
+ case c.report.failed:
+ c.say("the image failed the local check")
+ default:
+ c.say("the image passed the local check. a box also applies its egress fence and its own limits, so this is a convenience, not a guarantee")
+ }
+ return c.report
+}
+
+func (c *rootfsCheck) check(ctx context.Context) {
+ mem := checkMemory
+ c.say(fmt.Sprintf("checking the image with %s: read-only root, %s of memory, %d s for web to answer",
+ c.rt.name, humanBytes(&mem), int(release.RootfsHealthTimeout/time.Second)))
+ // R11: no local runtime applies MemoryDenyWriteExecute, so a JIT passes
+ // here and stops on a box. A warning: the report does not fail on it.
+ if !c.b.jit {
+ if name, prog, ok := rootfsimage.JITProgram(c.b.tree, c.b.procs); ok {
+ c.say(fmt.Sprintf("the %s process runs %s, which needs a JIT, and this release does not declare one. it passes here but stops at start on a box. ship it with --jit", name, prog))
+ }
+ }
+ if !c.checkArch(ctx) || !c.makeImageAndVolumes(ctx) || !c.writeEnvFiles() {
+ return
+ }
+ if _, ok := c.b.procs[release.ProcessRelease]; ok && !c.runRelease(ctx) {
+ return
+ }
+ if !c.runWeb(ctx) {
+ return
+ }
+ c.runWorkers(ctx)
+}
+
+// checkArch warns when this machine is not arm64 (RFS-1-C61).
+func (c *rootfsCheck) checkArch(ctx context.Context) bool {
+ format := "{{.Architecture}}"
+ if c.rt.name == "podman" {
+ format = "{{.Host.Arch}}"
+ }
+ out, err := c.rt.run(ctx, nil, "info", "--format", format)
+ if err != nil {
+ return c.runtimeFailed(ctx, "info", err)
+ }
+ if arch := strings.TrimSpace(string(out)); arch != "aarch64" && arch != "arm64" {
+ c.say(fmt.Sprintf("this machine is %s, so the arm64 image runs under emulation and starts slower than on a box", arch))
+ }
+ return true
+}
+
+// makeImageAndVolumes imports the tree and makes the instance's two
+// directories, each a tmpfs volume owned by the check's user. Each name is
+// recorded before the runtime is asked, so a call cut short is cleaned up.
+func (c *rootfsCheck) makeImageAndVolumes(ctx context.Context) bool {
+ c.image = c.prefix + ":latest"
+ if err := c.rt.importTree(ctx, c.b.tree, c.image); err != nil {
+ return c.runtimeFailed(ctx, "import", err)
+ }
+ for _, v := range []string{c.prefix + "-data", c.prefix + "-run"} {
+ c.volumes = append(c.volumes, v)
+ _, err := c.rt.run(ctx, nil, "volume", "create", "--driver", "local",
+ "--opt", "type=tmpfs", "--opt", "device=tmpfs",
+ "--opt", fmt.Sprintf("o=uid=%d,gid=%d,mode=0700", checkUID, checkUID), v)
+ if err != nil {
+ return c.runtimeFailed(ctx, "volume create", err)
+ }
+ }
+ return true
+}
+
+// writeEnvFiles writes one env file per process the check starts, 0600 in
+// the build's scratch: the process's env, then the instance's CARLOS_*
+// values, which win over nothing (validation keeps CARLOS_ names out of a
+// process's env). There is no CARLOS_CONTROL_SOCKET or CARLOS_PURGE:
+// nothing serves them here. An env file holds one variable per line, so a
+// value with a newline in it reaches the process cut at the newline.
+func (c *rootfsCheck) writeEnvFiles() bool {
+ for _, name := range slices.Sorted(maps.Keys(c.b.procs)) {
+ if name == release.ProcessConsole {
+ continue
+ }
+ var buf strings.Builder
+ env := c.b.procs[name].Env
+ for _, k := range slices.Sorted(maps.Keys(env)) {
+ fmt.Fprintf(&buf, "%s=%s\n", k, env[k])
+ }
+ fmt.Fprintf(&buf, "CARLOS_SOCKET=%s\nCARLOS_DB=%s\nCARLOS_HOST=%s\nCARLOS_REGION=local\n", checkSocket, checkDB, checkHost)
+ path := filepath.Join(c.b.scratch, "check-"+name+".env")
+ if err := os.WriteFile(path, []byte(buf.String()), 0o600); err != nil {
+ c.say(err.Error())
+ c.report.failed = true
+ return false
+ }
+ c.envFiles[name] = path
+ }
+ return true
+}
+
+// runArgs is the run argv of process name in container ctr: a box's launch
+// contract (spec §5.2) as a local runtime can give it.
+func (c *rootfsCheck) runArgs(ctr, name string) []string {
+ p := c.b.procs[name]
+ wd := p.WorkingDir
+ if wd == "" {
+ wd = "/"
+ }
+ mem := strconv.FormatInt(checkMemory, 10)
+ user := fmt.Sprintf("%d:%d", checkUID, checkUID)
+ args := []string{"run", "-d", "--name", ctr,
+ "--read-only", "--tmpfs", "/tmp",
+ "--memory", mem, "--memory-swap", mem,
+ "--cap-drop", "ALL", "--security-opt", "no-new-privileges",
+ "--platform", "linux/arm64", "--user", user,
+ "-v", c.volumes[0] + ":" + checkDataDir,
+ "-v", c.volumes[1] + ":" + checkRunDir,
+ "-v", c.probe + ":" + checkProbeMount + ":ro",
+ "--env-file", c.envFiles[name],
+ "-w", wd,
+ "--entrypoint", p.Command[0],
+ "--label", "carlos-check=" + c.id,
+ c.image,
+ }
+ return append(args, p.Command[1:]...)
+}
+
+// start runs process name detached and answers its container. A run the
+// runtime refused is reported: an arm64 program this machine cannot run is
+// RFS-1-C62.
+func (c *rootfsCheck) start(ctx context.Context, name string) (string, bool) {
+ ctr := c.prefix + "-" + name
+ c.containers = append(c.containers, ctr)
+ if _, err := c.rt.run(ctx, nil, c.runArgs(ctr, name)...); err != nil {
+ if ctx.Err() == nil && execFormatError(err.Error()) {
+ return ctr, c.fail(checkNoEmulation)
+ }
+ return ctr, c.runtimeFailed(ctx, "run", err)
+ }
+ return ctr, true
+}
+
+// runRelease runs release to its end, under its timeout. A failure stops
+// the check: a box would not start web either.
+func (c *rootfsCheck) runRelease(ctx context.Context) bool {
+ began := time.Now()
+ ctr, ok := c.start(ctx, release.ProcessRelease)
+ if !ok {
+ return false
+ }
+ timeout := time.Duration(release.DefaultReleaseTimeoutSec) * time.Second
+ if s := c.b.procs[release.ProcessRelease].TimeoutSec; s > 0 {
+ timeout = time.Duration(s) * time.Second
+ }
+ wctx, cancel := context.WithTimeout(ctx, timeout)
+ out, err := c.rt.run(wctx, nil, "wait", ctr)
+ timedOut := wctx.Err() != nil
+ cancel()
+ if err != nil {
+ if ctx.Err() == nil && timedOut {
+ return c.fail(fmt.Sprintf("release did not finish within %s", checkDuration(timeout)))
+ }
+ return c.runtimeFailed(ctx, "wait", err)
+ }
+ code, err := strconv.Atoi(lastLine(string(out)))
+ if err != nil {
+ return c.runtimeFailed(ctx, "wait", fmt.Errorf("%q is not an exit status", strings.TrimSpace(string(out))))
+ }
+ if code == 0 {
+ c.say(fmt.Sprintf("release finished in %s", checkDuration(time.Since(began))))
+ return true
+ }
+ logs := c.logs(ctx, ctr)
+ if execFormatError(logs...) {
+ return c.fail(checkNoEmulation)
+ }
+ c.fail(fmt.Sprintf("release exited with status %d. its last lines:", code))
+ c.sayLogs(logs)
+ return false
+}
+
+// runWeb starts web and probes GET / on its socket from inside it.
+func (c *rootfsCheck) runWeb(ctx context.Context) bool {
+ ctr, ok := c.start(ctx, release.ProcessWeb)
+ if !ok {
+ return false
+ }
+ // The probe ends itself at its timeout; the margin bounds a runtime
+ // that never returns.
+ pctx, cancel := context.WithTimeout(ctx, release.RootfsHealthTimeout+30*time.Second)
+ out, perr := c.rt.run(pctx, nil, "exec", ctr, checkProbeMount, "__rootfs-check-probe",
+ "--socket", checkSocket, "--timeout", fmt.Sprintf("%ds", int(release.RootfsHealthTimeout/time.Second)))
+ cancel()
+ if ctx.Err() != nil {
+ return false
+ }
+ var res checkProbeResult
+ parsed := json.Unmarshal([]byte(lastLine(string(out))), &res) == nil
+ if perr == nil && parsed {
+ c.say(fmt.Sprintf("web answered GET / with %d after %s", res.Status, checkDuration(time.Duration(res.AfterMS)*time.Millisecond)))
+ return true
+ }
+
+ st, err := c.inspect(ctx, ctr)
+ if err != nil {
+ return c.runtimeFailed(ctx, "inspect", err)
+ }
+ logs := c.logs(ctx, ctr)
+ probeSaid := string(out)
+ if perr != nil {
+ probeSaid += "\n" + perr.Error()
+ }
+ switch {
+ case execFormatError(probeSaid) || !st.Running && execFormatError(logs...):
+ return c.fail(checkNoEmulation)
+ case st.OOMKilled || !st.Running:
+ return c.exited(release.ProcessWeb, st, logs)
+ case !parsed:
+ // The runtime could not run the probe at all, or what it ran is
+ // not the probe (CARLOS_CHECK_PROBE naming another program).
+ if perr == nil {
+ perr = fmt.Errorf("the probe answered %q", strings.TrimSpace(string(out)))
+ }
+ return c.runtimeFailed(ctx, "exec", perr)
+ case len(res.Ports) > 0:
+ ports := make([]string, len(res.Ports))
+ for i, p := range res.Ports {
+ ports[i] = strconv.Itoa(p)
+ }
+ return c.fail(fmt.Sprintf("web listens on port %s instead of the socket. serve HTTP on the unix socket at CARLOS_SOCKET", strings.Join(ports, ", ")))
+ case res.Status >= 500:
+ return c.fail(fmt.Sprintf("web answered GET / with %d. health needs an answer below 500", res.Status))
+ default:
+ return c.fail(fmt.Sprintf("web did not answer GET / on CARLOS_SOCKET within %s", checkDuration(release.RootfsHealthTimeout)))
+ }
+}
+
+// runWorkers starts each worker, in name order, waits workerSettle, and
+// reports each as running or as why it stopped.
+func (c *rootfsCheck) runWorkers(ctx context.Context) {
+ var workers, ctrs []string
+ for _, name := range slices.Sorted(maps.Keys(c.b.procs)) {
+ switch name {
+ case release.ProcessWeb, release.ProcessRelease, release.ProcessConsole:
+ continue
+ }
+ ctr, ok := c.start(ctx, name)
+ if !ok {
+ return
+ }
+ workers, ctrs = append(workers, name), append(ctrs, ctr)
+ }
+ if len(workers) == 0 {
+ return
+ }
+ if checkPause(ctx, workerSettle) != nil {
+ return
+ }
+ for i, name := range workers {
+ st, err := c.inspect(ctx, ctrs[i])
+ if err != nil {
+ c.runtimeFailed(ctx, "inspect", err)
+ return
+ }
+ if st.Running && !st.OOMKilled {
+ c.say(fmt.Sprintf("%s is still running after %s", name, checkDuration(workerSettle)))
+ continue
+ }
+ c.exited(name, st, c.logs(ctx, ctrs[i]))
+ }
+}
+
+// exited reports a process that stopped: out of memory (RFS-1-C71), a
+// write to the read-only root (RFS-1-C72), or its status and last lines
+// (RFS-1-C70).
+func (c *rootfsCheck) exited(name string, st containerState, logs []string) bool {
+ mem := checkMemory
+ switch {
+ case st.OOMKilled:
+ return c.fail(fmt.Sprintf("%s ran out of memory. a rootfs instance's tier allows %s", name, humanBytes(&mem)))
+ case slices.ContainsFunc(logs, func(l string) bool { return strings.Contains(l, "Read-only file system") }):
+ return c.fail(fmt.Sprintf("%s tried to write outside its instance directory. the root is read-only, so write under %s or /tmp", name, checkDataDir))
+ }
+ c.fail(fmt.Sprintf("%s exited with status %d. its last lines:", name, st.ExitCode))
+ c.sayLogs(logs)
+ return false
+}
+
+func (c *rootfsCheck) inspect(ctx context.Context, ctr string) (containerState, error) {
+ var st containerState
+ out, err := c.rt.run(ctx, nil, "inspect", "--format", "{{json .State}}", ctr)
+ if err != nil {
+ return st, err
+ }
+ return st, json.Unmarshal([]byte(lastLine(string(out))), &st)
+}
+
+// logs is the container's last checkLogLines lines, both streams (the
+// runner joins them for logs), or none when the runtime cannot give them.
+func (c *rootfsCheck) logs(ctx context.Context, ctr string) []string {
+ out, err := c.rt.run(ctx, nil, "logs", "--tail", strconv.Itoa(checkLogLines), ctr)
+ if err != nil {
+ return nil
+ }
+ var lines []string
+ for _, l := range strings.Split(strings.TrimRight(string(out), "\r\n"), "\n") {
+ lines = append(lines, strings.TrimRight(l, "\r"))
+ }
+ if len(lines) > checkLogLines {
+ lines = lines[len(lines)-checkLogLines:]
+ }
+ if len(lines) == 1 && lines[0] == "" {
+ return nil
+ }
+ return lines
+}
+
+// cleanup removes, on every path, everything the check asked the runtime to
+// make: containers, then volumes, then the image. It runs under its own
+// bound, not ctx, so a Ctrl-C still cleans up. A removal of something the
+// runtime never made fails harmlessly and is not reported.
+func (c *rootfsCheck) cleanup(ctx context.Context) {
+ ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
+ defer cancel()
+ for _, ctr := range c.containers {
+ c.rt.run(ctx, nil, "rm", "-f", ctr)
+ }
+ for _, v := range c.volumes {
+ c.rt.run(ctx, nil, "volume", "rm", v)
+ }
+ if c.image != "" {
+ c.rt.run(ctx, nil, "rmi", c.image)
+ }
+ for _, p := range c.envFiles {
+ os.Remove(p)
+ }
+}
+
+func (c *rootfsCheck) say(line string) {
+ c.report.lines = append(c.report.lines, line)
+ fmt.Fprintln(c.out, line)
+}
+
+// fail reports line and fails the check. It answers false, so a step can
+// return it.
+func (c *rootfsCheck) fail(line string) bool {
+ c.say(line)
+ c.report.failed = true
+ return false
+}
+
+func (c *rootfsCheck) sayLogs(logs []string) {
+ for _, l := range logs {
+ c.say(" " + l)
+ }
+}
+
+// runtimeFailed reports a runtime call that failed, in the runtime's own
+// words (a diagnostic, not a sentence of ours), and fails the check. When
+// ctx has ended the failure is the stop itself and nothing is said.
+func (c *rootfsCheck) runtimeFailed(ctx context.Context, verb string, err error) bool {
+ c.report.failed = true
+ if ctx.Err() == nil {
+ c.say(fmt.Sprintf("%s %s: %v", c.rt.name, verb, err))
+ }
+ return false
+}
+
+// execFormatError reports whether any of texts carries the kernel's refusal
+// of a program for another architecture (ENOEXEC), in any case: runc writes
+// "exec format error", crun "Exec format error".
+func execFormatError(texts ...string) bool {
+ return slices.ContainsFunc(texts, func(s string) bool { return strings.Contains(strings.ToLower(s), "exec format error") })
+}
+
+// checkDuration is d for the report: tenths of a second under ten seconds,
+// whole seconds above.
+func checkDuration(d time.Duration) string {
+ if d < 10*time.Second {
+ return fmt.Sprintf("%.1f s", d.Seconds())
+ }
+ return fmt.Sprintf("%d s", int(d/time.Second))
+}
+
+func lastLine(s string) string {
+ s = strings.TrimSpace(s)
+ if i := strings.LastIndexByte(s, '\n'); i >= 0 {
+ s = s[i+1:]
+ }
+ return strings.TrimSpace(s)
+}
diff --git a/ship_check_probe.go b/ship_check_probe.go
new file mode 100644
index 000000000..cc291f1f2
--- /dev/null
+++ b/ship_check_probe.go
@@ -0,0 +1,283 @@
+package main
+
+import (
+ "bufio"
+ "context"
+ "crypto/sha256"
+ "debug/elf"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "flag"
+ "fmt"
+ "io"
+ "net"
+ "net/http"
+ "os"
+ "path/filepath"
+ "runtime"
+ "slices"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/carlosframework/platform/internal/publicrelease"
+ "github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/version"
+)
+
+// The health probe of carlos ship --check (rootfs apps Plan 1 Task 11). A
+// unix socket made inside a container cannot be dialed from the host when
+// the runtime is a VM (macOS, Windows, Docker Desktop on Linux), so the
+// probe runs inside the web container: a linux carlos binary bind-mounted
+// at checkProbeMount, run with `exec` as the hidden verb
+// __rootfs-check-probe. Its one JSON line on stdout is a diagnostic for
+// runRootfsCheck, not text a person reads.
+
+// checkProbeMount is where the probe binary is mounted in every check
+// container: the path where a box binds its trampoline.
+const checkProbeMount = "/opt/carlos/carlos"
+
+// checkProbeInterval is how often the probe asks GET / again.
+const checkProbeInterval = 250 * time.Millisecond
+
+// checkProbeProcNet are the kernel's TCP socket tables the probe reads for
+// the ports web listens on instead of the socket. A var so a test can point
+// it at a sample.
+var checkProbeProcNet = []string{"/proc/net/tcp", "/proc/net/tcp6"}
+
+// checkProbeAnswer is the probe's line when web answered below 500.
+type checkProbeAnswer struct {
+ Status int `json:"status"`
+ AfterMS int64 `json:"after_ms"`
+}
+
+// checkProbeTimeout is the probe's line when it gave up: the last status
+// web answered (0 for none) and the TCP ports in LISTEN.
+type checkProbeTimeout struct {
+ Status int `json:"status"`
+ Ports []int `json:"ports"`
+}
+
+// runRootfsCheckProbe is __rootfs-check-probe: it asks GET / over the unix
+// socket at --socket every checkProbeInterval until an answer below 500
+// (exit 0) or --timeout (exit 1). A flag error is exit 2.
+func runRootfsCheckProbe(args []string) int {
+ fs := flag.NewFlagSet("__rootfs-check-probe", flag.ContinueOnError)
+ socket := fs.String("socket", "", "the unix socket web serves on")
+ timeout := fs.Duration("timeout", release.RootfsHealthTimeout, "how long web has to answer")
+ if err := fs.Parse(args); err != nil || *socket == "" || fs.NArg() > 0 {
+ return 2
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), *timeout)
+ defer cancel()
+ client := &http.Client{Transport: &http.Transport{
+ DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
+ var d net.Dialer
+ return d.DialContext(ctx, "unix", *socket)
+ },
+ DisableKeepAlives: true,
+ }}
+
+ start := time.Now()
+ last := 0
+ tick := time.NewTicker(checkProbeInterval)
+ defer tick.Stop()
+ for {
+ if status, err := checkProbeGet(ctx, client); err == nil {
+ last = status
+ if status < 500 {
+ writeProbeLine(checkProbeAnswer{Status: status, AfterMS: time.Since(start).Milliseconds()})
+ return 0
+ }
+ }
+ select {
+ case <-ctx.Done():
+ writeProbeLine(checkProbeTimeout{Status: last, Ports: listeningPorts(checkProbeProcNet)})
+ return 1
+ case <-tick.C:
+ }
+ }
+}
+
+// checkProbeGet asks GET / once, for the check's host, and answers the
+// status.
+func checkProbeGet(ctx context.Context, client *http.Client) (int, error) {
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://"+checkHost+"/", nil)
+ if err != nil {
+ return 0, err
+ }
+ res, err := client.Do(req)
+ if err != nil {
+ return 0, err
+ }
+ io.Copy(io.Discard, io.LimitReader(res.Body, 1<<20))
+ res.Body.Close()
+ return res.StatusCode, nil
+}
+
+func writeProbeLine(v any) {
+ line, _ := json.Marshal(v)
+ fmt.Fprintf(stdout, "%s\n", line)
+}
+
+// listeningPorts is every local port in LISTEN (st 0A) in the given
+// /proc/net/tcp-format tables, sorted, each once. A table that cannot be
+// read adds nothing.
+func listeningPorts(tables []string) []int {
+ ports := []int{}
+ for _, table := range tables {
+ f, err := os.Open(table)
+ if err != nil {
+ continue
+ }
+ sc := bufio.NewScanner(f)
+ sc.Scan() // the header
+ for sc.Scan() {
+ // sl local_address rem_address st ...
+ fields := strings.Fields(sc.Text())
+ if len(fields) < 4 || fields[3] != "0A" {
+ continue
+ }
+ i := strings.LastIndexByte(fields[1], ':')
+ port, err := strconv.ParseUint(fields[1][i+1:], 16, 16)
+ if i < 0 || err != nil {
+ continue
+ }
+ if !slices.Contains(ports, int(port)) {
+ ports = append(ports, int(port))
+ }
+ }
+ f.Close()
+ }
+ slices.Sort(ports)
+ return ports
+}
+
+// The probe binary's seams: the tests set the platform, the running
+// executable and the cache directory. Production never changes them.
+var (
+ checkProbeGOOS = runtime.GOOS
+ checkProbeGOARCH = runtime.GOARCH
+ checkProbeExecutable = os.Executable
+ checkProbeCacheDir = os.UserCacheDir
+)
+
+// errCheckProbeDev is a development build with no linux carlos to probe
+// with (RFS-1-C76).
+var errCheckProbeDev = errors.New("--check needs a linux build of carlos to probe the socket from inside the container, and this one is a development build. use a released carlos, or set CARLOS_CHECK_PROBE to a linux carlos binary")
+
+// checkProbeBinary is the host path of the linux carlos binary --check
+// mounts at checkProbeMount, in order: CARLOS_CHECK_PROBE; on Linux this
+// binary itself when it is static (a dynamic one may not find its loader in
+// the image); else the signed release of this version for linux-arm64, or
+// for this machine's architecture on a Linux host, fetched once into the
+// user's cache.
+func checkProbeBinary(ctx context.Context) (string, error) {
+ if p := os.Getenv("CARLOS_CHECK_PROBE"); p != "" {
+ return filepath.Abs(p)
+ }
+ platform := "linux-arm64"
+ if checkProbeGOOS == "linux" {
+ if exe, err := checkProbeExecutable(); err == nil {
+ if exe, err = filepath.EvalSymlinks(exe); err == nil && staticELF(exe) {
+ return exe, nil
+ }
+ }
+ platform = "linux-" + checkProbeGOARCH
+ }
+ if !releaseBuild(version.Build) {
+ return "", errCheckProbeDev
+ }
+ p, err := cachedCheckProbe(ctx, version.Build, publicrelease.AssetName(platform))
+ if err != nil {
+ return "", fmt.Errorf("could not fetch the linux build of carlos for the check: %v", err)
+ }
+ return p, nil
+}
+
+// staticELF reports whether path is an ELF file with no program
+// interpreter (PT_INTERP), so it runs in an image without a loader.
+func staticELF(path string) bool {
+ f, err := elf.Open(path)
+ if err != nil {
+ return false
+ }
+ defer f.Close()
+ for _, p := range f.Progs {
+ if p.Type == elf.PT_INTERP {
+ return false
+ }
+ }
+ return true
+}
+
+// cachedCheckProbe is <cache>/carlos/check-probe/<tag>/<asset>: fetched from
+// the self-update feed (signature and checksum verified) when it is absent,
+// and hashed again against the digest recorded beside it on every use, so
+// a damaged copy is fetched again rather than mounted.
+func cachedCheckProbe(ctx context.Context, tag, asset string) (string, error) {
+ base, err := checkProbeCacheDir()
+ if err != nil {
+ return "", err
+ }
+ dir := filepath.Join(base, "carlos", "check-probe", tag)
+ bin := filepath.Join(dir, asset)
+ sumFile := bin + ".sha256"
+ if want, err := os.ReadFile(sumFile); err == nil {
+ if got, err := fileSHA256(bin); err == nil && got == strings.TrimSpace(string(want)) {
+ return bin, nil
+ }
+ }
+ data, sum, err := updateFeed().Asset(ctx, tag, asset, 256<<20)
+ if err != nil {
+ return "", err
+ }
+ if err := os.MkdirAll(dir, 0o755); err != nil {
+ return "", err
+ }
+ // The binary first, then its digest: a copy cut short between the two
+ // has no matching digest and is fetched again.
+ if err := writeFileAtomic(bin, data, 0o755); err != nil {
+ return "", err
+ }
+ if err := writeFileAtomic(sumFile, []byte(sum+"\n"), 0o644); err != nil {
+ return "", err
+ }
+ return bin, nil
+}
+
+func fileSHA256(path string) (string, error) {
+ f, err := os.Open(path)
+ if err != nil {
+ return "", err
+ }
+ defer f.Close()
+ h := sha256.New()
+ if _, err := io.Copy(h, f); err != nil {
+ return "", err
+ }
+ return hex.EncodeToString(h.Sum(nil)), nil
+}
+
+// writeFileAtomic writes data to a temp file beside path and renames it
+// over path.
+func writeFileAtomic(path string, data []byte, mode os.FileMode) error {
+ tmp, err := os.CreateTemp(filepath.Dir(path), ".carlos-check-probe-*")
+ if err != nil {
+ return err
+ }
+ defer os.Remove(tmp.Name())
+ if _, err := tmp.Write(data); err != nil {
+ tmp.Close()
+ return err
+ }
+ if err := tmp.Chmod(mode); err != nil {
+ tmp.Close()
+ return err
+ }
+ if err := tmp.Close(); err != nil {
+ return err
+ }
+ return os.Rename(tmp.Name(), path)
+}
diff --git a/ship_check_probe_test.go b/ship_check_probe_test.go
new file mode 100644
index 000000000..37e92c4f5
--- /dev/null
+++ b/ship_check_probe_test.go
@@ -0,0 +1,313 @@
+package main
+
+import (
+ "bytes"
+ "context"
+ "crypto/ed25519"
+ "crypto/rand"
+ "crypto/sha256"
+ "debug/elf"
+ "encoding/base64"
+ "encoding/binary"
+ "encoding/hex"
+ "encoding/json"
+ "fmt"
+ "net"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "runtime"
+ "slices"
+ "strings"
+ "sync/atomic"
+ "testing"
+)
+
+// probeSocketDir is a short directory for a unix socket: t.TempDir() can
+// pass the 104-byte limit of a socket path on macOS.
+func probeSocketDir(t *testing.T) string {
+ t.Helper()
+ if runtime.GOOS == "windows" {
+ t.Skip("the probe dials a unix socket inside a linux container")
+ }
+ dir, err := os.MkdirTemp("", "cp")
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { os.RemoveAll(dir) })
+ return dir
+}
+
+func TestCheckProbeAnswersOnTheFirstHealthyStatus(t *testing.T) {
+ sock := filepath.Join(probeSocketDir(t), "app.sock")
+ ln, err := net.Listen("unix", sock)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var hits atomic.Int32
+ srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Method != http.MethodGet || r.URL.Path != "/" || r.Host != checkHost {
+ t.Errorf("probe asked %s %s for %s", r.Method, r.URL.Path, r.Host)
+ }
+ if hits.Add(1) <= 2 {
+ w.WriteHeader(http.StatusServiceUnavailable)
+ return
+ }
+ w.WriteHeader(http.StatusOK)
+ })}
+ go srv.Serve(ln)
+ t.Cleanup(func() { srv.Close() })
+
+ out := captureStdout(t)
+ if code := runRootfsCheckProbe([]string{"--socket", sock, "--timeout", "10s"}); code != 0 {
+ t.Fatalf("exit %d: %s", code, out)
+ }
+ var got map[string]any
+ if err := json.Unmarshal(out.Bytes(), &got); err != nil || got["status"] != float64(200) {
+ t.Fatalf("probe said %q (%v)", out, err)
+ }
+ if _, ok := got["after_ms"]; !ok || len(got) != 2 {
+ t.Errorf("probe said %q, want status and after_ms only", out)
+ }
+ if n := hits.Load(); n != 3 {
+ t.Errorf("%d requests, want 3", n)
+ }
+}
+
+const sampleProcNetTCP = ` sl local_address rem_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode
+ 0: 00000000:0BB8 00000000:0000 0A 00000000:00000000 00:00000000 00000000 65534 0 12345 1 0000000000000000 100 0 0 10 0
+ 1: 0100007F:1F90 0100007F:D431 01 00000000:00000000 00:00000000 00000000 65534 0 12346 1 0000000000000000 20 4 30 10 -1
+`
+
+const sampleProcNetTCP6 = ` sl local_address remote_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode
+ 0: 00000000000000000000000000000000:0BB8 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 65534 0 22345 1 0000000000000000 100 0 0 10 0
+ 1: 00000000000000000000000000000000:0050 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 65534 0 22346 1 0000000000000000 100 0 0 10 0
+`
+
+func TestCheckProbeTimesOutAndNamesListeningPorts(t *testing.T) {
+ dir := probeSocketDir(t)
+ tcp, tcp6 := filepath.Join(dir, "tcp"), filepath.Join(dir, "tcp6")
+ if err := os.WriteFile(tcp, []byte(sampleProcNetTCP), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ if got := listeningPorts([]string{tcp}); !slices.Equal(got, []int{3000}) {
+ t.Errorf("ports = %v, want the LISTEN row's 3000 and not the ESTABLISHED 8080", got)
+ }
+ if err := os.WriteFile(tcp6, []byte(sampleProcNetTCP6), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ if got := listeningPorts([]string{tcp, tcp6, filepath.Join(dir, "missing")}); !slices.Equal(got, []int{80, 3000}) {
+ t.Errorf("ports = %v, want 80 and 3000, each once", got)
+ }
+
+ old := checkProbeProcNet
+ checkProbeProcNet = []string{tcp}
+ t.Cleanup(func() { checkProbeProcNet = old })
+
+ // Nothing on the socket.
+ out := captureStdout(t)
+ if code := runRootfsCheckProbe([]string{"--socket", filepath.Join(dir, "none.sock"), "--timeout", "300ms"}); code != 1 {
+ t.Fatalf("exit %d, want 1", code)
+ }
+ if got, want := out.String(), `{"status":0,"ports":[3000]}`+"\n"; got != want {
+ t.Errorf("probe said %q, want %q", got, want)
+ }
+
+ // A server that only fails: its last status is named.
+ sock := filepath.Join(dir, "app.sock")
+ ln, err := net.Listen("unix", sock)
+ if err != nil {
+ t.Fatal(err)
+ }
+ srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.WriteHeader(http.StatusBadGateway)
+ })}
+ go srv.Serve(ln)
+ t.Cleanup(func() { srv.Close() })
+ out.Reset()
+ if code := runRootfsCheckProbe([]string{"--socket", sock, "--timeout", "600ms"}); code != 1 {
+ t.Fatalf("exit %d, want 1", code)
+ }
+ if got, want := out.String(), `{"status":502,"ports":[3000]}`+"\n"; got != want {
+ t.Errorf("probe said %q, want %q", got, want)
+ }
+
+ if code := runRootfsCheckProbe(nil); code != 2 {
+ t.Errorf("no --socket: exit %d, want 2", code)
+ }
+}
+
+// probeTestELF is an ELF64 executable with one program header: PT_INTERP
+// for a dynamic build, PT_LOAD for a static one.
+func probeTestELF(t *testing.T, path string, dynamic bool) {
+ t.Helper()
+ h := make([]byte, 64+56)
+ copy(h, "\x7fELF")
+ h[4], h[5], h[6] = 2, 1, 1 // 64-bit, little-endian, version 1
+ le := binary.LittleEndian
+ le.PutUint16(h[16:], uint16(elf.ET_EXEC))
+ le.PutUint16(h[18:], uint16(elf.EM_AARCH64))
+ le.PutUint32(h[20:], 1)
+ le.PutUint64(h[32:], 64) // program headers right after this header
+ le.PutUint16(h[52:], 64)
+ le.PutUint16(h[54:], 56)
+ le.PutUint16(h[56:], 1)
+ typ := elf.PT_LOAD
+ if dynamic {
+ typ = elf.PT_INTERP
+ }
+ le.PutUint32(h[64:], uint32(typ))
+ if err := os.WriteFile(path, h, 0o755); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// probeSeams sets the platform, the running executable and the cache dir.
+func probeSeams(t *testing.T, goos, goarch, exe, cache string) {
+ t.Helper()
+ oldOS, oldArch, oldExe, oldCache := checkProbeGOOS, checkProbeGOARCH, checkProbeExecutable, checkProbeCacheDir
+ checkProbeGOOS, checkProbeGOARCH = goos, goarch
+ checkProbeExecutable = func() (string, error) { return exe, nil }
+ checkProbeCacheDir = func() (string, error) { return cache, nil }
+ t.Cleanup(func() {
+ checkProbeGOOS, checkProbeGOARCH, checkProbeExecutable, checkProbeCacheDir = oldOS, oldArch, oldExe, oldCache
+ })
+ t.Setenv("CARLOS_CHECK_PROBE", "")
+}
+
+// probeFeed serves v0.8.0's signed checksum list and the given assets, with
+// the update tests' seams (updateDownloadBase, cliUpdatePubKey) pointed at
+// it, and counts each asset's downloads.
+func probeFeed(t *testing.T, assets map[string][]byte) map[string]*atomic.Int32 {
+ t.Helper()
+ pub, priv, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var sums strings.Builder
+ hits := map[string]*atomic.Int32{}
+ for name, body := range assets {
+ s := sha256.Sum256(body)
+ fmt.Fprintf(&sums, "%s %s\n", hex.EncodeToString(s[:]), name)
+ hits[name] = &atomic.Int32{}
+ }
+ sig := ed25519.Sign(priv, []byte(sums.String()))
+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ name, ok := strings.CutPrefix(r.URL.Path, "/v0.8.0/")
+ switch {
+ case !ok:
+ http.NotFound(w, r)
+ case name == "SHA256SUMS.txt":
+ w.Write([]byte(sums.String()))
+ case name == "SHA256SUMS.txt.sig":
+ w.Write([]byte(base64.StdEncoding.EncodeToString(sig)))
+ case assets[name] != nil:
+ hits[name].Add(1)
+ w.Write(assets[name])
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ t.Cleanup(ts.Close)
+ oldBase, oldKey := updateDownloadBase, cliUpdatePubKey
+ updateDownloadBase, cliUpdatePubKey = ts.URL, base64.StdEncoding.EncodeToString(pub)
+ t.Cleanup(func() { updateDownloadBase, cliUpdatePubKey = oldBase, oldKey })
+ return hits
+}
+
+func TestCheckProbeBinaryOrder(t *testing.T) {
+ dir := t.TempDir()
+ static, dynamic := filepath.Join(dir, "static"), filepath.Join(dir, "dynamic")
+ probeTestELF(t, static, false)
+ probeTestELF(t, dynamic, true)
+ ctx := context.Background()
+ c76 := "--check needs a linux build of carlos to probe the socket from inside the container, and this one is a development build. use a released carlos, or set CARLOS_CHECK_PROBE to a linux carlos binary"
+
+ t.Run("CARLOS_CHECK_PROBE wins", func(t *testing.T) {
+ probeSeams(t, "linux", "arm64", static, t.TempDir())
+ t.Chdir(dir)
+ t.Setenv("CARLOS_CHECK_PROBE", "probe")
+ got, err := checkProbeBinary(ctx)
+ if want, _ := filepath.Abs("probe"); err != nil || got != want {
+ t.Errorf("got %q, %v; want %q", got, err, want)
+ }
+ })
+ t.Run("a static linux binary is used as is", func(t *testing.T) {
+ probeSeams(t, "linux", "arm64", static, t.TempDir())
+ useVersion(t, "dev")
+ got, err := checkProbeBinary(ctx)
+ if want, _ := filepath.EvalSymlinks(static); err != nil || got != want {
+ t.Errorf("got %q, %v; want %q", got, err, want)
+ }
+ })
+ t.Run("a dynamic development build is C76", func(t *testing.T) {
+ probeSeams(t, "linux", "arm64", dynamic, t.TempDir())
+ useVersion(t, "dev")
+ if _, err := checkProbeBinary(ctx); err == nil || err.Error() != c76 {
+ t.Errorf("err = %v, want C76", err)
+ }
+ })
+ t.Run("a development build off Linux is C76", func(t *testing.T) {
+ probeSeams(t, "darwin", "arm64", static, t.TempDir())
+ useVersion(t, "dev")
+ if _, err := checkProbeBinary(ctx); err == nil || err.Error() != c76 {
+ t.Errorf("err = %v, want C76", err)
+ }
+ })
+ t.Run("a release fetches linux-arm64 once", func(t *testing.T) {
+ cache := t.TempDir()
+ probeSeams(t, "darwin", "arm64", static, cache)
+ useVersion(t, "v0.8.0")
+ body := []byte("the arm64 carlos")
+ hits := probeFeed(t, map[string][]byte{"carlos-linux-arm64": body})
+ want := filepath.Join(cache, "carlos", "check-probe", "v0.8.0", "carlos-linux-arm64")
+ for i := range 2 {
+ got, err := checkProbeBinary(ctx)
+ if err != nil || got != want {
+ t.Fatalf("call %d: %q, %v; want %q", i+1, got, err, want)
+ }
+ }
+ if n := hits["carlos-linux-arm64"].Load(); n != 1 {
+ t.Errorf("%d downloads, want 1: the second use is the cache", n)
+ }
+ if b, err := os.ReadFile(want); err != nil || !bytes.Equal(b, body) {
+ t.Errorf("cached %q, %v", b, err)
+ }
+ if fi, err := os.Stat(want); err != nil || fi.Mode().Perm()&0o100 == 0 {
+ t.Errorf("the cached probe is not executable: %v, %v", fi, err)
+ }
+ // A damaged copy no longer matches its digest and is fetched again.
+ if err := os.WriteFile(want, []byte("damaged"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := checkProbeBinary(ctx); err != nil {
+ t.Fatal(err)
+ }
+ if b, _ := os.ReadFile(want); !bytes.Equal(b, body) || hits["carlos-linux-arm64"].Load() != 2 {
+ t.Errorf("a damaged copy was used: %q", b)
+ }
+ })
+ t.Run("a dynamic linux release fetches its own architecture", func(t *testing.T) {
+ probeSeams(t, "linux", "amd64", dynamic, t.TempDir())
+ useVersion(t, "v0.8.0")
+ hits := probeFeed(t, map[string][]byte{"carlos-linux-amd64": []byte("amd64"), "carlos-linux-arm64": []byte("arm64")})
+ got, err := checkProbeBinary(ctx)
+ if err != nil || filepath.Base(got) != "carlos-linux-amd64" {
+ t.Fatalf("%q, %v", got, err)
+ }
+ if hits["carlos-linux-arm64"].Load() != 0 {
+ t.Error("fetched arm64 for an amd64 Linux host")
+ }
+ })
+ t.Run("a failed fetch is C77", func(t *testing.T) {
+ probeSeams(t, "darwin", "arm64", static, t.TempDir())
+ useVersion(t, "v0.8.0")
+ probeFeed(t, map[string][]byte{"carlos-linux-amd64": []byte("amd64")})
+ _, err := checkProbeBinary(ctx)
+ want := "could not fetch the linux build of carlos for the check: no carlos-linux-arm64 in the release's checksum list"
+ if err == nil || err.Error() != want {
+ t.Errorf("err = %v, want %q", err, want)
+ }
+ })
+}
diff --git a/ship_check_test.go b/ship_check_test.go
new file mode 100644
index 000000000..f949241fe
--- /dev/null
+++ b/ship_check_test.go
@@ -0,0 +1,777 @@
+package main
+
+import (
+ "archive/tar"
+ "bytes"
+ "context"
+ "errors"
+ "io"
+ "maps"
+ "os"
+ "path"
+ "path/filepath"
+ "regexp"
+ "runtime"
+ "slices"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/rootfsimage"
+)
+
+// The check's tests never need Docker or Podman: the runtime is fakeCheck, a
+// recording runnerFunc that answers from a script and keeps the set of
+// objects the check made, which must end empty.
+
+// checkName is the shape of every name the check gives the runtime.
+var checkName = regexp.MustCompile(`^carlos-check-[0-9a-f]{8}(-[a-z][a-z0-9-]*|:latest)$`)
+
+type fakeCheck struct {
+ t *testing.T
+ calls [][]string
+ // made is every object the check asked for, live the ones that exist,
+ // keyed "container:<name>", "volume:<name>" or "image:<name>".
+ made, live map[string]bool
+ strays []string
+ cleaning bool
+
+ arch string
+ waitOut string
+ waitHang bool
+ probeOut string
+ probeErr error
+ runErr map[string]error // by process
+ state map[string]string // inspect's State JSON, by process
+ logs map[string]string // by process
+
+ failAt, cancelAt int // the index of the call to fail or cancel at; -1 for none
+ cancel context.CancelFunc
+
+ out *bytes.Buffer
+ atImport string // what the report had said when import ran
+ env map[string]string // each process's env file, read at its run
+ envMode map[string]os.FileMode
+}
+
+func newFakeCheck(t *testing.T) *fakeCheck {
+ return &fakeCheck{
+ t: t, made: map[string]bool{}, live: map[string]bool{},
+ arch: "aarch64", waitOut: "0\n", probeOut: `{"status":200,"after_ms":42}` + "\n",
+ runErr: map[string]error{}, state: map[string]string{}, logs: map[string]string{},
+ failAt: -1, cancelAt: -1, out: &bytes.Buffer{},
+ env: map[string]string{}, envMode: map[string]os.FileMode{},
+ }
+}
+
+func (f *fakeCheck) rt() containerRuntime { return containerRuntime{name: "docker", run: f.run} }
+
+// procOf is the process a container is named for.
+func procOf(ctr string) string {
+ return strings.TrimPrefix(ctr, ctr[:len("carlos-check-")+9])
+}
+
+func flagValue(args []string, flag string) string {
+ if i := slices.Index(args, flag); i >= 0 && i+1 < len(args) {
+ return args[i+1]
+ }
+ return ""
+}
+
+func (f *fakeCheck) run(ctx context.Context, stdin io.Reader, args ...string) ([]byte, error) {
+ if stdin != nil {
+ io.Copy(io.Discard, stdin)
+ }
+ i := len(f.calls)
+ f.calls = append(f.calls, slices.Clone(args))
+ last := args[len(args)-1]
+
+ var removes string
+ switch {
+ case args[0] == "rm":
+ removes = "container:" + last
+ case args[0] == "volume" && args[1] == "rm":
+ removes = "volume:" + last
+ case args[0] == "rmi":
+ removes = "image:" + last
+ }
+ if removes != "" {
+ f.cleaning = true
+ if ctx.Err() != nil {
+ f.t.Errorf("%v ran under an ended context", args)
+ }
+ if !f.made[removes] {
+ f.strays = append(f.strays, removes)
+ }
+ delete(f.live, removes)
+ return nil, nil
+ }
+ if f.cleaning {
+ f.t.Errorf("%v after the cleanup began", args)
+ }
+
+ var makes string
+ switch {
+ case args[0] == "import":
+ makes = "image:" + last
+ case args[0] == "volume" && args[1] == "create":
+ makes = "volume:" + last
+ case args[0] == "run":
+ ctr := flagValue(args, "--name")
+ makes = "container:" + ctr
+ proc := procOf(ctr)
+ if b, err := os.ReadFile(flagValue(args, "--env-file")); err == nil {
+ f.env[proc] = string(b)
+ }
+ if fi, err := os.Stat(flagValue(args, "--env-file")); err == nil {
+ f.envMode[proc] = fi.Mode().Perm()
+ }
+ }
+ if makes != "" {
+ if name := makes[strings.IndexByte(makes, ':')+1:]; !checkName.MatchString(name) {
+ f.t.Errorf("%v names %q", args, name)
+ }
+ f.made[makes] = true
+ }
+ if args[0] == "import" {
+ f.atImport = f.out.String()
+ }
+
+ if i == f.failAt {
+ // A run that fails may still have made its container, as docker
+ // run does when the start fails after the create.
+ if args[0] == "run" {
+ f.live[makes] = true
+ }
+ return nil, errors.New("injected failure")
+ }
+ if makes != "" {
+ f.live[makes] = true
+ }
+ if i == f.cancelAt {
+ f.cancel()
+ return nil, ctx.Err()
+ }
+
+ switch args[0] {
+ case "info":
+ return []byte(f.arch + "\n"), nil
+ case "run":
+ return []byte("0123abcd\n"), f.runErr[procOf(flagValue(args, "--name"))]
+ case "wait":
+ if f.waitHang {
+ <-ctx.Done()
+ return nil, ctx.Err()
+ }
+ return []byte(f.waitOut), nil
+ case "exec":
+ return []byte(f.probeOut), f.probeErr
+ case "inspect":
+ if s, ok := f.state[procOf(last)]; ok {
+ return []byte(s + "\n"), nil
+ }
+ return []byte(`{"Status":"running","Running":true,"OOMKilled":false,"ExitCode":0}` + "\n"), nil
+ case "logs":
+ return []byte(f.logs[procOf(last)]), nil
+ }
+ return nil, nil
+}
+
+// summary is each call as verb and object: "run web", "volume rm data".
+func (f *fakeCheck) summary() []string {
+ var out []string
+ for _, c := range f.calls {
+ obj := func(name string) string {
+ if strings.HasSuffix(name, ":latest") {
+ return "image"
+ }
+ return procOf(name)
+ }
+ switch c[0] {
+ case "info", "import", "rmi":
+ out = append(out, c[0])
+ case "volume":
+ out = append(out, "volume "+c[1]+" "+obj(c[len(c)-1]))
+ case "run":
+ out = append(out, "run "+obj(flagValue(c, "--name")))
+ case "exec":
+ out = append(out, "exec "+obj(c[1]))
+ case "rm":
+ out = append(out, "rm -f "+obj(c[len(c)-1]))
+ default:
+ out = append(out, c[0]+" "+obj(c[len(c)-1]))
+ }
+ }
+ return out
+}
+
+// checkLeftNothing fails t when the check left an object behind or removed
+// one it never asked for.
+func (f *fakeCheck) checkLeftNothing(t *testing.T) {
+ t.Helper()
+ if len(f.live) != 0 {
+ t.Errorf("left behind: %v", f.live)
+ }
+ if len(f.strays) != 0 {
+ t.Errorf("removed what it never made: %v", f.strays)
+ }
+}
+
+// noSettle makes workerSettle cost nothing.
+func noSettle(t *testing.T) {
+ t.Helper()
+ old := checkPause
+ checkPause = func(ctx context.Context, _ time.Duration) error { return ctx.Err() }
+ t.Cleanup(func() { checkPause = old })
+}
+
+// checkTree is a tree holding the given files, each executable.
+func checkTree(t *testing.T, files map[string]string) *rootfsimage.Tree {
+ t.Helper()
+ var buf bytes.Buffer
+ tw := tar.NewWriter(&buf)
+ mt := time.Unix(1_700_000_000, 0)
+ dirs := map[string]bool{}
+ for name := range files {
+ for d := path.Dir(name); d != "."; d = path.Dir(d) {
+ dirs[d] = true
+ }
+ }
+ for _, d := range slices.Sorted(maps.Keys(dirs)) {
+ if err := tw.WriteHeader(&tar.Header{Name: d + "/", Typeflag: tar.TypeDir, Mode: 0o755, ModTime: mt}); err != nil {
+ t.Fatal(err)
+ }
+ }
+ for _, name := range slices.Sorted(maps.Keys(files)) {
+ body := files[name]
+ if err := tw.WriteHeader(&tar.Header{Name: name, Typeflag: tar.TypeReg, Mode: 0o755, Size: int64(len(body)), ModTime: mt}); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := tw.Write([]byte(body)); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if err := tw.Close(); err != nil {
+ t.Fatal(err)
+ }
+ tree := rootfsimage.NewTree(t.TempDir())
+ if err := tree.ApplyLayer(&buf); err != nil {
+ t.Fatal(err)
+ }
+ return tree
+}
+
+// shopProcs is a release, a web and one worker, jobs.
+func shopProcs() map[string]release.Process {
+ return map[string]release.Process{
+ release.ProcessRelease: {Command: []string{"/srv/migrate"}},
+ release.ProcessWeb: {Command: []string{"/srv/server", "--port", "x"}, WorkingDir: "/srv",
+ Env: map[string]string{"SHOP_MODE": "production", "A_FIRST": "1"}},
+ "jobs": {Command: []string{"/srv/jobs"}},
+ }
+}
+
+func shopBuild(t *testing.T, procs map[string]release.Process) *builtRootfs {
+ t.Helper()
+ tree := checkTree(t, map[string]string{"srv/server": "x", "srv/migrate": "x", "srv/jobs": "x"})
+ return &builtRootfs{tree: tree, procs: procs, scratch: t.TempDir()}
+}
+
+const (
+ checkC60 = "checking the image with docker: read-only root, 1.0GB of memory, 120 s for web to answer"
+ checkC74 = "the image passed the local check. a box also applies its egress fence and its own limits, so this is a convenience, not a guarantee"
+ checkC75 = "the image failed the local check"
+)
+
+func TestRootfsCheckHappyPath(t *testing.T) {
+ noSettle(t)
+ f := newFakeCheck(t)
+ r := runRootfsCheck(context.Background(), f.rt(), shopBuild(t, shopProcs()), "/probe/carlos", f.out)
+
+ want := []string{
+ "info", "import", "volume create data", "volume create run",
+ "run release", "wait release", "run web", "exec web", "run jobs", "inspect jobs",
+ "rm -f release", "rm -f web", "rm -f jobs", "volume rm data", "volume rm run", "rmi",
+ }
+ if got := f.summary(); !slices.Equal(got, want) {
+ t.Errorf("calls:\n got %q\nwant %q", got, want)
+ }
+ if r.failed {
+ t.Errorf("the report failed: %q", r.lines)
+ }
+ if len(r.lines) != 5 {
+ t.Fatalf("report = %q", r.lines)
+ }
+ if r.lines[0] != checkC60 {
+ t.Errorf("first line %q, want C60", r.lines[0])
+ }
+ for i, prefix := range []string{"release finished in ", "web answered GET / with 200 after 0.0 s", "jobs is still running after 5.0 s", checkC74} {
+ if !strings.HasPrefix(r.lines[i+1], prefix) {
+ t.Errorf("line %d = %q, want %q", i+1, r.lines[i+1], prefix)
+ }
+ }
+ if got := f.out.String(); got != strings.Join(r.lines, "\n")+"\n" {
+ t.Errorf("out = %q, want the report's lines", got)
+ }
+ f.checkLeftNothing(t)
+}
+
+func TestRootfsCheckWarnsOnAnotherArchitecture(t *testing.T) {
+ noSettle(t)
+ for _, c := range []struct{ runtime, arch, format string }{
+ {"docker", "x86_64", "{{.Architecture}}"},
+ {"podman", "amd64", "{{.Host.Arch}}"},
+ } {
+ f := newFakeCheck(t)
+ f.arch = c.arch
+ rt := f.rt()
+ rt.name = c.runtime
+ r := runRootfsCheck(context.Background(), rt, shopBuild(t, shopProcs()), "/probe/carlos", f.out)
+ if got := f.calls[0]; !slices.Equal(got, []string{"info", "--format", c.format}) {
+ t.Errorf("%s: info argv = %q", c.runtime, got)
+ }
+ want := "this machine is " + c.arch + ", so the arm64 image runs under emulation and starts slower than on a box"
+ if !slices.Contains(r.lines, want) || r.failed {
+ t.Errorf("%s: report = %q", c.runtime, r.lines)
+ }
+ }
+}
+
+func TestRootfsCheckRunArgs(t *testing.T) {
+ noSettle(t)
+ f := newFakeCheck(t)
+ r := runRootfsCheck(context.Background(), f.rt(), shopBuild(t, shopProcs()), "/probe/carlos", f.out)
+ if r.failed {
+ t.Fatalf("report = %q", r.lines)
+ }
+ var web []string
+ for _, c := range f.calls {
+ if c[0] == "run" && procOf(flagValue(c, "--name")) == "web" {
+ web = c
+ }
+ }
+ if web == nil {
+ t.Fatal("web was never run")
+ }
+ name := flagValue(web, "--name")
+ prefix := strings.TrimSuffix(name, "-web")
+ id := strings.TrimPrefix(prefix, "carlos-check-")
+ if !slices.Equal(web[:4], []string{"run", "-d", "--name", name}) {
+ t.Errorf("argv starts %q", web[:4])
+ }
+ for _, pair := range [][2]string{
+ {"--tmpfs", "/tmp"},
+ {"--memory", "1073741824"},
+ {"--memory-swap", "1073741824"},
+ {"--cap-drop", "ALL"},
+ {"--security-opt", "no-new-privileges"},
+ {"--platform", "linux/arm64"},
+ {"--user", "65534:65534"},
+ {"-w", "/srv"},
+ {"--entrypoint", "/srv/server"},
+ {"--label", "carlos-check=" + id},
+ } {
+ if got := flagValue(web, pair[0]); got != pair[1] {
+ t.Errorf("%s = %q, want %q", pair[0], got, pair[1])
+ }
+ }
+ if !slices.Contains(web, "--read-only") {
+ t.Error("no --read-only")
+ }
+ var mounts []string
+ for i, a := range web {
+ if a == "-v" {
+ mounts = append(mounts, web[i+1])
+ }
+ }
+ wantMounts := []string{
+ prefix + "-data:/data/instances/check",
+ prefix + "-run:/run/carlos/instances/check",
+ "/probe/carlos:/opt/carlos/carlos:ro",
+ }
+ if !slices.Equal(mounts, wantMounts) {
+ t.Errorf("mounts = %q, want %q", mounts, wantMounts)
+ }
+ if tail := web[len(web)-3:]; !slices.Equal(tail, []string{prefix + ":latest", "--port", "x"}) {
+ t.Errorf("argv ends %q, want the image then the command's arguments", tail)
+ }
+
+ wantEnv := "A_FIRST=1\nSHOP_MODE=production\n" +
+ "CARLOS_SOCKET=/run/carlos/instances/check/app.sock\n" +
+ "CARLOS_DB=/data/instances/check/shop.example.com.db\n" +
+ "CARLOS_HOST=shop.example.com\nCARLOS_REGION=local\n"
+ if got := f.env["web"]; got != wantEnv {
+ t.Errorf("web's env file:\n%s\nwant:\n%s", got, wantEnv)
+ }
+ for _, unwanted := range []string{"CARLOS_CONTROL_SOCKET", "CARLOS_PURGE"} {
+ if strings.Contains(f.env["web"], unwanted) {
+ t.Errorf("the env file sets %s", unwanted)
+ }
+ }
+ if runtime.GOOS != "windows" && f.envMode["web"] != 0o600 {
+ t.Errorf("the env file's mode is %v, want 0600", f.envMode["web"])
+ }
+
+ // The probe runs inside web, from the mounted binary.
+ var exec []string
+ for _, c := range f.calls {
+ if c[0] == "exec" {
+ exec = c
+ }
+ }
+ if want := []string{"exec", name, "/opt/carlos/carlos", "__rootfs-check-probe", "--socket", "/run/carlos/instances/check/app.sock", "--timeout", "120s"}; !slices.Equal(exec, want) {
+ t.Errorf("exec = %q, want %q", exec, want)
+ }
+ // Volumes are tmpfs, owned by the check's user.
+ if want := []string{"volume", "create", "--driver", "local", "--opt", "type=tmpfs", "--opt", "device=tmpfs", "--opt", "o=uid=65534,gid=65534,mode=0700", prefix + "-data"}; !slices.Equal(f.calls[2], want) {
+ t.Errorf("volume create = %q, want %q", f.calls[2], want)
+ }
+ if want := []string{"inspect", "--format", "{{json .State}}", prefix + "-jobs"}; !slices.Equal(f.calls[9], want) {
+ t.Errorf("inspect = %q, want %q", f.calls[9], want)
+ }
+}
+
+func TestRootfsCheckClassifiesWebFailures(t *testing.T) {
+ noSettle(t)
+ exit1 := errors.New("exit status 1")
+ cases := []struct {
+ name string
+ setup func(f *fakeCheck)
+ want []string
+ }{
+ {"a port instead of the socket", func(f *fakeCheck) {
+ f.probeOut, f.probeErr = `{"status":0,"ports":[3000]}`+"\n", exit1
+ }, []string{"web listens on port 3000 instead of the socket. serve HTTP on the unix socket at CARLOS_SOCKET"}},
+ {"a 503", func(f *fakeCheck) {
+ f.probeOut, f.probeErr = `{"status":503,"ports":[]}`+"\n", exit1
+ }, []string{"web answered GET / with 503. health needs an answer below 500"}},
+ {"out of memory", func(f *fakeCheck) {
+ f.probeOut, f.probeErr = "", errors.New("container is not running")
+ f.state["web"] = `{"Running":false,"OOMKilled":true,"ExitCode":137}`
+ }, []string{"web ran out of memory. a rootfs instance's tier allows 1.0GB"}},
+ {"a write to the root", func(f *fakeCheck) {
+ f.probeOut, f.probeErr = "", errors.New("container is not running")
+ f.state["web"] = `{"Running":false,"ExitCode":1}`
+ f.logs["web"] = "booting\nopen /srv/log/shop.log: Read-only file system\n"
+ }, []string{"web tried to write outside its instance directory. the root is read-only, so write under /data/instances/check or /tmp"}},
+ {"an exit", func(f *fakeCheck) {
+ f.probeOut, f.probeErr = "", errors.New("container is not running")
+ f.state["web"] = `{"Running":false,"ExitCode":1}`
+ f.logs["web"] = "booting\r\nno config for shop\n"
+ }, []string{"web exited with status 1. its last lines:", " booting", " no config for shop"}},
+ {"an amd64 program", func(f *fakeCheck) {
+ f.runErr["web"] = errors.New(`docker: Error response from daemon: failed to create task for container: exec: "/srv/server": exec format error`)
+ }, []string{"the arm64 image cannot run here. install emulation for arm64 (qemu and binfmt), or run the check on an arm64 machine"}},
+ {"a probe that is not the probe", func(f *fakeCheck) {
+ f.probeOut, f.probeErr = "hello\n", nil
+ }, []string{`docker exec: the probe answered "hello"`}},
+ {"an amd64 program, as crun says it", func(f *fakeCheck) {
+ f.probeOut, f.probeErr = "", errors.New("container is not running")
+ f.state["web"] = `{"Running":false,"ExitCode":1}`
+ f.logs["web"] = `{"msg":"exec container process` + " `/srv/server`" + `: Exec format error","level":"error"}` + "\n"
+ }, []string{"the arm64 image cannot run here. install emulation for arm64 (qemu and binfmt), or run the check on an arm64 machine"}},
+ {"nothing listening", func(f *fakeCheck) {
+ f.probeOut, f.probeErr = `{"status":0,"ports":[]}`+"\n", exit1
+ }, []string{"web did not answer GET / on CARLOS_SOCKET within 120 s"}},
+ }
+ for _, c := range cases {
+ t.Run(c.name, func(t *testing.T) {
+ f := newFakeCheck(t)
+ c.setup(f)
+ r := runRootfsCheck(context.Background(), f.rt(), shopBuild(t, shopProcs()), "/probe/carlos", f.out)
+ if !r.failed {
+ t.Error("the report passed")
+ }
+ want := append(append([]string{}, c.want...), checkC75)
+ if got := r.lines[len(r.lines)-len(want):]; !slices.Equal(got, want) {
+ t.Errorf("report ends %q, want %q", got, want)
+ }
+ for _, s := range f.summary() {
+ if s == "run jobs" {
+ t.Error("a worker started after web failed")
+ }
+ }
+ f.checkLeftNothing(t)
+ })
+ }
+}
+
+func TestRootfsCheckReleaseFailureStopsTheCheck(t *testing.T) {
+ noSettle(t)
+ t.Run("an exit", func(t *testing.T) {
+ f := newFakeCheck(t)
+ f.waitOut = "3\n"
+ f.logs["release"] = "migrating\nno such table: orders\n"
+ r := runRootfsCheck(context.Background(), f.rt(), shopBuild(t, shopProcs()), "/probe/carlos", f.out)
+ want := []string{"release exited with status 3. its last lines:", " migrating", " no such table: orders", checkC75}
+ if !r.failed || !slices.Equal(r.lines[1:], want) {
+ t.Errorf("report = %q (failed %v), want C60 then %q", r.lines, r.failed, want)
+ }
+ if slices.Contains(f.summary(), "run web") {
+ t.Error("web ran after release failed")
+ }
+ f.checkLeftNothing(t)
+ })
+ t.Run("a timeout", func(t *testing.T) {
+ f := newFakeCheck(t)
+ f.waitHang = true
+ procs := shopProcs()
+ procs[release.ProcessRelease] = release.Process{Command: []string{"/srv/migrate"}, TimeoutSec: 1}
+ r := runRootfsCheck(context.Background(), f.rt(), shopBuild(t, procs), "/probe/carlos", f.out)
+ want := []string{"release did not finish within 1.0 s", checkC75}
+ if !r.failed || !slices.Equal(r.lines[1:], want) {
+ t.Errorf("report = %q, want C60 then %q", r.lines, want)
+ }
+ if slices.Contains(f.summary(), "run web") {
+ t.Error("web ran after release timed out")
+ }
+ f.checkLeftNothing(t)
+ })
+}
+
+func TestRootfsCheckWorkerThatExits(t *testing.T) {
+ noSettle(t)
+ f := newFakeCheck(t)
+ f.state["jobs"] = `{"Running":false,"ExitCode":2}`
+ f.logs["jobs"] = "jobs: no queue\n"
+ r := runRootfsCheck(context.Background(), f.rt(), shopBuild(t, shopProcs()), "/probe/carlos", f.out)
+ want := []string{"jobs exited with status 2. its last lines:", " jobs: no queue", checkC75}
+ if !r.failed || !slices.Equal(r.lines[len(r.lines)-3:], want) {
+ t.Errorf("report = %q, want it to end %q", r.lines, want)
+ }
+ f.checkLeftNothing(t)
+}
+
+func TestRootfsCheckNeverStartsConsole(t *testing.T) {
+ noSettle(t)
+ f := newFakeCheck(t)
+ procs := shopProcs()
+ procs[release.ProcessConsole] = release.Process{Command: []string{"/srv/server", "console"}}
+ b := shopBuild(t, procs)
+ r := runRootfsCheck(context.Background(), f.rt(), b, "/probe/carlos", f.out)
+ if r.failed {
+ t.Fatalf("report = %q", r.lines)
+ }
+ for _, s := range f.summary() {
+ if strings.HasSuffix(s, " console") {
+ t.Errorf("the check called %q", s)
+ }
+ }
+ if _, ok := f.env["console"]; ok {
+ t.Error("console got an env file")
+ }
+ // The env files go with the check.
+ if ents, _ := os.ReadDir(b.scratch); len(ents) != 0 {
+ t.Errorf("the scratch still holds %v", ents)
+ }
+}
+
+func TestRootfsCheckWarnsAboutAJITWithoutTheFlag(t *testing.T) {
+ noSettle(t)
+ c85 := "the web process runs node, which needs a JIT, and this release does not declare one. it passes here but stops at start on a box. ship it with --jit"
+ nodeBuild := func(t *testing.T, jit bool) *builtRootfs {
+ tree := checkTree(t, map[string]string{"usr/local/bin/node": "x", "srv/server.js": "x"})
+ procs := map[string]release.Process{
+ release.ProcessWeb: {Command: []string{"node", "server.js"}, WorkingDir: "/srv"},
+ }
+ return &builtRootfs{tree: tree, procs: procs, scratch: t.TempDir(), jit: jit}
+ }
+
+ f := newFakeCheck(t)
+ r := runRootfsCheck(context.Background(), f.rt(), nodeBuild(t, false), "/probe/carlos", f.out)
+ if !slices.Contains(r.lines, c85) {
+ t.Errorf("no C85 in %q", r.lines)
+ }
+ if !strings.Contains(f.atImport, c85) {
+ t.Errorf("C85 came after the import: the report had said %q", f.atImport)
+ }
+ if r.failed || r.lines[len(r.lines)-1] != checkC74 {
+ t.Errorf("a JIT warning failed the check: %q", r.lines)
+ }
+
+ f = newFakeCheck(t)
+ r = runRootfsCheck(context.Background(), f.rt(), nodeBuild(t, true), "/probe/carlos", f.out)
+ if slices.Contains(r.lines, c85) {
+ t.Errorf("--jit still warned: %q", r.lines)
+ }
+
+ f = newFakeCheck(t)
+ rails := &builtRootfs{
+ tree: checkTree(t, map[string]string{"rails/bin/rails": "#!/usr/bin/env ruby\n"}),
+ procs: map[string]release.Process{
+ release.ProcessWeb: {Command: []string{"bin/rails", "server"}, WorkingDir: "/rails"},
+ },
+ scratch: t.TempDir(),
+ }
+ r = runRootfsCheck(context.Background(), f.rt(), rails, "/probe/carlos", f.out)
+ for _, l := range r.lines {
+ if strings.Contains(l, "needs a JIT") {
+ t.Errorf("a Rails web warned: %q", l)
+ }
+ }
+}
+
+// Review Focus 5: a failure, or a stop, at any step removes everything made
+// before it, and nothing else.
+func TestRootfsCheckCleansUpOnEveryPath(t *testing.T) {
+ noSettle(t)
+ happy := newFakeCheck(t)
+ runRootfsCheck(context.Background(), happy.rt(), shopBuild(t, shopProcs()), "/probe/carlos", happy.out)
+ steps := slices.IndexFunc(happy.calls, func(c []string) bool { return c[0] == "rm" })
+ if steps < 1 {
+ t.Fatalf("no cleanup in %q", happy.summary())
+ }
+
+ for i := range steps {
+ step := happy.summary()[i]
+ t.Run("failure at "+step, func(t *testing.T) {
+ f := newFakeCheck(t)
+ f.failAt = i
+ r := runRootfsCheck(context.Background(), f.rt(), shopBuild(t, shopProcs()), "/probe/carlos", f.out)
+ if !r.failed || r.lines[len(r.lines)-1] != checkC75 {
+ t.Errorf("report = %q, want it to fail", r.lines)
+ }
+ f.checkLeftNothing(t)
+ })
+ t.Run("stop at "+step, func(t *testing.T) {
+ f := newFakeCheck(t)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ f.cancelAt, f.cancel = i, cancel
+ r := runRootfsCheck(ctx, f.rt(), shopBuild(t, shopProcs()), "/probe/carlos", f.out)
+ if !r.failed {
+ t.Error("a stopped check passed")
+ }
+ if l := r.lines[len(r.lines)-1]; l == checkC74 || l == checkC75 {
+ t.Errorf("a stopped check ended %q", l)
+ }
+ for _, c := range f.calls[i+1:] {
+ if c[0] != "rm" && c[0] != "rmi" && !(c[0] == "volume" && c[1] == "rm") {
+ t.Errorf("%q after the stop", c)
+ }
+ }
+ f.checkLeftNothing(t)
+ })
+ }
+
+ t.Run("stop while the workers settle", func(t *testing.T) {
+ f := newFakeCheck(t)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ old := checkPause
+ checkPause = func(context.Context, time.Duration) error { cancel(); return context.Canceled }
+ defer func() { checkPause = old }()
+ r := runRootfsCheck(ctx, f.rt(), shopBuild(t, shopProcs()), "/probe/carlos", f.out)
+ if !r.failed {
+ t.Error("a stopped check passed")
+ }
+ f.checkLeftNothing(t)
+ })
+}
+
+// The production runner keeps a failed command's stdout, and gives logs
+// both of the container's streams.
+func TestExecRunnerKeepsStdoutAndJoinsLogs(t *testing.T) {
+ bin := t.TempDir()
+ stubRuntime(t, bin, "docker", `case "$1" in
+exec) echo '{"status":0,"ports":[3000]}'; echo 'probe gave up' >&2; exit 1 ;;
+logs) echo out-1; echo err-1 >&2; echo out-2 ;;
+esac
+`)
+ onlyPath(t, bin)
+ rt, err := findContainerRuntime()
+ if err != nil {
+ t.Fatal(err)
+ }
+ out, err := rt.run(context.Background(), nil, "exec", "x")
+ if err == nil || err.Error() != "probe gave up" || string(out) != `{"status":0,"ports":[3000]}`+"\n" {
+ t.Errorf("exec: %q, %v", out, err)
+ }
+ out, err = rt.run(context.Background(), nil, "logs", "x")
+ if err != nil || string(out) != "out-1\nerr-1\nout-2\n" {
+ t.Errorf("logs: %q, %v", out, err)
+ }
+}
+
+// Through the binary: carlos ship --rootfs --check with a stub docker, from
+// the flag rules to the exit status, and nothing left in the temp dir.
+func TestShipRootfsCheckThroughTheBinary(t *testing.T) {
+ if testing.Short() {
+ t.Skip("runs the carlos binary")
+ }
+ run, bucket := versioningCLI(t)
+ dir, procs := rootfsRailsDir(t)
+ bin, work := t.TempDir(), t.TempDir()
+ stubRuntime(t, bin, "docker", `echo "$@" >> "$FAKE_LOG"
+case "$1" in
+info) echo aarch64 ;;
+import) cat > /dev/null ;;
+exec) echo "$FAKE_PROBE"; exit "$FAKE_PROBE_EXIT" ;;
+inspect) echo '{"Running":true}' ;;
+esac
+exit 0
+`)
+ probe := filepath.Join(work, "probe")
+ if err := os.WriteFile(probe, []byte("x"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ tmp := t.TempDir()
+ t.Setenv("PATH", bin)
+ t.Setenv("CARLOS_CONTAINER_RUNTIME", "")
+ t.Setenv("CARLOS_CHECK_PROBE", probe)
+ t.Setenv("FAKE_LOG", filepath.Join(work, "log"))
+ t.Setenv("TMPDIR", tmp)
+
+ t.Setenv("FAKE_PROBE", `{"status":200,"after_ms":7}`)
+ t.Setenv("FAKE_PROBE_EXIT", "0")
+ out, err := run("ship", "--rootfs", dir, "--processes", procs, "--check")
+ if err != nil {
+ t.Fatalf("ship --check: %v\n%s", err, out)
+ }
+ for _, want := range []string{"web answered GET / with 200 after 0.0 s\n", checkC74 + "\n"} {
+ if !strings.Contains(out, want) {
+ t.Errorf("ship --check does not say %q:\n%s", want, out)
+ }
+ }
+ log, _ := os.ReadFile(filepath.Join(work, "log"))
+ for _, verb := range []string{"rm -f carlos-check-", "volume rm carlos-check-", "rmi carlos-check-"} {
+ if !strings.Contains(string(log), verb) {
+ t.Errorf("the runtime never saw %q:\n%s", verb, log)
+ }
+ }
+ if strings.Contains(string(log), "-console") {
+ t.Errorf("the filled-in console was started:\n%s", log)
+ }
+
+ t.Setenv("FAKE_PROBE", `{"status":0,"ports":[]}`)
+ t.Setenv("FAKE_PROBE_EXIT", "1")
+ out, err = run("ship", "--rootfs", dir, "--processes", procs, "--check")
+ var ee interface{ ExitCode() int }
+ if !errors.As(err, &ee) || ee.ExitCode() != 1 {
+ t.Fatalf("a failed check: %v\n%s", err, out)
+ }
+ for _, want := range []string{"web did not answer GET / on CARLOS_SOCKET within 120 s\n", checkC75 + "\n"} {
+ if !strings.Contains(out, want) {
+ t.Errorf("a failed check does not say %q:\n%s", want, out)
+ }
+ }
+
+ for _, flag := range [][]string{{"--app", "shop"}, {"-a", "shop"}, {"--version", "0.1.0-1"}, {"--console", "https://console.example.com"}, {"--message", "x"}} {
+ out, err := run(append([]string{"ship", "--rootfs", dir, "--processes", procs, "--check"}, flag...)...)
+ name := strings.TrimLeft(flag[0], "-")
+ if name == "a" {
+ name = "app"
+ }
+ if want := "--check ships nothing, so it cannot be combined with --" + name; err == nil || !strings.Contains(out, want) {
+ t.Errorf("--check %v: %v, want %q:\n%s", flag, err, want, out)
+ }
+ }
+
+ if left := rootfsScratchLeft(t, tmp); len(left) != 0 {
+ t.Errorf("--check left its scratch: %v", left)
+ }
+ if files := rootfsStoredFiles(t, bucket); len(files) != 0 {
+ t.Errorf("--check stored %v", files)
+ }
+}
diff --git a/ship_rootfs.go b/ship_rootfs.go
index 43d2ceb4b..3f1436775 100644
--- a/ship_rootfs.go
+++ b/ship_rootfs.go
@@ -10,6 +10,7 @@ import (
"os/signal"
"path/filepath"
"strings"
+ "sync"
"syscall"
"time"
@@ -34,6 +35,15 @@ var shipRootfsOnlyFlags = []string{"check", "hibernate", "jit", "processes"}
// (RFS-1-C35).
var shipRootfsExcludedFlags = []string{"kind", "target", "capabilities", "program-change"}
+// shipCheckExcludedFlags are the flags only a ship uses, which --check
+// refuses rather than ignores (RFS-1-C89): a check ships nothing, and a flag
+// that seems to take effect but does not would say otherwise.
+var shipCheckExcludedFlags = []string{
+ "app", "version", "label", "notes", "notes-file", "swap",
+ "console", "destination", "account", "account-id", "registry",
+ "message", "no-prompt",
+}
+
// shipRootfsMaxBytes is the largest image a ship sends (RFS-1-C43). It is
// release.MaxRootfsBytes, the bound WriteSquashfs already writes under, so
// with this value the check never fires for an image this binary built;
@@ -63,6 +73,10 @@ type rootfsShipOpts struct {
acctOf func() string
messageFlags mutationMessageFlags
+ // Resolved by runShipRootfs for --check.
+ rt containerRuntime
+ probe string
+
// Resolved by runShipRootfs.
procs map[string]release.Process
sess *apiSession // API mode
@@ -75,19 +89,26 @@ type rootfsShipOpts struct {
// builtRootfs is a rootfs image built on this machine: the tree it was
// built from, the processes with the image's config merged in and console
// filled, the squashfs file (empty for a --check build, which writes none)
-// and its size, and --jit, carried for the manifest and --check's warning.
+// and its size, --jit, carried for the manifest and --check's warning, and
+// the scratch directory it was built in, where --check writes its env
+// files.
type builtRootfs struct {
- tree *rootfsimage.Tree
- procs map[string]release.Process
- image string
- size int64
- jit bool
+ tree *rootfsimage.Tree
+ procs map[string]release.Process
+ image string
+ size int64
+ jit bool
+ scratch string
}
-// errRootfsCheckPending is what a --check build answers until the check
-// itself lands (Plan 1 Task 11): runShipRootfs exits 2 on it, saying
-// nothing more.
-var errRootfsCheckPending = errors.New("ship: --check is not built yet")
+var (
+ // errRootfsCheckFailed is a --check whose report failed: the report
+ // said why, and runShipRootfs exits 1.
+ errRootfsCheckFailed = errors.New("ship: the local check failed")
+ // errRootfsCheckStopped is a --check ended by a stop signal after it
+ // cleaned up: runShipRootfs exits 130, as removeOnStopSignal does.
+ errRootfsCheckStopped = errors.New("ship: the local check was stopped")
+)
// runShipRootfs is runShip's rootfs body. Everything that can exit (the
// flag rules, mode and account resolution, the version mint) runs here,
@@ -107,9 +128,15 @@ func runShipRootfs(fs *flag.FlagSet, o rootfsShipOpts) {
}
// --check runs the image here and ships nothing: it needs no app, no
- // login and no console.
+ // login and no console, and refuses the flags that would name them.
var sess *apiSession
- if !o.check {
+ if o.check {
+ for _, name := range shipCheckExcludedFlags {
+ if flagWasSet(fs, name) {
+ log.Fatalf("--check ships nothing, so it cannot be combined with --%s", name)
+ }
+ }
+ } else {
var err error
if sess, err = resolveMode(o.console); err != nil {
log.Fatal(err)
@@ -124,7 +151,16 @@ func runShipRootfs(fs *flag.FlagSet, o rootfsShipOpts) {
log.Fatal(err)
}
o.procs = procs
- if !o.check {
+ if o.check {
+ // Found before the build, so a machine that cannot run the check
+ // says so before any time goes into the image.
+ if o.rt, err = findContainerRuntime(); err != nil {
+ log.Fatal(err)
+ }
+ if o.probe, err = checkProbeBinary(context.Background()); err != nil {
+ log.Fatal(err)
+ }
+ } else {
if o.notes, err = shipNoteText(o.notes, o.notesFile); err != nil {
log.Fatal(err)
}
@@ -153,17 +189,19 @@ func runShipRootfs(fs *flag.FlagSet, o rootfsShipOpts) {
}
err = shipRootfsRun(context.Background(), o)
- if errors.Is(err, errRootfsCheckPending) {
- os.Exit(2)
- }
- if err != nil {
+ switch {
+ case errors.Is(err, errRootfsCheckFailed):
+ os.Exit(1)
+ case errors.Is(err, errRootfsCheckStopped):
+ os.Exit(130)
+ case err != nil:
log.Fatal(err)
}
}
// shipRootfsRun builds the image in a scratch directory it removes on
// every path, then ships it: through o.sess when set, else to o.st. A
-// --check build writes no image and answers errRootfsCheckPending.
+// --check build writes no image and runs the check instead.
func shipRootfsRun(ctx context.Context, o rootfsShipOpts) error {
scratch, err := os.MkdirTemp("", "carlos-rootfs-*")
if err != nil {
@@ -181,7 +219,21 @@ func shipRootfsRun(ctx context.Context, o rootfsShipOpts) error {
}
b.jit = o.jit
if o.check {
- return errRootfsCheckPending
+ // From here a stop signal cancels the check, which removes its
+ // containers, volumes and image before it returns; the defer above
+ // then removes the scratch. The signal context is made before the
+ // scratch handler stops, so no signal falls between the two.
+ cctx, cancel := signal.NotifyContext(ctx, shipRootfsStopSignals...)
+ defer cancel()
+ stop()
+ r := runRootfsCheck(cctx, o.rt, b, o.probe, stdout)
+ switch {
+ case cctx.Err() != nil && ctx.Err() == nil:
+ return errRootfsCheckStopped
+ case r.failed:
+ return errRootfsCheckFailed
+ }
+ return nil
}
if b.size > shipRootfsMaxBytes {
return fmt.Errorf("the rootfs image is %s, over the 2 GiB one release can carry. make the image smaller and ship again", humanBytes(&b.size))
@@ -229,11 +281,11 @@ func shipRootfsRun(ctx context.Context, o rootfsShipOpts) error {
var shipRootfsStopSignals = []os.Signal{os.Interrupt, syscall.SIGTERM, syscall.SIGHUP}
// removeOnStopSignal removes scratch and exits 130 when a stop signal
-// arrives, until the returned stop is called. Left to the runtime, a signal
-// ends the process without running its defers, and the scratch (up to the
-// spool plus a 2 GiB image) would stay in the temp directory. A goroutine
-// rather than a cancelled context, because the build (WriteSquashfs) does
-// not watch one.
+// arrives, until the returned stop is called (once or more). Left to the
+// runtime, a signal ends the process without running its defers, and the
+// scratch (up to the spool plus a 2 GiB image) would stay in the temp
+// directory. A goroutine rather than a cancelled context, because the
+// build (WriteSquashfs) does not watch one.
func removeOnStopSignal(scratch string) (stop func()) {
ch := make(chan os.Signal, 1)
done := make(chan struct{})
@@ -246,9 +298,12 @@ func removeOnStopSignal(scratch string) (stop func()) {
case <-done:
}
}()
+ var once sync.Once
return func() {
- signal.Stop(ch)
- close(done)
+ once.Do(func() {
+ signal.Stop(ch)
+ close(done)
+ })
}
}
@@ -310,7 +365,7 @@ func buildRootfs(ctx context.Context, src string, procs map[string]release.Proce
if err := rootfsimage.ResolveCommands(t, procs); err != nil {
return nil, err
}
- b := &builtRootfs{tree: t, procs: procs}
+ b := &builtRootfs{tree: t, procs: procs, scratch: scratch}
if !write {
return b, nil
}
diff --git a/ship_rootfs_unix_test.go b/ship_rootfs_unix_test.go
index e8eaa91a7..8b151140e 100644
--- a/ship_rootfs_unix_test.go
+++ b/ship_rootfs_unix_test.go
@@ -9,6 +9,8 @@ import (
"net/http/httptest"
"os"
"os/exec"
+ "path/filepath"
+ "strings"
"testing"
"time"
)
@@ -91,3 +93,78 @@ func TestShipRootfsRemovesItsScratchOnInterrupt(t *testing.T) {
t.Errorf("an interrupted ship left its scratch: %v", left)
}
}
+
+// A Ctrl-C while --check probes web still removes what the check made: the
+// stub runtime holds the probe's exec, the child is sent SIGINT there, and
+// it must remove its container, volumes and image, then its scratch, and
+// exit 130.
+func TestShipRootfsCheckCleansUpOnInterrupt(t *testing.T) {
+ if testing.Short() {
+ t.Skip("runs the carlos binary")
+ }
+ bin := buildCarlos(t)
+ dir, procs := rootfsRailsDir(t)
+ stubs, work, tmp := t.TempDir(), t.TempDir(), t.TempDir()
+ stubRuntime(t, stubs, "docker", `echo "$@" >> "$FAKE_LOG"
+case "$1" in
+info) echo aarch64 ;;
+import) cat > /dev/null ;;
+exec) : > "$FAKE_READY"; exec sleep 60 ;;
+esac
+exit 0
+`)
+ probe := filepath.Join(work, "probe")
+ if err := os.WriteFile(probe, []byte("x"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ logPath, ready := filepath.Join(work, "log"), filepath.Join(work, "ready")
+ cmd := exec.Command(bin, "ship", "--rootfs", dir, "--processes", procs, "--check")
+ cmd.Env = carlosTestEnv(t, "PATH="+stubs, "CARLOS_CONTAINER_RUNTIME=", "CARLOS_CHECK_PROBE="+probe,
+ "FAKE_LOG="+logPath, "FAKE_READY="+ready, "TMPDIR="+tmp)
+ cmd.Dir = t.TempDir()
+ cmd.Stdout, cmd.Stderr = io.Discard, io.Discard
+ if err := cmd.Start(); err != nil {
+ t.Fatal(err)
+ }
+ exited := make(chan error, 1)
+ go func() { exited <- cmd.Wait() }()
+
+ deadline := time.Now().Add(60 * time.Second)
+ for {
+ if _, err := os.Stat(ready); err == nil {
+ break
+ }
+ select {
+ case err := <-exited:
+ t.Fatalf("the check ended before its probe: %v", err)
+ case <-time.After(50 * time.Millisecond):
+ }
+ if time.Now().After(deadline) {
+ cmd.Process.Kill()
+ t.Fatal("the check never probed web")
+ }
+ }
+ if err := cmd.Process.Signal(os.Interrupt); err != nil {
+ t.Fatal(err)
+ }
+ var err error
+ select {
+ case err = <-exited:
+ case <-time.After(30 * time.Second):
+ cmd.Process.Kill()
+ t.Fatal("the check did not stop on SIGINT")
+ }
+ var ee *exec.ExitError
+ if !errors.As(err, &ee) || ee.ExitCode() != 130 {
+ t.Errorf("exit after SIGINT: %v, want status 130", err)
+ }
+ log, _ := os.ReadFile(logPath)
+ for _, want := range []string{"\nrm -f carlos-check-", "-web\n", "\nvolume rm carlos-check-", "-data\n", "-run\n", "\nrmi carlos-check-"} {
+ if !strings.Contains(string(log), want) {
+ t.Errorf("the runtime never saw %q:\n%s", want, log)
+ }
+ }
+ if left := rootfsScratchLeft(t, tmp); len(left) != 0 {
+ t.Errorf("an interrupted check left its scratch: %v", left)
+ }
+}
diff --git a/ship_runtime.go b/ship_runtime.go
index e29933cea..e205b546b 100644
--- a/ship_runtime.go
+++ b/ship_runtime.go
@@ -26,7 +26,10 @@ import (
var errNoRuntime = errors.New("--check runs the image with Docker or Podman, and neither is installed here")
// runnerFunc runs the runtime's binary with args, feeding it stdin (nil for
-// none), and answers its stdout. A failure carries the runtime's stderr.
+// none), and answers its stdout, also when it fails (--check's probe exits 1
+// with its findings on stdout). A failure carries the runtime's stderr. For
+// logs, whose stdout and stderr are the container's two streams, the answer
+// is both streams in one.
type runnerFunc func(ctx context.Context, stdin io.Reader, args ...string) ([]byte, error)
// containerRuntime is the runtime found: name is "docker" or "podman", and
@@ -63,11 +66,16 @@ func execRunner(bin string) runnerFunc {
cmd.Stdin = stdin
var stdout, stderr bytes.Buffer
cmd.Stdout, cmd.Stderr = &stdout, &stderr
+ if len(args) > 0 && args[0] == "logs" {
+ // One writer for both, so the lines keep the order the
+ // runtime wrote them in.
+ cmd.Stderr = &stdout
+ }
if err := cmd.Run(); err != nil {
if msg := strings.TrimSpace(stderr.String()); msg != "" {
- return nil, errors.New(msg)
+ return stdout.Bytes(), errors.New(msg)
}
- return nil, err
+ return stdout.Bytes(), err
}
return stdout.Bytes(), nil
}